mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-30 04:27:40 +08:00
373 lines
13 KiB
Python
373 lines
13 KiB
Python
import re
|
|
import traceback
|
|
import xml.dom.minidom
|
|
from typing import List, Tuple, Union
|
|
from urllib.parse import urljoin
|
|
|
|
import chardet
|
|
from lxml import etree
|
|
|
|
from app.core.config import settings
|
|
from app.helper.browser import PlaywrightHelper
|
|
from app.log import logger
|
|
from app.utils.dom import DomUtils
|
|
from app.utils.http import RequestUtils
|
|
from app.utils.string import StringUtils
|
|
|
|
|
|
class RssHelper:
|
|
"""
|
|
RSS帮助类，解析RSS报文、获取RSS地址等
|
|
"""
|
|
# 各站点RSS链接获取配置
|
|
rss_link_conf = {
|
|
"default": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"hares.top": {
|
|
"xpath": "//*[@id='layui-layer100001']/div[2]/div/p[4]/a/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"et8.org": {
|
|
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/a[2]/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"pttime.org": {
|
|
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/text()[5]",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"showrows": 10,
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1
|
|
}
|
|
},
|
|
"ourbits.club": {
|
|
"xpath": "//a[@class='gen_rsslink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"totheglory.im": {
|
|
"xpath": "//textarea/text()",
|
|
"url": "rsstools.php?c51=51&c52=52&c53=53&c54=54&c108=108&c109=109&c62=62&c63=63&c67=67&c69=69&c70=70&c73=73&c76=76&c75=75&c74=74&c87=87&c88=88&c99=99&c90=90&c58=58&c103=103&c101=101&c60=60",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"monikadesign.uk": {
|
|
"xpath": "//a/@href",
|
|
"url": "rss",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"zhuque.in": {
|
|
"xpath": "//a/@href",
|
|
"url": "user/rss",
|
|
"render": True,
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"hdchina.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"rsscart": 0
|
|
}
|
|
},
|
|
"audiences.me": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"torrent_type": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"shadowflow.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"paid": 0,
|
|
"search_mode": 0,
|
|
"showrows": 30
|
|
}
|
|
},
|
|
"hddolby.com": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"hdhome.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"pthome.net": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"ptsbao.club": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"size": 0
|
|
}
|
|
},
|
|
"leaves.red": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 0,
|
|
"paid": 2
|
|
}
|
|
},
|
|
"hdtime.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 0,
|
|
}
|
|
},
|
|
"m-team.io": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"showrows": 50,
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"https": 1
|
|
}
|
|
},
|
|
"u2.dmhy.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"inclautochecked": 1,
|
|
"trackerssl": 1
|
|
}
|
|
},
|
|
}
|
|
|
|
@staticmethod
|
|
def parse(url, proxy: bool = False, timeout: int = 15) -> Union[List[dict], None]:
|
|
"""
|
|
解析RSS订阅URL，获取RSS中的种子信息
|
|
:param url: RSS地址
|
|
:param proxy: 是否使用代理
|
|
:param timeout: 请求超时
|
|
:return: 种子信息列表，如为None代表Rss过期
|
|
"""
|
|
# 开始处理
|
|
ret_array: list = []
|
|
if not url:
|
|
return []
|
|
try:
|
|
ret = RequestUtils(proxies=settings.PROXY if proxy else None, timeout=timeout).get_res(url)
|
|
if not ret:
|
|
return []
|
|
except Exception as err:
|
|
logger.error(f"获取RSS失败：{str(err)} - {traceback.format_exc()}")
|
|
return []
|
|
if ret:
|
|
ret_xml = ""
|
|
try:
|
|
# 使用chardet检测字符编码
|
|
raw_data = ret.content
|
|
if raw_data:
|
|
try:
|
|
result = chardet.detect(raw_data)
|
|
encoding = result['encoding']
|
|
# 解码为字符串
|
|
ret_xml = raw_data.decode(encoding)
|
|
except Exception as e:
|
|
logger.debug(f"chardet解码失败：{str(e)}")
|
|
# 探测utf-8解码
|
|
match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text)
|
|
if match:
|
|
encoding = match.group(1)
|
|
if encoding:
|
|
ret_xml = raw_data.decode(encoding)
|
|
else:
|
|
ret.encoding = ret.apparent_encoding
|
|
if not ret_xml:
|
|
ret_xml = ret.text
|
|
# 解析XML
|
|
dom_tree = xml.dom.minidom.parseString(ret_xml)
|
|
rootNode = dom_tree.documentElement
|
|
items = rootNode.getElementsByTagName("item")
|
|
for item in items:
|
|
try:
|
|
# 标题
|
|
title = DomUtils.tag_value(item, "title", default="")
|
|
if not title:
|
|
continue
|
|
# 描述
|
|
description = DomUtils.tag_value(item, "description", default="")
|
|
# 种子页面
|
|
link = DomUtils.tag_value(item, "link", default="")
|
|
# 种子链接
|
|
enclosure = DomUtils.tag_value(item, "enclosure", "url", default="")
|
|
if not enclosure and not link:
|
|
continue
|
|
# 部分RSS只有link没有enclosure
|
|
if not enclosure and link:
|
|
enclosure = link
|
|
# 大小
|
|
size = DomUtils.tag_value(item, "enclosure", "length", default=0)
|
|
if size and str(size).isdigit():
|
|
size = int(size)
|
|
else:
|
|
size = 0
|
|
# 发布日期
|
|
pubdate = DomUtils.tag_value(item, "pubDate", default="")
|
|
if pubdate:
|
|
# 转换为时间
|
|
pubdate = StringUtils.get_time(pubdate)
|
|
# 返回对象
|
|
tmp_dict = {'title': title,
|
|
'enclosure': enclosure,
|
|
'size': size,
|
|
'description': description,
|
|
'link': link,
|
|
'pubdate': pubdate}
|
|
ret_array.append(tmp_dict)
|
|
except Exception as e1:
|
|
logger.debug(f"解析RSS失败：{str(e1)} - {traceback.format_exc()}")
|
|
continue
|
|
except Exception as e2:
|
|
logger.error(f"解析RSS失败：{str(e2)} - {traceback.format_exc()}")
|
|
# RSS过期 观众RSS 链接已过期，您需要获得一个新的！ pthome RSS Link has expired, You need to get a new one!
|
|
_rss_expired_msg = [
|
|
"RSS 链接已过期, 您需要获得一个新的!",
|
|
"RSS Link has expired, You need to get a new one!",
|
|
"RSS Link has expired, You need to get new!"
|
|
]
|
|
if ret_xml in _rss_expired_msg:
|
|
return None
|
|
return ret_array
|
|
|
|
def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False) -> Tuple[str, str]:
|
|
"""
|
|
获取站点rss地址
|
|
:param url: 站点地址
|
|
:param cookie: 站点cookie
|
|
:param ua: 站点ua
|
|
:param proxy: 是否使用代理
|
|
:return: rss地址、错误信息
|
|
"""
|
|
try:
|
|
# 获取站点域名
|
|
domain = StringUtils.get_url_domain(url)
|
|
# 获取配置
|
|
site_conf = self.rss_link_conf.get(domain) or self.rss_link_conf.get("default")
|
|
# RSS地址
|
|
rss_url = urljoin(url, site_conf.get("url"))
|
|
# RSS请求参数
|
|
rss_params = site_conf.get("params")
|
|
# 请求RSS页面
|
|
if site_conf.get("render"):
|
|
html_text = PlaywrightHelper().get_page_source(
|
|
url=rss_url,
|
|
cookies=cookie,
|
|
ua=ua,
|
|
proxies=settings.PROXY if proxy else None
|
|
)
|
|
else:
|
|
res = RequestUtils(
|
|
cookies=cookie,
|
|
timeout=60,
|
|
ua=ua,
|
|
proxies=settings.PROXY if proxy else None
|
|
).post_res(url=rss_url, data=rss_params)
|
|
if res:
|
|
html_text = res.text
|
|
elif res is not None:
|
|
return "", f"获取 {url} RSS链接失败，错误码：{res.status_code}，错误原因：{res.reason}"
|
|
else:
|
|
return "", f"获取RSS链接失败：无法连接 {url} "
|
|
# 解析HTML
|
|
html = etree.HTML(html_text)
|
|
if html:
|
|
rss_link = html.xpath(site_conf.get("xpath"))
|
|
if rss_link:
|
|
return str(rss_link[-1]), ""
|
|
return "", f"获取RSS链接失败：{url}"
|
|
except Exception as e:
|
|
return "", f"获取 {url} RSS链接失败：{str(e)}"
|