fix etree

This commit is contained in:
jxxghp
2025-07-08 11:00:38 +08:00
parent a8aedba6ff
commit d336bcbf1f
17 changed files with 897 additions and 749 deletions
+4
View File
@@ -271,6 +271,7 @@ class SiteChain(ChainBase):
logger.error(f"获取站点页面失败:{url}")
return favicon_url, None
html = etree.HTML(html_text)
try:
if StringUtils.is_valid_html_element(html):
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
if fav_link:
@@ -281,6 +282,9 @@ class SiteChain(ChainBase):
return favicon_url, base64.b64encode(res.content).decode()
else:
logger.error(f"获取站点图标失败:{favicon_url}")
finally:
if html is not None:
del html
return favicon_url, None
def sync_cookies(self, manual=False) -> Tuple[bool, str]:
-1
View File
@@ -312,4 +312,3 @@ class StreamingPlatforms(metaclass=Singleton):
if name is None:
return False
return name.upper() in self._lookup_cache
+9
View File
@@ -96,6 +96,7 @@ class CookieHelper:
return None, None, "获取源码失败"
# 查找用户名输入框
html = etree.HTML(html_text)
try:
username_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("username"):
if html.xpath(xpath):
@@ -143,6 +144,10 @@ class CookieHelper:
break
if not submit_xpath:
return None, None, "未找到登录按钮"
finally:
if html is not None:
del html
# 点击登录按钮
try:
# 等待登录按钮准备好
@@ -185,6 +190,7 @@ class CookieHelper:
if not otp_code:
return None, None, "需要二次验证码"
html = etree.HTML(page.content())
try:
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
if html.xpath(xpath):
try:
@@ -198,6 +204,9 @@ class CookieHelper:
logger.error(f"二次验证码输入失败:{str(e)}")
return None, None, f"二次验证码输入失败:{str(e)}"
break
finally:
if html is not None:
del html
# 登录后的源码
html_text = page.content()
if not html_text:
+12 -1
View File
@@ -14,7 +14,7 @@ class DiscuzUserInfo(SiteParserBase):
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
user_info = html.xpath('//a[contains(@href, "&uid=")]')
if user_info:
user_id_match = re.search(r"&uid=(\d+)", user_info[0].attrib['href'])
@@ -23,6 +23,9 @@ class DiscuzUserInfo(SiteParserBase):
self._torrent_seeding_page = f"forum.php?&mod=torrents&cat_5up=on"
self._user_detail_page = user_info[0].attrib['href']
self.username = user_info[0].text.strip()
finally:
if html is not None:
del html
def _parse_site_page(self, html_text: str):
pass
@@ -34,6 +37,7 @@ class DiscuzUserInfo(SiteParserBase):
:return:
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -68,6 +72,9 @@ class DiscuzUserInfo(SiteParserBase):
download_text = html.xpath('//li[em[contains(text(),"下载量")]]/text()')
if download_text:
self.download = StringUtils.num_filesize(download_text[0].strip().split('/')[-1])
finally:
if html is not None:
del html
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]:
"""
@@ -77,6 +84,7 @@ class DiscuzUserInfo(SiteParserBase):
:return: 下页地址
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -115,6 +123,9 @@ class DiscuzUserInfo(SiteParserBase):
next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href')
if next_page_text:
next_page = next_page_text[-1].strip()
finally:
if html is not None:
del html
return next_page
+12 -3
View File
@@ -24,10 +24,13 @@ class FileListSiteUserInfo(SiteParserBase):
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()')
if ret:
self.username = str(ret[0])
finally:
if html is not None:
del html
def _parse_user_traffic_info(self, html_text: str):
"""
@@ -40,7 +43,7 @@ class FileListSiteUserInfo(SiteParserBase):
def _parse_user_detail_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
upload_html = html.xpath('//table//tr/td[text()="Uploaded"]/following-sibling::td//text()')
if upload_html:
self.upload = StringUtils.num_filesize(upload_html[0])
@@ -72,7 +75,9 @@ class FileListSiteUserInfo(SiteParserBase):
bonus_html = html.xpath('//a[contains(@href, "shop.php")]')
if bonus_html:
self.bonus = StringUtils.str_float(bonus_html[0].xpath("string(.)").strip())
pass
finally:
if html is not None:
del html
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
"""
@@ -82,6 +87,7 @@ class FileListSiteUserInfo(SiteParserBase):
:return: 下页地址
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -104,6 +110,9 @@ class FileListSiteUserInfo(SiteParserBase):
# 是否存在下页数据
next_page = None
finally:
if html is not None:
del html
return next_page
+12 -1
View File
@@ -14,7 +14,7 @@ class GazelleSiteUserInfo(SiteParserBase):
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
tmps = html.xpath('//a[contains(@href, "user.php?id=")]')
if tmps:
user_id_match = re.search(r"user.php\?id=(\d+)", tmps[0].attrib['href'])
@@ -54,6 +54,9 @@ class GazelleSiteUserInfo(SiteParserBase):
bonus_match = re.search(r"([\d,.]+)", bonus_text)
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1))
finally:
if html is not None:
del html
def _parse_site_page(self, html_text: str):
pass
@@ -65,6 +68,7 @@ class GazelleSiteUserInfo(SiteParserBase):
:return:
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -86,6 +90,9 @@ class GazelleSiteUserInfo(SiteParserBase):
'//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
finally:
if html is not None:
del html
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
"""
@@ -95,6 +102,7 @@ class GazelleSiteUserInfo(SiteParserBase):
:return: 下页地址
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -137,6 +145,9 @@ class GazelleSiteUserInfo(SiteParserBase):
next_page_text = html.xpath('//a[contains(.//text(), "Next") or contains(.//text(), "下一页")]/@href')
if next_page_text:
next_page = next_page_text[-1].strip()
finally:
if html is not None:
del html
return next_page
+12
View File
@@ -14,6 +14,7 @@ class IptSiteUserInfo(SiteParserBase):
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
tmps = html.xpath('//a[contains(@href, "/u/")]//text()')
tmps_id = html.xpath('//a[contains(@href, "/u/")]/@href')
if tmps:
@@ -33,12 +34,16 @@ class IptSiteUserInfo(SiteParserBase):
self.leeching = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[1])
self.ratio = StringUtils.str_float(str(tmps[0].xpath('span/text()')[0]).strip().replace('-', '0'))
self.bonus = StringUtils.str_float(tmps[0].xpath('a')[3].xpath('text()')[0])
finally:
if html is not None:
del html
def _parse_site_page(self, html_text: str):
pass
def _parse_user_detail_info(self, html_text: str):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
@@ -50,9 +55,13 @@ class IptSiteUserInfo(SiteParserBase):
join_at_text = html.xpath('//tr/th[text()="Join date"]/following-sibling::td[1]/text()')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0])
finally:
if html is not None:
del html
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]:
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
# seeding start
@@ -75,6 +84,9 @@ class IptSiteUserInfo(SiteParserBase):
self.seeding = page_seeding
self.seeding_size = page_seeding_size
finally:
if html is not None:
del html
def _parse_user_traffic_info(self, html_text: str):
pass
@@ -23,6 +23,7 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
if not html_text:
return
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
total_row = html.xpath('//table[@class="table table-bordered"]//tr[td[1][normalize-space()="Total"]]')
@@ -32,3 +33,6 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
seeding_size = total_row[0].xpath('./td[3]/text()')
self.seeding = StringUtils.str_int(seeding_count[0]) if seeding_count else 0
self.seeding_size = StringUtils.num_filesize(seeding_size[0].strip()) if seeding_size else 0
finally:
if html is not None:
del html
@@ -17,6 +17,7 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
# 上传、下载、分享率
upload_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)",
html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[4]/text()')[0])
@@ -32,6 +33,9 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
self.ratio = StringUtils.str_float(ratio_match.group(1)) if (
ratio_match and ratio_match.group(1).strip()) else calc_ratio
finally:
if html is not None:
del html
def _parse_user_detail_info(self, html_text: str):
"""
@@ -42,12 +46,16 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
super()._parse_user_detail_info(html_text)
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
# 加入时间
join_at_text = html.xpath('//*[@id="mainContent"]/div/div[2]/div[4]/div[3]/span[2]/text()[1]')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip())
finally:
if html is not None:
del html
def _get_user_level(self, html):
super()._get_user_level(html)
+29
View File
@@ -34,6 +34,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
:return:
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
@@ -49,6 +50,9 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self.message_unread = StringUtils.str_int(message_unread_match[-1][1])
elif message_text.isdigit():
self.message_unread = StringUtils.str_int(message_text)
finally:
if html is not None:
del html
def _parse_user_base_info(self, html_text: str):
"""
@@ -61,6 +65,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self._parse_message_unread(html_text)
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
@@ -73,6 +78,10 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self.username = str(ret[0])
ret = html.xpath('//a[contains(@href, "userdetails")]//strong//text()')
finally:
if html is not None:
del html
if ret:
self.username = str(ret[0])
return
@@ -98,6 +107,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self.leeching = StringUtils.str_int(leeching_match.group(2)) if leeching_match and leeching_match.group(
2).strip() else 0
html = etree.HTML(html_text)
try:
has_ucoin, self.bonus = self._parse_ucoin(html)
if has_ucoin:
return
@@ -120,6 +130,9 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self.bonus = StringUtils.str_float(bonus_match.group(1).strip('"'))
except Exception as err:
logger.error(f"{self._site_name} 解析魔力值出错, 错误信息: {str(err)}")
finally:
if html is not None:
del html
@staticmethod
def _parse_ucoin(html):
@@ -155,6 +168,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
:return: 下页地址
"""
html = etree.HTML(str(html_text).replace(r'\/', '/'))
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -221,6 +235,9 @@ class NexusPhpSiteUserInfo(SiteParserBase):
if next_page:
if self.userid not in next_page:
next_page = f'{next_page}&userid={self.userid}&type=seeding'
finally:
if html is not None:
del html
return next_page
@@ -231,6 +248,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
:return:
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
@@ -282,6 +300,9 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self.seeding = tmp_seeding
self._fixup_torrent_seeding_page(html)
finally:
if html is not None:
del html
def _fixup_torrent_seeding_page(self, html):
"""
@@ -348,6 +369,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -358,11 +380,15 @@ class NexusPhpSiteUserInfo(SiteParserBase):
next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href')
if next_page_text:
next_page = next_page_text[-1].strip()
finally:
if html is not None:
del html
return next_page
def _parse_message_content(self, html_text):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None, None, None
# 标题
@@ -385,6 +411,9 @@ class NexusPhpSiteUserInfo(SiteParserBase):
'|//div[contains(@class,"layui-card-body")]')
if message_content:
message_content_text = message_content[0].xpath("string(.)").strip()
finally:
if html is not None:
del html
return message_head_text, message_date_text, message_content_text
@@ -114,6 +114,7 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
def _parse_user_base_info(self, html_text: str):
"""只有奶糖余额才需要在 base 中获取,其它均可以在详情页拿到"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
bonus = html.xpath(
@@ -121,9 +122,13 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
)
if bonus:
self.bonus = StringUtils.str_float(bonus[0].strip())
finally:
if html is not None:
del html
def _parse_user_detail_info(self, html_text: str):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
# 缩小一下查找范围,所有的信息都在这个 div 里
@@ -156,6 +161,9 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
# 分享率
if ratio := user_info.xpath('.//span[contains(text(), "分享率")]/em/text()'):
self.ratio = StringUtils.str_float(ratio[0].strip())
finally:
if html is not None:
del html
def _parse_message_content(self, html_text):
"""
+12 -1
View File
@@ -24,9 +24,13 @@ class SmallHorseSiteUserInfo(SiteParserBase):
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
ret = html.xpath('//a[contains(@href, "user.php")]//text()')
if ret:
self.username = str(ret[0])
finally:
if html is not None:
del html
def _parse_user_traffic_info(self, html_text: str):
"""
@@ -36,6 +40,7 @@ class SmallHorseSiteUserInfo(SiteParserBase):
"""
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
tmps = html.xpath('//ul[@class = "stats nobullet"]')
if tmps:
if tmps[1].xpath("li") and tmps[1].xpath("li")[0].xpath("span//text()"):
@@ -51,6 +56,9 @@ class SmallHorseSiteUserInfo(SiteParserBase):
self.user_level = str(tmps[3].xpath("li")[0].xpath("text()")[0]).split(":")[1].strip()
self.leeching = StringUtils.str_int(
(tmps[4].xpath("li")[6].xpath("text()")[0]).split(":")[1].replace("[", ""))
finally:
if html is not None:
del html
def _parse_user_detail_info(self, html_text: str):
pass
@@ -63,6 +71,7 @@ class SmallHorseSiteUserInfo(SiteParserBase):
:return: 下页地址
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -95,7 +104,9 @@ class SmallHorseSiteUserInfo(SiteParserBase):
page_num = next_pages[0].xpath("string(.)").strip()
if page_num.isdigit():
next_page = f"{self._torrent_seeding_page}&page={page_num}"
finally:
if html is not None:
del html
return next_page
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
@@ -32,6 +32,7 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
"""
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
upload_html = html.xpath('//div[contains(@class,"profile-uploaded")]//span/text()')
if upload_html:
self.upload = StringUtils.num_filesize(upload_html[0])
@@ -55,6 +56,9 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
bonus_html = html.xpath('//span[contains(@class, "total-TL-points")]/text()')
if bonus_html:
self.bonus = StringUtils.str_float(bonus_html[0].strip())
finally:
if html is not None:
del html
def _parse_user_detail_info(self, html_text: str):
pass
@@ -67,6 +71,7 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
:return: 下页地址
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -94,6 +99,9 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
# 是否存在下页数据
next_page = None
finally:
if html is not None:
del html
return next_page
+12 -1
View File
@@ -14,7 +14,7 @@ class Unit3dSiteUserInfo(SiteParserBase):
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
tmps = html.xpath('//a[contains(@href, "/users/") and contains(@href, "settings")]/@href')
if tmps:
user_name_match = re.search(r"/users/(.+)/settings", tmps[0])
@@ -29,6 +29,9 @@ class Unit3dSiteUserInfo(SiteParserBase):
bonus_match = re.search(r"([\d,.]+)", bonus_text)
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1))
finally:
if html is not None:
del html
def _parse_site_page(self, html_text: str):
pass
@@ -40,6 +43,7 @@ class Unit3dSiteUserInfo(SiteParserBase):
:return:
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -55,6 +59,9 @@ class Unit3dSiteUserInfo(SiteParserBase):
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(
join_at_text[0].replace('注册日期', '').replace('註冊日期', '').replace('Registration date', ''))
finally:
if html is not None:
del html
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
"""
@@ -64,6 +71,7 @@ class Unit3dSiteUserInfo(SiteParserBase):
:return: 下页地址
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return None
@@ -102,6 +110,9 @@ class Unit3dSiteUserInfo(SiteParserBase):
page_num = next_pages[0].xpath("string(.)").strip()
if page_num.isdigit():
next_page = f"{self._torrent_seeding_page}&page={page_num}"
finally:
if html is not None:
del html
return next_page
+4
View File
@@ -104,6 +104,7 @@ class SubtitleModule(_ModuleBase):
logger.warn(f"读取页面代码失败:{torrent.page_url}")
return
html = etree.HTML(res.text)
try:
sublink_list = []
for xpath in self._SITE_SUBTITLE_XPATH:
sublinks = html.xpath(xpath)
@@ -118,6 +119,9 @@ class SubtitleModule(_ModuleBase):
else:
sublink = "%s/%s" % (base_url, sublink)
sublink_list.append(sublink)
finally:
if html is not None:
del html
# 下载所有字幕文件
for sublink in sublink_list:
logger.info(f"找到字幕下载链接:{sublink},开始下载...")
+3
View File
@@ -563,6 +563,9 @@ class TmdbApi:
except Exception as err:
logger.error(f"从TheDbMovie网站查询出错:{str(err)}")
return {}
finally:
if html is not None:
del html
return {}
def get_info(self,
+8 -1
View File
@@ -13,6 +13,7 @@ class SiteUtils:
:return:
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return False
# 存在明显的密码输入框,说明未登录
@@ -34,6 +35,9 @@ class SiteUtils:
if html.xpath(xpath):
return True
return False
finally:
if html is not None:
del html
@classmethod
def is_checkin(cls, html_text: str) -> bool:
@@ -42,6 +46,7 @@ class SiteUtils:
:return True已签到 False未签到
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return False
# 站点签到支持的识别XPATH
@@ -61,5 +66,7 @@ class SiteUtils:
for xpath in xpaths:
if html.xpath(xpath):
return False
return True
finally:
if html is not None:
del html