GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-05 15:38:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(string): split utilities by responsibility

Browse Source
This commit is contained in:
jxxghp
2026-08-15 08:03:55 +08:00
parent a2117bafc8
commit 96118e73e7
99 changed files with 1608 additions and 1424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/modules/indexer/parser/__init__.py
+2 -2
View File
@@ -13,7 +13,7 @@ from app.adapters.network.cloudflare import under_challenge
from app.runtime.log import logger
from app.adapters.network.http import RequestUtils
from app.domain.site import SiteUtils
from app.domain.string import StringUtils
from app.foundation import size as size_tools
# 站点框架
@@ -183,7 +183,7 @@ class SiteParserBase(metaclass=ABCMeta):
"""
将站点页面中的文件大小文本转换为字节。
"""
return StringUtils.num_filesize(text)
return size_tools.parse_size(text)
def parse(self):
"""
app/modules/indexer/parser/bitpt.py
+7 -6
View File
@@ -10,7 +10,8 @@ from urllib.parse import urljoin, urlencode
from bs4 import BeautifulSoup
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
class BitptSiteUserInfo(SiteParserBase):
schema = SiteSchema.Bitpt
@@ -52,10 +53,10 @@ class BitptSiteUserInfo(SiteParserBase):
self.userid = info_dict.get('UID')
self.username = info_dict.get('用户名').split('\xa0')[0] if '用户名' in info_dict else None
self.user_level = info_dict.get('用户级别') if '用户级别' in info_dict else None
self.join_at = StringUtils.unify_datetime_str(info_dict.get('注册时间')) if '注册时间' in info_dict else None
self.join_at = time_tools.normalize_datetime(info_dict.get('注册时间')) if '注册时间' in info_dict else None
self.upload = StringUtils.num_filesize(info_dict.get('上传流量')) if '上传流量' in info_dict else 0
self.download = StringUtils.num_filesize(info_dict.get('下载流量')) if '下载流量' in info_dict else 0
self.upload = size_tools.parse_size(info_dict.get('上传流量')) if '上传流量' in info_dict else 0
self.download = size_tools.parse_size(info_dict.get('下载流量')) if '下载流量' in info_dict else 0
self.ratio = float(info_dict.get('共享率')) if '共享率' in info_dict else 0
bonus_str = info_dict.get('星辰', '')
self.bonus = float(re.search(r'累计([\d\.]+)', bonus_str).group(1)) if re.search(r'累计([\d\.]+)', bonus_str) else 0
@@ -71,7 +72,7 @@ class BitptSiteUserInfo(SiteParserBase):
match = re.search(r'当前上传的种子\((\d+)个, 共([\d\.]+ [KMGT]B)\)', seeding_link)
if match:
self.seeding = int(match.group(1))
self.seeding_size = StringUtils.num_filesize(match.group(2))
self.seeding_size = size_tools.parse_size(match.group(2))
else:
self.seeding = 0
self.seeding_size = 0
@@ -102,7 +103,7 @@ class BitptSiteUserInfo(SiteParserBase):
size_text = size_a.text.strip() if size_a else size_td.text.strip()
if size_text:
page_seeding += 1
page_seeding_size += StringUtils.num_filesize(size_text)
page_seeding_size += size_tools.parse_size(size_text)
return page_seeding, page_seeding_size
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
app/modules/indexer/parser/discuz.py
+13 -10
View File
@@ -5,7 +5,10 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class DiscuzUserInfo(SiteParserBase):
@@ -38,7 +41,7 @@ class DiscuzUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
# 用户等级
@@ -49,29 +52,29 @@ class DiscuzUserInfo(SiteParserBase):
# 加入日期
join_at_text = html.xpath('//li[em[text()="注册时间"]]/text()')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
self.join_at = time_tools.normalize_datetime(join_at_text[0].strip())
# 分享率
ratio_text = html.xpath('//li[contains(.//text(), "分享率")]//text()')
if ratio_text:
ratio_match = re.search(r"\(([\d,.]+)\)", ratio_text[0])
if ratio_match and ratio_match.group(1).strip():
self.bonus = StringUtils.str_float(ratio_match.group(1))
self.bonus = text_tools.parse_float(ratio_match.group(1))
# 积分
bouns_text = html.xpath('//li[em[text()="积分"]]/text()')
if bouns_text:
self.bonus = StringUtils.str_float(bouns_text[0].strip())
self.bonus = text_tools.parse_float(bouns_text[0].strip())
# 上传
upload_text = html.xpath('//li[em[contains(text(),"上传量")]]/text()')
if upload_text:
self.upload = StringUtils.num_filesize(upload_text[0].strip().split('/')[-1])
self.upload = size_tools.parse_size(upload_text[0].strip().split('/')[-1])
# 下载
download_text = html.xpath('//li[em[contains(text(),"下载量")]]/text()')
if download_text:
self.download = StringUtils.num_filesize(download_text[0].strip().split('/')[-1])
self.download = size_tools.parse_size(download_text[0].strip().split('/')[-1])
finally:
if html is not None:
del html
@@ -85,7 +88,7 @@ class DiscuzUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
size_col = 3
@@ -108,8 +111,8 @@ class DiscuzUserInfo(SiteParserBase):
page_seeding = len(seeding_sizes)
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i])
size = size_tools.parse_size(seeding_sizes[i].xpath("string(.)").strip())
seeders = text_tools.parse_int(seeding_seeders[i])
page_seeding_size += size
page_seeding_info.append([seeders, size])
app/modules/indexer/parser/file_list.py
+14 -11
View File
@@ -5,7 +5,10 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class FileListSiteUserInfo(SiteParserBase):
@@ -46,22 +49,22 @@ class FileListSiteUserInfo(SiteParserBase):
try:
upload_html = html.xpath('//table//tr/td[text()="Uploaded"]/following-sibling::td//text()')
if upload_html:
self.upload = StringUtils.num_filesize(upload_html[0])
self.upload = size_tools.parse_size(upload_html[0])
download_html = html.xpath('//table//tr/td[text()="Downloaded"]/following-sibling::td//text()')
if download_html:
self.download = StringUtils.num_filesize(download_html[0])
self.download = size_tools.parse_size(download_html[0])
ratio_html = html.xpath('//table//tr/td[text()="Share ratio"]/following-sibling::td//text()')
if ratio_html:
share_ratio = StringUtils.str_float(ratio_html[0])
share_ratio = text_tools.parse_float(ratio_html[0])
else:
share_ratio = 0
self.ratio = 0 if self.download == 0 else share_ratio
seed_html = html.xpath('//table//tr/td[text()="Seed bonus"]/following-sibling::td//text()')
if seed_html:
self.seeding = StringUtils.str_int(seed_html[1])
self.seeding_size = StringUtils.num_filesize(seed_html[3])
self.seeding = text_tools.parse_int(seed_html[1])
self.seeding_size = size_tools.parse_size(seed_html[3])
user_level_html = html.xpath('//table//tr/td[text()="Class"]/following-sibling::td//text()')
if user_level_html:
@@ -70,11 +73,11 @@ class FileListSiteUserInfo(SiteParserBase):
join_at_html = html.xpath('//table//tr/td[contains(text(), "Join")]/following-sibling::td//text()')
if join_at_html:
join_at = (join_at_html[0].split("("))[0].strip()
self.join_at = StringUtils.unify_datetime_str(join_at)
self.join_at = time_tools.normalize_datetime(join_at)
bonus_html = html.xpath('//a[contains(@href, "shop.php")]')
if bonus_html:
self.bonus = StringUtils.str_float(bonus_html[0].xpath("string(.)").strip())
self.bonus = text_tools.parse_float(bonus_html[0].xpath("string(.)").strip())
finally:
if html is not None:
del html
@@ -88,7 +91,7 @@ class FileListSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
size_col = 6
@@ -100,8 +103,8 @@ class FileListSiteUserInfo(SiteParserBase):
seeding_seeders = html.xpath(f'//table/tr[position()>1]/td[{seeders_col}]')
if seeding_sizes and seeding_seeders:
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
size = size_tools.parse_size(seeding_sizes[i].xpath("string(.)").strip())
seeders = text_tools.parse_int(seeding_seeders[i].xpath("string(.)").strip())
page_seeding_size += size
page_seeding_info.append([seeders, size])
app/modules/indexer/parser/gazelle.py
+15 -12
View File
@@ -5,7 +5,10 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class GazelleSiteUserInfo(SiteParserBase):
@@ -26,19 +29,19 @@ class GazelleSiteUserInfo(SiteParserBase):
tmps = html.xpath('//*[@id="header-uploaded-value"]/@data-value')
if tmps:
self.upload = StringUtils.num_filesize(tmps[0])
self.upload = size_tools.parse_size(tmps[0])
else:
tmps = html.xpath('//li[@id="stats_seeding"]/span/text()')
if tmps:
self.upload = StringUtils.num_filesize(tmps[0])
self.upload = size_tools.parse_size(tmps[0])
tmps = html.xpath('//*[@id="header-downloaded-value"]/@data-value')
if tmps:
self.download = StringUtils.num_filesize(tmps[0])
self.download = size_tools.parse_size(tmps[0])
else:
tmps = html.xpath('//li[@id="stats_leeching"]/span/text()')
if tmps:
self.download = StringUtils.num_filesize(tmps[0])
self.download = size_tools.parse_size(tmps[0])
self.ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
@@ -46,14 +49,14 @@ class GazelleSiteUserInfo(SiteParserBase):
if tmps:
bonus_match = re.search(r"([\d,.]+)", tmps[0])
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1))
self.bonus = text_tools.parse_float(bonus_match.group(1))
else:
tmps = html.xpath('//a[contains(@href, "bonus")]')
if tmps:
bonus_text = tmps[0].xpath("string(.)")
bonus_match = re.search(r"([\d,.]+)", bonus_text)
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1))
self.bonus = text_tools.parse_float(bonus_match.group(1))
finally:
if html is not None:
del html
@@ -69,7 +72,7 @@ class GazelleSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
# 用户等级
@@ -84,12 +87,12 @@ class GazelleSiteUserInfo(SiteParserBase):
# 加入日期
join_at_text = html.xpath('//*[@id="join-date-value"]/@data-value')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
self.join_at = time_tools.normalize_datetime(join_at_text[0].strip())
else:
join_at_text = html.xpath(
'//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
self.join_at = time_tools.normalize_datetime(join_at_text[0].strip())
finally:
if html is not None:
del html
@@ -103,7 +106,7 @@ class GazelleSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
size_col = 3
@@ -122,7 +125,7 @@ class GazelleSiteUserInfo(SiteParserBase):
page_seeding = len(seeding_sizes)
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
size = size_tools.parse_size(seeding_sizes[i].xpath("string(.)").strip())
seeders = int(seeding_seeders[i])
page_seeding_size += size
app/modules/indexer/parser/hddolby.py
+2 -2
View File
@@ -5,7 +5,7 @@ from typing import Optional, Tuple
from app.runtime.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
from app.domain.string import StringUtils
from app.domain import site as site_rules
class HDDolbySiteUserInfo(SiteParserBase):
@@ -44,7 +44,7 @@ class HDDolbySiteUserInfo(SiteParserBase):
获取站点页面地址
"""
# 更换api地址
self._base_url = f"https://api.{StringUtils.get_url_domain(self._base_url)}"
self._base_url = f"https://api.{site_rules.extract_domain(self._base_url)}"
self._user_traffic_page = None
self._user_detail_page = None
self._user_basic_page = "api/v1/user/data"
app/modules/indexer/parser/ipt_project.py
+14 -11
View File
@@ -5,7 +5,10 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class IptSiteUserInfo(SiteParserBase):
@@ -28,12 +31,12 @@ class IptSiteUserInfo(SiteParserBase):
tmps = html.xpath('//div[@class = "stats"]/div/div')
if tmps:
self.upload = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[1]).strip())
self.download = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[2]).strip())
self.seeding = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[0])
self.leeching = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[1])
self.ratio = StringUtils.str_float(str(tmps[0].xpath('span/text()')[0]).strip().replace('-', '0'))
self.bonus = StringUtils.str_float(tmps[0].xpath('a')[3].xpath('text()')[0])
self.upload = size_tools.parse_size(str(tmps[0].xpath('span/text()')[1]).strip())
self.download = size_tools.parse_size(str(tmps[0].xpath('span/text()')[2]).strip())
self.seeding = text_tools.parse_int(tmps[0].xpath('a')[2].xpath('text()')[0])
self.leeching = text_tools.parse_int(tmps[0].xpath('a')[2].xpath('text()')[1])
self.ratio = text_tools.parse_float(str(tmps[0].xpath('span/text()')[0]).strip().replace('-', '0'))
self.bonus = text_tools.parse_float(tmps[0].xpath('a')[3].xpath('text()')[0])
finally:
if html is not None:
del html
@@ -44,7 +47,7 @@ class IptSiteUserInfo(SiteParserBase):
def _parse_user_detail_info(self, html_text: str):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
user_levels_text = html.xpath('//tr/th[text()="Class"]/following-sibling::td[1]/text()')
@@ -54,7 +57,7 @@ class IptSiteUserInfo(SiteParserBase):
# 加入日期
join_at_text = html.xpath('//tr/th[text()="Join date"]/following-sibling::td[1]/text()')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0])
self.join_at = time_tools.normalize_datetime(join_at_text[0].split(' (')[0])
finally:
if html is not None:
del html
@@ -62,7 +65,7 @@ class IptSiteUserInfo(SiteParserBase):
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]:
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
# seeding start
seeding_end_pos = 3
@@ -80,7 +83,7 @@ class IptSiteUserInfo(SiteParserBase):
per_size = per_size.split('(')[-1]
per_size = per_size.split(')')[0]
page_seeding_size += StringUtils.num_filesize(per_size)
page_seeding_size += size_tools.parse_size(per_size)
self.seeding = page_seeding
self.seeding_size = page_seeding_size
app/modules/indexer/parser/mtorrent.py
+2 -2
View File
@@ -5,7 +5,7 @@ from urllib.parse import urljoin
from app.runtime.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.domain import site as site_rules
class MTorrentSiteUserInfo(SiteParserBase):
@@ -39,7 +39,7 @@ class MTorrentSiteUserInfo(SiteParserBase):
获取站点页面地址
"""
# 更换api地址
self._base_url = f"https://api.{StringUtils.get_url_domain(self._base_url)}"
self._base_url = f"https://api.{site_rules.extract_domain(self._base_url)}"
self._user_traffic_page = None
self._user_detail_page = None
self._user_basic_page = "api/member/profile"
app/modules/indexer/parser/nexus_audiences.py
+18 -16
View File
@@ -8,7 +8,9 @@ from lxml import etree
from app.runtime.log import logger
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
@@ -32,7 +34,7 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
super()._parse_message_unread(html_text)
return
@@ -62,7 +64,7 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
message_links = self.__parse_table_unread_message_links(html)
@@ -86,7 +88,7 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
"""
html = etree.HTML(html_text)
try:
if StringUtils.is_valid_html_element(html):
if DomUtils.has_child_elements(html):
head = self.__extract_first_text(
html,
'//*[contains(concat(" ", normalize-space(@class), " "), " pm-hero__title ")]'
@@ -350,7 +352,7 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
for user_node in html.xpath('//*[@data-uploader-url or @data-uploader-stats]'):
@@ -432,18 +434,18 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
metric_key = field or tone or label
if metric_key in {"uploaded", "上传量", "upload"}:
self.upload = StringUtils.num_filesize(value)
self.upload = size_tools.parse_size(value)
elif metric_key in {"downloaded", "下载量", "download"}:
self.download = StringUtils.num_filesize(value)
self.download = size_tools.parse_size(value)
elif metric_key in {"bonus", "爆米花"}:
self.bonus = StringUtils.str_float(value)
self.bonus = text_tools.parse_float(value)
elif metric_key == "ratio":
self.ratio = StringUtils.str_float(value)
self.ratio = text_tools.parse_float(value)
elif metric_key in {"active", "活跃"}:
active_match = re.search(r"↑\s*(\d+)\s*/\s*↓\s*(\d+)", value)
if active_match:
self.seeding = StringUtils.str_int(active_match.group(1))
self.leeching = StringUtils.str_int(active_match.group(2))
self.seeding = text_tools.parse_int(active_match.group(1))
self.leeching = text_tools.parse_int(active_match.group(2))
def __parse_inbox_unread(self, message_link):
"""
@@ -482,7 +484,7 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
inbox_count = re.search(r"(?:收件箱\s*)?(\d[\d,]*)\s*/\s*(\d[\d,]*)", text)
if inbox_count:
return StringUtils.str_int(inbox_count.group(2))
return text_tools.parse_int(inbox_count.group(2))
return None
@@ -500,7 +502,7 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
text = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
single_count = re.fullmatch(r"(\d[\d,]*)", text)
if single_count:
return StringUtils.str_int(single_count.group(1))
return text_tools.parse_int(single_count.group(1))
return None
@staticmethod
@@ -526,15 +528,15 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
return
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
total_row = html.xpath('//table[@class="table table-bordered"]//tr[td[1][normalize-space()="Total"]]')
if not total_row:
return
seeding_count = total_row[0].xpath('./td[2]/text()')
seeding_size = total_row[0].xpath('./td[3]/text()')
self.seeding = StringUtils.str_int(seeding_count[0]) if seeding_count else 0
self.seeding_size = StringUtils.num_filesize(seeding_size[0].strip()) if seeding_size else 0
self.seeding = text_tools.parse_int(seeding_count[0]) if seeding_count else 0
self.seeding_size = size_tools.parse_size(seeding_size[0].strip()) if seeding_size else 0
finally:
if html is not None:
del html
app/modules/indexer/parser/nexus_hhanclub.py
+9 -6
View File
@@ -5,7 +5,10 @@ from lxml import etree
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
@@ -27,11 +30,11 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
html.xpath('//*[@id="user-info-panel"]/div[2]/div[1]/div[1]/div/text()')[0])
# 计算分享率
self.upload = StringUtils.num_filesize(upload_match.group(1).strip()) if upload_match else 0
self.download = StringUtils.num_filesize(download_match.group(1).strip()) if download_match else 0
self.upload = size_tools.parse_size(upload_match.group(1).strip()) if upload_match else 0
self.download = size_tools.parse_size(download_match.group(1).strip()) if download_match else 0
# 优先使用页面上的分享率
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
self.ratio = StringUtils.str_float(ratio_match.group(1)) if (
self.ratio = text_tools.parse_float(ratio_match.group(1)) if (
ratio_match and ratio_match.group(1).strip()) else calc_ratio
finally:
if html is not None:
@@ -47,12 +50,12 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
# 加入时间
join_at_text = html.xpath('//span[contains(text(), "加入日期")]/following-sibling::span/span/@title')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
self.join_at = time_tools.normalize_datetime(join_at_text[0].strip())
finally:
if html is not None:
del html
app/modules/indexer/parser/nexus_php.py
+25 -23
View File
@@ -7,7 +7,9 @@ from lxml import etree
from app.runtime.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class NexusPhpSiteUserInfo(SiteParserBase):
@@ -40,7 +42,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
message_labels = html.xpath('//a[@href="messages.php"]/..')
@@ -52,9 +54,9 @@ class NexusPhpSiteUserInfo(SiteParserBase):
message_unread_match = re.findall(r"[^Date](信息箱\s*|\((?![^)]*:)|你有\xa0)(\d+)", message_text)
if message_unread_match and len(message_unread_match[-1]) == 2:
self.message_unread = StringUtils.str_int(message_unread_match[-1][1])
self.message_unread = text_tools.parse_int(message_unread_match[-1][1])
elif message_text.isdigit():
self.message_unread = StringUtils.str_int(message_text)
self.message_unread = text_tools.parse_int(message_text)
finally:
if html is not None:
del html
@@ -71,7 +73,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//b//text()')
@@ -106,10 +108,10 @@ class NexusPhpSiteUserInfo(SiteParserBase):
# 计算分享率
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
# 优先使用页面上的分享率
self.ratio = StringUtils.str_float(ratio_match.group(1)) if (
self.ratio = text_tools.parse_float(ratio_match.group(1)) if (
ratio_match and ratio_match.group(1).strip()) else calc_ratio
leeching_match = re.search(r"(Torrents leeching|下载中)[\u4E00-\u9FA5\D\s]+(\d+)[\s\S]+<", html_text)
self.leeching = StringUtils.str_int(leeching_match.group(2)) if leeching_match and leeching_match.group(
self.leeching = text_tools.parse_int(leeching_match.group(2)) if leeching_match and leeching_match.group(
2).strip() else 0
html = etree.HTML(html_text)
try:
@@ -121,18 +123,18 @@ class NexusPhpSiteUserInfo(SiteParserBase):
bonus_text = str(tmps[0]).strip()
bonus_match = re.search(r"([\d,.]+)", bonus_text)
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1))
self.bonus = text_tools.parse_float(bonus_match.group(1))
return
bonus_match = re.search(r"mybonus.[\[\]:：<>/a-zA-Z_\-=\"'\s#;.(使用魔力值豆]+\s*([\d,.]+)[<()&\s]", html_text)
try:
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1))
self.bonus = text_tools.parse_float(bonus_match.group(1))
return
bonus_match = re.search(r"[魔力值|\]][\[\]:：<>/a-zA-Z_\-=\"'\s#;]+\s*([\d,.]+|\"[\d,.]+\")[<>()&\s]",
html_text,
flags=re.S)
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1).strip('"'))
self.bonus = text_tools.parse_float(bonus_match.group(1).strip('"'))
except Exception as err:
logger.error(f"{self._site_name} 解析魔力值出错, 错误信息: {str(err)}")
finally:
@@ -146,18 +148,18 @@ class NexusPhpSiteUserInfo(SiteParserBase):
:param html:
:return:
"""
if StringUtils.is_valid_html_element(html):
if DomUtils.has_child_elements(html):
gold, silver, copper = None, None, None
golds = html.xpath('//span[@class = "ucoin-symbol ucoin-gold"]//text()')
if golds:
gold = StringUtils.str_float(str(golds[-1]))
gold = text_tools.parse_float(str(golds[-1]))
silvers = html.xpath('//span[@class = "ucoin-symbol ucoin-silver"]//text()')
if silvers:
silver = StringUtils.str_float(str(silvers[-1]))
silver = text_tools.parse_float(str(silvers[-1]))
coppers = html.xpath('//span[@class = "ucoin-symbol ucoin-copper"]//text()')
if coppers:
copper = StringUtils.str_float(str(coppers[-1]))
copper = text_tools.parse_float(str(coppers[-1]))
if gold or silver or copper:
gold = gold if gold else 0
silver = silver if silver else 0
@@ -174,7 +176,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(str(html_text).replace(r'\/', '/'))
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
# 首页存在扩展链接，使用扩展链接
@@ -215,7 +217,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
for i in range(0, len(seeding_sizes)):
size = self.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i])
seeders = text_tools.parse_int(seeding_seeders[i])
page_seeding_size += size
page_seeding_info.append([seeders, size])
@@ -274,7 +276,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
self._get_user_level(html)
@@ -287,7 +289,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
'|//div/b[text()="加入日期"]/../text()'
'|//*[@id="outer"]/table/tr/td/div/div[1]/div[2]/div[3]/span[1]/span/@title')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip())
self.join_at = time_tools.normalize_datetime(join_at_text[0].split(' (')[0].strip())
# 做种体积 & 做种数
# seeding 页面获取不到的话，此处再获取一次
@@ -300,7 +302,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
tmp_seeding_info = []
for i in range(0, len(seeding_sizes)):
size = self.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i])
seeders = text_tools.parse_int(seeding_seeders[i])
tmp_seeding_size += size
tmp_seeding_info.append([seeders, size])
@@ -316,7 +318,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
if seeding_sizes:
seeding_match = re.search(r"总做种数:\s+(\d+)", seeding_sizes[0], re.IGNORECASE)
seeding_size_match = re.search(r"总做种体积:\s+([\d,.\s]+[KMGTPI]*B)", seeding_sizes[0], re.IGNORECASE)
tmp_seeding = StringUtils.str_int(seeding_match.group(1)) if (
tmp_seeding = text_tools.parse_int(seeding_match.group(1)) if (
seeding_match and seeding_match.group(1)) else 0
tmp_seeding_size = self.num_filesize(
seeding_size_match.group(1).strip()) if seeding_size_match else 0
@@ -396,7 +398,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
message_links = html.xpath('//tr[not(./td/img[@alt="Read"])]/td/a[contains(@href, "viewmessage")]/@href')
@@ -415,7 +417,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
def _parse_message_content(self, html_text):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None, None, None
# 标题
message_head_text = None
@@ -448,4 +450,4 @@ class NexusPhpSiteUserInfo(SiteParserBase):
if not self.bonus:
bonus_text = html.xpath('//tr/td[text()="魔力值" or text()="猫粮"]/following-sibling::td[1]/text()')
if bonus_text:
self.bonus = StringUtils.str_float(bonus_text[0].strip())
self.bonus = text_tools.parse_float(bonus_text[0].strip())
app/modules/indexer/parser/nexus_rabbit.py
+12 -9
View File
@@ -7,7 +7,10 @@ from urllib.parse import urljoin
from app.runtime.log import logger
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser import SiteParserBase
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class NexusRabbitSiteUserInfo(SiteParserBase):
@@ -73,7 +76,7 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
for torrent in torrents:
seeders = int(torrent.get("seeders", 0))
size = StringUtils.num_filesize(torrent.get("size"))
size = size_tools.parse_size(torrent.get("size"))
seeding_size += size
seeding_info.append([seeders, size])
@@ -115,13 +118,13 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
"""只有奶糖余额才需要在 base 中获取，其它均可以在详情页拿到"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
bonus = html.xpath(
'//div[contains(text(), "奶糖余额")]/following-sibling::div[1]/text()'
)
if bonus:
self.bonus = StringUtils.str_float(bonus[0].strip())
self.bonus = text_tools.parse_float(bonus[0].strip())
finally:
if html is not None:
del html
@@ -129,7 +132,7 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
def _parse_user_detail_info(self, html_text: str):
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
# 缩小一下查找范围，所有的信息都在这个 div 里
user_info = html.xpath('//div[contains(@class, "layui-hares-user-info-right")]')
@@ -147,20 +150,20 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
# 加入日期
if join_date := user_info.xpath('.//span[contains(text(), "注册日期")]/text()'):
join_date = join_date[0].strip().split("\r")[0].removeprefix("注册日期：")
self.join_at = StringUtils.unify_datetime_str(join_date)
self.join_at = time_tools.normalize_datetime(join_date)
# 上传量
if upload := user_info.xpath('.//span[contains(text(), "上传量")]/text()'):
self.upload = StringUtils.num_filesize(
self.upload = size_tools.parse_size(
upload[0].strip().removeprefix("上传量：")
)
# 下载量
if download := user_info.xpath('.//span[contains(text(), "下载量")]/text()'):
self.download = StringUtils.num_filesize(
self.download = size_tools.parse_size(
download[0].strip().removeprefix("下载量：")
)
# 分享率
if ratio := user_info.xpath('.//span[contains(text(), "分享率")]/em/text()'):
self.ratio = StringUtils.str_float(ratio[0].strip())
self.ratio = text_tools.parse_float(ratio[0].strip())
finally:
if html is not None:
del html
app/modules/indexer/parser/rousi.py
+5 -4
View File
@@ -6,7 +6,8 @@ from typing import Optional, Tuple
from app.runtime.log import logger
from app.runtime.config import settings
from app.adapters.network.http import RequestUtils
from app.domain.string import StringUtils
from app.domain import site as site_rules
from app.foundation import temporal as time_tools
from app.modules.indexer.parser import SiteParserBase, SiteSchema
@@ -23,7 +24,7 @@ class RousiSiteUserInfo(SiteParserBase):
配置 API 请求地址和请求头
使用 API v1 的 /profile 接口获取用户信息
"""
self._base_url = f"https://{StringUtils.get_url_domain(self._site_url)}"
self._base_url = f"https://{site_rules.extract_domain(self._site_url)}"
self._user_basic_page = "api/v1/profile?include_fields[user]=seeding_leeching_data"
self._user_basic_params = {}
self._user_basic_headers = {
@@ -97,7 +98,7 @@ class RousiSiteUserInfo(SiteParserBase):
self.user_level = user_info.get("level_text") or user_info.get("role_text")
# 注册时间：统一格式为 YYYY-MM-DD HH:MM:SS
join_at = StringUtils.unify_datetime_str(user_info.get("registered_at"))
join_at = time_tools.normalize_datetime(user_info.get("registered_at"))
if join_at:
# 确保格式为 YYYY-MM-DD HH:MM:SS (19位)
if len(join_at) >= 19:
@@ -219,7 +220,7 @@ class RousiSiteUserInfo(SiteParserBase):
self.message_unread = len(messages)
for messsage in messages:
head = messsage.get("title")
date = StringUtils.unify_datetime_str(messsage.get("created_at"))
date = time_tools.normalize_datetime(messsage.get("created_at"))
content = messsage.get("content")
logger.debug(f"{self._site_name} 标题 {head} 时间 {date} 内容 {content}")
self.message_unread_contents.append((head, date, content))
app/modules/indexer/parser/small_horse.py
+14 -11
View File
@@ -5,7 +5,10 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class SmallHorseSiteUserInfo(SiteParserBase):
@@ -44,17 +47,17 @@ class SmallHorseSiteUserInfo(SiteParserBase):
tmps = html.xpath('//ul[@class = "stats nobullet"]')
if tmps:
if tmps[1].xpath("li") and tmps[1].xpath("li")[0].xpath("span//text()"):
self.join_at = StringUtils.unify_datetime_str(tmps[1].xpath("li")[0].xpath("span//text()")[0])
self.upload = StringUtils.num_filesize(str(tmps[1].xpath("li")[2].xpath("text()")[0]).split(":")[1].strip())
self.download = StringUtils.num_filesize(
self.join_at = time_tools.normalize_datetime(tmps[1].xpath("li")[0].xpath("span//text()")[0])
self.upload = size_tools.parse_size(str(tmps[1].xpath("li")[2].xpath("text()")[0]).split(":")[1].strip())
self.download = size_tools.parse_size(
str(tmps[1].xpath("li")[3].xpath("text()")[0]).split(":")[1].strip())
if tmps[1].xpath("li")[4].xpath("span//text()"):
self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[4].xpath("span//text()")[0]).replace('∞', '0'))
self.ratio = text_tools.parse_float(str(tmps[1].xpath("li")[4].xpath("span//text()")[0]).replace('∞', '0'))
else:
self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
self.bonus = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
self.ratio = text_tools.parse_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
self.bonus = text_tools.parse_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
self.user_level = str(tmps[3].xpath("li")[0].xpath("text()")[0]).split(":")[1].strip()
self.leeching = StringUtils.str_int(
self.leeching = text_tools.parse_int(
(tmps[4].xpath("li")[6].xpath("text()")[0]).split(":")[1].replace("[", ""))
finally:
if html is not None:
@@ -72,7 +75,7 @@ class SmallHorseSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
size_col = 6
@@ -87,8 +90,8 @@ class SmallHorseSiteUserInfo(SiteParserBase):
page_seeding = len(seeding_sizes)
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
size = size_tools.parse_size(seeding_sizes[i].xpath("string(.)").strip())
seeders = text_tools.parse_int(seeding_seeders[i].xpath("string(.)").strip())
page_seeding_size += size
page_seeding_info.append([seeders, size])
app/modules/indexer/parser/sunnypt.py
+3 -3
View File
@@ -5,7 +5,7 @@ from urllib.parse import urlencode, urljoin
from app.runtime.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import temporal as time_tools
class SunnyPTSiteUserInfo(SiteParserBase):
@@ -75,7 +75,7 @@ class SunnyPTSiteUserInfo(SiteParserBase):
self.userid = user_info.get("id")
self.username = user_info.get("username")
self.user_level = user_info.get("level") or str(user_info.get("class") or "")
self.join_at = StringUtils.unify_datetime_str(user_info.get("registered_at"))
self.join_at = time_tools.normalize_datetime(user_info.get("registered_at"))
self.upload = int(user_info.get("uploaded") or 0)
self.download = int(user_info.get("downloaded") or 0)
self.ratio = float(user_info.get("ratio") or 0)
@@ -122,7 +122,7 @@ class SunnyPTSiteUserInfo(SiteParserBase):
continue
title = message.get("title")
content = message.get("content")
created_at = StringUtils.unify_datetime_str(message.get("created_at"))
created_at = time_tools.normalize_datetime(message.get("created_at"))
message_id = message.get("id")
if title and content and created_at:
message_source = f"sunnypt-message:{message_id}" if message_id is not None else None
app/modules/indexer/parser/tnode.py
+2 -2
View File
@@ -5,7 +5,7 @@ from typing import Optional
from app.runtime.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import temporal as time_tools
class TNodeSiteUserInfo(SiteParserBase):
@@ -49,7 +49,7 @@ class TNodeSiteUserInfo(SiteParserBase):
self.username = user_info.get("username")
self.user_level = user_info.get("class", {}).get("name")
self.join_at = user_info.get("regTime", 0)
self.join_at = StringUtils.unify_datetime_str(str(self.join_at))
self.join_at = time_tools.normalize_datetime(str(self.join_at))
self.upload = user_info.get("upload")
self.download = user_info.get("download")
app/modules/indexer/parser/torrent_leech.py
+14 -11
View File
@@ -5,7 +5,10 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class TorrentLeechSiteUserInfo(SiteParserBase):
@@ -26,7 +29,7 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
html = etree.HTML(html_text)
current_userid = None
try:
if StringUtils.is_valid_html_element(html):
if DomUtils.has_child_elements(html):
profile_routes = html.xpath(
'//span[contains(concat(" ", normalize-space(@class), " "), " centerTopBar ")]'
'//*[@onclick]/@onclick'
@@ -71,7 +74,7 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
html_text = self._prepare_html_text(html_text)
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return
username_html = html.xpath('//div[contains(concat(" ", normalize-space(@class), " "), '
@@ -85,13 +88,13 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
upload_html = html.xpath('//div[contains(@class,"profile-uploaded")]//span/text()')
if upload_html:
self.upload = StringUtils.num_filesize(upload_html[0])
self.upload = size_tools.parse_size(upload_html[0])
download_html = html.xpath('//div[contains(@class,"profile-downloaded")]//span/text()')
if download_html:
self.download = StringUtils.num_filesize(download_html[0])
self.download = size_tools.parse_size(download_html[0])
ratio_html = html.xpath('//div[contains(@class,"profile-ratio")]//span/text()')
if ratio_html:
self.ratio = StringUtils.str_float(ratio_html[0].replace('∞', '0'))
self.ratio = text_tools.parse_float(ratio_html[0].replace('∞', '0'))
user_level_html = html.xpath('//table[contains(@class, "profileViewTable")]'
'//tr/td[normalize-space()="Class"]/'
@@ -103,11 +106,11 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
'//tr/td[normalize-space()="Registration date"]/'
'following-sibling::td[1]/text()')
if join_at_html:
self.join_at = StringUtils.unify_datetime_str(join_at_html[0].strip())
self.join_at = time_tools.normalize_datetime(join_at_html[0].strip())
bonus_html = html.xpath('//span[contains(@class, "total-TL-points")]/text()')
if bonus_html:
self.bonus = StringUtils.str_float(bonus_html[0].strip())
self.bonus = text_tools.parse_float(bonus_html[0].strip())
finally:
if html is not None:
del html
@@ -129,7 +132,7 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
size_col = 2
@@ -144,8 +147,8 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
page_seeding = len(seeding_sizes)
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i])
size = size_tools.parse_size(seeding_sizes[i].xpath("string(.)").strip())
seeders = text_tools.parse_int(seeding_seeders[i])
page_seeding_size += size
page_seeding_info.append([seeders, size])
app/modules/indexer/parser/unit3d.py
+13 -10
View File
@@ -5,7 +5,10 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import text as text_tools
from app.foundation.dom import DomUtils
class Unit3dSiteUserInfo(SiteParserBase):
@@ -28,7 +31,7 @@ class Unit3dSiteUserInfo(SiteParserBase):
bonus_text = tmps[0].xpath("string(.)")
bonus_match = re.search(r"([\d,.]+)", bonus_text)
if bonus_match and bonus_match.group(1).strip():
self.bonus = StringUtils.str_float(bonus_match.group(1))
self.bonus = text_tools.parse_float(bonus_match.group(1))
finally:
if html is not None:
del html
@@ -44,7 +47,7 @@ class Unit3dSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
# 用户等级
@@ -57,7 +60,7 @@ class Unit3dSiteUserInfo(SiteParserBase):
'or contains(text(), "註冊日期") '
'or contains(text(), "Registration date")]/text()')
if join_at_text:
self.join_at = StringUtils.unify_datetime_str(
self.join_at = time_tools.normalize_datetime(
join_at_text[0].replace('注册日期', '').replace('註冊日期', '').replace('Registration date', ''))
finally:
if html is not None:
@@ -72,7 +75,7 @@ class Unit3dSiteUserInfo(SiteParserBase):
"""
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
if not DomUtils.has_child_elements(html):
return None
size_col = 9
@@ -93,8 +96,8 @@ class Unit3dSiteUserInfo(SiteParserBase):
page_seeding = len(seeding_sizes)
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
size = size_tools.parse_size(seeding_sizes[i].xpath("string(.)").strip())
seeders = text_tools.parse_int(seeding_seeders[i].xpath("string(.)").strip())
page_seeding_size += size
page_seeding_info.append([seeders, size])
@@ -120,12 +123,12 @@ class Unit3dSiteUserInfo(SiteParserBase):
html_text = self._prepare_html_text(html_text)
upload_match = re.search(r"[^总]上[传傳]量?[:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
self.upload = StringUtils.num_filesize(upload_match.group(1).strip()) if upload_match else 0
self.upload = size_tools.parse_size(upload_match.group(1).strip()) if upload_match else 0
download_match = re.search(r"[^总子影力]下[载載]量?[:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
self.download = StringUtils.num_filesize(download_match.group(1).strip()) if download_match else 0
self.download = size_tools.parse_size(download_match.group(1).strip()) if download_match else 0
ratio_match = re.search(r"分享率[:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)", html_text)
self.ratio = StringUtils.str_float(ratio_match.group(1)) if (
self.ratio = text_tools.parse_float(ratio_match.group(1)) if (
ratio_match and ratio_match.group(1).strip()) else 0.0
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
app/modules/indexer/parser/yema.py
+2 -2
View File
@@ -4,7 +4,7 @@ from typing import Optional, Tuple
from app.runtime.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import temporal as time_tools
class YemaSiteUserInfo(SiteParserBase):
@@ -65,7 +65,7 @@ class YemaSiteUserInfo(SiteParserBase):
self.username = user_info.get("name")
self.user_level = str(user_info.get("level")) \
if user_info.get("level") is not None else None
self.join_at = StringUtils.unify_datetime_str(user_info.get("registerTime"))
self.join_at = time_tools.normalize_datetime(user_info.get("registerTime"))
self.upload = int(user_info.get("promotionUploadSize") or 0)
self.download = int(user_info.get("promotionDownloadSize") or 0)
self.ratio = round(self.upload / (self.download or 1), 2)
app/modules/indexer/parser/zhixing.py
+2 -2
View File
@@ -8,7 +8,7 @@ import re
from typing import Optional, Tuple
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.domain.string import StringUtils
from app.foundation import temporal as time_tools
from bs4 import BeautifulSoup
from urllib.parse import urljoin
@@ -68,7 +68,7 @@ class ZhixingSiteUserInfo(SiteParserBase):
self.userid = info_dict.get('UID')
self.username = info_dict.get('用户名')
self.user_level = info_dict.get('用户组')
self.join_at = StringUtils.unify_datetime_str(info_dict.get('注册时间')) if '注册时间' in info_dict else None
self.join_at = time_tools.normalize_datetime(info_dict.get('注册时间')) if '注册时间' in info_dict else None
def num_filesize_safe(s: str):
if s: