GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-08-05 05:07:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add Rust acceleration for core parsing

Browse Source
This commit is contained in:
jxxghp
2026-05-22 19:57:26 +08:00
parent 7daeb17d85
commit bd4d493f34
28 changed files with 5012 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/modules/filter/RuleParser.py
View File

@@ -2,6 +2,8 @@ import threading
from pyparsing import Forward, Literal, Word, alphas, infixNotation, opAssoc, alphanums, Combine, nums, ParseResults
from app.utils import rust_accel
class RuleParser:
@@ -48,9 +50,30 @@ class RuleParser:
返回:
解析结果
"""
rust_result = rust_accel.parse_filter_rule(expression)
if rust_result is not None:
return _RustParseResults(rust_result)
return self.expr.parseString(expression)
class _RustParseResults(list):
"""
包装 Rust 解析结果，提供本模块调用方使用的 as_list/asList 接口。
"""
def as_list(self) -> list:
"""
返回兼容 pyparsing.ParseResults.as_list 的列表结构。
"""
return list(self)
def asList(self) -> list: # noqa: N802
"""
返回兼容 pyparsing.ParseResults.asList 的列表结构。
"""
return self.as_list()
if __name__ == '__main__':
# 测试代码
expression_str = """

44
app/modules/filter/__init__.py
View File

@@ -11,6 +11,7 @@ from app.modules import _ModuleBase
from app.modules.filter.RuleParser import RuleParser
from app.modules.filter.builtin_rules import BUILTIN_RULE_SET
from app.schemas.types import ModuleType, OtherModulesType, SystemConfigKey
from app.utils import rust_accel
from app.utils.string import StringUtils
@@ -138,6 +139,9 @@ class FilterModule(_ModuleBase):
# 查询规则表详情
groups = self.rulehelper.get_rule_group_by_media(media=mediainfo, group_names=rule_groups)
if groups:
rust_filtered = self.__filter_torrents_by_rust(groups, torrent_list, mediainfo)
if rust_filtered is not None:
return rust_filtered
for group in groups:
# 过滤种子
torrent_list = self.__filter_torrents(
@@ -150,6 +154,46 @@ class FilterModule(_ModuleBase):
)
return torrent_list
def __filter_torrents_by_rust(self, groups: list, torrent_list: List[TorrentInfo],
mediainfo: MediaInfo) -> Optional[List[TorrentInfo]]:
"""
使用 Rust 批量过滤种子；遇到不可支持的规则时返回 None 交由 Python 逻辑处理。
"""
if not torrent_list:
return []
payloads = [self.__build_rust_torrent_payload(torrent) for torrent in torrent_list]
media_payload = mediainfo.to_dict() if mediainfo and hasattr(mediainfo, "to_dict") else (
vars(mediainfo).copy() if mediainfo else None
)
result = rust_accel.filter_torrents(
rule_set=self.rule_set,
rule_strings=[group.rule_string for group in groups],
torrents=payloads,
media_info=media_payload,
)
if result is None:
return None
filtered_torrents = []
for index, pri_order in result:
torrent = torrent_list[int(index)]
torrent.pri_order = int(pri_order)
filtered_torrents.append(torrent)
return filtered_torrents
@staticmethod
def __build_rust_torrent_payload(torrent: TorrentInfo) -> dict:
"""
组装 Rust 过滤器需要的纯数据载荷，避免 Rust 直接依赖 Python 业务对象。
"""
payload = torrent.to_dict() if hasattr(torrent, "to_dict") else vars(torrent).copy()
payload["pub_minutes"] = torrent.pub_minutes()
if payload.get("size"):
meta = MetaInfo(title=torrent.title, subtitle=torrent.description)
payload["episode_count"] = meta.total_episode or 1
else:
payload["episode_count"] = 1
return payload
def __filter_torrents(self, rule_string: str, rule_name: str,
torrent_list: List[TorrentInfo],
mediainfo: MediaInfo,

12
app/modules/indexer/parser/__init__.py
View File

@@ -11,8 +11,10 @@ from requests import Session
from app.core.config import settings
from app.helper.cloudflare import under_challenge
from app.log import logger
from app.utils import rust_accel
from app.utils.http import RequestUtils
from app.utils.site import SiteUtils
from app.utils.string import StringUtils
# 站点框架
@@ -154,6 +156,16 @@ class SiteParserBase(metaclass=ABCMeta):
"""
return self.schema
@staticmethod
def num_filesize(text) -> int:
"""
将站点页面中的文件大小文本转换为字节，优先使用 Rust 快路径。
"""
rust_value = rust_accel.parse_filesize(text)
if rust_value is not None:
return rust_value
return StringUtils.num_filesize(text)
def parse(self):
"""
解析站点信息

10
app/modules/indexer/parser/nexus_php.py
View File

@@ -93,10 +93,10 @@ class NexusPhpSiteUserInfo(SiteParserBase):
html_text = self._prepare_html_text(html_text)
upload_match = re.search(r"[^总]上[传傳]量?[:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
self.upload = StringUtils.num_filesize(upload_match.group(1).strip()) if upload_match else 0
self.upload = self.num_filesize(upload_match.group(1).strip()) if upload_match else 0
download_match = re.search(r"[^总子影力]下[载載]量?[:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
self.download = StringUtils.num_filesize(download_match.group(1).strip()) if download_match else 0
self.download = self.num_filesize(download_match.group(1).strip()) if download_match else 0
ratio_match = re.search(r"分享率[:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)", html_text)
# 计算分享率
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
@@ -209,7 +209,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
page_seeding = len(seeding_sizes)
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
size = self.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i])
page_seeding_size += size
@@ -273,7 +273,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
tmp_seeding_size = 0
tmp_seeding_info = []
for i in range(0, len(seeding_sizes)):
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
size = self.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
seeders = StringUtils.str_int(seeding_seeders[i])
tmp_seeding_size += size
@@ -292,7 +292,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
seeding_size_match = re.search(r"总做种体积:\s+([\d,.\s]+[KMGTPI]*B)", seeding_sizes[0], re.IGNORECASE)
tmp_seeding = StringUtils.str_int(seeding_match.group(1)) if (
seeding_match and seeding_match.group(1)) else 0
tmp_seeding_size = StringUtils.num_filesize(
tmp_seeding_size = self.num_filesize(
seeding_size_match.group(1).strip()) if seeding_size_match else 0
if not self.seeding_size:
self.seeding_size = tmp_seeding_size

8
app/modules/indexer/parser/zhixing.py
View File

@@ -75,7 +75,7 @@ class ZhixingSiteUserInfo(SiteParserBase):
s = s.strip()
if re.match(r'^\d+(\.\d+)?$', s):
s += ' B'
return StringUtils.num_filesize(s) if s else 0
return self.num_filesize(s) if s else 0
self.upload = num_filesize_safe(info_dict.get('上传流量')) if '上传流量' in info_dict else 0
self.download = num_filesize_safe(info_dict.get('下载流量')) if '下载流量' in info_dict else 0
@@ -108,7 +108,7 @@ class ZhixingSiteUserInfo(SiteParserBase):
if size_td:
size_text = size_td.find('a').text if size_td.find('a') else size_td.text.strip()
page_seeding += 1
page_seeding_size += StringUtils.num_filesize(size_text)
page_seeding_size += self.num_filesize(size_text)
return page_seeding, page_seeding_size
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
@@ -164,7 +164,7 @@ class ZhixingSiteUserInfo(SiteParserBase):
s = s.strip()
if re.match(r'^\d+(\.\d+)?$', s):
s += ' B'
return StringUtils.num_filesize(s) if s else 0
return self.num_filesize(s) if s else 0
self.seeding = int(self._basic_info.get('当前保种数量', 0))
self.seeding_size = num_filesize_safe(self._basic_info.get('当前保种容量', ''))
@@ -181,4 +181,4 @@ class ZhixingSiteUserInfo(SiteParserBase):
self.message_unread = str(self.message_unread or 0)
self.seeding = str(self.seeding or 0)
self.seeding_size = str(self.seeding_size or 0)
self.seeding_size = str(self.seeding_size or 0)

27
app/modules/indexer/spider/__init__.py
View File

@@ -12,6 +12,7 @@ from pyquery import PyQuery
from app.core.config import settings
from app.log import logger
from app.schemas.types import MediaType
from app.utils import rust_accel
from app.utils.http import RequestUtils, AsyncRequestUtils
from app.utils.string import StringUtils
from app.utils.url import UrlUtils
@@ -95,6 +96,19 @@ class SiteSpider:
"""
获取搜索URL
"""
rust_url = rust_accel.build_indexer_search_url({
"search": self.search,
"batch": self.batch,
"browse": self.browse,
"category": self.category,
"domain": self.domain,
"keyword": self.keyword,
"mtype": self.mtype.value if self.mtype else None,
"cat": self.cat,
"page": self.page,
})
if rust_url:
return rust_url
# 种子搜索相对路径
paths = self.search.get('paths', [])
torrentspath = ""
@@ -658,6 +672,9 @@ class SiteSpider:
"""
if not text or not filters or not isinstance(filters, list):
return text
rust_text = rust_accel.apply_indexer_text_filters(text, filters)
if rust_text is not None:
return rust_text
if not isinstance(text, str):
text = str(text)
for filter_item in filters:
@@ -739,6 +756,16 @@ class SiteSpider:
# 清空旧结果
self.torrents_info_array = []
rust_torrents = rust_accel.parse_indexer_torrents(
html_text=html_text,
domain=self.domain,
list_config=self.list,
fields=self.fields,
category=self.category,
result_num=int(self.result_num),
)
if rust_torrents is not None:
return rust_torrents
html_doc = None
try:
# 解析站点文本对象