mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 07:27:15 +08:00
perf: optimize rust acceleration paths
Rust vs Python benchmark results: - RSS: Rust 0.299 ms/loop vs Python 7.913 ms/loop, 26.47x faster - Filter: Rust 12.740 ms/loop vs Python 57.187 ms/loop, 4.49x faster - MetaInfo: Rust 64.680 ms/loop vs Python 316.158 ms/loop, 4.89x faster - Indexer agsvpt: Rust 145.76 ms vs Python 3686.50 ms, 25.29x faster - Indexer pttime: Rust 166.51 ms vs Python 4019.87 ms, 24.14x faster - Indexer chdbits: Rust 161.17 ms vs Python 3604.28 ms, 22.36x faster - Indexer iptorrents: Rust 77.82 ms vs Python 17615.52 ms, 226.36x faster Validation: - cargo fmt/check/test for rust/moviepilot_rust - pytest Rust-related coverage: tests/test_rust_accel.py tests/test_torrent_filter.py tests/test_metainfo.py tests/test_indexer_spider_search_url.py tests/test_workflow_fetch_rss.py - tests/run.py legacy suite - pylint app/ --errors-only
This commit is contained in:
+53
-19
@@ -228,6 +228,37 @@ class RssHelper:
|
||||
},
|
||||
}
|
||||
|
||||
def __decode_fast_text(self, raw_data: bytes, ret) -> Optional[str]:
|
||||
"""
|
||||
使用响应声明编码或 UTF-8 快速解码,优先服务 Rust 解析快路径。
|
||||
"""
|
||||
seen_encodings = set()
|
||||
for encoding in (getattr(ret, "encoding", None), "utf-8"):
|
||||
if not encoding or encoding in seen_encodings:
|
||||
continue
|
||||
seen_encodings.add(encoding)
|
||||
try:
|
||||
return raw_data.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return None
|
||||
|
||||
def __parse_with_rust(self, ret_xml: Optional[str]) -> Optional[list]:
|
||||
"""
|
||||
调用 Rust RSS 解析器,并统一处理基础 XML 校验和最大条目限制。
|
||||
"""
|
||||
if not ret_xml or not ret_xml.strip():
|
||||
return None
|
||||
ret_xml_stripped = ret_xml.strip()
|
||||
if not ret_xml_stripped.startswith('<'):
|
||||
return None
|
||||
rust_items = rust_accel.parse_rss_items(ret_xml, self.MAX_RSS_ITEMS + 1)
|
||||
if rust_items is None:
|
||||
return None
|
||||
if len(rust_items) > self.MAX_RSS_ITEMS:
|
||||
logger.warning(f"RSS条目过多: 超过{self.MAX_RSS_ITEMS},仅处理前{self.MAX_RSS_ITEMS}个")
|
||||
return rust_items[:self.MAX_RSS_ITEMS]
|
||||
|
||||
def parse(self, url, proxy: bool = False,
|
||||
timeout: Optional[int] = 15, headers: dict = None, ua: str = None) -> Union[List[dict], None, bool]:
|
||||
"""
|
||||
@@ -270,21 +301,26 @@ class RssHelper:
|
||||
return False
|
||||
|
||||
if raw_data:
|
||||
try:
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
# 解码为字符串
|
||||
ret_xml = raw_data.decode(encoding)
|
||||
except Exception as e:
|
||||
logger.debug(f"chardet解码失败:{str(e)}")
|
||||
# 探测utf-8解码
|
||||
match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text)
|
||||
if match:
|
||||
encoding = match.group(1)
|
||||
if encoding:
|
||||
ret_xml = raw_data.decode(encoding)
|
||||
else:
|
||||
ret.encoding = ret.apparent_encoding
|
||||
ret_xml = self.__decode_fast_text(raw_data, ret)
|
||||
rust_items = self.__parse_with_rust(ret_xml)
|
||||
if rust_items is not None:
|
||||
return rust_items
|
||||
if not ret_xml:
|
||||
try:
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
# 解码为字符串
|
||||
ret_xml = raw_data.decode(encoding)
|
||||
except Exception as e:
|
||||
logger.debug(f"chardet解码失败:{str(e)}")
|
||||
# 探测utf-8解码
|
||||
match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text)
|
||||
if match:
|
||||
encoding = match.group(1)
|
||||
if encoding:
|
||||
ret_xml = raw_data.decode(encoding)
|
||||
else:
|
||||
ret.encoding = ret.apparent_encoding
|
||||
if not ret_xml:
|
||||
ret_xml = ret.text
|
||||
|
||||
@@ -299,11 +335,9 @@ class RssHelper:
|
||||
logger.error("RSS内容不是有效的XML格式")
|
||||
return False
|
||||
|
||||
rust_items = rust_accel.parse_rss_items(ret_xml, self.MAX_RSS_ITEMS + 1)
|
||||
rust_items = self.__parse_with_rust(ret_xml)
|
||||
if rust_items is not None:
|
||||
if len(rust_items) > self.MAX_RSS_ITEMS:
|
||||
logger.warning(f"RSS条目过多: 超过{self.MAX_RSS_ITEMS},仅处理前{self.MAX_RSS_ITEMS}个")
|
||||
return rust_items[:self.MAX_RSS_ITEMS]
|
||||
return rust_items
|
||||
|
||||
# 使用lxml.etree解析XML
|
||||
parser = None
|
||||
|
||||
Reference in New Issue
Block a user