perf: optimize rust acceleration paths

Rust vs Python benchmark results:

- RSS: Rust 0.299 ms/loop vs Python 7.913 ms/loop, 26.47x faster

- Filter: Rust 12.740 ms/loop vs Python 57.187 ms/loop, 4.49x faster

- MetaInfo: Rust 64.680 ms/loop vs Python 316.158 ms/loop, 4.89x faster

- Indexer agsvpt: Rust 145.76 ms vs Python 3686.50 ms, 25.29x faster

- Indexer pttime: Rust 166.51 ms vs Python 4019.87 ms, 24.14x faster

- Indexer chdbits: Rust 161.17 ms vs Python 3604.28 ms, 22.36x faster

- Indexer iptorrents: Rust 77.82 ms vs Python 17615.52 ms, 226.36x faster

Validation:

- cargo fmt/check/test for rust/moviepilot_rust

- pytest Rust-related coverage: tests/test_rust_accel.py tests/test_torrent_filter.py tests/test_metainfo.py tests/test_indexer_spider_search_url.py tests/test_workflow_fetch_rss.py

- tests/run.py legacy suite

- pylint app/ --errors-only
This commit is contained in:
jxxghp
2026-05-23 19:41:18 +08:00
parent a6826e6a4e
commit 0bf228d29d
15 changed files with 1727 additions and 392 deletions
+53 -19
View File
@@ -228,6 +228,37 @@ class RssHelper:
},
}
def __decode_fast_text(self, raw_data: bytes, ret) -> Optional[str]:
"""
使用响应声明编码或 UTF-8 快速解码,优先服务 Rust 解析快路径。
"""
seen_encodings = set()
for encoding in (getattr(ret, "encoding", None), "utf-8"):
if not encoding or encoding in seen_encodings:
continue
seen_encodings.add(encoding)
try:
return raw_data.decode(encoding)
except UnicodeDecodeError:
continue
return None
def __parse_with_rust(self, ret_xml: Optional[str]) -> Optional[list]:
"""
调用 Rust RSS 解析器,并统一处理基础 XML 校验和最大条目限制。
"""
if not ret_xml or not ret_xml.strip():
return None
ret_xml_stripped = ret_xml.strip()
if not ret_xml_stripped.startswith('<'):
return None
rust_items = rust_accel.parse_rss_items(ret_xml, self.MAX_RSS_ITEMS + 1)
if rust_items is None:
return None
if len(rust_items) > self.MAX_RSS_ITEMS:
logger.warning(f"RSS条目过多: 超过{self.MAX_RSS_ITEMS},仅处理前{self.MAX_RSS_ITEMS}")
return rust_items[:self.MAX_RSS_ITEMS]
def parse(self, url, proxy: bool = False,
timeout: Optional[int] = 15, headers: dict = None, ua: str = None) -> Union[List[dict], None, bool]:
"""
@@ -270,21 +301,26 @@ class RssHelper:
return False
if raw_data:
try:
result = chardet.detect(raw_data)
encoding = result['encoding']
# 解码为字符串
ret_xml = raw_data.decode(encoding)
except Exception as e:
logger.debug(f"chardet解码失败:{str(e)}")
# 探测utf-8解码
match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text)
if match:
encoding = match.group(1)
if encoding:
ret_xml = raw_data.decode(encoding)
else:
ret.encoding = ret.apparent_encoding
ret_xml = self.__decode_fast_text(raw_data, ret)
rust_items = self.__parse_with_rust(ret_xml)
if rust_items is not None:
return rust_items
if not ret_xml:
try:
result = chardet.detect(raw_data)
encoding = result['encoding']
# 解码为字符串
ret_xml = raw_data.decode(encoding)
except Exception as e:
logger.debug(f"chardet解码失败:{str(e)}")
# 探测utf-8解码
match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text)
if match:
encoding = match.group(1)
if encoding:
ret_xml = raw_data.decode(encoding)
else:
ret.encoding = ret.apparent_encoding
if not ret_xml:
ret_xml = ret.text
@@ -299,11 +335,9 @@ class RssHelper:
logger.error("RSS内容不是有效的XML格式")
return False
rust_items = rust_accel.parse_rss_items(ret_xml, self.MAX_RSS_ITEMS + 1)
rust_items = self.__parse_with_rust(ret_xml)
if rust_items is not None:
if len(rust_items) > self.MAX_RSS_ITEMS:
logger.warning(f"RSS条目过多: 超过{self.MAX_RSS_ITEMS},仅处理前{self.MAX_RSS_ITEMS}")
return rust_items[:self.MAX_RSS_ITEMS]
return rust_items
# 使用lxml.etree解析XML
parser = None