feat: accelerate rss parsing with rust

rust/moviepilot_rust/Cargo.lock

@@ -487,6 +487,7 @@ dependencies = [
  "minijinja",
  "once_cell",
  "pyo3",
+ "quick-xml",
  "regex",
  "scraper",
  "url",
@@ -693,6 +694,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.38.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.45"

rust/moviepilot_rust/Cargo.toml

@@ -12,6 +12,7 @@ minijinja = "2.20"
 chrono = "0.4"
 once_cell = "1.20"
 pyo3 = { version = "0.23", features = ["abi3-py311", "extension-module"] }
+quick-xml = "0.38"
 regex = "1.11"
 scraper = "0.24"
 url = "2.5"

rust/moviepilot_rust/src/lib.rs

@@ -1,5 +1,6 @@
 mod filter;
 mod indexer;
+mod rss;
 mod utils;
 
 use pyo3::prelude::*;
@@ -16,5 +17,6 @@ fn moviepilot_rust(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(is_available, m)?)?;
     m.add_function(wrap_pyfunction!(filter::parse_filter_rule_fast, m)?)?;
     m.add_function(wrap_pyfunction!(indexer::parse_indexer_torrents_fast, m)?)?;
+    m.add_function(wrap_pyfunction!(rss::parse_rss_items_fast, m)?)?;
     Ok(())
 }

rust/moviepilot_rust/src/rss.rs

@@ -0,0 +1,384 @@
+use chrono::{DateTime, Local, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc};
+use pyo3::prelude::*;
+use pyo3::types::{PyAny, PyDict, PyList};
+use quick_xml::events::{BytesRef, BytesStart, Event};
+use quick_xml::name::QName;
+use quick_xml::Reader;
+
+#[derive(Default)]
+struct RssItem {
+    title: String,
+    description: String,
+    link: String,
+    enclosure: String,
+    size: i64,
+    pubdate: String,
+    nickname: String,
+}
+
+#[derive(Clone, Copy)]
+enum TextField {
+    Title,
+    Description,
+    Link,
+    Pubdate,
+    Nickname,
+}
+
+/// 解析 RSS/Atom 文本并返回 MoviePilot 现有调用方兼容的条目字典。
+#[pyfunction]
+#[pyo3(signature = (xml_text, max_items=1000))]
+pub(crate) fn parse_rss_items_fast(
+    py: Python<'_>,
+    xml_text: &str,
+    max_items: usize,
+) -> PyResult<Option<PyObject>> {
+    let parsed = parse_rss_items(xml_text, max_items)?;
+    let result = PyList::empty(py);
+    for item in parsed {
+        result.append(item_to_py(py, &item)?)?;
+    }
+    Ok(Some(result.into()))
+}
+
+/// 使用 quick-xml 流式读取 RSS/Atom，避免 lxml XPath 对每个 item 的重复遍历。
+fn parse_rss_items(xml_text: &str, max_items: usize) -> PyResult<Vec<RssItem>> {
+    let mut reader = Reader::from_str(xml_text);
+
+    let mut results = Vec::new();
+    let mut current_item: Option<RssItem> = None;
+    let mut item_depth = 0usize;
+    let mut current_field: Option<(TextField, usize)> = None;
+
+    loop {
+        match reader.read_event() {
+            Ok(Event::Start(event)) => {
+                let local = local_name(event.name());
+                if current_item.is_none() && is_item_node(&local) {
+                    current_item = Some(RssItem::default());
+                    item_depth = 1;
+                    current_field = None;
+                    continue;
+                }
+
+                if let Some(item) = current_item.as_mut() {
+                    item_depth += 1;
+                    handle_start_field(&event, &local, item, item_depth, &mut current_field)?;
+                }
+            }
+            Ok(Event::Empty(event)) => {
+                let local = local_name(event.name());
+                if let Some(item) = current_item.as_mut() {
+                    handle_empty_field(&event, &local, item)?;
+                }
+            }
+            Ok(Event::Text(event)) => {
+                if let (Some(item), Some((field, _))) = (current_item.as_mut(), current_field) {
+                    let text = event.decode().map_err(to_py_value_error)?.to_string();
+                    append_text_field(item, field, &text);
+                }
+            }
+            Ok(Event::CData(event)) => {
+                if let (Some(item), Some((field, _))) = (current_item.as_mut(), current_field) {
+                    let text = event.decode().map_err(to_py_value_error)?.to_string();
+                    append_text_field(item, field, &text);
+                }
+            }
+            Ok(Event::GeneralRef(event)) => {
+                if let (Some(item), Some((field, _))) = (current_item.as_mut(), current_field) {
+                    let text = resolve_general_ref(&event)?;
+                    append_text_field(item, field, &text);
+                }
+            }
+            Ok(Event::End(event)) => {
+                let local = local_name(event.name());
+                if current_item.is_some() && item_depth == 1 && is_item_node(&local) {
+                    if let Some(item) = current_item.take() {
+                        if let Some(item) = finalize_item(item) {
+                            results.push(item);
+                            if results.len() >= max_items {
+                                break;
+                            }
+                        }
+                    }
+                    item_depth = 0;
+                    current_field = None;
+                    continue;
+                }
+
+                if current_item.is_some() && item_depth > 0 {
+                    if current_field
+                        .map(|(_, depth)| depth == item_depth)
+                        .unwrap_or(false)
+                    {
+                        current_field = None;
+                    }
+                    item_depth = item_depth.saturating_sub(1);
+                }
+            }
+            Ok(Event::Eof) => break,
+            Err(err) => {
+                return Err(to_py_value_error(err));
+            }
+            _ => {}
+        }
+    }
+
+    Ok(results)
+}
+
+/// 处理开始标签，记录当前需要采集文本的字段和链接属性。
+fn handle_start_field(
+    event: &BytesStart<'_>,
+    local: &str,
+    item: &mut RssItem,
+    depth: usize,
+    current_field: &mut Option<(TextField, usize)>,
+) -> PyResult<()> {
+    if local == "enclosure" {
+        fill_enclosure(event, item)?;
+        return Ok(());
+    }
+
+    if local == "link" {
+        fill_link_from_href(event, item)?;
+    }
+
+    if current_field.is_none() {
+        if let Some(field) = pick_text_field(local, item) {
+            *current_field = Some((field, depth));
+        }
+    }
+    Ok(())
+}
+
+/// 处理空标签，覆盖 Atom 的 link href 和 RSS 的 enclosure。
+fn handle_empty_field(event: &BytesStart<'_>, local: &str, item: &mut RssItem) -> PyResult<()> {
+    match local {
+        "enclosure" => fill_enclosure(event, item)?,
+        "link" => fill_link_from_href(event, item)?,
+        _ => {}
+    }
+    Ok(())
+}
+
+/// 根据标签名和已采集状态选择当前文本字段。
+fn pick_text_field(local: &str, item: &RssItem) -> Option<TextField> {
+    match local {
+        "title" if item.title.is_empty() => Some(TextField::Title),
+        "description" | "summary" if item.description.is_empty() => Some(TextField::Description),
+        "link" if item.link.is_empty() => Some(TextField::Link),
+        "pubDate" | "published" | "updated" if item.pubdate.is_empty() => Some(TextField::Pubdate),
+        "creator" if item.nickname.is_empty() => Some(TextField::Nickname),
+        _ => None,
+    }
+}
+
+/// 追加文本字段内容，兼容 CDATA 和带内联标签的描述。
+fn append_text_field(item: &mut RssItem, field: TextField, text: &str) {
+    if text.is_empty() {
+        return;
+    }
+    match field {
+        TextField::Title => item.title.push_str(text),
+        TextField::Description => item.description.push_str(text),
+        TextField::Link => item.link.push_str(text),
+        TextField::Pubdate => item.pubdate.push_str(text),
+        TextField::Nickname => item.nickname.push_str(text),
+    }
+}
+
+/// 解析 XML 通用实体，保留未识别实体的原始文本以便 Python 兜底时可复查。
+fn resolve_general_ref(event: &BytesRef<'_>) -> PyResult<String> {
+    if let Some(value) = event.resolve_char_ref().map_err(to_py_value_error)? {
+        return Ok(value.to_string());
+    }
+    let name = event.decode().map_err(to_py_value_error)?;
+    let resolved = match name.as_ref() {
+        "amp" => "&".to_string(),
+        "lt" => "<".to_string(),
+        "gt" => ">".to_string(),
+        "apos" => "'".to_string(),
+        "quot" => "\"".to_string(),
+        other => format!("&{other};"),
+    };
+    Ok(resolved)
+}
+
+/// 从 enclosure 标签读取下载链接和大小。
+fn fill_enclosure(event: &BytesStart<'_>, item: &mut RssItem) -> PyResult<()> {
+    if !item.enclosure.is_empty() {
+        return Ok(());
+    }
+    if let Some(url) = attr_value(event, b"url")? {
+        item.enclosure = url;
+    }
+    if let Some(length) = attr_value(event, b"length")? {
+        item.size = length.trim().parse::<i64>().unwrap_or(0);
+    }
+    Ok(())
+}
+
+/// 从 Atom link 的 href 属性读取页面地址。
+fn fill_link_from_href(event: &BytesStart<'_>, item: &mut RssItem) -> PyResult<()> {
+    if !item.link.is_empty() {
+        return Ok(());
+    }
+    if let Some(href) = attr_value(event, b"href")? {
+        item.link = href;
+    }
+    Ok(())
+}
+
+/// 读取并反转义指定属性值。
+fn attr_value(event: &BytesStart<'_>, name: &[u8]) -> PyResult<Option<String>> {
+    for attr in event.attributes().with_checks(false) {
+        let attr = attr.map_err(to_py_value_error)?;
+        if attr.key.as_ref().eq_ignore_ascii_case(name) {
+            let value = attr
+                .decode_and_unescape_value(event.decoder())
+                .map_err(to_py_value_error)?;
+            return Ok(Some(value.trim().to_string()));
+        }
+    }
+    Ok(None)
+}
+
+/// 完成单条 RSS item 的兼容性整理，保留原 Python 逻辑的跳过条件。
+fn finalize_item(mut item: RssItem) -> Option<RssItem> {
+    item.title = item.title.trim().to_string();
+    item.description = item.description.trim().to_string();
+    item.link = item.link.trim().to_string();
+    item.enclosure = item.enclosure.trim().to_string();
+    item.pubdate = item.pubdate.trim().to_string();
+    item.nickname = item.nickname.trim().to_string();
+
+    if item.title.is_empty() {
+        return None;
+    }
+    if item.enclosure.is_empty() {
+        if item.link.is_empty() {
+            return None;
+        }
+        item.enclosure = item.link.clone();
+    }
+    Some(item)
+}
+
+/// 将 Rust 条目转换为 Python dict，字段名保持与 RssHelper.parse 原返回一致。
+fn item_to_py(py: Python<'_>, item: &RssItem) -> PyResult<PyObject> {
+    let dict = PyDict::new(py);
+    dict.set_item("title", &item.title)?;
+    dict.set_item("enclosure", &item.enclosure)?;
+    dict.set_item("size", item.size)?;
+    dict.set_item("description", &item.description)?;
+    dict.set_item("link", &item.link)?;
+    if let Some(timestamp) = parse_pubdate_timestamp(&item.pubdate) {
+        dict.set_item("pubdate", py_datetime_from_timestamp(py, timestamp)?)?;
+    } else {
+        dict.set_item("pubdate", "")?;
+    }
+    if !item.nickname.is_empty() {
+        dict.set_item("nickname", &item.nickname)?;
+    }
+    Ok(dict.into())
+}
+
+/// 将 Unix 时间戳转换为本地时区 Python datetime，匹配原 astimezone(tz=None) 语义。
+fn py_datetime_from_timestamp<'py>(py: Python<'py>, timestamp: i64) -> PyResult<Bound<'py, PyAny>> {
+    let datetime_mod = py.import("datetime")?;
+    let datetime_cls = datetime_mod.getattr("datetime")?;
+    let timezone_cls = datetime_mod.getattr("timezone")?;
+    let utc = timezone_cls.getattr("utc")?;
+    let utc_dt = datetime_cls.call_method1("fromtimestamp", (timestamp, utc))?;
+    utc_dt.call_method0("astimezone")
+}
+
+/// 解析 RSS/Atom 常见日期格式并返回时间戳。
+fn parse_pubdate_timestamp(value: &str) -> Option<i64> {
+    let trimmed = value.trim();
+    if trimmed.is_empty() {
+        return None;
+    }
+    if let Ok(datetime) = DateTime::parse_from_rfc2822(trimmed) {
+        return Some(datetime.timestamp());
+    }
+    if let Ok(datetime) = DateTime::parse_from_rfc3339(trimmed) {
+        return Some(datetime.timestamp());
+    }
+    if let Some(timestamp) = parse_utc_suffix_datetime(trimmed) {
+        return Some(timestamp);
+    }
+    parse_local_naive_datetime(trimmed)
+}
+
+/// 兼容部分站点输出的 UTC/GMT 文本后缀。
+fn parse_utc_suffix_datetime(value: &str) -> Option<i64> {
+    for suffix in [" UTC", " GMT"] {
+        let Some(stripped) = value.strip_suffix(suffix) else {
+            continue;
+        };
+        for format in [
+            "%a, %d %b %Y %H:%M:%S",
+            "%d %b %Y %H:%M:%S",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%dT%H:%M:%S",
+        ] {
+            if let Ok(naive) = NaiveDateTime::parse_from_str(stripped.trim(), format) {
+                return Some(Utc.from_utc_datetime(&naive).timestamp());
+            }
+        }
+    }
+    None
+}
+
+/// 解析不带时区的日期格式，并按系统本地时区解释。
+fn parse_local_naive_datetime(value: &str) -> Option<i64> {
+    for format in [
+        "%Y-%m-%d %H:%M:%S",
+        "%Y-%m-%dT%H:%M:%S",
+        "%Y-%m-%d %H:%M",
+        "%Y/%m/%d %H:%M:%S",
+        "%Y/%m/%d %H:%M",
+        "%d %b %Y %H:%M:%S",
+        "%a, %d %b %Y %H:%M:%S",
+    ] {
+        if let Ok(naive) = NaiveDateTime::parse_from_str(value, format) {
+            return local_timestamp(naive);
+        }
+    }
+    for format in ["%Y-%m-%d", "%Y/%m/%d", "%d %b %Y"] {
+        if let Ok(date) = NaiveDate::parse_from_str(value, format) {
+            return local_timestamp(NaiveDateTime::new(date, NaiveTime::MIN));
+        }
+    }
+    None
+}
+
+/// 将本地无时区时间转换为时间戳，处理夏令时歧义时取较早值。
+fn local_timestamp(naive: NaiveDateTime) -> Option<i64> {
+    Local
+        .from_local_datetime(&naive)
+        .single()
+        .or_else(|| Local.from_local_datetime(&naive).earliest())
+        .map(|datetime| datetime.timestamp())
+}
+
+/// 判断当前标签是否为 RSS item 或 Atom entry。
+fn is_item_node(local: &str) -> bool {
+    matches!(local, "item" | "entry")
+}
+
+/// 提取 XML 名称的本地部分，用于兼容 dc:creator 这类命名空间字段。
+fn local_name(name: QName<'_>) -> String {
+    let raw = std::str::from_utf8(name.as_ref()).unwrap_or_default();
+    raw.rsplit_once(':')
+        .map(|(_, local)| local)
+        .unwrap_or(raw)
+        .to_string()
+}
+
+/// 将 quick-xml 错误转换为 Python ValueError 交给 Python 包装层判断是否兜底。
+fn to_py_value_error<E: std::fmt::Display>(err: E) -> PyErr {
+    pyo3::exceptions::PyValueError::new_err(err.to_string())
+}