refactor(text): unify Chinese conversion on Rust (#6481)

This commit is contained in:
InfinityPacer
2026-08-27 17:17:06 +08:00
committed by GitHub
parent fcdef31ecf
commit 2b1f590c4d
12 changed files with 71 additions and 80 deletions
+4 -7
View File
@@ -35,16 +35,13 @@ NATIVE_MODULES = (
)
def _verify_text_capabilities(*, free_threaded: bool) -> None:
def _verify_text_capabilities() -> None:
"""验证标准与 free-threaded profile 共同依赖的文本能力。"""
moviepilot_rust = import_module("moviepilot_rust")
if not moviepilot_rust.is_available() or not moviepilot_rust.jieba_cut("中文分词"):
raise RuntimeError("中文分词运行依赖不可用")
if free_threaded:
converted = moviepilot_rust.zhconv_fast("后台", "zh-hant")
else:
converted = import_module("zhconv_rs").zhconv("后台", "zh-hant")
converted = moviepilot_rust.zhconv_fast("后台", "zh-hant")
if not converted:
raise RuntimeError("中文转换运行依赖不可用")
@@ -60,7 +57,7 @@ def _verify_native_profile(*, free_threaded: bool) -> None:
profile_modules = (
(("psycopg", "psycopg"),)
if free_threaded
else (("psycopg2", "psycopg2"), ("zhconv-rs", "zhconv_rs"))
else (("psycopg2", "psycopg2"),)
)
for name, module_name in (*NATIVE_MODULES, *profile_modules):
imported[name] = import_module(module_name)
@@ -79,7 +76,7 @@ def main(*, full: bool = False) -> None:
for module_name in CORE_MODULES:
import_module(module_name)
_verify_text_capabilities(free_threaded=free_threaded)
_verify_text_capabilities()
if full:
_verify_native_profile(free_threaded=free_threaded)
if free_threaded and sys._is_gil_enabled():
+1 -10
View File
@@ -6,15 +6,6 @@ from typing import Generator, List, Optional, Union
import moviepilot_rust
from app.foundation.environment import is_free_threaded_runtime
if is_free_threaded_runtime():
_zhconv = moviepilot_rust.zhconv_fast
else:
import zhconv_rs
_zhconv = zhconv_rs.zhconv
def cut(text: str, HMM: bool = True, cut_all: bool = False) -> list[str]:
"""通过统一原生入口执行中文分词。"""
@@ -23,7 +14,7 @@ def cut(text: str, HMM: bool = True, cut_all: bool = False) -> list[str]:
def convert(text: str, target: str) -> str:
"""通过统一入口执行 MediaWiki 中文简繁转换。"""
return _zhconv(text, target)
return moviepilot_rust.zhconv_fast(text, target)
def contains_chinese(value: Union[str, list]) -> bool: