chore: update text processing dependencies

This commit is contained in:
jxxghp
2026-05-23 11:51:57 +08:00
parent 5f0ae3a75e
commit 00fc8b2f53
12 changed files with 87 additions and 215 deletions

10
app/utils/jieba.py Normal file
View File

@@ -0,0 +1,10 @@
"""中文分词工具。"""
from fast_jieba import cut as fast_jieba_cut
def cut(text: str, HMM: bool = True, cut_all: bool = False) -> list[str]:
"""
使用 fast-jieba 执行中文分词,并兼容 jieba.cut 的常用参数名。
"""
return fast_jieba_cut(text, hmm=HMM, cut_all=cut_all)

10
app/utils/zhconv.py Normal file
View File

@@ -0,0 +1,10 @@
"""中文简繁转换工具。"""
from zhconv_rs import zhconv as _zhconv # pylint: disable=no-name-in-module
def convert(text: str, target: str) -> str:
"""
使用 zhconv-rs 执行中文简繁转换,并隔离第三方包的函数名差异。
"""
return _zhconv(text, target)