From 62019d3d238319bda909e776524403005fabd343 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Tue, 11 Aug 2026 09:28:40 +0800 Subject: [PATCH] =?UTF-8?q?fix(music):=20CJK=20=E6=A0=87=E9=A2=98=E9=80=82?= =?UTF-8?q?=E9=85=8D=20Lucene=20=E5=88=86=E8=AF=8D=E7=9A=84=E9=80=90?= =?UTF-8?q?=E5=AD=97=20OR=20=E6=A3=80=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MusicBrainz 索引将连续 CJK 视为单一词元，短语检索对中文标题永远零命中， 新增 _query_phrase 将 CJK 文本拆为逐字 OR 检索式（OR 组带括号避免 AND 优先级歧义）， 拉丁文本保持短语检索 - 逐字 OR 召回宽，候选挑选收紧：标题未命中的候选不能仅凭艺术家署名得分 - 汉字数字与阿拉伯数字写法差异归一比对（茹此精彩十三首/茹此精彩13首） - 补充逐字 OR 检索式、数字归一、艺术家独占候选拒绝等测试案例 --- app/modules/musicbrainz/__init__.py | 107 +++++++++++++++++++++++----- tests/test_musicbrainz_module.py | 48 +++++++++++-- 2 files changed, 130 insertions(+), 25 deletions(-) diff --git a/app/modules/musicbrainz/__init__.py b/app/modules/musicbrainz/__init__.py index 11d524593..84bcbb1d7 100644 --- a/app/modules/musicbrainz/__init__.py +++ b/app/modules/musicbrainz/__init__.py @@ -154,12 +154,12 @@ class MusicBrainzModule(_ModuleBase): queries: list[str] = [] for query in [ cls._build_query(meta), - f'recording:"{cls._escape_query(title)}"', - f'recording:"{cls._escape_query(bare_title)}" AND artist:"{cls._escape_query(artist)}"' + f"recording:{cls._query_phrase(title)}" if title else None, + f'recording:{cls._query_phrase(bare_title)} AND artist:"{cls._escape_query(artist)}"' if artist and bare_title and bare_title != title else None, # 艺术家署名变体（外文艺名等）导致 AND 条件零命中时，仅按主体曲名检索， # 候选挑选阶段要求艺术家命中兜住同名异曲 - f'recording:"{cls._escape_query(bare_title)}"' if bare_title else None, + f"recording:{cls._query_phrase(bare_title)}" if bare_title else None, ]: if query and query not in queries: queries.append(query) @@ -216,13 +216,13 @@ class MusicBrainzModule(_ModuleBase): bare_title = cls._normalize_text(bare_title) queries: list[str] = [] for query in [ - f'releasegroup:"{cls._escape_query(title)}" AND artist:"{cls._escape_query(artist)}"' + f'releasegroup:{cls._query_phrase(title)} AND artist:"{cls._escape_query(artist)}"' if artist else None, - f'releasegroup:"{cls._escape_query(title)}"', - f'releasegroup:"{cls._escape_query(bare_title)}" AND artist:"{cls._escape_query(artist)}"' + f"releasegroup:{cls._query_phrase(title)}" if title else None, + f'releasegroup:{cls._query_phrase(bare_title)} AND artist:"{cls._escape_query(artist)}"' if artist and bare_title and bare_title != title else None, # 署名变体兜底：仅按去注释专辑名检索，挑选阶段要求艺术家同时命中 - f'releasegroup:"{cls._escape_query(bare_title)}"' if bare_title else None, + f"releasegroup:{cls._query_phrase(bare_title)}" if bare_title else None, ]: if query and query not in queries: queries.append(query) @@ -231,15 +231,12 @@ class MusicBrainzModule(_ModuleBase): def _search_artists(self, meta: MetaMusic, limit: int) -> list[MusicInfo]: """按用户输入中的艺术家部分搜索 Artist 浏览候选。""" artist_name = meta.artists[0] if meta.artists else meta.title - if not artist_name: + phrase = self._query_phrase(artist_name) + if not phrase: return [] payload = self._request_json( "/artist", - params={ - "query": f'artist:"{self._escape_query(artist_name)}"', - "limit": max(1, min(limit, 100)), - "fmt": "json", - }, + params={"query": f"artist:{phrase}", "limit": max(1, min(limit, 100)), "fmt": "json"}, ) return [ artist.to_music_info() @@ -582,6 +579,7 @@ class MusicBrainzModule(_ModuleBase): if normalized_source and (candidate.source or "").casefold() != normalized_source: continue score = 0 + title_match = False # 多艺术家资源任一命中即可，联名候选不会因主艺术家顺序失配 artist_match = bool(meta.artists) and any( cls._same_text(artist_name, candidate_artist) @@ -590,6 +588,7 @@ class MusicBrainzModule(_ModuleBase): ) if clean_title and cls._same_text(clean_title, candidate.title): score += 4 + title_match = True elif ( bare_title and artist_match @@ -602,6 +601,7 @@ class MusicBrainzModule(_ModuleBase): ) ): score += 2 + title_match = True if artist_match: score += 3 if meta.album and cls._same_text(meta.album, candidate.album): @@ -613,8 +613,11 @@ class MusicBrainzModule(_ModuleBase): if meta.year and candidate.year and int(meta.year) == int(candidate.year): score += 1 # 已知艺术家时，艺术家未命中的候选不能采信（ISRC 精确身份除外）， - # 兜住宽检索阶梯下同名异曲的误配 - if meta.artists and not artist_match and not isrc_match: + # 兜住宽检索阶梯下同名异曲的误配；CJK 逐字 OR 检索召回宽， + # 标题未命中的候选同样不能仅凭艺术家署名得分（ISRC 除外） + if (meta.artists and not artist_match and not isrc_match) or ( + not title_match and not isrc_match + ): score = 0 ranked.append((score, candidate)) if not ranked: @@ -696,8 +699,35 @@ class MusicBrainzModule(_ModuleBase): """忽略大小写、空白与标点差异比较两个音乐文本字段。""" compact_left = cls._match_text(left) compact_right = cls._match_text(right) - # 空文本不参与比对，避免缺失字段被误判为相等 - return bool(compact_left) and compact_left == compact_right + if compact_left and compact_left == compact_right: + return True + # 条目与资源标题的汉字数字写法不一致（十三首/13首），归一后再比对 + return bool(compact_left) and cls._convert_cjk_numerals( + compact_left) == cls._convert_cjk_numerals(compact_right) + + # 常见汉字数字归一为阿拉伯数字：十三首/13首、二十周年/20周年 在条目与资源标题间混用 + _CJK_DIGIT_MAP = {"零": 0, "一": 1, "二": 2, "两": 2, "三": 3, "四": 4, + "五": 5, "六": 6, "七": 7, "八": 8, "九": 9} + _CJK_NUMERAL_RUN_RE = re.compile(r"[零一二两三四五六七八九十百]+") + + @classmethod + def _convert_cjk_numerals(cls, value: str) -> str: + """将文本中连续的汉字数字转换为阿拉伯数字，支持十位与百位组合。""" + + def _convert(run: str) -> str: + total, current = 0, 0 + for char in run: + if char in cls._CJK_DIGIT_MAP: + current = cls._CJK_DIGIT_MAP[char] + elif char == "十": + total += (current or 1) * 10 + current = 0 + elif char == "百": + total += (current or 1) * 100 + current = 0 + return str(total + current) + + return cls._CJK_NUMERAL_RUN_RE.sub(lambda m: _convert(m.group()), str(value or "")) @classmethod def _normalize_text(cls, value: Optional[str]) -> str: @@ -839,7 +869,7 @@ class MusicBrainzModule(_ModuleBase): # 资源标题先剥离音质标记，避免规格文本污染检索式导致零命中 title = cls._search_title(meta.title) if title: - clauses.append(f'recording:"{cls._escape_query(title)}"') + clauses.append(f"recording:{cls._query_phrase(title)}") if meta.artists: clauses.append(f'artist:"{cls._escape_query(meta.artists[0])}"') if meta.album: @@ -853,6 +883,47 @@ class MusicBrainzModule(_ModuleBase): """转义 MusicBrainz 查询中的引号和反斜线。""" return value.replace("\\", "\\\\").replace('"', '\\"').strip() + # 中日韩字符：Lucene 标准分词器不会切分连续 CJK，短语检索对中文标题永远零命中 + _QUERY_CJK_RE = re.compile( + r"[\u3040-\u30FF\u3400-\u4DBF\u4E00-\u9FFF\uAC00-\uD7AF]") + # 检索词元切分：按空白、标点与括号拆分，保留 CJK 串与拉丁词（括号对逐字检索无意义） + _QUERY_TOKEN_SPLIT_RE = re.compile( + r"[\s\-–—−－。，、；：！？·．…()（）「」『』【】\[\]《》]+") + + @classmethod + def _query_phrase(cls, value: Optional[str]) -> Optional[str]: + """构造适配 Lucene 分词的检索表达式。 + + 无 CJK 的普通文本返回带引号短语；含 CJK 的文本拆为词元后用 OR 交集检索， + MusicBrainz 索引中连续 CJK 是单一词元，逐字 OR 才能命中（「茹此精彩十三首」）； + 过宽的召回由候选挑选阶段的标题与艺术家比对收紧。 + """ + text = str(value or "").strip() + if not text: + return None + if not cls._QUERY_CJK_RE.search(text): + return f'"{cls._escape_query(text)}"' + tokens = [ + token for token in cls._QUERY_TOKEN_SPLIT_RE.split(text) if token.strip() + ] + if not tokens: + return None + parts = [ + f'"{cls._escape_query(token)}"' + if not cls._QUERY_CJK_RE.search(token) else cls._or_group( + [f'"{cls._escape_query(char)}"' for char in token] + ) + for token in tokens + ] + return cls._or_group(parts) + + @staticmethod + def _or_group(parts: list[str]) -> str: + """拼接 OR 检索表达式，多项时用括号包裹避免与外层 AND 产生优先级歧义。""" + if len(parts) == 1: + return parts[0] + return "(" + " OR ".join(parts) + ")" + @classmethod def _recording_to_info(cls, recording: dict[str, Any]) -> Optional[MusicInfo]: """将 MusicBrainz Recording 响应转换为标准音乐信息。""" diff --git a/tests/test_musicbrainz_module.py b/tests/test_musicbrainz_module.py index 957fd9d5c..d07162c2a 100644 --- a/tests/test_musicbrainz_module.py +++ b/tests/test_musicbrainz_module.py @@ -35,7 +35,8 @@ def test_build_query_strips_audio_quality_tokens(): ) ) - assert query == 'recording:"永远是朋友" AND artist:"毛阿敏"' + # CJK 短语在 Lucene 索引中是单一词元，检索式拆为逐字 OR，OR 组带括号避免 AND 优先级歧义 + assert query == 'recording:("永" OR "远" OR "是" OR "朋" OR "友") AND artist:"毛阿敏"' def test_select_candidate_matches_traditional_chinese_title(): @@ -165,8 +166,8 @@ def test_search_music_interleaves_recordings_albums_and_artists(monkeypatch): assert results[1].album == "叶惠美" assert results[2].title == "周杰伦" assert results[2].artists == [] - assert requested[1][1]["query"] == 'releasegroup:"晴天" AND artist:"周杰伦"' - assert requested[2][1]["query"] == 'artist:"周杰伦"' + assert requested[1][1]["query"] == 'releasegroup:("晴" OR "天") AND artist:"周杰伦"' + assert requested[2][1]["query"] == 'artist:("周" OR "杰" OR "伦")' def test_file_recognition_searches_recordings_only(monkeypatch): @@ -608,11 +609,29 @@ def test_recording_queries_ladder_relaxes_to_bare_title_last(): MetaMusic(title="晴天 (电影版)", artists=["周杰伦"]) ) - assert queries[0] == 'recording:"晴天 (电影版)" AND artist:"周杰伦"' - assert queries[1] == 'recording:"晴天 (电影版)"' - assert queries[2] == 'recording:"晴天" AND artist:"周杰伦"' + full = '("晴" OR "天") OR ("电" OR "影" OR "版")' + assert queries[0] == f'recording:({full}) AND artist:"周杰伦"' + assert queries[1] == f'recording:({full})' + assert queries[2] == 'recording:("晴" OR "天") AND artist:"周杰伦"' # 署名变体兜底放在最后，由候选挑选的艺术家要求收紧 - assert queries[-1] == 'recording:"晴天"' + assert queries[-1] == 'recording:("晴" OR "天")' + + +def test_query_phrase_latin_unchanged_and_cjk_char_or(): + """拉丁文本保持短语检索，CJK 文本拆为逐字 OR，混合文本按词元拆分。""" + assert MusicBrainzModule._query_phrase("Fearless") == '"Fearless"' + assert MusicBrainzModule._query_phrase("晴天") == '("晴" OR "天")' + assert MusicBrainzModule._query_phrase("好歌茹芸 Vol. 3") == ( + '(("好" OR "歌" OR "茹" OR "芸") OR "Vol." OR "3")' + ) + assert MusicBrainzModule._query_phrase("") is None + + +def test_same_text_normalizes_cjk_numerals(): + """汉字数字与阿拉伯数字写法差异不应阻断候选比对。""" + assert MusicBrainzModule._same_text("茹此精彩十三首", "茹此精彩13首") + assert MusicBrainzModule._same_text("二十周年演唱会", "20周年演唱会") + assert not MusicBrainzModule._same_text("茹此精彩十三首", "茹此精彩14首") def test_select_candidate_rejects_wrong_artist_same_title(): @@ -629,6 +648,21 @@ def test_select_candidate_rejects_wrong_artist_same_title(): assert MusicBrainzModule._select_candidate(meta, [wrong_artist], source="musicbrainz") is None +def test_select_candidate_rejects_artist_only_match(): + """CJK 逐字 OR 检索召回宽，标题未命中的候选不能仅凭艺术家署名被采信。""" + meta = MetaMusic(title="茹此精彩十三首", artists=["许茹芸"]) + same_artist_other_song = MusicInfo( + source="musicbrainz", + music_type="recording", + media_id="recording-wrong", + title="半首歌", + artists=["许茹芸"], + ) + + assert MusicBrainzModule._select_candidate( + meta, [same_artist_other_song], source="musicbrainz") is None + + def test_select_album_candidate_requires_title_and_artist(): """专辑候选需标题（含去括号弱匹配）与艺术家同时命中才采信。""" meta = MetaMusic(title="我爱夜 (新歌+精选)", artists=["许茹芸"], year=2003)