diff --git a/app/api/endpoints/music.py b/app/api/endpoints/music.py index b72e0f805..b6e22cbb7 100644 --- a/app/api/endpoints/music.py +++ b/app/api/endpoints/music.py @@ -8,11 +8,14 @@ from app.chain.music import MusicChain from app.schemas.types import MediaType from app.core.context import MusicAlbumInfo, MusicArtistInfo, MusicInfo from app.core.security import verify_token +from app.db.models.user import User +from app.db.user_oper import get_current_active_superuser_async from app.modules.listenbrainz import ( LISTENBRAINZ_CHART_RANGES, LISTENBRAINZ_FRESH_MAX_DAYS, LISTENBRAINZ_FRESH_SORTS, ) +from app.modules.musicbrainz.music_cache import MusicBrainzCache router = APIRouter() @@ -67,6 +70,53 @@ async def recognize_music( return _serialize_music(info) +@router.get( + "/cache", summary="查询音乐识别缓存", response_model=schemas.Response +) +async def music_recognition_cache( + _: User = Depends(get_current_active_superuser_async), +) -> schemas.Response: + """查询可管理的 MusicBrainz 识别缓存。""" + cache_items = MusicBrainzCache().list_items() + recognized_count = sum(1 for item in cache_items if item["media_id"]) + return schemas.Response( + success=True, + data={ + "count": len(cache_items), + "recognized": recognized_count, + "unrecognized": len(cache_items) - recognized_count, + "data": cache_items, + }, + ) + + +@router.delete( + "/cache/{cache_key:path}", + summary="删除指定音乐识别缓存", + response_model=schemas.Response, +) +async def delete_music_recognition_cache( + cache_key: str, + _: User = Depends(get_current_active_superuser_async), +) -> schemas.Response: + """按缓存键删除单条 MusicBrainz 识别缓存。""" + deleted_item = MusicBrainzCache().delete(cache_key) + if not deleted_item: + return schemas.Response(success=False, message="音乐识别缓存不存在") + return schemas.Response(success=True, message="音乐识别缓存删除成功") + + +@router.delete( + "/cache", summary="清空音乐识别缓存", response_model=schemas.Response +) +async def clear_music_recognition_cache( + _: User = Depends(get_current_active_superuser_async), +) -> schemas.Response: + """清空全部 MusicBrainz 识别缓存。""" + MusicBrainzCache().clear() + return schemas.Response(success=True, message="音乐识别缓存清理完成") + + @router.get( "/explore", summary="探索音乐", diff --git a/app/api/endpoints/torrent.py b/app/api/endpoints/torrent.py index 49c6c98ce..4fa7e19d7 100644 --- a/app/api/endpoints/torrent.py +++ b/app/api/endpoints/torrent.py @@ -118,8 +118,11 @@ async def delete_cache( if len(cache_data[domain]) == original_count: return schemas.Response(success=False, message="未找到指定的种子") - # 保存更新后的缓存 - await torrents_chain.async_save_cache(cache_data, torrents_chain.cache_file) + # 保存更新后的缓存：影视与音乐分别回写各自存储文件 + video_cache, music_cache = torrents_chain.split_cache_contexts(cache_data) + video_file, music_file = torrents_chain.cache_files() + await torrents_chain.async_save_cache(video_cache, video_file) + await torrents_chain.async_save_cache(music_cache, music_file) return schemas.Response(success=True, message="种子删除成功") except Exception as e: @@ -248,8 +251,11 @@ async def reidentify_cache( # 更新上下文中的媒体信息 target_context.media_info = mediainfo - # 保存更新后的缓存 - await torrents_chain.async_save_cache(cache_data, TorrentsChain().cache_file) + # 保存更新后的缓存：影视与音乐分别回写各自存储文件 + video_cache, music_cache = torrents_chain.split_cache_contexts(cache_data) + video_file, music_file = torrents_chain.cache_files() + await torrents_chain.async_save_cache(video_cache, video_file) + await torrents_chain.async_save_cache(music_cache, music_file) return schemas.Response( success=True, diff --git a/app/chain/subscribe.py b/app/chain/subscribe.py index 1dcd96c63..652b0b6db 100644 --- a/app/chain/subscribe.py +++ b/app/chain/subscribe.py @@ -2081,6 +2081,8 @@ class SubscribeChain(ChainBase): torrents = TorrentsChain().refresh( sites=sites, progress_callback=_update_refresh_progress if progress_callback else None, + # 存在音乐订阅时额外抓取站点音乐专用入口，音乐不一定在默认种子首页 + include_music=self.has_music_subscribe(), ) self.match( torrents, @@ -2132,6 +2134,13 @@ class SubscribeChain(ChainBase): return ret_sites + def has_music_subscribe(self) -> bool: + """判断是否存在可搜索状态的音乐订阅，用于决定是否额外刷新站点音乐入口。""" + return any( + subscribe.type == MediaType.MUSIC.value + for subscribe in SubscribeOper().list(self.get_states_for_search('R')) or [] + ) + def match( self, torrents: Dict[str, List[Context]], diff --git a/app/chain/torrents.py b/app/chain/torrents.py index 4fc3cf7e9..ef686b2fa 100644 --- a/app/chain/torrents.py +++ b/app/chain/torrents.py @@ -30,6 +30,9 @@ class TorrentsChain(ChainBase): _spider_file = "__torrents_cache__" _rss_file = "__rss_cache__" + # 音乐资源独立缓存，与影视种子分开计算配额与存储，避免被影视资源挤出 + _music_spider_file = "__torrents_music_cache__" + _music_rss_file = "__rss_music_cache__" @property def cache_file(self) -> str: @@ -58,7 +61,7 @@ class TorrentsChain(ChainBase): def get_torrents(self, stype: Optional[str] = None) -> Dict[str, List[Context]]: """ - 获取当前缓存的种子 + 获取当前缓存的种子，包含独立缓存的音乐资源 :param stype: 强制指定缓存类型，spider:爬虫缓存，rss:rss缓存 """ @@ -74,11 +77,59 @@ class TorrentsChain(ChainBase): # 兼容性处理：为旧版本的Context对象补齐新增候选识别字段 self._ensure_context_compatibility(torrents_cache, stype=stype) + # 合并音乐独立缓存，供订阅匹配等消费方按站点读取完整候选 + music_cache = self.get_music_torrents(stype=stype) + for domain, contexts in music_cache.items(): + if contexts: + torrents_cache.setdefault(domain, []).extend(contexts) + return torrents_cache + def get_music_torrents(self, stype: Optional[str] = None) -> Dict[str, List[Context]]: + """ + 获取音乐独立缓存的种子 + :param stype: 强制指定缓存类型，spider:爬虫缓存，rss:rss缓存 + """ + if not stype: + stype = settings.SUBSCRIBE_MODE + music_file = self._music_spider_file if stype == 'spider' else self._music_rss_file + music_cache = self.load_cache(music_file) or {} + # 兼容性处理：为旧版本的Context对象补齐新增候选识别字段 + self._ensure_context_compatibility(music_cache, stype=stype) + return music_cache + + def cache_files(self, stype: Optional[str] = None) -> tuple: + """ + 返回影视与音乐缓存文件名，供按当前订阅模式回写各自缓存 + :param stype: 强制指定缓存类型，spider:爬虫缓存，rss:rss缓存 + """ + if not stype: + stype = settings.SUBSCRIBE_MODE + if stype == 'spider': + return self._spider_file, self._music_spider_file + return self._rss_file, self._music_rss_file + + @staticmethod + def split_cache_contexts( + torrents_cache: Dict[str, List[Context]], + ) -> tuple: + """ + 将合并读取的缓存按种子分类拆分为影视缓存与音乐缓存，用于分别回写各自存储文件。 + """ + video_cache: Dict[str, List[Context]] = {} + music_cache: Dict[str, List[Context]] = {} + for domain, contexts in torrents_cache.items(): + for context in contexts: + torrent = context.torrent_info + if torrent and torrent.category in (MediaType.MUSIC, MediaType.MUSIC.value): + music_cache.setdefault(domain, []).append(context) + else: + video_cache.setdefault(domain, []).append(context) + return video_cache, music_cache + async def async_get_torrents(self, stype: Optional[str] = None) -> Dict[str, List[Context]]: """ - 异步获取当前缓存的种子 + 异步获取当前缓存的种子，包含独立缓存的音乐资源 :param stype: 强制指定缓存类型，spider:爬虫缓存，rss:rss缓存 """ @@ -88,11 +139,19 @@ class TorrentsChain(ChainBase): # 异步读取缓存 if stype == 'spider': torrents_cache = await self.async_load_cache(self._spider_file) or {} + music_cache = await self.async_load_cache(self._music_spider_file) or {} else: torrents_cache = await self.async_load_cache(self._rss_file) or {} + music_cache = await self.async_load_cache(self._music_rss_file) or {} # 兼容性处理：为旧版本的Context对象补齐新增候选识别字段 self._ensure_context_compatibility(torrents_cache, stype=stype) + self._ensure_context_compatibility(music_cache, stype=stype) + + # 合并音乐独立缓存，供订阅匹配等消费方按站点读取完整候选 + for domain, contexts in music_cache.items(): + if contexts: + torrents_cache.setdefault(domain, []).extend(contexts) return torrents_cache @@ -385,20 +444,24 @@ class TorrentsChain(ChainBase): def clear_torrents(self): """ - 清理种子缓存数据 + 清理种子缓存数据，包含音乐独立缓存 """ logger.info(f'开始清理种子缓存数据 ...') self.remove_cache(self._spider_file) self.remove_cache(self._rss_file) + self.remove_cache(self._music_spider_file) + self.remove_cache(self._music_rss_file) logger.info(f'种子缓存数据清理完成') async def async_clear_torrents(self): """ - 异步清理种子缓存数据 + 异步清理种子缓存数据，包含音乐独立缓存 """ logger.info(f'开始异步清理种子缓存数据 ...') await self.async_remove_cache(self._spider_file) await self.async_remove_cache(self._rss_file) + await self.async_remove_cache(self._music_spider_file) + await self.async_remove_cache(self._music_rss_file) logger.info(f'异步种子缓存数据清理完成') def browse(self, domain: str, keyword: Optional[str] = None, cat: Optional[str] = None, @@ -465,6 +528,8 @@ class TorrentsChain(ChainBase): if not rss_items: logger.error(f'站点 {domain} 未获取到RSS数据！') return [] + # 站点级媒体类型，用于给缺少分类信息的 RSS 种子补充分类 + site_media_type = MediaType.from_agent(site.get("media_type")) # 组装种子 ret_torrents: List[TorrentInfo] = [] try: @@ -484,6 +549,8 @@ class TorrentsChain(ChainBase): page_url=item.get("link"), size=item.get("size"), pubdate=item["pubdate"].strftime("%Y-%m-%d %H:%M:%S") if item.get("pubdate") else None, + # RSS 报文不带站点分类，按站点媒体类型补充，否则音乐资源无法进入音乐订阅匹配 + category=site_media_type.value if site_media_type else None, ) ret_torrents.append(torrentinfo) finally: @@ -491,17 +558,71 @@ class TorrentsChain(ChainBase): del rss_items return ret_torrents + @staticmethod + def _music_browse_paths(site: dict) -> List[str]: + """ + 返回站点独立于默认浏览入口的音乐种子页面路径。 + + 部分站点的默认种子列表只显示电影和电视剧，音乐需要单独的菜单页面进入； + 这类站点在索引配置中用 type=music 的搜索路径声明音乐入口。 + 默认入口已覆盖音乐（音乐站点或未定义独立入口）时返回空列表。 + """ + # 音乐站点全站都是音乐资源，默认浏览入口已经覆盖 + if MediaType.from_agent(site.get("media_type")) == MediaType.MUSIC: + return [] + paths = (site.get("search") or {}).get("paths") or [] + if len(paths) <= 1: + return [] + # 计算默认浏览使用的路径，与其相同的音乐入口无需重复抓取 + browse_conf = site.get("browse") or {} + default_path = browse_conf.get("path") + if not default_path: + default_path = next( + (item.get("path") for item in paths if item.get("type") in (None, "all")), + paths[0].get("path"), + ) + return [ + item.get("path") for item in paths + if item.get("type") == "music" + and item.get("path") + and item.get("path") != default_path + ] + + def __append_music_browse_torrents( + self, + domain: str, + torrents: List[TorrentInfo], + ) -> List[TorrentInfo]: + """ + 追加抓取站点音乐专用入口的最新种子，并按种子链接去重后返回合并结果。 + """ + seen = {torrent.enclosure for torrent in torrents if torrent.enclosure} + for page in range(2): + page_torrents = self.browse(domain=domain, page=page, mtype=MediaType.MUSIC) + if not page_torrents: + # 某一页没有数据，说明已经到最后一页，停止获取 + break + for torrent in page_torrents: + if torrent.enclosure and torrent.enclosure in seen: + continue + if torrent.enclosure: + seen.add(torrent.enclosure) + torrents.append(torrent) + return torrents + def refresh( self, stype: Optional[str] = None, sites: List[int] = None, progress_callback: Optional[Callable[..., None]] = None, + include_music: bool = False, ) -> Dict[str, List[Context]]: """ 刷新站点最新资源，识别并缓存起来 :param stype: 强制指定缓存类型，spider:爬虫缓存，rss:rss缓存 :param sites: 强制指定站点ID列表，为空则读取设置的订阅站点 :param progress_callback: 资源刷新进度更新回调 + :param include_music: 是否额外抓取站点的音乐专用浏览入口，服务音乐订阅 """ def __is_no_cache_site(_domain: str) -> bool: @@ -521,13 +642,21 @@ class TorrentsChain(ChainBase): if not sites: sites = SystemConfigOper().get(SystemConfigKey.RssSites) or [] - # 读取缓存 - torrents_cache = self.get_torrents() + # 读取缓存，影视与音乐分别独立存储 + if stype == 'spider': + torrents_cache = self.load_cache(self._spider_file) or {} + music_cache = self.load_cache(self._music_spider_file) or {} + else: + torrents_cache = self.load_cache(self._rss_file) or {} + music_cache = self.load_cache(self._music_rss_file) or {} + self._ensure_context_compatibility(torrents_cache, stype=stype) + self._ensure_context_compatibility(music_cache, stype=stype) - # 缓存过滤掉无效种子 - for _domain, _torrents in torrents_cache.items(): - torrents_cache[_domain] = [_torrent for _torrent in _torrents - if not TorrentHelper().is_invalid(_torrent.torrent_info.enclosure)] + # 缓存过滤掉无效种子（影视与音乐缓存分别处理） + for _cache in (torrents_cache, music_cache): + for _domain, _torrents in _cache.items(): + _cache[_domain] = [_torrent for _torrent in _torrents + if not TorrentHelper().is_invalid(_torrent.torrent_info.enclosure)] # 需要刷新的站点domain domains = [] @@ -572,31 +701,47 @@ class TorrentsChain(ChainBase): else: # 如果某一页没有数据，说明已经到最后一页，停止获取 break + # 存在音乐订阅时，默认首页可能不包含音乐资源，需要额外抓取音乐专用入口 + if include_music and self._music_browse_paths(indexer): + torrents = self.__append_music_browse_torrents( + domain=domain, torrents=torrents + ) else: # 刷新RSS种子 torrents: List[TorrentInfo] = self.rss(domain=domain) # 按pubdate降序排列 torrents.sort(key=lambda x: x.pubdate or '', reverse=True) - # 取前N条 - torrents = torrents[:settings.CONF.refresh] - if torrents: + # 音乐与影视按同一公共参数独立计算刷新配额，并分别写入各自缓存，音乐不会被影视资源挤出 + music_torrents = [ + t for t in torrents if t.category == MediaType.MUSIC.value + ][:settings.CONF.refresh] + torrents = [ + t for t in torrents if t.category != MediaType.MUSIC.value + ][:settings.CONF.refresh] + if torrents or music_torrents: if __is_no_cache_site(domain): # 不需要缓存的站点，直接处理 - logger.info(f'{indexer.get("name")} 有 {len(torrents)} 个种子 (不缓存)') + logger.info(f'{indexer.get("name")} 有 {len(torrents) + len(music_torrents)} 个种子 (不缓存)') torrents_cache[domain] = [] + music_cache[domain] = [] else: # 过滤出没有处理过的种子 - 优化：使用集合查找，避免重复创建字符串列表 cached_signatures = {f'{t.torrent_info.title}{t.torrent_info.description}' for t in torrents_cache.get(domain) or []} torrents = [torrent for torrent in torrents if f'{torrent.title}{torrent.description}' not in cached_signatures] - if torrents: - logger.info(f'{indexer.get("name")} 有 {len(torrents)} 个新种子') + # 音乐种子对照音乐独立缓存去重 + music_signatures = {f'{t.torrent_info.title}{t.torrent_info.description}' + for t in music_cache.get(domain) or []} + music_torrents = [torrent for torrent in music_torrents + if f'{torrent.title}{torrent.description}' not in music_signatures] + if torrents or music_torrents: + logger.info(f'{indexer.get("name")} 有 {len(torrents) + len(music_torrents)} 个新种子') else: logger.info(f'{indexer.get("name")} 没有新种子') continue try: - for torrent in torrents: + for torrent in torrents + music_torrents: if global_vars.is_system_stopped: break if not torrent.enclosure: @@ -647,29 +792,39 @@ class TorrentsChain(ChainBase): # 如果未识别到媒体信息，设置初始失败次数为1 if not mediainfo or not all(resolve_media_identity(media=mediainfo)): context.media_recognize_fail_count = 1 - # 添加到缓存 - if not torrents_cache.get(domain): - torrents_cache[domain] = [context] + # 添加到缓存：音乐进入独立缓存，与影视分开存储 + if torrent.category == MediaType.MUSIC.value: + target_cache = music_cache else: - torrents_cache[domain].append(context) - # 如果超过了限制条数则移除掉前面的 - if len(torrents_cache[domain]) > settings.CONF.torrents: - torrents_cache[domain] = torrents_cache[domain][-settings.CONF.torrents:] + target_cache = torrents_cache + if not target_cache.get(domain): + target_cache[domain] = [context] + else: + target_cache[domain].append(context) + # 如果超过了限制条数则移除掉前面的，音乐与影视各自独立计算配额 + if len(target_cache[domain]) > settings.CONF.torrents: + target_cache[domain] = target_cache[domain][-settings.CONF.torrents:] finally: torrents.clear() + music_torrents.clear() del torrents + del music_torrents else: logger.info(f'{indexer.get("name")} 没有获取到种子') - # 保存缓存到本地 + # 保存缓存到本地，影视与音乐分别存储 if stype == "spider": self.save_cache(torrents_cache, self._spider_file) + self.save_cache(music_cache, self._music_spider_file) else: self.save_cache(torrents_cache, self._rss_file) + self.save_cache(music_cache, self._music_rss_file) # 去除不在站点范围内的缓存种子 if sites and torrents_cache: torrents_cache = {k: v for k, v in torrents_cache.items() if k in domains} + if sites and music_cache: + music_cache = {k: v for k, v in music_cache.items() if k in domains} if progress_callback: progress_callback( @@ -678,6 +833,11 @@ class TorrentsChain(ChainBase): data={"total": total_indexers, "finished": total_indexers}, ) + # 订阅匹配需要完整候选，音乐独立缓存在返回值中按站点合并 + for _domain, _contexts in music_cache.items(): + if _contexts: + torrents_cache.setdefault(_domain, []).extend(_contexts) + return torrents_cache @staticmethod diff --git a/app/modules/indexer/spider/__init__.py b/app/modules/indexer/spider/__init__.py index 1cddd0ba8..28224feb1 100644 --- a/app/modules/indexer/spider/__init__.py +++ b/app/modules/indexer/spider/__init__.py @@ -165,6 +165,8 @@ class SiteSpider: # 种子搜索相对路径 paths = self.search.get('paths', []) torrentspath = "" + # 是否选中了媒体类型专用路径，浏览模式下专用路径优先于 browse 配置 + typed_path_selected = False if len(paths) == 1: torrentspath = paths[0].get('path', '') else: @@ -183,6 +185,7 @@ class SiteSpider: not expected_type and path_type == "all" ): torrentspath = path.get('path', '') + typed_path_selected = bool(expected_type and path_type == expected_type) break if not torrentspath: torrentspath = fallback_path @@ -282,16 +285,18 @@ class SiteSpider: "page": self.page or 0, "keyword": "" } - # 有单独浏览路径 - if self.browse: + # 有单独浏览路径；指定了媒体类型专用路径时不覆盖，确保音乐等专用入口可达 + if self.browse and not typed_path_selected: torrentspath = self.browse.get("path") if self.browse.get("start"): start_page = int(self.browse.get("start")) + int(self.page or 0) inputs_dict.update({ "page": start_page }) - elif self.page: - torrentspath = torrentspath + f"?page={self.page}" + elif self.page and "{page}" not in str(torrentspath): + # 按路径是否已带查询参数选择连接符，避免拼出两个问号的非法地址 + separator = "&" if "?" in str(torrentspath) else "?" + torrentspath = torrentspath + f"{separator}page={self.page}" # 搜索Url searchurl = self.domain + str(torrentspath).format(**inputs_dict) diff --git a/app/modules/musicbrainz/__init__.py b/app/modules/musicbrainz/__init__.py index 75fc5fdc3..907fc8a97 100644 --- a/app/modules/musicbrainz/__init__.py +++ b/app/modules/musicbrainz/__init__.py @@ -19,6 +19,7 @@ from app.core.context import ( from app.core.meta import MetaBase, MetaMusic from app.log import logger from app.modules import _ModuleBase +from app.modules.musicbrainz.music_cache import MusicBrainzCache from app.schemas.types import MediaRecognizeType, MediaType, ModuleType from app.utils.http import RequestUtils from app.utils.zhconv import convert as zhconv_convert @@ -36,6 +37,8 @@ class MusicBrainzModule(_ModuleBase): _request_interval = 1.0 _request_lock = threading.Lock() _last_request_at = 0.0 + # 本地识别缓存，由模块管理器初始化时挂载 + cache: MusicBrainzCache = None # 全局复用 HTTP 会话：keep-alive 省去每次请求的 DNS+TLS 握手（约 6s → 0.4s） _session: Optional[Session] = None _session_lock = threading.Lock() @@ -72,14 +75,32 @@ class MusicBrainzModule(_ModuleBase): ) def init_module(self) -> None: - """初始化无状态的 MusicBrainz 模块。""" + """初始化 MusicBrainz 模块并挂载本地识别缓存。""" + self.cache = MusicBrainzCache() def init_setting(self) -> Optional[Tuple[str, Union[str, bool]]]: """MusicBrainz 无需独立密钥或启用开关。""" return None def stop(self) -> None: - """停止模块；当前实现没有需要释放的持久资源。""" + """停止模块，退出前持久化识别缓存。""" + if self.cache: + try: + self.cache.save() + except Exception as err: + logger.error(f"保存音乐识别缓存失败：{str(err)}") + + def scheduler_job(self) -> None: + """定时任务，每10分钟持久化一次音乐识别缓存。""" + if self.cache: + self.cache.save() + + def clear_cache(self) -> None: + """响应全局缓存清理事件，清空音乐识别缓存。""" + logger.info("开始清除音乐识别缓存 ...") + if self.cache: + self.cache.clear() + logger.info("音乐识别缓存清除完成") def test(self) -> Tuple[bool, str]: """测试 MusicBrainz 搜索接口连通性。""" @@ -610,11 +631,23 @@ class MusicBrainzModule(_ModuleBase): if source == self._source and mediaid: return self.recognize_music(source, str(mediaid)) return None + # 识别缓存命中直接响应，避免重复搜索占用 MusicBrainz 限流配额 + cache_enabled = bool(kwargs.get("cache", True)) + if cache_enabled and self.cache: + cached_info = self.cache.get(meta) + if cached_info: + if cached_info.media_id: + logger.info(f"{meta.title} 使用音乐识别缓存：{cached_info.title}") + else: + logger.info(f"{meta.title} 使用音乐识别缓存：无法识别") + cached_info.recognize_cache_hit = True + return cached_info # 携带数据源与原生 ID 的请求优先按详情识别 resolved_source = source or meta.media_source if resolved_source and (mediaid or meta.media_id): info = self.recognize_music(resolved_source, str(mediaid or meta.media_id)) if info: + self._update_recognize_cache(meta, info) return info # 无身份时按标题搜索并挑选可信候选，检索不到时返回元数据兑底 # 文件识别只能从 Recording 中挑选，专辑或艺术家同名结果不能成为音轨身份。 @@ -626,7 +659,38 @@ class MusicBrainzModule(_ModuleBase): if not matched and meta.artists: albums = self._search_albums(meta, limit=10) matched = self._select_album_candidate(meta, albums) - return matched or self._info_from_meta(meta) + result = matched or self._info_from_meta(meta) + # 无远端身份的兑底结果同样入缓存，避免批量识别时反复搜索同一文件 + self._update_recognize_cache(meta, result) + return result + + def _update_recognize_cache(self, meta: MetaMusic, info: Optional[MusicInfo]) -> None: + """识别完成后把结果写入本地识别缓存，未挂载缓存时静默跳过。""" + if self.cache: + self.cache.update(meta, info) + + def update_recognize_cache( + self, + meta: MetaBase, + mediainfo: MusicInfo, + ) -> Optional[bool]: + """回填音乐本地识别缓存，共享识别成功后避免重复回查。""" + if not meta or not mediainfo: + return None + if not isinstance(meta, MetaMusic) or not isinstance(mediainfo, MusicInfo): + return None + if mediainfo.source != self._source: + return None + self._update_recognize_cache(meta, mediainfo) + return True + + async def async_update_recognize_cache( + self, + meta: MetaBase, + mediainfo: MusicInfo, + ) -> Optional[bool]: + """异步回填音乐本地识别缓存。""" + return self.update_recognize_cache(meta=meta, mediainfo=mediainfo) async def async_recognize_media( self, diff --git a/app/modules/musicbrainz/music_cache.py b/app/modules/musicbrainz/music_cache.py new file mode 100644 index 000000000..3a4a11db4 --- /dev/null +++ b/app/modules/musicbrainz/music_cache.py @@ -0,0 +1,234 @@ +import pickle +import traceback +from math import ceil +from threading import RLock +from time import time +from typing import Optional + +from app.core.cache import FileCache, TTLCache +from app.core.config import settings +from app.core.context import MusicInfo +from app.core.meta import MetaMusic +from app.log import logger +from app.utils.singleton import WeakSingleton + +lock = RLock() +PERSISTENCE_VERSION = 1 +PERSISTENCE_REGION = "recognize" +PERSISTENCE_KEY = "musicbrainz" + + +class MusicBrainzCache(metaclass=WeakSingleton): + """ + MusicBrainz识别缓存数据 + { + "source": '', + "media_id": '', + "title": '', + "artists": [], + "album": '', + "year": '', + "music_type": '' + } + """ + + def __init__(self): + """初始化音乐识别缓存并恢复未过期的持久化数据。""" + self.maxsize = settings.CONF.musicbrainz + self.ttl = settings.CONF.meta + self.region = "__musicbrainz_cache__" + self._cache = TTLCache(region=self.region, maxsize=self.maxsize, ttl=self.ttl) + self._expires_at: dict[str, float] = {} + self._dirty = False + self._file_cache = None + if not self._cache.is_redis(): + self._file_cache = FileCache(base=settings.CACHE_PATH, ttl=self.ttl) + self._restore() + + def _restore(self) -> None: + """从统一文件缓存恢复仍在有效期内的音乐识别数据。""" + try: + content = self._file_cache.get(PERSISTENCE_KEY, region=PERSISTENCE_REGION) + if not content: + return + payload = pickle.loads(content) + now = time() + if ( + not isinstance(payload, dict) + or payload.get("version") != PERSISTENCE_VERSION + or not isinstance(payload.get("items"), dict) + ): + return + + for key, item in payload["items"].items(): + if not isinstance(item, dict): + self._dirty = True + continue + value = item.get("value") + expires_at = item.get("expires_at") + if not isinstance(value, dict) or not isinstance(expires_at, (int, float)): + self._dirty = True + continue + remaining_ttl = expires_at - now + if remaining_ttl <= 0: + self._dirty = True + continue + self._cache.set(key, value, ttl=ceil(remaining_ttl)) + self._expires_at[key] = expires_at + except Exception as err: + logger.error(f"加载音乐识别缓存失败：{str(err)} - {traceback.format_exc()}") + + def _set(self, key: str, value: dict) -> None: + """写入单条音乐识别缓存并记录其独立过期时间。""" + self._cache.set(key, value) + if not self._cache.is_redis(): + self._expires_at[key] = time() + self.ttl + self._dirty = True + + def clear(self): + """ + 清空所有音乐识别缓存 + """ + with lock: + self._cache.clear() + self._expires_at.clear() + self._dirty = True + self.save(force=True) + + def list_items(self) -> list[dict]: + """ + 返回可供管理界面展示的音乐识别缓存列表。 + """ + with lock: + cache_items = [] + for key, value in self._cache.items(): + if not isinstance(value, dict): + continue + cache_items.append({ + "key": key, + "media_id": value.get("media_id") or "", + "title": value.get("title") or "", + "artists": value.get("artists") or [], + "album": value.get("album") or "", + "year": value.get("year") or "", + "music_type": value.get("music_type") or "recording", + "cover_url": value.get("cover_url") or "", + }) + return sorted(cache_items, key=lambda item: item["key"]) + + @staticmethod + def __get_key(meta: MetaMusic) -> str: + """ + 获取缓存KEY，携带数据源原生 ID 时以 ID 为准身份 + """ + artists = "/".join(meta.artists or []) + return f"[音乐]{meta.media_id or meta.title}-{artists}-{meta.album}-{meta.year}" + + def get(self, meta: MetaMusic) -> Optional[MusicInfo]: + """ + 根据元数据获取缓存的音乐识别结果 + @param meta: 音乐元数据 + @return: 缓存命中的音乐信息，未命中返回 None + """ + key = self.__get_key(meta) + with lock: + cache_data = self._cache.get(key) + if not cache_data and self._expires_at.pop(key, None) is not None: + self._dirty = True + if not cache_data: + return None + try: + return MusicInfo.from_dict(cache_data) + except Exception as err: + logger.error(f"解析音乐识别缓存失败：{str(err)}") + return None + + def delete(self, key: str) -> dict: + """ + 删除缓存信息 + @param key: 缓存key + @return: 被删除的缓存内容 + """ + with lock: + cache_data = self._cache.get(key) + if cache_data: + self._cache.delete(key) + self._expires_at.pop(key, None) + self._dirty = True + self.save(force=True) + return cache_data + return {} + + def update(self, meta: MetaMusic, info: Optional[MusicInfo]) -> None: + """ + 新增或更新缓存条目，无远端身份的兜底结果也写入内存负缓存， + 避免批量识别时反复请求 MusicBrainz 触发限流 + """ + if not meta or not info: + return + key = self.__get_key(meta) + cache_data = info.to_dict() + # 上游原始响应体积大且不参与身份恢复，不入缓存 + cache_data.pop("raw_data", None) + with lock: + self._set(key, cache_data) + + def save(self, force: bool = False) -> None: + """ + 使用统一文件缓存保存未过期的音乐识别数据。 + """ + if self._cache.is_redis(): + return + if not self._file_cache: + return + with lock: + now = time() + cache_items = dict(self._cache.items()) + active_keys = set(cache_items) + stale_keys = set(self._expires_at) - active_keys + if stale_keys: + for key in stale_keys: + self._expires_at.pop(key, None) + self._dirty = True + + persisted_items = {} + for key, value in cache_items.items(): + expires_at = self._expires_at.get(key) + if expires_at is None: + expires_at = now + self.ttl + self._expires_at[key] = expires_at + self._dirty = True + # 负缓存只留在内存，重启后允许重新尝试识别 + if expires_at <= now or not value.get("media_id"): + continue + persisted_items[key] = { + "value": value, + "expires_at": expires_at, + } + + if not force and not self._dirty: + return + + try: + if persisted_items: + payload = { + "version": PERSISTENCE_VERSION, + "items": persisted_items, + } + self._file_cache.set( + PERSISTENCE_KEY, + pickle.dumps(payload, pickle.HIGHEST_PROTOCOL), + region=PERSISTENCE_REGION, + ) + else: + self._file_cache.delete(PERSISTENCE_KEY, region=PERSISTENCE_REGION) + self._dirty = False + except Exception as err: + logger.error(f"保存音乐识别缓存失败：{str(err)} - {traceback.format_exc()}") + + def __del__(self): + """实例释放前保存非 Redis 缓存。""" + try: + self.save() + except Exception: + pass diff --git a/docs/mcp-api.md b/docs/mcp-api.md index 23cc52747..874bad9e7 100644 --- a/docs/mcp-api.md +++ b/docs/mcp-api.md @@ -254,10 +254,15 @@ AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-ch | GET | `/api/v1/tmdb/cache` | 查询 TheMovieDb 识别缓存统计、共享识别累计成功命中次数及开关状态 | | DELETE | `/api/v1/tmdb/cache/{cache_key}` | 按缓存键删除单条 TheMovieDb 识别缓存，缓存键需要进行 URL 编码 | | DELETE | `/api/v1/tmdb/cache` | 清空全部 TheMovieDb 识别缓存 | +| GET | `/api/v1/music/cache` | 查询 MusicBrainz 音乐识别缓存统计及条目列表 | +| DELETE | `/api/v1/music/cache/{cache_key}` | 按缓存键删除单条音乐识别缓存，缓存键需要进行 URL 编码 | +| DELETE | `/api/v1/music/cache` | 清空全部音乐识别缓存 | TMDB 缓存查询响应的 `data` 包含 `count`、`recognized`、`unrecognized`、`data`，以及共享识别统计字段 `shared_recognized` 和开关字段 `shared_recognize_enabled`。共享命中次数仅在共享结果驱动的二次媒体识别成功后累计。 +音乐识别缓存查询响应的 `data` 包含 `count`、`recognized`、`unrecognized` 和 `data`；条目字段包括缓存键、`media_id`、`title`、`artists`、`album`、`year`、`music_type` 和 `cover_url`。未携带远端身份的兜底负缓存仅保留在内存，不参与持久化。 + ### 插件补充接口 **GET** `/api/v1/plugin/history/{plugin_id}` diff --git a/tests/test_indexer_spider_search_url.py b/tests/test_indexer_spider_search_url.py index a1edf871d..d569a9b6e 100644 --- a/tests/test_indexer_spider_search_url.py +++ b/tests/test_indexer_spider_search_url.py @@ -32,6 +32,14 @@ def _get_search_url(indexer: dict, keyword: str | list[str], mtype: MediaType = return spider._SiteSpider__get_search_url() +def _get_browse_url(indexer: dict, mtype: MediaType = None, page: int = 0) -> str: + """ + 调用 SiteSpider 无关键词浏览的 URL 构造逻辑，模拟订阅刷新抓取首页。 + """ + spider = SiteSpider(indexer=indexer, keyword=None, mtype=mtype, page=page) + return spider._SiteSpider__get_search_url() + + def _get_haidan_params(keyword: str | None, mtype: MediaType = None) -> dict: """ 调用 HaiDanSpider 私有参数构造逻辑，避免真实请求站点。 @@ -217,6 +225,64 @@ def test_typed_search_path_falls_back_to_all_path(): assert parsed_url.path == "/torrents.php" +def test_music_browse_uses_dedicated_music_entry(): + """ + 订阅刷新浏览音乐资源时应使用站点的音乐专用入口，而不是默认首页。 + """ + indexer = _build_indexer( + id="hhanclub", + domain="https://hhanclub.net/", + search={ + "paths": [ + {"path": "torrents.php", "type": "all"}, + {"path": "special.php", "type": "music"}, + ], + "params": {"search": "{keyword}"}, + }, + ) + + assert urlparse(_get_browse_url(indexer, MediaType.MUSIC)).path == "/special.php" + # 未指定媒体类型时仍浏览默认首页，保持影视刷新行为不变 + assert urlparse(_get_browse_url(indexer)).path == "/torrents.php" + + +def test_music_browse_overrides_browse_config_path(): + """ + 同时配置 browse 路径和音乐专用路径时，音乐浏览应优先使用专用入口。 + """ + indexer = _build_indexer( + search={ + "paths": [ + {"path": "torrents.php", "type": "all"}, + {"path": "music.php", "type": "music"}, + ], + "params": {"search": "{keyword}"}, + }, + browse={"path": "browse.php"}, + ) + + assert urlparse(_get_browse_url(indexer, MediaType.MUSIC)).path == "/music.php" + assert urlparse(_get_browse_url(indexer)).path == "/browse.php" + + +def test_browse_pagination_appends_existing_query_string(): + """ + 浏览路径自带查询参数时翻页应追加 & 连接符，不能拼出两个问号。 + """ + indexer = _build_indexer( + search={ + "paths": [{"path": "torrents.php?action=advanced&searchstr={keyword}"}], + }, + ) + + browse_url = _get_browse_url(indexer, page=1) + + assert browse_url.count("?") == 1 + query = parse_qs(urlparse(browse_url).query) + assert query["page"] == ["1"] + assert query["action"] == ["advanced"] + + def test_category_item_can_use_distinct_search_parameter_value(): """ DiscuzX 子分类可以用展示分类 ID 解析结果，同时用父分类值构造搜索参数。 diff --git a/tests/test_music_recognize_cache.py b/tests/test_music_recognize_cache.py new file mode 100644 index 000000000..691a54e91 --- /dev/null +++ b/tests/test_music_recognize_cache.py @@ -0,0 +1,436 @@ +"""MusicBrainz 音乐识别本地持久化缓存测试。 + +覆盖缓存键生成、读写回环、负缓存、持久化恢复与保存、管理端点 +统计与权限，以及 MusicBrainzModule 识别流程对缓存的命中与回填。 +""" +import asyncio +import inspect +import pickle +from unittest.mock import Mock + +from app.api.endpoints import music as music_endpoint +from app.core.context import MusicInfo +from app.core.meta import MetaMusic +from app.db.user_oper import get_current_active_superuser_async +from app.modules.musicbrainz import music_cache as music_cache_module +from app.modules.musicbrainz import MusicBrainzModule +from app.modules.musicbrainz.music_cache import MusicBrainzCache + + +class _MemoryCacheStub: + """提供音乐缓存管理测试所需的最小内存后端。""" + + def __init__(self, data: dict): + """使用给定字典初始化测试缓存。""" + self.data = data + + @staticmethod + def is_redis() -> bool: + """测试替身固定使用非 Redis 后端。""" + return False + + def items(self): + """返回全部缓存条目。""" + return self.data.items() + + def get(self, key: str): + """读取指定缓存条目。""" + return self.data.get(key) + + def delete(self, key: str): + """删除指定缓存条目。""" + self.data.pop(key, None) + + def set(self, key: str, value, ttl=None): + """写入指定缓存条目。""" + self.data[key] = value + + def clear(self): + """清空全部缓存条目。""" + self.data.clear() + + +class _TTLCacheStub(_MemoryCacheStub): + """记录每条数据恢复时剩余 TTL 的内存缓存替身。""" + + def __init__(self): + """初始化空缓存和 TTL 记录。""" + super().__init__({}) + self.ttls = {} + + def set(self, key: str, value, ttl=None): + """写入缓存并记录本次设置的 TTL。""" + super().set(key, value, ttl=ttl) + self.ttls[key] = ttl + + +class _FileCacheStub: + """提供音乐缓存持久化测试所需的统一文件缓存替身。""" + + def __init__(self, content: bytes = None): + """使用预置序列化内容初始化文件缓存。""" + self.content = content + self.set_calls = [] + self.delete_calls = [] + + def get(self, key: str, region: str): + """读取预置缓存内容。""" + return self.content + + def set(self, key: str, value: bytes, region: str): + """记录统一文件缓存写入。""" + self.content = value + self.set_calls.append((key, region)) + + def delete(self, key: str, region: str): + """记录统一文件缓存删除。""" + self.content = None + self.delete_calls.append((key, region)) + + +def _build_music_cache(data: dict) -> MusicBrainzCache: + """构造绕过单例初始化的音乐识别缓存测试实例。""" + cache = object.__new__(MusicBrainzCache) + cache.maxsize = 256 + cache.ttl = 3600 + cache.region = "__musicbrainz_cache__" + cache._cache = _MemoryCacheStub(data) + cache._expires_at = {key: float("inf") for key in data} + cache._dirty = False + cache._file_cache = None + cache.save = lambda force=False: None + return cache + + +def _build_initialized_music_cache(monkeypatch, file_cache: _FileCacheStub, + runtime_cache: _TTLCacheStub, + now: float = 1000) -> MusicBrainzCache: + """使用可控时间和缓存替身初始化完整音乐识别缓存实例。""" + monkeypatch.setattr(music_cache_module, "time", lambda: now) + monkeypatch.setattr(music_cache_module, "TTLCache", lambda **kwargs: runtime_cache) + monkeypatch.setattr(music_cache_module, "FileCache", lambda **kwargs: file_cache) + cache = object.__new__(MusicBrainzCache) + cache.__init__() + return cache + + +def _music_info(**kwargs) -> MusicInfo: + """构造标准音乐识别结果，默认携带远端身份。""" + defaults = { + "source": "musicbrainz", + "media_id": "rec-1", + "title": "晴天", + "artists": ["周杰伦"], + "album": "叶惠美", + "year": 2003, + } + defaults.update(kwargs) + return MusicInfo(**defaults) + + +def test_music_cache_endpoints_require_superuser(): + """音乐识别缓存管理接口必须仅允许超级管理员访问。""" + endpoints = [ + music_endpoint.music_recognition_cache, + music_endpoint.delete_music_recognition_cache, + music_endpoint.clear_music_recognition_cache, + ] + + for endpoint in endpoints: + dependency = inspect.signature(endpoint).parameters["_"].default.dependency + assert dependency is get_current_active_superuser_async + + +def test_music_cache_key_prefers_media_id(): + """携带数据源原生 ID 的元数据应以 ID 作为缓存键主身份。""" + cache = _build_music_cache({}) + meta = MetaMusic(title="晴天", artists=["周杰伦"], media_id="rec-1") + + cache.update(meta, _music_info()) + + # 缓存键取自请求元数据，携带原生 ID 时以 ID 为主身份 + assert list(cache._cache.data.keys()) == ["[音乐]rec-1-周杰伦-None-None"] + + +def test_music_cache_update_and_get_roundtrip(): + """识别结果入缓存后应能还原出标准音乐信息，且不保存上游原始响应。""" + cache = _build_music_cache({}) + meta = MetaMusic(title="晴天", artists=["周杰伦"], album="叶惠美", year=2003) + info = _music_info(raw_data={"payload": "large"}) + + cache.update(meta, info) + hit = cache.get(meta) + + assert hit is not None + assert hit.media_id == "rec-1" + assert hit.title == "晴天" + assert hit.artists == ["周杰伦"] + stored = next(iter(cache._cache.data.values())) + assert "raw_data" not in stored + + +def test_music_cache_get_miss_returns_none(): + """未命中的缓存查询应返回 None 而不是抛错。""" + cache = _build_music_cache({}) + meta = MetaMusic(title="未知曲目") + + assert cache.get(meta) is None + + +def test_music_cache_list_items_normalizes_and_sorts(): + """管理列表应输出稳定顺序和前端展示所需字段。""" + cache = _build_music_cache({ + "z-key": { + "media_id": "rec-2", + "title": "Zulu", + "artists": ["歌手B"], + "album": "专辑B", + "year": 2024, + "music_type": "recording", + }, + "a-key": { + "media_id": "", + "title": "Alpha", + "year": 2023, + }, + }) + + items = cache.list_items() + + assert [item["key"] for item in items] == ["a-key", "z-key"] + assert items[0]["media_id"] == "" + assert items[0]["artists"] == [] + assert items[0]["music_type"] == "recording" + assert items[1]["artists"] == ["歌手B"] + + +def test_music_cache_delete_and_clear_persist_immediately(monkeypatch): + """管理操作应修改运行时缓存并立即触发本地持久化。""" + cache = _build_music_cache({"first": {"media_id": "rec-1"}, "second": {"media_id": "rec-2"}}) + saved_forces = [] + monkeypatch.setattr(cache, "save", lambda force=False: saved_forces.append(force)) + + assert cache.delete("first") == {"media_id": "rec-1"} + assert cache.delete("missing") == {} + cache.clear() + + assert cache.list_items() == [] + assert saved_forces == [True, True] + + +def test_music_cache_restores_only_unexpired_persisted_items(monkeypatch): + """持久化恢复应保留每条数据原有期限并跳过已过期条目。""" + payload = { + "version": music_cache_module.PERSISTENCE_VERSION, + "items": { + "fresh": { + "value": {"media_id": "rec-1", "title": "有效"}, + "expires_at": 1030, + }, + "expired": { + "value": {"media_id": "rec-2", "title": "过期"}, + "expires_at": 999, + }, + }, + } + file_cache = _FileCacheStub(pickle.dumps(payload)) + runtime_cache = _TTLCacheStub() + + cache = _build_initialized_music_cache( + monkeypatch=monkeypatch, + file_cache=file_cache, + runtime_cache=runtime_cache, + ) + + assert runtime_cache.data == {"fresh": {"media_id": "rec-1", "title": "有效"}} + assert runtime_cache.ttls == {"fresh": 30} + assert cache._expires_at == {"fresh": 1030} + + +def test_music_cache_persists_only_items_with_media_id(monkeypatch): + """持久化应跳过未识别的负缓存条目，只保存携带远端身份的结果。""" + file_cache = _FileCacheStub() + runtime_cache = _TTLCacheStub() + cache = _build_initialized_music_cache( + monkeypatch=monkeypatch, + file_cache=file_cache, + runtime_cache=runtime_cache, + ) + runtime_cache.data = { + "recognized": {"media_id": "rec-1", "title": "晴天"}, + "negative": {"media_id": "", "title": "未知曲目"}, + } + cache._expires_at = { + "recognized": 1060, + "negative": 1070, + } + cache._dirty = True + + cache.save() + + payload = pickle.loads(file_cache.content) + assert file_cache.set_calls == [( + music_cache_module.PERSISTENCE_KEY, + music_cache_module.PERSISTENCE_REGION, + )] + assert payload == { + "version": music_cache_module.PERSISTENCE_VERSION, + "items": { + "recognized": { + "value": {"media_id": "rec-1", "title": "晴天"}, + "expires_at": 1060, + }, + }, + } + + +def test_music_cache_save_removes_file_when_empty(monkeypatch): + """全部条目失效后保存应删除持久化文件。""" + file_cache = _FileCacheStub(pickle.dumps({"version": 1, "items": {}})) + runtime_cache = _TTLCacheStub() + cache = _build_initialized_music_cache( + monkeypatch=monkeypatch, + file_cache=file_cache, + runtime_cache=runtime_cache, + ) + cache._dirty = True + + cache.save() + + assert file_cache.delete_calls == [( + music_cache_module.PERSISTENCE_KEY, + music_cache_module.PERSISTENCE_REGION, + )] + + +def test_music_cache_endpoint_returns_management_statistics(monkeypatch): + """查询接口应返回识别成功和失败条目的统计。""" + cache = _build_music_cache({ + "recognized": {"media_id": "rec-1", "title": "晴天"}, + "unrecognized": {"media_id": "", "title": "未知曲目"}, + }) + monkeypatch.setattr(music_endpoint, "MusicBrainzCache", lambda: cache) + + response = asyncio.run(music_endpoint.music_recognition_cache(None)) + + assert response.success is True + assert response.data["count"] == 2 + assert response.data["recognized"] == 1 + assert response.data["unrecognized"] == 1 + assert len(response.data["data"]) == 2 + + +def test_music_cache_delete_endpoint_reports_missing_item(monkeypatch): + """删除接口应区分成功删除与缓存不存在。""" + cache = _build_music_cache({"existing": {"media_id": "rec-1"}}) + monkeypatch.setattr(music_endpoint, "MusicBrainzCache", lambda: cache) + + deleted_response = asyncio.run( + music_endpoint.delete_music_recognition_cache("existing", None) + ) + missing_response = asyncio.run( + music_endpoint.delete_music_recognition_cache("missing", None) + ) + + assert deleted_response.success is True + assert missing_response.success is False + + +def test_music_cache_clear_endpoint_removes_all_items(monkeypatch): + """清空接口应删除全部音乐识别缓存。""" + cache = _build_music_cache({"existing": {"media_id": "rec-1"}}) + monkeypatch.setattr(music_endpoint, "MusicBrainzCache", lambda: cache) + + response = asyncio.run(music_endpoint.clear_music_recognition_cache(None)) + + assert response.success is True + assert cache.list_items() == [] + + +def _build_module_with_cache(cache: MusicBrainzCache) -> MusicBrainzModule: + """构造挂载测试缓存的 MusicBrainz 模块实例。""" + module = MusicBrainzModule() + module.cache = cache + return module + + +def test_module_recognize_media_hits_cache_without_search(monkeypatch): + """识别缓存命中时应直接返回缓存结果，不再触发 MusicBrainz 搜索。""" + cache = _build_music_cache({}) + module = _build_module_with_cache(cache) + meta = MetaMusic(title="晴天", artists=["周杰伦"], album="叶惠美", year=2003) + cache.update(meta, _music_info()) + search_mock = Mock() + monkeypatch.setattr(module, "_search_recordings", search_mock) + + result = module.recognize_media(meta=meta) + + assert result is not None + assert result.media_id == "rec-1" + assert getattr(result, "recognize_cache_hit") is True + search_mock.assert_not_called() + + +def test_module_recognize_media_bypasses_cache_when_disabled(monkeypatch): + """cache=False 时不读取缓存，重新走搜索识别流程。""" + cache = _build_music_cache({}) + module = _build_module_with_cache(cache) + meta = MetaMusic(title="晴天", artists=["周杰伦"], album="叶惠美", year=2003) + cache.update(meta, _music_info()) + fresh = _music_info(media_id="rec-2") + monkeypatch.setattr(module, "_search_recordings", Mock(return_value=[fresh])) + monkeypatch.setattr(module, "_select_candidate", Mock(return_value=fresh)) + + result = module.recognize_media(meta=meta, cache=False) + + assert result is fresh + assert getattr(result, "recognize_cache_hit", False) is False + + +def test_module_recognize_media_writes_search_result_to_cache(monkeypatch): + """搜索识别成功后应回填本地识别缓存。""" + cache = _build_music_cache({}) + module = _build_module_with_cache(cache) + meta = MetaMusic(title="晴天", artists=["周杰伦"], album="叶惠美", year=2003) + matched = _music_info() + monkeypatch.setattr(module, "_search_recordings", Mock(return_value=[matched])) + monkeypatch.setattr(module, "_select_candidate", Mock(return_value=matched)) + + result = module.recognize_media(meta=meta) + + assert result is matched + hit = cache.get(meta) + assert hit is not None + assert hit.media_id == "rec-1" + + +def test_module_recognize_media_caches_offline_fallback(monkeypatch): + """搜索无结果的兜底信息也应进入负缓存，避免重复请求。""" + cache = _build_music_cache({}) + module = _build_module_with_cache(cache) + meta = MetaMusic(title="未知曲目", artists=["未知艺术家"]) + monkeypatch.setattr(module, "_search_recordings", Mock(return_value=[])) + monkeypatch.setattr(module, "_search_albums", Mock(return_value=[])) + + result = module.recognize_media(meta=meta) + + assert result is not None + assert result.media_id is None + cached = cache.get(meta) + assert cached is not None + assert cached.media_id is None + assert cached.title == "未知曲目" + + +def test_module_update_recognize_cache_only_for_musicbrainz_music(): + """共享识别回填仅处理本数据源的音乐结果。""" + cache = _build_music_cache({}) + module = _build_module_with_cache(cache) + meta = MetaMusic(title="晴天", artists=["周杰伦"]) + + assert module.update_recognize_cache(meta=meta, mediainfo=_music_info()) is True + assert cache.get(meta) is not None + + other_source = _music_info(source="other", media_id="x-1") + assert module.update_recognize_cache(meta=meta, mediainfo=other_source) is None + assert module.update_recognize_cache(meta=None, mediainfo=_music_info()) is None diff --git a/tests/test_music_subscribe.py b/tests/test_music_subscribe.py index 53379a993..87e250f90 100644 --- a/tests/test_music_subscribe.py +++ b/tests/test_music_subscribe.py @@ -593,3 +593,21 @@ def test_follow_preserves_album_entity_and_track_count(): assert subscribe_oper.exist_history.call_args.kwargs["music_type"] == MUSIC_ENTITY_ALBUM assert add.call_args.kwargs["music_type"] == MUSIC_ENTITY_ALBUM assert add.call_args.kwargs["total_tracks"] == 11 + + +def test_refresh_enables_music_entry_fetch_when_music_subscribe_exists(): + """存在音乐订阅时，订阅刷新应要求种子链额外抓取站点音乐专用入口。""" + chain = SubscribeChain() + subscribe_oper = Mock() + # get_subscribed_sites 不带状态查询，has_music_subscribe 按可搜索状态查询 + subscribe_oper.list.side_effect = lambda state=None: [_subscribe(state="R")] + torrents_chain = Mock() + torrents_chain.refresh.return_value = {} + + with patch("app.chain.subscribe.SubscribeOper", return_value=subscribe_oper), \ + patch("app.chain.subscribe.SystemConfigOper") as system_config, \ + patch("app.chain.subscribe.TorrentsChain", return_value=torrents_chain): + system_config.return_value.get.return_value = [] + chain.refresh() + + assert torrents_chain.refresh.call_args.kwargs["include_music"] is True diff --git a/tests/test_music_torrents.py b/tests/test_music_torrents.py index 337eb952b..1597d77a9 100644 --- a/tests/test_music_torrents.py +++ b/tests/test_music_torrents.py @@ -1,9 +1,12 @@ import asyncio +import copy +from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch from app.chain.torrents import TorrentsChain from app.core.meta import MetaMusic -from app.core.context import MusicInfo +from app.core.context import Context, MusicInfo, TorrentInfo +from app.modules.indexer.spider import SiteSpider from app.schemas.types import MediaType @@ -67,3 +70,274 @@ def test_music_cache_context_uses_music_models(): assert context.media_info.title == "Get Lucky" assert context.candidate_recognized is False media_chain.assert_not_called() + + +def test_rss_sets_music_category_from_site_media_type(): + """RSS 报文不带分类，音乐站点的种子应按站点媒体类型补充音乐分类。""" + chain = TorrentsChain() + site = { + "id": 9, + "name": "音乐站", + "domain": "https://music.example.com", + "media_type": "music", + "rss": "https://music.example.com/rss.php?passkey=key", + "proxy": False, + "timeout": 30, + "ua": None, + } + rss_items = [{ + "title": "Daft Punk - Random Access Memories [FLAC]", + "enclosure": "https://music.example.com/download.php?id=1", + "link": "https://music.example.com/details.php?id=1", + "size": 1024, + "pubdate": None, + }] + sites_helper = Mock() + sites_helper.get_indexer.return_value = site + + with ( + patch("app.chain.torrents.SitesHelper", return_value=sites_helper), + patch("app.chain.torrents.RssHelper") as rss_helper, + ): + rss_helper.return_value.parse.return_value = rss_items + torrents = chain.rss("music.example.com") + + assert len(torrents) == 1 + assert torrents[0].category == MediaType.MUSIC.value + + +def test_music_browse_paths_detects_dedicated_entry(): + """站点用 type=music 声明的独立音乐入口应被识别，默认入口已覆盖时不重复抓取。""" + mixed_site = { + "search": { + "paths": [ + {"path": "torrents.php", "type": "all"}, + {"path": "special.php", "type": "music"}, + ] + } + } + assert TorrentsChain._music_browse_paths(mixed_site) == ["special.php"] + + # 音乐站点全站都是音乐，无需额外入口 + music_site = {"media_type": "music", "search": {"paths": [ + {"path": "torrents.php", "type": "all"}, + {"path": "torrents.php", "type": "music"}, + ]}} + assert TorrentsChain._music_browse_paths(music_site) == [] + + # 音乐入口与默认入口相同无需重复抓取 + same_entry_site = {"search": {"paths": [ + {"path": "torrents.php", "type": "all"}, + {"path": "torrents.php", "type": "music"}, + ]}} + assert TorrentsChain._music_browse_paths(same_entry_site) == [] + + +def test_spider_music_entry_browse_resolves_music_category(): + """音乐入口浏览应请求专用页面，并把音乐分类种子解析为音乐类型。""" + indexer = { + "id": "hhanclub", + "name": "憨憨", + "domain": "https://hhanclub.net/", + "search": { + "paths": [ + {"path": "torrents.php", "type": "all"}, + {"path": "special.php", "type": "music"}, + ], + "params": {"search": "{keyword}"}, + }, + "category": { + "param": "cat", + "movie": [{"id": "401", "cat": "Movies"}], + "music": [{"id": "410", "cat": "Music"}], + }, + "torrents": { + "list": {"selector": "table tr.torrent"}, + "fields": { + "title": {"selector": "a.t"}, + "category": { + "selector": "a.c", + "attribute": "href", + "filters": [{"name": "querystring", "args": "cat"}], + }, + "download": {"selector": "a.d", "attribute": "href"}, + }, + }, + } + html = """ + + + + + + +
MusicDaft Punk - Random Access Memories [FLAC]下载
+ """ + request_utils = Mock() + request_utils.return_value.get_res.return_value = Mock() + request_utils.get_decoded_html_content.return_value = html + + with ( + patch("app.modules.indexer.spider.RequestUtils", request_utils), + patch( + "app.modules.indexer.spider.rust_accel.parse_indexer_torrents", + return_value=None, + ), + ): + torrents = SiteSpider(indexer=indexer, mtype=MediaType.MUSIC).get_torrents() + + # 请求必须命中音乐专用入口而不是默认首页 + requested_url = request_utils.return_value.get_res.call_args[0][0] + assert "/special.php" in requested_url + assert len(torrents) == 1 + assert torrents[0]["category"] == MediaType.MUSIC.value + assert torrents[0]["title"] == "Daft Punk - Random Access Memories [FLAC]" + + +def test_refresh_include_music_fetches_music_entry(): + """存在音乐订阅时，spider 刷新应额外抓取音乐专用入口，并把音乐写入独立缓存。""" + chain = TorrentsChain() + default_torrent = TorrentInfo( + site=1, site_name="Test", + title="Some.Movie.2026.1080p", + enclosure="https://example.com/download?id=1", + category=MediaType.MOVIE.value, + pubdate="2026-08-07 00:00:00", + ) + music_torrent = TorrentInfo( + site=1, site_name="Test", + title="Daft Punk - Get Lucky [FLAC]", + enclosure="https://example.com/download?id=2", + category=MediaType.MUSIC.value, + pubdate="2026-08-08 00:00:00", + ) + duplicated_torrent = TorrentInfo( + site=1, site_name="Test", + title="Daft Punk - Get Lucky [FLAC]", + enclosure="https://example.com/download?id=2", + category=MediaType.MUSIC.value, + pubdate="2026-08-08 00:00:00", + ) + + def _fake_browse(domain, keyword=None, cat=None, page=None, mtype=None): + if mtype == MediaType.MUSIC: + # 第二页返回空，验证分页提前结束；首页返回音乐种子 + return [] if page else [music_torrent, duplicated_torrent] + return [default_torrent] if not page else [] + + sites_helper = Mock() + sites_helper.get_indexers.return_value = [{ + "id": 1, + "name": "Test", + "domain": "https://example.com", + "search": { + "paths": [ + {"path": "torrents.php", "type": "all"}, + {"path": "special.php", "type": "music"}, + ] + }, + }] + # 保存时对入参做快照，避免后续合并返回值修改同一字典引用 + saved = {} + save_cache = Mock(side_effect=lambda data, filename: saved.__setitem__(filename, copy.deepcopy(data))) + + with ( + patch.object(chain, "load_cache", return_value=None), + patch.object(chain, "browse", side_effect=_fake_browse), + patch.object(chain, "save_cache", save_cache), + patch("app.chain.torrents.SitesHelper", return_value=sites_helper), + patch("app.chain.torrents.MediaChain"), + ): + result = chain.refresh(stype="spider", sites=[1], include_music=True) + + # 返回值供订阅匹配，包含影视与音乐完整候选且去重 + contexts = result["example.com"] + titles = {context.torrent_info.title for context in contexts} + assert titles == {"Some.Movie.2026.1080p", "Daft Punk - Get Lucky [FLAC]"} + music_context = next( + context for context in contexts + if context.torrent_info.title.startswith("Daft Punk") + ) + assert isinstance(music_context.meta_info, MetaMusic) + assert isinstance(music_context.media_info, MusicInfo) + # 音乐不应经过影视识别链的媒体回填 + assert music_context.candidate_recognized is False + + # 影视与音乐分别写入各自缓存文件，音乐不占用影视缓存空间 + assert set(saved) == {TorrentsChain._spider_file, TorrentsChain._music_spider_file} + video_titles = { + context.torrent_info.title for context in saved[TorrentsChain._spider_file]["example.com"] + } + music_titles = { + context.torrent_info.title for context in saved[TorrentsChain._music_spider_file]["example.com"] + } + assert video_titles == {"Some.Movie.2026.1080p"} + assert music_titles == {"Daft Punk - Get Lucky [FLAC]"} + + +def test_music_cache_not_evicted_by_video_torrents(): + """影视缓存按配额裁剪时，音乐独立缓存中的资源不应被挤出。""" + chain = TorrentsChain() + + def _music_context(title, enclosure, pubdate): + torrent = TorrentInfo( + site=1, site_name="Test", title=title, enclosure=enclosure, + category=MediaType.MUSIC.value, pubdate=pubdate, + ) + return Context( + meta_info=MetaMusic(org_string=title, title=title), + media_info=MusicInfo(title=title), + torrent_info=torrent, + ) + + existing_music = [ + _music_context(f"Album {i}", f"https://example.com/download?id=m{i}", + f"2026-08-0{i + 1} 00:00:00") + for i in range(2) + ] + video_torrents = [ + TorrentInfo( + site=1, site_name="Test", title=f"Movie.{i}.1080p", + enclosure=f"https://example.com/download?id=v{i}", + category=MediaType.MOVIE.value, pubdate=f"2026-08-1{i} 00:00:00", + ) + for i in range(3) + ] + + def _fake_load(filename): + if filename == TorrentsChain._music_spider_file: + return {"example.com": existing_music} + return None + + def _fake_browse(domain, keyword=None, cat=None, page=None, mtype=None): + return video_torrents if not page else [] + + sites_helper = Mock() + sites_helper.get_indexers.return_value = [{ + "id": 1, "name": "Test", "domain": "https://example.com", + }] + saved = {} + save_cache = Mock(side_effect=lambda data, filename: saved.__setitem__(filename, copy.deepcopy(data))) + fake_settings = Mock() + # 公共参数：缓存上限 2，刷新配额 5，音乐与影视各自独立计算 + fake_settings.CONF = SimpleNamespace(torrents=2, refresh=5) + fake_settings.NO_CACHE_SITE_KEY = "no-cache-site.invalid" + + with ( + patch("app.chain.torrents.settings", fake_settings), + patch.object(chain, "load_cache", side_effect=_fake_load), + patch.object(chain, "browse", side_effect=_fake_browse), + patch.object(chain, "save_cache", save_cache), + patch("app.chain.torrents.SitesHelper", return_value=sites_helper), + patch("app.chain.torrents.MediaChain"), + ): + chain.refresh(stype="spider", sites=[1]) + + # 影视缓存独立按配额裁剪，仅保留最新的两条 + assert len(saved[TorrentsChain._spider_file]["example.com"]) == 2 + # 音乐独立缓存不受影视种子大量涌入影响，既有的两条音乐完整保留 + music_titles = { + context.torrent_info.title + for context in saved[TorrentsChain._music_spider_file]["example.com"] + } + assert music_titles == {"Album 0", "Album 1"}