diff --git a/app/chain/torrents.py b/app/chain/torrents.py index c6e07365c..ce0a71e81 100644 --- a/app/chain/torrents.py +++ b/app/chain/torrents.py @@ -526,6 +526,118 @@ class TorrentsChain(ChainBase): torrents.append(torrent) return torrents + def _refresh_indexer( + self, + indexer: dict, + stype: str, + include_music: bool, + torrents_cache: Dict[str, List[Context]], + music_cache: Dict[str, List[Context]], + ) -> str: + """抓取并写入单个站点的影视、音乐资源缓存。""" + domain = site_rules.extract_domain(indexer.get("domain")) + if stype == "spider": + torrents: List[TorrentInfo] = [] + for page in range(2): + page_torrents = self.browse(domain=domain, page=page) + if not page_torrents: + break + torrents.extend(page_torrents) + else: + torrents = self.rss(domain=domain) + if include_music and self._music_browse_paths(indexer): + torrents = self.__append_music_browse_torrents(domain=domain, torrents=torrents) + torrents.sort(key=lambda item: item.pubdate or "", reverse=True) + music_torrents = [ + item for item in torrents if item.category == MediaType.MUSIC.value + ][:self.runtime_config.refresh_batch_size] + torrents = [ + item for item in torrents if item.category != MediaType.MUSIC.value + ][:self.runtime_config.refresh_batch_size] + if not torrents and not music_torrents: + logger.info(f'{indexer.get("name")} 没有获取到种子') + return domain + if self._is_no_cache_site(domain): + logger.info( + f'{indexer.get("name")} 有 {len(torrents) + len(music_torrents)} 个种子 (不缓存)' + ) + torrents_cache[domain] = [] + music_cache[domain] = [] + else: + cached_signatures = { + f'{item.torrent_info.title}{item.torrent_info.description}' + for item in torrents_cache.get(domain) or [] + } + torrents = [ + item for item in torrents + if f'{item.title}{item.description}' not in cached_signatures + ] + music_signatures = { + f'{item.torrent_info.title}{item.torrent_info.description}' + for item in music_cache.get(domain) or [] + } + music_torrents = [ + item for item in music_torrents + if f'{item.title}{item.description}' not in music_signatures + ] + if not torrents and not music_torrents: + logger.info(f'{indexer.get("name")} 没有新种子') + return domain + logger.info(f'{indexer.get("name")} 有 {len(torrents) + len(music_torrents)} 个新种子') + for torrent in torrents + music_torrents: + if global_vars.is_system_stopped: + break + if not torrent.enclosure: + logger.warning(f"缺少种子链接，忽略处理: {torrent.title}") + continue + context = self._build_refresh_context(torrent, stype) + target_cache = music_cache if torrent.category == MediaType.MUSIC.value else torrents_cache + target_cache.setdefault(domain, []).append(context) + if len(target_cache[domain]) > self.runtime_config.torrent_cache_size: + target_cache[domain] = target_cache[domain][-self.runtime_config.torrent_cache_size:] + return domain + + def _is_no_cache_site(self, domain: str) -> bool: + """判断站点是否配置为不缓存资源。""" + return any(key in domain for key in self.runtime_config.no_cache_site_key.split(",")) + + def _build_refresh_context(self, torrent: TorrentInfo, stype: str) -> Context: + """识别单个种子并构造缓存上下文。""" + logger.info(f'处理资源：{torrent.title} ...') + if torrent.category == MediaType.MUSIC.value: + meta = MetaMusic.parse_query(torrent.title) + mediainfo = MusicInfo( + title=meta.title, + artists=list(meta.artists), + album=meta.album, + year=meta.year, + names=[meta.title] if meta.title else [], + ) + candidate_recognized = False + match_source = "unknown" + else: + meta = MetaInfo(title=torrent.title, subtitle=torrent.description) + if torrent.title != meta.org_string: + logger.info(f'种子名称应用识别词后发生改变：{torrent.title} => {meta.org_string}') + if meta.type != MediaType.TV and torrent.category == MediaType.TV.value: + meta.type = MediaType.TV + mediainfo = MediaChain().recognize_by_meta(meta, obtain_images=False) or MediaInfo() + mediainfo.clear() + candidate_recognized = bool(mediainfo and all(resolve_media_identity(media=mediainfo))) + match_source = self._get_media_id_match_source(mediainfo) + context = Context( + meta_info=meta, + media_info=mediainfo, + torrent_info=torrent, + resource_source="spider" if stype == "spider" else "rss", + match_source=match_source if candidate_recognized else "unknown", + candidate_recognized=candidate_recognized, + media_info_is_target=False, + ) + if not mediainfo or not all(resolve_media_identity(media=mediainfo)): + context.media_recognize_fail_count = 1 + return context + def refresh( self, stype: Optional[str] = None, @@ -541,15 +653,6 @@ class TorrentsChain(ChainBase): :param include_music: 是否额外抓取站点的音乐专用浏览入口，服务音乐订阅 """ - def __is_no_cache_site(_domain: str) -> bool: - """ - 判断站点是否不需要缓存 - """ - for url_key in self.runtime_config.no_cache_site_key.split(','): - if url_key in _domain: - return True - return False - # 刷新类型 if not stype: stype = self.runtime_config.subscribe_mode @@ -594,145 +697,20 @@ class TorrentsChain(ChainBase): if progress_callback: progress_callback( value=(index - 1) / total_indexers * 100 if total_indexers else 100, - text=( - f"正在刷新站点资源（{index}/{total_indexers}）" - f"{indexer.get('name')} ..." - ), + text=f"正在刷新站点资源（{index}/{total_indexers}）{indexer.get('name')} ...", data={ "total": total_indexers, "finished": index - 1, "current": indexer.get("id"), }, ) - domain = site_rules.extract_domain(indexer.get("domain")) - domains.append(domain) - if stype == "spider": - # 刷新首页种子 - torrents: List[TorrentInfo] = [] - # 读取第0页和第1页 - for page in range(2): - page_torrents = self.browse(domain=domain, page=page) - if page_torrents: - torrents.extend(page_torrents) - else: - # 如果某一页没有数据，说明已经到最后一页，停止获取 - break - # 存在音乐订阅时，默认首页可能不包含音乐资源，需要额外抓取音乐专用入口 - if include_music and self._music_browse_paths(indexer): - torrents = self.__append_music_browse_torrents( - domain=domain, torrents=torrents - ) - else: - # 刷新RSS种子 - torrents: List[TorrentInfo] = self.rss(domain=domain) - # 混合站点的 RSS 通常不提供媒体分类；有音乐订阅时补抓专用入口， - # 后续仍按与 spider 相同的独立缓存和去重规则处理。 - if include_music and self._music_browse_paths(indexer): - torrents = self.__append_music_browse_torrents( - domain=domain, torrents=torrents - ) - # 按pubdate降序排列 - torrents.sort(key=lambda x: x.pubdate or '', reverse=True) - # 音乐与影视按同一公共参数独立计算刷新配额，并分别写入各自缓存，音乐不会被影视资源挤出 - music_torrents = [ - t for t in torrents if t.category == MediaType.MUSIC.value - ][:self.runtime_config.refresh_batch_size] - torrents = [ - t for t in torrents if t.category != MediaType.MUSIC.value - ][:self.runtime_config.refresh_batch_size] - if torrents or music_torrents: - if __is_no_cache_site(domain): - # 不需要缓存的站点，直接处理 - logger.info(f'{indexer.get("name")} 有 {len(torrents) + len(music_torrents)} 个种子 (不缓存)') - torrents_cache[domain] = [] - music_cache[domain] = [] - else: - # 过滤出没有处理过的种子 - 优化：使用集合查找，避免重复创建字符串列表 - cached_signatures = {f'{t.torrent_info.title}{t.torrent_info.description}' - for t in torrents_cache.get(domain) or []} - torrents = [torrent for torrent in torrents - if f'{torrent.title}{torrent.description}' not in cached_signatures] - # 音乐种子对照音乐独立缓存去重 - music_signatures = {f'{t.torrent_info.title}{t.torrent_info.description}' - for t in music_cache.get(domain) or []} - music_torrents = [torrent for torrent in music_torrents - if f'{torrent.title}{torrent.description}' not in music_signatures] - if torrents or music_torrents: - logger.info(f'{indexer.get("name")} 有 {len(torrents) + len(music_torrents)} 个新种子') - else: - logger.info(f'{indexer.get("name")} 没有新种子') - continue - try: - for torrent in torrents + music_torrents: - if global_vars.is_system_stopped: - break - if not torrent.enclosure: - logger.warn(f"缺少种子链接，忽略处理: {torrent.title}") - continue - logger.info(f'处理资源：{torrent.title} ...') - if torrent.category == MediaType.MUSIC.value: - meta = MetaMusic.parse_query(torrent.title) - mediainfo = MusicInfo( - title=meta.title, - artists=list(meta.artists), - album=meta.album, - year=meta.year, - names=[meta.title] if meta.title else [], - ) - candidate_recognized = False - match_source = "unknown" - else: - meta = MetaInfo(title=torrent.title, subtitle=torrent.description) - if torrent.title != meta.org_string: - logger.info(f'种子名称应用识别词后发生改变：{torrent.title} => {meta.org_string}') - # 使用站点种子分类，校正类型识别 - if meta.type != MediaType.TV \ - and torrent.category == MediaType.TV.value: - meta.type = MediaType.TV - mediainfo = MediaChain().recognize_by_meta( - meta, - obtain_images=False, - ) - if not mediainfo: - logger.warn(f'{torrent.title} 未识别到媒体信息') - mediainfo = MediaInfo() - mediainfo.clear() - candidate_recognized = bool( - mediainfo and all(resolve_media_identity(media=mediainfo)) - ) - match_source = self._get_media_id_match_source(mediainfo) - # 上下文 - context = Context( - meta_info=meta, - media_info=mediainfo, - torrent_info=torrent, - resource_source="spider" if stype == "spider" else "rss", - match_source=match_source if candidate_recognized else "unknown", - candidate_recognized=candidate_recognized, - media_info_is_target=False, - ) - # 如果未识别到媒体信息，设置初始失败次数为1 - if not mediainfo or not all(resolve_media_identity(media=mediainfo)): - context.media_recognize_fail_count = 1 - # 添加到缓存：音乐进入独立缓存，与影视分开存储 - if torrent.category == MediaType.MUSIC.value: - target_cache = music_cache - else: - target_cache = torrents_cache - if not target_cache.get(domain): - target_cache[domain] = [context] - else: - target_cache[domain].append(context) - # 如果超过了限制条数则移除掉前面的，音乐与影视各自独立计算配额 - if len(target_cache[domain]) > self.runtime_config.torrent_cache_size: - target_cache[domain] = target_cache[domain][-self.runtime_config.torrent_cache_size:] - finally: - torrents.clear() - music_torrents.clear() - del torrents - del music_torrents - else: - logger.info(f'{indexer.get("name")} 没有获取到种子') + domains.append(self._refresh_indexer( + indexer=indexer, + stype=stype, + include_music=include_music, + torrents_cache=torrents_cache, + music_cache=music_cache, + )) # 保存缓存到本地，影视与音乐分别存储 if stype == "spider": diff --git a/docs/refactor/backend-architecture-next-stage.md b/docs/refactor/backend-architecture-next-stage.md index c6ad95d28..5a4a1a36b 100644 --- a/docs/refactor/backend-architecture-next-stage.md +++ b/docs/refactor/backend-architecture-next-stage.md @@ -971,6 +971,8 @@ Outbox adapter、DB 装饰器、Base 与 UoW，strict 清单扩大到 37 个源 对应 `mcp_jsonrpc`、`download.add`、`DownloadChain.get_no_exists_info` 退出超限清单，总债务从 28 降到 25。 - 2026-08-22 将 `SiteChain.sync_cookies` 拆为单域名处理、黑名单判断、索引器地址解析和连接重试阶段，入口降至预算内； 保留已有站点健康、黑名单、失败重试时的事件与进度回调语义，站点专项测试通过。 +- 继续将 `TorrentsChain.refresh` 拆为单站点抓取、上下文构造和缓存写入阶段，入口退出超限清单； + 音乐双缓存、去重、停止信号和订阅匹配专项测试通过，当前复杂度债务由 25 项降至 21 项。 配置债务继续按模块族收敛：`app/application/image.py` 的壁纸模式、图片缓存、代理和安全后缀读取已接入 `ChainRuntimeConfig`，canonical `settings` 直接读取文件数从 137 降至 136；配置/依赖基线已更新，壁纸与图片专项测试通过。 diff --git a/tests/fixtures/architecture/complexity-baseline.json b/tests/fixtures/architecture/complexity-baseline.json index 834625c2e..2ddb39da3 100644 --- a/tests/fixtures/architecture/complexity-baseline.json +++ b/tests/fixtures/architecture/complexity-baseline.json @@ -2,8 +2,8 @@ "api_endpoint": { "app/api/endpoints/agent.py:web_agent_stream": 346, "app/api/endpoints/media.py:scrape": 105, - "app/api/endpoints/openai.py:chat_completions": 107, - "app/api/endpoints/openai.py:responses": 105, + "app/api/endpoints/openai.py:chat_completions": 95, + "app/api/endpoints/openai.py:responses": 93, "app/api/endpoints/system.py:get_logging": 111, "app/api/endpoints/system.py:nettest": 87, "app/api/endpoints/torrent.py:reidentify_cache": 147 @@ -20,14 +20,10 @@ "app/chain/download.py:DownloadChain.batch_download": 572, "app/chain/download.py:DownloadChain.download_single": 255, "app/chain/mediaserver.py:MediaServerChain.sync": 292, - "app/chain/site.py:SiteChain.sync_cookies": 180, "app/chain/subscribe.py:SubscribeChain.add": 183, "app/chain/subscribe.py:SubscribeChain.async_add": 186, - "app/chain/subscribe.py:SubscribeChain.match": 417, - "app/chain/subscribe.py:SubscribeChain.search": 249, - "app/chain/subscribe.py:SubscribeChain.subscribe_files_info": 199, - "app/chain/torrents.py:TorrentsChain.refresh": 235, - "app/chain/transfer.py:TransferChain.do_transfer": 885, - "app/chain/transfer.py:TransferChain.process": 154 + "app/chain/subscribe.py:SubscribeChain.match": 415, + "app/chain/subscribe.py:SubscribeChain.search": 246, + "app/chain/transfer.py:TransferChain.do_transfer": 885 } }