mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-28 19:47:41 +08:00
feat: add subtitle search functionality and related data handling
This commit is contained in:
+35
-1
@@ -14,7 +14,7 @@ from transmission_rpc import File
|
||||
|
||||
from app.core.cache import FileCache, AsyncFileCache, fresh, async_fresh
|
||||
from app.core.config import settings
|
||||
from app.core.context import Context, MediaInfo, TorrentInfo
|
||||
from app.core.context import Context, MediaInfo, SubtitleInfo, TorrentInfo
|
||||
from app.core.event import EventManager
|
||||
from app.core.meta import MetaBase
|
||||
from app.core.module import ModuleManager
|
||||
@@ -1053,6 +1053,23 @@ class ChainBase(metaclass=ABCMeta):
|
||||
"search_torrents", site=site, keyword=keyword, mtype=mtype, page=page
|
||||
)
|
||||
|
||||
def search_subtitles(
|
||||
self,
|
||||
site: dict,
|
||||
keyword: str,
|
||||
page: Optional[int] = 0,
|
||||
) -> List[SubtitleInfo]:
|
||||
"""
|
||||
搜索一个站点的字幕资源。
|
||||
:param site: 站点
|
||||
:param keyword: 搜索关键词
|
||||
:param page: 页码
|
||||
:return: 字幕列表
|
||||
"""
|
||||
return self.run_module(
|
||||
"search_subtitles", site=site, keyword=keyword, page=page
|
||||
)
|
||||
|
||||
async def async_search_torrents(
|
||||
self,
|
||||
site: dict,
|
||||
@@ -1072,6 +1089,23 @@ class ChainBase(metaclass=ABCMeta):
|
||||
"async_search_torrents", site=site, keyword=keyword, mtype=mtype, page=page
|
||||
)
|
||||
|
||||
async def async_search_subtitles(
|
||||
self,
|
||||
site: dict,
|
||||
keyword: str,
|
||||
page: Optional[int] = 0,
|
||||
) -> List[SubtitleInfo]:
|
||||
"""
|
||||
异步搜索一个站点的字幕资源。
|
||||
:param site: 站点
|
||||
:param keyword: 搜索关键词
|
||||
:param page: 页码
|
||||
:return: 字幕列表
|
||||
"""
|
||||
return await self.async_run_module(
|
||||
"async_search_subtitles", site=site, keyword=keyword, page=page
|
||||
)
|
||||
|
||||
def refresh_torrents(
|
||||
self,
|
||||
site: dict,
|
||||
|
||||
+209
-1
@@ -2,15 +2,17 @@ import base64
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Set, Dict, Union
|
||||
|
||||
from app import schemas
|
||||
from app.chain import ChainBase
|
||||
from app.chain.storage import StorageChain
|
||||
from app.core.cache import FileCache
|
||||
from app.core.config import settings, global_vars
|
||||
from app.core.context import MediaInfo, TorrentInfo, Context
|
||||
from app.core.context import MediaInfo, SubtitleInfo, TorrentInfo, Context
|
||||
from app.core.event import eventmanager, Event
|
||||
from app.core.meta import MetaBase
|
||||
from app.core.metainfo import MetaInfo
|
||||
@@ -26,6 +28,7 @@ from app.schemas.types import MediaType, TorrentStatus, EventType, MessageChanne
|
||||
ChainEventType
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
class DownloadChain(ChainBase):
|
||||
@@ -33,6 +36,211 @@ class DownloadChain(ChainBase):
|
||||
下载处理链
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _safe_subtitle_file_name(file_name: str, fallback_name: str) -> str:
|
||||
"""
|
||||
生成安全的字幕文件名。
|
||||
"""
|
||||
file_name = Path(file_name or fallback_name).name
|
||||
if not Path(file_name).suffix and Path(fallback_name).suffix:
|
||||
file_name = f"{file_name}{Path(fallback_name).suffix}"
|
||||
return file_name
|
||||
|
||||
@staticmethod
|
||||
def _is_subtitle_archive(file_name: str) -> bool:
|
||||
"""
|
||||
判断是否为字幕压缩包。
|
||||
"""
|
||||
return Path(file_name).suffix.lower() == ".zip"
|
||||
|
||||
@staticmethod
|
||||
def _is_subtitle_file(file_name: str) -> bool:
|
||||
"""
|
||||
判断是否为支持的字幕文件。
|
||||
"""
|
||||
return Path(file_name).suffix.lower() in settings.RMT_SUBEXT
|
||||
|
||||
@staticmethod
|
||||
def _detect_subtitle_fallback_name(subtitle: SubtitleInfo, content: bytes) -> str:
|
||||
"""
|
||||
根据响应内容生成兜底字幕文件名。
|
||||
"""
|
||||
suffix = ".zip" if content.startswith(b"PK") else ".srt"
|
||||
return f"{subtitle.title or subtitle.subtitle_id or 'subtitle'}{suffix}"
|
||||
|
||||
@staticmethod
|
||||
def _resolve_media_download_dir(
|
||||
media_info: MediaInfo,
|
||||
save_path: Optional[str] = None,
|
||||
) -> Optional[Path]:
|
||||
"""
|
||||
根据媒体信息解析下载目录。
|
||||
"""
|
||||
storage = 'local'
|
||||
if save_path:
|
||||
return Path(save_path)
|
||||
|
||||
dir_info = DirectoryHelper().get_dir(media_info, include_unsorted=True)
|
||||
storage = dir_info.storage if dir_info else storage
|
||||
if not dir_info:
|
||||
logger.error(f"未找到下载目录:{media_info.type.value} {media_info.title_year}")
|
||||
return None
|
||||
|
||||
if not dir_info.media_type and dir_info.download_type_folder:
|
||||
download_dir = Path(dir_info.download_path) / media_info.type.value
|
||||
else:
|
||||
download_dir = Path(dir_info.download_path)
|
||||
|
||||
if not dir_info.media_category and dir_info.download_category_folder and media_info.category:
|
||||
download_dir = download_dir / media_info.category
|
||||
|
||||
file_uri = FileURI(storage=storage, path=download_dir.as_posix())
|
||||
return Path(file_uri.uri)
|
||||
|
||||
@staticmethod
|
||||
def _upload_subtitle_file(
|
||||
storage_chain: StorageChain,
|
||||
storage: str,
|
||||
working_dir_item: schemas.FileItem,
|
||||
subtitle_file: Path,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
上传单个字幕文件到目标目录。
|
||||
"""
|
||||
target_sub_file = Path(working_dir_item.path) / subtitle_file.name
|
||||
if storage_chain.get_file_item(storage, target_sub_file):
|
||||
logger.info(f"字幕文件已存在:{target_sub_file}")
|
||||
return target_sub_file.as_posix()
|
||||
logger.info(f"转移字幕 {subtitle_file} 到 {target_sub_file} ...")
|
||||
uploaded = storage_chain.upload_file(working_dir_item, subtitle_file)
|
||||
if uploaded:
|
||||
return uploaded.path
|
||||
return None
|
||||
|
||||
def _save_subtitle_response(
|
||||
self,
|
||||
subtitle: SubtitleInfo,
|
||||
response,
|
||||
target_dir: Path,
|
||||
) -> List[str]:
|
||||
"""
|
||||
保存字幕下载响应到目标目录。
|
||||
"""
|
||||
fallback_name = self._detect_subtitle_fallback_name(subtitle, response.content)
|
||||
file_name = subtitle.file_name or TorrentHelper.get_url_filename(response, subtitle.enclosure)
|
||||
if not Path(file_name).suffix:
|
||||
file_name = fallback_name
|
||||
file_name = self._safe_subtitle_file_name(
|
||||
file_name=file_name,
|
||||
fallback_name=fallback_name,
|
||||
)
|
||||
if not self._is_subtitle_archive(file_name) and not self._is_subtitle_file(file_name):
|
||||
logger.warn(f"下载链接不是支持的字幕文件:{subtitle.enclosure} - {file_name}")
|
||||
return []
|
||||
|
||||
file_uri = FileURI.from_uri(target_dir.as_posix())
|
||||
storage = file_uri.storage
|
||||
target_path = Path(file_uri.path)
|
||||
storage_chain = StorageChain()
|
||||
working_dir_item = storage_chain.get_folder(storage, target_path)
|
||||
if not working_dir_item:
|
||||
logger.error(f"下载目录不存在,无法保存字幕:{target_path}")
|
||||
return []
|
||||
|
||||
saved_files = []
|
||||
temp_file = settings.TEMP_PATH / file_name
|
||||
temp_extract_dir = temp_file.with_name(temp_file.stem)
|
||||
try:
|
||||
temp_file.write_bytes(response.content)
|
||||
if self._is_subtitle_archive(file_name):
|
||||
shutil.unpack_archive(temp_file, temp_extract_dir, format='zip')
|
||||
for sub_file in SystemUtils.list_files(temp_extract_dir, settings.RMT_SUBEXT):
|
||||
uploaded_path = self._upload_subtitle_file(
|
||||
storage_chain=storage_chain,
|
||||
storage=storage,
|
||||
working_dir_item=working_dir_item,
|
||||
subtitle_file=sub_file,
|
||||
)
|
||||
if uploaded_path:
|
||||
saved_files.append(uploaded_path)
|
||||
else:
|
||||
uploaded_path = self._upload_subtitle_file(
|
||||
storage_chain=storage_chain,
|
||||
storage=storage,
|
||||
working_dir_item=working_dir_item,
|
||||
subtitle_file=temp_file,
|
||||
)
|
||||
if uploaded_path:
|
||||
saved_files.append(uploaded_path)
|
||||
return saved_files
|
||||
finally:
|
||||
try:
|
||||
if temp_extract_dir.exists():
|
||||
shutil.rmtree(temp_extract_dir)
|
||||
if temp_file.exists():
|
||||
temp_file.unlink()
|
||||
except Exception as err:
|
||||
logger.error(f"删除临时字幕文件失败:{str(err)}")
|
||||
|
||||
def download_subtitle(
|
||||
self,
|
||||
subtitle: SubtitleInfo,
|
||||
tmdbid: Optional[int] = None,
|
||||
doubanid: Optional[str] = None,
|
||||
save_path: Optional[str] = None,
|
||||
username: Optional[str] = None,
|
||||
) -> Tuple[bool, str, List[str]]:
|
||||
"""
|
||||
下载字幕文件并保存到媒体对应的下载目录。
|
||||
|
||||
:param subtitle: 字幕搜索结果
|
||||
:param tmdbid: TMDB ID
|
||||
:param doubanid: 豆瓣 ID
|
||||
:param save_path: 保存路径
|
||||
:param username: 调用下载的用户名
|
||||
:return: 成功状态、提示消息、保存文件列表
|
||||
"""
|
||||
if not subtitle or not subtitle.enclosure:
|
||||
return False, "字幕下载链接为空", []
|
||||
|
||||
metainfo = MetaInfo(title=subtitle.title, subtitle=subtitle.description)
|
||||
mediainfo = self.recognize_media(
|
||||
meta=metainfo,
|
||||
tmdbid=tmdbid,
|
||||
doubanid=doubanid,
|
||||
)
|
||||
if not mediainfo:
|
||||
return False, "无法识别媒体信息", []
|
||||
|
||||
target_dir = self._resolve_media_download_dir(
|
||||
media_info=mediainfo,
|
||||
save_path=save_path,
|
||||
)
|
||||
if not target_dir:
|
||||
return False, "未找到下载目录", []
|
||||
|
||||
request = RequestUtils(
|
||||
cookies=subtitle.site_cookie,
|
||||
ua=subtitle.site_ua or settings.USER_AGENT,
|
||||
proxies=settings.PROXY if subtitle.site_proxy else None,
|
||||
)
|
||||
response = request.get_res(subtitle.enclosure)
|
||||
if not response or response.status_code != 200:
|
||||
return False, "下载字幕文件失败", []
|
||||
|
||||
saved_files = self._save_subtitle_response(
|
||||
subtitle=subtitle,
|
||||
response=response,
|
||||
target_dir=target_dir,
|
||||
)
|
||||
if not saved_files:
|
||||
return False, "未保存任何字幕文件", []
|
||||
|
||||
logger.info(
|
||||
f"{mediainfo.title_year} 字幕下载完成:{subtitle.site_name} - {subtitle.title},用户:{username}"
|
||||
)
|
||||
return True, "字幕下载成功", saved_files
|
||||
|
||||
def _submit_download_added_task(
|
||||
self,
|
||||
context: Context,
|
||||
|
||||
+321
-1
@@ -14,7 +14,7 @@ from fastapi.concurrency import run_in_threadpool
|
||||
from app.chain import ChainBase
|
||||
from app.core.config import global_vars, settings
|
||||
from app.core.context import Context
|
||||
from app.core.context import MediaInfo, TorrentInfo
|
||||
from app.core.context import MediaInfo, SubtitleInfo, TorrentInfo
|
||||
from app.core.event import eventmanager, Event
|
||||
from app.core.metainfo import MetaInfo
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
@@ -33,6 +33,7 @@ class SearchChain(ChainBase):
|
||||
"""
|
||||
|
||||
__result_temp_file = "__search_result__"
|
||||
__subtitle_result_temp_file = "__subtitle_search_result__"
|
||||
__search_params_temp_file = "__search_params__"
|
||||
__ai_indices_cache_file = "__ai_recommend_indices__"
|
||||
|
||||
@@ -76,6 +77,18 @@ class SearchChain(ChainBase):
|
||||
page_size = self.get_search_page_size(site=site, keyword=keyword)
|
||||
return page_size is not None and len(page_results or []) >= page_size
|
||||
|
||||
@staticmethod
|
||||
def _should_continue_subtitle_search_pages(site: dict, page_results: Optional[List[Any]]) -> bool:
|
||||
"""
|
||||
判断字幕搜索是否继续抓取下一页。
|
||||
"""
|
||||
subtitle_conf = (site or {}).get("subtitles") or {}
|
||||
try:
|
||||
page_size = int(subtitle_conf.get("result_num") or site.get("result_num") or 100)
|
||||
except (TypeError, ValueError):
|
||||
page_size = 100
|
||||
return page_size > 0 and len(page_results or []) >= page_size
|
||||
|
||||
@property
|
||||
def is_ai_recommend_enabled(self) -> bool:
|
||||
"""
|
||||
@@ -192,6 +205,7 @@ class SearchChain(ChainBase):
|
||||
"year": str(params.get("year") or ""),
|
||||
"season": str(params.get("season") or ""),
|
||||
"sites": str(params.get("sites") or ""),
|
||||
"result_type": str(params.get("result_type") or "torrent"),
|
||||
}
|
||||
return normalized if normalized["keyword"] else None
|
||||
|
||||
@@ -205,6 +219,7 @@ class SearchChain(ChainBase):
|
||||
year: Optional[str] = None,
|
||||
season: Optional[int] = None,
|
||||
sites: Optional[List[int]] = None,
|
||||
result_type: Optional[str] = "torrent",
|
||||
) -> None:
|
||||
"""
|
||||
保存最后一次资源搜索参数。
|
||||
@@ -218,6 +233,7 @@ class SearchChain(ChainBase):
|
||||
"year": year,
|
||||
"season": season,
|
||||
"sites": self._stringify_sites(sites),
|
||||
"result_type": result_type or "torrent",
|
||||
}
|
||||
)
|
||||
if params:
|
||||
@@ -233,6 +249,7 @@ class SearchChain(ChainBase):
|
||||
year: Optional[str] = None,
|
||||
season: Optional[int] = None,
|
||||
sites: Optional[List[int]] = None,
|
||||
result_type: Optional[str] = "torrent",
|
||||
) -> None:
|
||||
"""
|
||||
异步保存最后一次资源搜索参数。
|
||||
@@ -246,6 +263,7 @@ class SearchChain(ChainBase):
|
||||
"year": year,
|
||||
"season": season,
|
||||
"sites": self._stringify_sites(sites),
|
||||
"result_type": result_type or "torrent",
|
||||
}
|
||||
)
|
||||
if params:
|
||||
@@ -555,6 +573,83 @@ class SearchChain(ChainBase):
|
||||
"""
|
||||
return await self.async_load_cache(self.__result_temp_file)
|
||||
|
||||
async def async_last_subtitle_search_results(self) -> Optional[List[SubtitleInfo]]:
|
||||
"""
|
||||
异步获取上次字幕搜索结果。
|
||||
"""
|
||||
return await self.async_load_cache(self.__subtitle_result_temp_file)
|
||||
|
||||
async def async_search_subtitles_by_title(self, title: str, page: Optional[int] = 0,
|
||||
sites: List[int] = None,
|
||||
cache_local: Optional[bool] = False) -> List[SubtitleInfo]:
|
||||
"""
|
||||
根据标题异步搜索字幕,不识别不过滤,直接返回站点字幕内容。
|
||||
:param title: 标题关键词
|
||||
:param page: 页码
|
||||
:param sites: 站点ID列表
|
||||
:param cache_local: 是否缓存到本地
|
||||
"""
|
||||
if cache_local:
|
||||
self.cancel_ai_recommend()
|
||||
await self.async_save_last_search_params(
|
||||
keyword=title,
|
||||
area="title",
|
||||
sites=sites,
|
||||
result_type="subtitle",
|
||||
)
|
||||
logger.info(f'开始搜索字幕,关键词:{title} ...')
|
||||
subtitles = await self.__async_search_subtitles_all_sites(
|
||||
keyword=title, sites=sites, page=page
|
||||
) or []
|
||||
if not subtitles:
|
||||
logger.warn(f'{title} 未搜索到字幕')
|
||||
return []
|
||||
if cache_local:
|
||||
await self.async_save_cache(subtitles, self.__subtitle_result_temp_file)
|
||||
return subtitles
|
||||
|
||||
async def async_search_subtitles_by_title_stream(self, title: str, page: Optional[int] = 0,
|
||||
sites: List[int] = None,
|
||||
cache_local: Optional[bool] = False) -> AsyncIterator[dict]:
|
||||
"""
|
||||
根据标题渐进式搜索字幕,不识别不过滤,按站点完成顺序返回结果。
|
||||
"""
|
||||
if cache_local:
|
||||
self.cancel_ai_recommend()
|
||||
await self.async_save_last_search_params(
|
||||
keyword=title,
|
||||
area="title",
|
||||
sites=sites,
|
||||
result_type="subtitle",
|
||||
)
|
||||
logger.info(f'开始渐进式搜索字幕,关键词:{title} ...')
|
||||
|
||||
subtitles: List[SubtitleInfo] = []
|
||||
async for event in self.__async_search_subtitles_all_sites_stream(
|
||||
keyword=title, sites=sites, page=page):
|
||||
result = event.pop("items", []) or []
|
||||
if result:
|
||||
subtitles.extend(result)
|
||||
yield {
|
||||
**event,
|
||||
"type": "append",
|
||||
"items": [subtitle.to_dict() for subtitle in result],
|
||||
"total_items": len(subtitles)
|
||||
}
|
||||
|
||||
if cache_local:
|
||||
await self.async_save_cache(subtitles, self.__subtitle_result_temp_file)
|
||||
|
||||
if not subtitles:
|
||||
logger.warn(f'{title} 未搜索到字幕')
|
||||
yield {
|
||||
"type": "done",
|
||||
"stage": "done",
|
||||
"text": f"搜索完成,共 {len(subtitles)} 个字幕",
|
||||
"items": [subtitle.to_dict() for subtitle in subtitles],
|
||||
"total_items": len(subtitles)
|
||||
}
|
||||
|
||||
async def async_search_by_id(self, tmdbid: Optional[int] = None, doubanid: Optional[str] = None,
|
||||
mtype: MediaType = None, area: Optional[str] = "title", season: Optional[int] = None,
|
||||
sites: List[int] = None, cache_local: bool = False) -> List[Context]:
|
||||
@@ -1622,6 +1717,231 @@ class SearchChain(ChainBase):
|
||||
logger.info(f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
progress.end()
|
||||
|
||||
async def __async_search_subtitles_all_sites(self, keyword: str,
|
||||
sites: List[int] = None,
|
||||
page: Optional[int] = 0) -> Optional[List[SubtitleInfo]]:
|
||||
"""
|
||||
异步搜索多个站点的字幕资源。
|
||||
:param keyword: 搜索关键词
|
||||
:param sites: 指定站点ID列表,如有则只搜索指定站点,否则搜索所有站点
|
||||
:param page: 搜索页码
|
||||
:reutrn: 字幕资源列表
|
||||
"""
|
||||
indexer_sites = []
|
||||
|
||||
if not sites:
|
||||
sites = SystemConfigOper().get(SystemConfigKey.IndexerSites) or []
|
||||
|
||||
for indexer in await SitesHelper().async_get_indexers():
|
||||
if not indexer.get("subtitles"):
|
||||
continue
|
||||
if not sites or indexer.get("id") in sites:
|
||||
indexer_sites.append(indexer)
|
||||
if not indexer_sites:
|
||||
logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕')
|
||||
return []
|
||||
|
||||
progress = ProgressHelper(ProgressKey.Search)
|
||||
progress.start()
|
||||
start_time = datetime.now()
|
||||
search_pages = self._build_search_pages(page)
|
||||
total_num = len(indexer_sites) * len(search_pages)
|
||||
finish_count = 0
|
||||
progress.update(value=0,
|
||||
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
||||
results = []
|
||||
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
|
||||
|
||||
async def search_site_page(site: dict, search_page: int) -> List[SubtitleInfo]:
|
||||
"""
|
||||
控制单次字幕站点页请求的并发量,并返回该页的字幕列表。
|
||||
"""
|
||||
async with semaphore:
|
||||
return await self.async_search_subtitles(
|
||||
site=site, keyword=keyword, page=search_page
|
||||
)
|
||||
|
||||
pending_tasks = {}
|
||||
|
||||
def submit_site_page(site: dict, page_index: int):
|
||||
"""
|
||||
提交异步字幕站点页搜索任务,并记录站点和页码位置。
|
||||
"""
|
||||
search_page = search_pages[page_index]
|
||||
task = asyncio.create_task(search_site_page(site=site, search_page=search_page))
|
||||
pending_tasks[task] = (site, page_index, search_page)
|
||||
|
||||
for site in indexer_sites:
|
||||
submit_site_page(site=site, page_index=0)
|
||||
|
||||
try:
|
||||
while pending_tasks:
|
||||
if global_vars.is_system_stopped:
|
||||
break
|
||||
done_tasks, _ = await asyncio.wait(
|
||||
pending_tasks.keys(),
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for future in done_tasks:
|
||||
site, page_index, search_page = pending_tasks.pop(future)
|
||||
finish_count += 1
|
||||
result = await future
|
||||
if result:
|
||||
results.extend(result)
|
||||
if (
|
||||
self._should_continue_subtitle_search_pages(site=site, page_results=result)
|
||||
and page_index + 1 < len(search_pages)
|
||||
):
|
||||
submit_site_page(site=site, page_index=page_index + 1)
|
||||
else:
|
||||
logger.debug(
|
||||
f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页"
|
||||
)
|
||||
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
|
||||
progress.update(value=finish_count / total_num * 100,
|
||||
text=f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
|
||||
finally:
|
||||
for task in pending_tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
if pending_tasks:
|
||||
await asyncio.gather(*pending_tasks.keys(), return_exceptions=True)
|
||||
|
||||
end_time = datetime.now()
|
||||
progress.update(value=100,
|
||||
text=f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
logger.info(f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
progress.end()
|
||||
return results
|
||||
|
||||
async def __async_search_subtitles_all_sites_stream(self, keyword: str,
|
||||
sites: List[int] = None,
|
||||
page: Optional[int] = 0) -> AsyncIterator[Dict[str, Any]]:
|
||||
"""
|
||||
异步搜索多个站点的字幕资源,按站点完成顺序渐进式返回结果。
|
||||
:param keyword: 搜索关键词
|
||||
:param sites: 指定站点ID列表,如有则只搜索指定站点,否则搜索所有站点
|
||||
:param page: 搜索页码
|
||||
"""
|
||||
indexer_sites = []
|
||||
|
||||
if not sites:
|
||||
sites = SystemConfigOper().get(SystemConfigKey.IndexerSites) or []
|
||||
|
||||
for indexer in await SitesHelper().async_get_indexers():
|
||||
if not indexer.get("subtitles"):
|
||||
continue
|
||||
if not sites or indexer.get("id") in sites:
|
||||
indexer_sites.append(indexer)
|
||||
if not indexer_sites:
|
||||
logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕')
|
||||
yield {
|
||||
"type": "done",
|
||||
"stage": "searching",
|
||||
"value": 100,
|
||||
"text": "未开启任何支持字幕搜索的有效站点,无法搜索字幕",
|
||||
"items": [],
|
||||
"finished": 0,
|
||||
"total": 0
|
||||
}
|
||||
return
|
||||
|
||||
progress = ProgressHelper(ProgressKey.Search)
|
||||
progress.start()
|
||||
start_time = datetime.now()
|
||||
search_pages = self._build_search_pages(page)
|
||||
total_num = len(indexer_sites) * len(search_pages)
|
||||
finish_count = 0
|
||||
progress.update(value=0,
|
||||
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
||||
yield {
|
||||
"type": "progress",
|
||||
"stage": "searching",
|
||||
"value": 0,
|
||||
"text": f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...",
|
||||
"items": [],
|
||||
"finished": 0,
|
||||
"total": total_num
|
||||
}
|
||||
|
||||
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
|
||||
|
||||
async def search_site(site: dict, search_page: int) -> List[SubtitleInfo]:
|
||||
"""
|
||||
搜索单个站点字幕页,用于渐进式返回入口。
|
||||
"""
|
||||
async with semaphore:
|
||||
site_result = await self.async_search_subtitles(
|
||||
site=site, keyword=keyword, page=search_page
|
||||
)
|
||||
return site_result or []
|
||||
|
||||
tasks = {}
|
||||
|
||||
def submit_site_page(site: dict, page_index: int):
|
||||
"""
|
||||
提交渐进式字幕站点页搜索任务,并保留站点和页码上下文。
|
||||
"""
|
||||
search_page = search_pages[page_index]
|
||||
task = asyncio.create_task(search_site(site=site, search_page=search_page))
|
||||
tasks[task] = (site, page_index, search_page)
|
||||
|
||||
for site in indexer_sites:
|
||||
submit_site_page(site=site, page_index=0)
|
||||
|
||||
results_count = 0
|
||||
try:
|
||||
while tasks:
|
||||
if global_vars.is_system_stopped:
|
||||
break
|
||||
done_tasks, _ = await asyncio.wait(
|
||||
tasks.keys(),
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for future in done_tasks:
|
||||
site, page_index, search_page = tasks.pop(future)
|
||||
finish_count += 1
|
||||
result = await future
|
||||
results_count += len(result)
|
||||
if (
|
||||
self._should_continue_subtitle_search_pages(site=site, page_results=result)
|
||||
and page_index + 1 < len(search_pages)
|
||||
):
|
||||
submit_site_page(site=site, page_index=page_index + 1)
|
||||
else:
|
||||
logger.debug(
|
||||
f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result)} 条,停止继续翻页"
|
||||
)
|
||||
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
|
||||
progress_value = finish_count / total_num * 100
|
||||
progress_text = f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
|
||||
progress.update(value=progress_value, text=progress_text)
|
||||
yield {
|
||||
"type": "append",
|
||||
"stage": "searching",
|
||||
"value": progress_value,
|
||||
"text": progress_text,
|
||||
"items": result,
|
||||
"site": site.get("name"),
|
||||
"site_id": site.get("id"),
|
||||
"page": search_page,
|
||||
"finished": finish_count,
|
||||
"total": total_num,
|
||||
"total_items": results_count
|
||||
}
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks.keys(), return_exceptions=True)
|
||||
|
||||
end_time = datetime.now()
|
||||
progress.update(value=100,
|
||||
text=f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
logger.info(f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
progress.end()
|
||||
|
||||
@eventmanager.register(EventType.SiteDeleted)
|
||||
def remove_site(self, event: Event):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user