feat: add subtitle search functionality and related data handling

This commit is contained in:
jxxghp
2026-06-09 06:46:26 +08:00
parent 738d92445a
commit e3c5a94c52
12 changed files with 1105 additions and 36 deletions
+35 -1
View File
@@ -14,7 +14,7 @@ from transmission_rpc import File
from app.core.cache import FileCache, AsyncFileCache, fresh, async_fresh
from app.core.config import settings
from app.core.context import Context, MediaInfo, TorrentInfo
from app.core.context import Context, MediaInfo, SubtitleInfo, TorrentInfo
from app.core.event import EventManager
from app.core.meta import MetaBase
from app.core.module import ModuleManager
@@ -1053,6 +1053,23 @@ class ChainBase(metaclass=ABCMeta):
"search_torrents", site=site, keyword=keyword, mtype=mtype, page=page
)
def search_subtitles(
self,
site: dict,
keyword: str,
page: Optional[int] = 0,
) -> List[SubtitleInfo]:
"""
搜索一个站点的字幕资源。
:param site: 站点
:param keyword: 搜索关键词
:param page: 页码
:return: 字幕列表
"""
return self.run_module(
"search_subtitles", site=site, keyword=keyword, page=page
)
async def async_search_torrents(
self,
site: dict,
@@ -1072,6 +1089,23 @@ class ChainBase(metaclass=ABCMeta):
"async_search_torrents", site=site, keyword=keyword, mtype=mtype, page=page
)
async def async_search_subtitles(
self,
site: dict,
keyword: str,
page: Optional[int] = 0,
) -> List[SubtitleInfo]:
"""
异步搜索一个站点的字幕资源。
:param site: 站点
:param keyword: 搜索关键词
:param page: 页码
:return: 字幕列表
"""
return await self.async_run_module(
"async_search_subtitles", site=site, keyword=keyword, page=page
)
def refresh_torrents(
self,
site: dict,
+209 -1
View File
@@ -2,15 +2,17 @@ import base64
import copy
import json
import re
import shutil
import time
from pathlib import Path
from typing import List, Optional, Tuple, Set, Dict, Union
from app import schemas
from app.chain import ChainBase
from app.chain.storage import StorageChain
from app.core.cache import FileCache
from app.core.config import settings, global_vars
from app.core.context import MediaInfo, TorrentInfo, Context
from app.core.context import MediaInfo, SubtitleInfo, TorrentInfo, Context
from app.core.event import eventmanager, Event
from app.core.meta import MetaBase
from app.core.metainfo import MetaInfo
@@ -26,6 +28,7 @@ from app.schemas.types import MediaType, TorrentStatus, EventType, MessageChanne
ChainEventType
from app.utils.http import RequestUtils
from app.utils.string import StringUtils
from app.utils.system import SystemUtils
class DownloadChain(ChainBase):
@@ -33,6 +36,211 @@ class DownloadChain(ChainBase):
下载处理链
"""
@staticmethod
def _safe_subtitle_file_name(file_name: str, fallback_name: str) -> str:
"""
生成安全的字幕文件名。
"""
file_name = Path(file_name or fallback_name).name
if not Path(file_name).suffix and Path(fallback_name).suffix:
file_name = f"{file_name}{Path(fallback_name).suffix}"
return file_name
@staticmethod
def _is_subtitle_archive(file_name: str) -> bool:
"""
判断是否为字幕压缩包。
"""
return Path(file_name).suffix.lower() == ".zip"
@staticmethod
def _is_subtitle_file(file_name: str) -> bool:
"""
判断是否为支持的字幕文件。
"""
return Path(file_name).suffix.lower() in settings.RMT_SUBEXT
@staticmethod
def _detect_subtitle_fallback_name(subtitle: SubtitleInfo, content: bytes) -> str:
"""
根据响应内容生成兜底字幕文件名。
"""
suffix = ".zip" if content.startswith(b"PK") else ".srt"
return f"{subtitle.title or subtitle.subtitle_id or 'subtitle'}{suffix}"
@staticmethod
def _resolve_media_download_dir(
media_info: MediaInfo,
save_path: Optional[str] = None,
) -> Optional[Path]:
"""
根据媒体信息解析下载目录。
"""
storage = 'local'
if save_path:
return Path(save_path)
dir_info = DirectoryHelper().get_dir(media_info, include_unsorted=True)
storage = dir_info.storage if dir_info else storage
if not dir_info:
logger.error(f"未找到下载目录:{media_info.type.value} {media_info.title_year}")
return None
if not dir_info.media_type and dir_info.download_type_folder:
download_dir = Path(dir_info.download_path) / media_info.type.value
else:
download_dir = Path(dir_info.download_path)
if not dir_info.media_category and dir_info.download_category_folder and media_info.category:
download_dir = download_dir / media_info.category
file_uri = FileURI(storage=storage, path=download_dir.as_posix())
return Path(file_uri.uri)
@staticmethod
def _upload_subtitle_file(
storage_chain: StorageChain,
storage: str,
working_dir_item: schemas.FileItem,
subtitle_file: Path,
) -> Optional[str]:
"""
上传单个字幕文件到目标目录。
"""
target_sub_file = Path(working_dir_item.path) / subtitle_file.name
if storage_chain.get_file_item(storage, target_sub_file):
logger.info(f"字幕文件已存在:{target_sub_file}")
return target_sub_file.as_posix()
logger.info(f"转移字幕 {subtitle_file}{target_sub_file} ...")
uploaded = storage_chain.upload_file(working_dir_item, subtitle_file)
if uploaded:
return uploaded.path
return None
def _save_subtitle_response(
self,
subtitle: SubtitleInfo,
response,
target_dir: Path,
) -> List[str]:
"""
保存字幕下载响应到目标目录。
"""
fallback_name = self._detect_subtitle_fallback_name(subtitle, response.content)
file_name = subtitle.file_name or TorrentHelper.get_url_filename(response, subtitle.enclosure)
if not Path(file_name).suffix:
file_name = fallback_name
file_name = self._safe_subtitle_file_name(
file_name=file_name,
fallback_name=fallback_name,
)
if not self._is_subtitle_archive(file_name) and not self._is_subtitle_file(file_name):
logger.warn(f"下载链接不是支持的字幕文件:{subtitle.enclosure} - {file_name}")
return []
file_uri = FileURI.from_uri(target_dir.as_posix())
storage = file_uri.storage
target_path = Path(file_uri.path)
storage_chain = StorageChain()
working_dir_item = storage_chain.get_folder(storage, target_path)
if not working_dir_item:
logger.error(f"下载目录不存在,无法保存字幕:{target_path}")
return []
saved_files = []
temp_file = settings.TEMP_PATH / file_name
temp_extract_dir = temp_file.with_name(temp_file.stem)
try:
temp_file.write_bytes(response.content)
if self._is_subtitle_archive(file_name):
shutil.unpack_archive(temp_file, temp_extract_dir, format='zip')
for sub_file in SystemUtils.list_files(temp_extract_dir, settings.RMT_SUBEXT):
uploaded_path = self._upload_subtitle_file(
storage_chain=storage_chain,
storage=storage,
working_dir_item=working_dir_item,
subtitle_file=sub_file,
)
if uploaded_path:
saved_files.append(uploaded_path)
else:
uploaded_path = self._upload_subtitle_file(
storage_chain=storage_chain,
storage=storage,
working_dir_item=working_dir_item,
subtitle_file=temp_file,
)
if uploaded_path:
saved_files.append(uploaded_path)
return saved_files
finally:
try:
if temp_extract_dir.exists():
shutil.rmtree(temp_extract_dir)
if temp_file.exists():
temp_file.unlink()
except Exception as err:
logger.error(f"删除临时字幕文件失败:{str(err)}")
def download_subtitle(
self,
subtitle: SubtitleInfo,
tmdbid: Optional[int] = None,
doubanid: Optional[str] = None,
save_path: Optional[str] = None,
username: Optional[str] = None,
) -> Tuple[bool, str, List[str]]:
"""
下载字幕文件并保存到媒体对应的下载目录。
:param subtitle: 字幕搜索结果
:param tmdbid: TMDB ID
:param doubanid: 豆瓣 ID
:param save_path: 保存路径
:param username: 调用下载的用户名
:return: 成功状态、提示消息、保存文件列表
"""
if not subtitle or not subtitle.enclosure:
return False, "字幕下载链接为空", []
metainfo = MetaInfo(title=subtitle.title, subtitle=subtitle.description)
mediainfo = self.recognize_media(
meta=metainfo,
tmdbid=tmdbid,
doubanid=doubanid,
)
if not mediainfo:
return False, "无法识别媒体信息", []
target_dir = self._resolve_media_download_dir(
media_info=mediainfo,
save_path=save_path,
)
if not target_dir:
return False, "未找到下载目录", []
request = RequestUtils(
cookies=subtitle.site_cookie,
ua=subtitle.site_ua or settings.USER_AGENT,
proxies=settings.PROXY if subtitle.site_proxy else None,
)
response = request.get_res(subtitle.enclosure)
if not response or response.status_code != 200:
return False, "下载字幕文件失败", []
saved_files = self._save_subtitle_response(
subtitle=subtitle,
response=response,
target_dir=target_dir,
)
if not saved_files:
return False, "未保存任何字幕文件", []
logger.info(
f"{mediainfo.title_year} 字幕下载完成:{subtitle.site_name} - {subtitle.title},用户:{username}"
)
return True, "字幕下载成功", saved_files
def _submit_download_added_task(
self,
context: Context,
+321 -1
View File
@@ -14,7 +14,7 @@ from fastapi.concurrency import run_in_threadpool
from app.chain import ChainBase
from app.core.config import global_vars, settings
from app.core.context import Context
from app.core.context import MediaInfo, TorrentInfo
from app.core.context import MediaInfo, SubtitleInfo, TorrentInfo
from app.core.event import eventmanager, Event
from app.core.metainfo import MetaInfo
from app.db.systemconfig_oper import SystemConfigOper
@@ -33,6 +33,7 @@ class SearchChain(ChainBase):
"""
__result_temp_file = "__search_result__"
__subtitle_result_temp_file = "__subtitle_search_result__"
__search_params_temp_file = "__search_params__"
__ai_indices_cache_file = "__ai_recommend_indices__"
@@ -76,6 +77,18 @@ class SearchChain(ChainBase):
page_size = self.get_search_page_size(site=site, keyword=keyword)
return page_size is not None and len(page_results or []) >= page_size
@staticmethod
def _should_continue_subtitle_search_pages(site: dict, page_results: Optional[List[Any]]) -> bool:
"""
判断字幕搜索是否继续抓取下一页。
"""
subtitle_conf = (site or {}).get("subtitles") or {}
try:
page_size = int(subtitle_conf.get("result_num") or site.get("result_num") or 100)
except (TypeError, ValueError):
page_size = 100
return page_size > 0 and len(page_results or []) >= page_size
@property
def is_ai_recommend_enabled(self) -> bool:
"""
@@ -192,6 +205,7 @@ class SearchChain(ChainBase):
"year": str(params.get("year") or ""),
"season": str(params.get("season") or ""),
"sites": str(params.get("sites") or ""),
"result_type": str(params.get("result_type") or "torrent"),
}
return normalized if normalized["keyword"] else None
@@ -205,6 +219,7 @@ class SearchChain(ChainBase):
year: Optional[str] = None,
season: Optional[int] = None,
sites: Optional[List[int]] = None,
result_type: Optional[str] = "torrent",
) -> None:
"""
保存最后一次资源搜索参数。
@@ -218,6 +233,7 @@ class SearchChain(ChainBase):
"year": year,
"season": season,
"sites": self._stringify_sites(sites),
"result_type": result_type or "torrent",
}
)
if params:
@@ -233,6 +249,7 @@ class SearchChain(ChainBase):
year: Optional[str] = None,
season: Optional[int] = None,
sites: Optional[List[int]] = None,
result_type: Optional[str] = "torrent",
) -> None:
"""
异步保存最后一次资源搜索参数。
@@ -246,6 +263,7 @@ class SearchChain(ChainBase):
"year": year,
"season": season,
"sites": self._stringify_sites(sites),
"result_type": result_type or "torrent",
}
)
if params:
@@ -555,6 +573,83 @@ class SearchChain(ChainBase):
"""
return await self.async_load_cache(self.__result_temp_file)
async def async_last_subtitle_search_results(self) -> Optional[List[SubtitleInfo]]:
"""
异步获取上次字幕搜索结果。
"""
return await self.async_load_cache(self.__subtitle_result_temp_file)
async def async_search_subtitles_by_title(self, title: str, page: Optional[int] = 0,
sites: List[int] = None,
cache_local: Optional[bool] = False) -> List[SubtitleInfo]:
"""
根据标题异步搜索字幕,不识别不过滤,直接返回站点字幕内容。
:param title: 标题关键词
:param page: 页码
:param sites: 站点ID列表
:param cache_local: 是否缓存到本地
"""
if cache_local:
self.cancel_ai_recommend()
await self.async_save_last_search_params(
keyword=title,
area="title",
sites=sites,
result_type="subtitle",
)
logger.info(f'开始搜索字幕,关键词:{title} ...')
subtitles = await self.__async_search_subtitles_all_sites(
keyword=title, sites=sites, page=page
) or []
if not subtitles:
logger.warn(f'{title} 未搜索到字幕')
return []
if cache_local:
await self.async_save_cache(subtitles, self.__subtitle_result_temp_file)
return subtitles
async def async_search_subtitles_by_title_stream(self, title: str, page: Optional[int] = 0,
sites: List[int] = None,
cache_local: Optional[bool] = False) -> AsyncIterator[dict]:
"""
根据标题渐进式搜索字幕,不识别不过滤,按站点完成顺序返回结果。
"""
if cache_local:
self.cancel_ai_recommend()
await self.async_save_last_search_params(
keyword=title,
area="title",
sites=sites,
result_type="subtitle",
)
logger.info(f'开始渐进式搜索字幕,关键词:{title} ...')
subtitles: List[SubtitleInfo] = []
async for event in self.__async_search_subtitles_all_sites_stream(
keyword=title, sites=sites, page=page):
result = event.pop("items", []) or []
if result:
subtitles.extend(result)
yield {
**event,
"type": "append",
"items": [subtitle.to_dict() for subtitle in result],
"total_items": len(subtitles)
}
if cache_local:
await self.async_save_cache(subtitles, self.__subtitle_result_temp_file)
if not subtitles:
logger.warn(f'{title} 未搜索到字幕')
yield {
"type": "done",
"stage": "done",
"text": f"搜索完成,共 {len(subtitles)} 个字幕",
"items": [subtitle.to_dict() for subtitle in subtitles],
"total_items": len(subtitles)
}
async def async_search_by_id(self, tmdbid: Optional[int] = None, doubanid: Optional[str] = None,
mtype: MediaType = None, area: Optional[str] = "title", season: Optional[int] = None,
sites: List[int] = None, cache_local: bool = False) -> List[Context]:
@@ -1622,6 +1717,231 @@ class SearchChain(ChainBase):
logger.info(f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds}")
progress.end()
async def __async_search_subtitles_all_sites(self, keyword: str,
sites: List[int] = None,
page: Optional[int] = 0) -> Optional[List[SubtitleInfo]]:
"""
异步搜索多个站点的字幕资源。
:param keyword: 搜索关键词
:param sites: 指定站点ID列表,如有则只搜索指定站点,否则搜索所有站点
:param page: 搜索页码
:reutrn: 字幕资源列表
"""
indexer_sites = []
if not sites:
sites = SystemConfigOper().get(SystemConfigKey.IndexerSites) or []
for indexer in await SitesHelper().async_get_indexers():
if not indexer.get("subtitles"):
continue
if not sites or indexer.get("id") in sites:
indexer_sites.append(indexer)
if not indexer_sites:
logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕')
return []
progress = ProgressHelper(ProgressKey.Search)
progress.start()
start_time = datetime.now()
search_pages = self._build_search_pages(page)
total_num = len(indexer_sites) * len(search_pages)
finish_count = 0
progress.update(value=0,
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
results = []
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
async def search_site_page(site: dict, search_page: int) -> List[SubtitleInfo]:
"""
控制单次字幕站点页请求的并发量,并返回该页的字幕列表。
"""
async with semaphore:
return await self.async_search_subtitles(
site=site, keyword=keyword, page=search_page
)
pending_tasks = {}
def submit_site_page(site: dict, page_index: int):
"""
提交异步字幕站点页搜索任务,并记录站点和页码位置。
"""
search_page = search_pages[page_index]
task = asyncio.create_task(search_site_page(site=site, search_page=search_page))
pending_tasks[task] = (site, page_index, search_page)
for site in indexer_sites:
submit_site_page(site=site, page_index=0)
try:
while pending_tasks:
if global_vars.is_system_stopped:
break
done_tasks, _ = await asyncio.wait(
pending_tasks.keys(),
return_when=asyncio.FIRST_COMPLETED,
)
for future in done_tasks:
site, page_index, search_page = pending_tasks.pop(future)
finish_count += 1
result = await future
if result:
results.extend(result)
if (
self._should_continue_subtitle_search_pages(site=site, page_results=result)
and page_index + 1 < len(search_pages)
):
submit_site_page(site=site, page_index=page_index + 1)
else:
logger.debug(
f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页"
)
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
progress.update(value=finish_count / total_num * 100,
text=f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
finally:
for task in pending_tasks:
if not task.done():
task.cancel()
if pending_tasks:
await asyncio.gather(*pending_tasks.keys(), return_exceptions=True)
end_time = datetime.now()
progress.update(value=100,
text=f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds}")
logger.info(f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds}")
progress.end()
return results
async def __async_search_subtitles_all_sites_stream(self, keyword: str,
sites: List[int] = None,
page: Optional[int] = 0) -> AsyncIterator[Dict[str, Any]]:
"""
异步搜索多个站点的字幕资源,按站点完成顺序渐进式返回结果。
:param keyword: 搜索关键词
:param sites: 指定站点ID列表,如有则只搜索指定站点,否则搜索所有站点
:param page: 搜索页码
"""
indexer_sites = []
if not sites:
sites = SystemConfigOper().get(SystemConfigKey.IndexerSites) or []
for indexer in await SitesHelper().async_get_indexers():
if not indexer.get("subtitles"):
continue
if not sites or indexer.get("id") in sites:
indexer_sites.append(indexer)
if not indexer_sites:
logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕')
yield {
"type": "done",
"stage": "searching",
"value": 100,
"text": "未开启任何支持字幕搜索的有效站点,无法搜索字幕",
"items": [],
"finished": 0,
"total": 0
}
return
progress = ProgressHelper(ProgressKey.Search)
progress.start()
start_time = datetime.now()
search_pages = self._build_search_pages(page)
total_num = len(indexer_sites) * len(search_pages)
finish_count = 0
progress.update(value=0,
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
yield {
"type": "progress",
"stage": "searching",
"value": 0,
"text": f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...",
"items": [],
"finished": 0,
"total": total_num
}
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
async def search_site(site: dict, search_page: int) -> List[SubtitleInfo]:
"""
搜索单个站点字幕页,用于渐进式返回入口。
"""
async with semaphore:
site_result = await self.async_search_subtitles(
site=site, keyword=keyword, page=search_page
)
return site_result or []
tasks = {}
def submit_site_page(site: dict, page_index: int):
"""
提交渐进式字幕站点页搜索任务,并保留站点和页码上下文。
"""
search_page = search_pages[page_index]
task = asyncio.create_task(search_site(site=site, search_page=search_page))
tasks[task] = (site, page_index, search_page)
for site in indexer_sites:
submit_site_page(site=site, page_index=0)
results_count = 0
try:
while tasks:
if global_vars.is_system_stopped:
break
done_tasks, _ = await asyncio.wait(
tasks.keys(),
return_when=asyncio.FIRST_COMPLETED,
)
for future in done_tasks:
site, page_index, search_page = tasks.pop(future)
finish_count += 1
result = await future
results_count += len(result)
if (
self._should_continue_subtitle_search_pages(site=site, page_results=result)
and page_index + 1 < len(search_pages)
):
submit_site_page(site=site, page_index=page_index + 1)
else:
logger.debug(
f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result)} 条,停止继续翻页"
)
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
progress_value = finish_count / total_num * 100
progress_text = f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
progress.update(value=progress_value, text=progress_text)
yield {
"type": "append",
"stage": "searching",
"value": progress_value,
"text": progress_text,
"items": result,
"site": site.get("name"),
"site_id": site.get("id"),
"page": search_page,
"finished": finish_count,
"total": total_num,
"total_items": results_count
}
finally:
for task in tasks:
if not task.done():
task.cancel()
if tasks:
await asyncio.gather(*tasks.keys(), return_exceptions=True)
end_time = datetime.now()
progress.update(value=100,
text=f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds}")
logger.info(f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds}")
progress.end()
@eventmanager.register(EventType.SiteDeleted)
def remove_site(self, event: Event):
"""