GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-05 23:47:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/v3' into pr-6401-resolve

Browse Source 
# Conflicts:
#	app/startup/modules_initializer.py
#	tests/test_configuration_ports.py
This commit is contained in:
jxxghp
2026-08-23 09:32:02 +08:00
parent 4145979e22 7205f8d9e2
commit fc6cc990ad
14 changed files with 438 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/adapters/network/browser.py
+27 -3
View File
@@ -1118,6 +1118,7 @@ class PlaywrightHelper:
:param headless: 是否无头模式
:param timeout: 超时时间
"""
timeout = timeout or 60
source = None
# 如果配置为 FlareSolverr，则直接调用获取页面源码
if self.__browser_emulation() == "flaresolverr":
@@ -1140,10 +1141,33 @@ class PlaywrightHelper:
if cookies:
page.set_extra_http_headers({"cookie": cookies})
page.goto(url)
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
page.goto(url, wait_until="load", timeout=timeout * 1000)
source = page.content()
# 修复: 部分站点(如 Cloudflare 质询页)会持续轮询请求,
# 导致 networkidle 永不触发而超时。改为等待页面加载完成后
# 轮询检查标题, 直到不再停留在质询/加载页。
challenge_titles = ("just a moment", "请稍候", "loading")
deadline = time.time() + timeout
while time.time() < deadline:
try:
current_title = (page.title() or "").strip().lower()
except Exception:
current_title = ""
if current_title and not any(
t in current_title for t in challenge_titles):
break
time.sleep(2)
# 页面跳转中 content() 可能失败, 重试几次
source = None
for _attempt in range(5):
try:
source = page.content()
if source:
break
except Exception:
source = None
time.sleep(2)
except Exception as e:
logger.error(f"获取网页源码失败: {str(e)}")
app/chain/__init__.py
+48
View File
@@ -613,6 +613,29 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
"search_torrents", site=site, keyword=keyword, mtype=mtype, page=page
)
def search_plugin_torrents(
self,
keyword: str,
mtype: Optional[MediaType] = None,
page: Optional[int] = 0,
) -> List[TorrentInfo]:
"""仅搜索插件提供的资源源，避免依赖或重复绑定站点索引器。"""
return self._module_dispatcher.execute_plugin_modules(
"search_torrents", None, site={}, keyword=keyword, mtype=mtype, page=page
) or []
def search_site_torrents(
self,
site: dict,
keyword: str,
mtype: Optional[MediaType] = None,
page: Optional[int] = 0,
) -> List[TorrentInfo]:
"""仅搜索指定站点索引器；插件资源源由搜索链统一调用一次。"""
return self._module_dispatcher.execute_system_modules(
"search_torrents", None, site=site, keyword=keyword, mtype=mtype, page=page
) or []
def search_subtitles(
self,
site: dict,
@@ -649,6 +672,31 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
"async_search_torrents", site=site, keyword=keyword, mtype=mtype, page=page
)
async def async_search_plugin_torrents(
self,
keyword: str,
mtype: Optional[MediaType] = None,
page: Optional[int] = 0,
) -> List[TorrentInfo]:
"""异步搜索插件提供的资源源。"""
return await self._module_dispatcher.async_execute_plugin_modules(
"async_search_torrents", None,
site={}, keyword=keyword, mtype=mtype, page=page
) or []
async def async_search_site_torrents(
self,
site: dict,
keyword: str,
mtype: Optional[MediaType] = None,
page: Optional[int] = 0,
) -> List[TorrentInfo]:
"""异步搜索指定站点索引器。"""
return await self._module_dispatcher.async_execute_system_modules(
"async_search_torrents", None,
site=site, keyword=keyword, mtype=mtype, page=page
) or []
async def async_search_subtitles(
self,
site: dict,
app/chain/search.py
+49 -16
View File
@@ -2321,9 +2321,15 @@ class SearchChain(ChainBase):
# 检查站点索引开关
if not sites or indexer.get("id") in sites:
indexer_sites.append(indexer)
plugin_results = self.search_plugin_torrents(
keyword=keyword,
mtype=mediainfo.type if mediainfo else mtype,
page=page,
)
if not indexer_sites:
logger.warn('未开启任何有效站点，无法搜索资源')
return []
logger.info(f'未开启有效站点，插件资源源返回 {len(plugin_results)} 条资源')
return plugin_results
# 开始进度
progress = ProgressHelper(ProgressKey.Search)
@@ -2339,7 +2345,7 @@ class SearchChain(ChainBase):
progress.update(value=0,
text=f"开始搜索，共 {len(indexer_sites)} 个站点，{len(search_pages)} 页 ...")
# 结果集
results = []
results = list(plugin_results)
# 同一站点按页顺序抓取，避免空页后仍继续请求该站点的后续页。
max_workers = min(
len(indexer_sites),
@@ -2356,13 +2362,13 @@ class SearchChain(ChainBase):
search_keyword = mediainfo.imdb_id if area == "imdbid" and mediainfo else keyword
if area == "imdbid":
# 搜索IMDBID
task = executor.submit(self.search_torrents, site=site,
task = executor.submit(self.search_site_torrents, site=site,
keyword=search_keyword,
mtype=mediainfo.type if mediainfo else mtype,
page=search_page)
else:
# 搜索标题
task = executor.submit(self.search_torrents, site=site,
task = executor.submit(self.search_site_torrents, site=site,
keyword=search_keyword,
mtype=mediainfo.type if mediainfo else mtype,
page=search_page)
@@ -2438,9 +2444,15 @@ class SearchChain(ChainBase):
# 检查站点索引开关
if not sites or indexer.get("id") in sites:
indexer_sites.append(indexer)
plugin_results = await self.async_search_plugin_torrents(
keyword=keyword,
mtype=mediainfo.type if mediainfo else mtype,
page=page,
)
if not indexer_sites:
logger.warn('未开启任何有效站点，无法搜索资源')
return []
logger.info(f'未开启有效站点，插件资源源返回 {len(plugin_results)} 条资源')
return plugin_results
# 开始进度（异步后端，避免同步 Redis 在事件循环上阻塞）
progress = AsyncProgressHelper(ProgressKey.Search)
@@ -2456,7 +2468,7 @@ class SearchChain(ChainBase):
await progress.update(value=0,
text=f"开始搜索，共 {len(indexer_sites)} 个站点，{len(search_pages)} 页 ...")
# 结果集
results = []
results = list(plugin_results)
semaphore = asyncio.Semaphore(
self.runtime_config.search_threadpool_size or total_num
)
@@ -2468,12 +2480,12 @@ class SearchChain(ChainBase):
async with semaphore:
if area == "imdbid":
# 搜索IMDBID
return await self.async_search_torrents(site=site,
return await self.async_search_site_torrents(site=site,
keyword=mediainfo.imdb_id if mediainfo else None,
mtype=mediainfo.type if mediainfo else mtype,
page=search_page)
# 搜索标题
return await self.async_search_torrents(site=site,
return await self.async_search_site_torrents(site=site,
keyword=keyword,
mtype=mediainfo.type if mediainfo else mtype,
page=search_page)
@@ -2562,16 +2574,37 @@ class SearchChain(ChainBase):
for indexer in await SitesHelper().async_get_indexers():
if not sites or indexer.get("id") in sites:
indexer_sites.append(indexer)
plugin_results = await self.async_search_plugin_torrents(
keyword=keyword,
mtype=mediainfo.type if mediainfo else mtype,
page=page,
)
if plugin_results:
yield {
"type": "append",
"stage": "searching",
"value": 100 if not indexer_sites else 0,
"text": f"插件资源源返回 {len(plugin_results)} 条资源",
"items": plugin_results,
"site": "插件资源源",
"site_id": None,
"page": page,
"finished": 0,
"total": len(indexer_sites),
"total_items": len(plugin_results),
}
if not indexer_sites:
logger.warn('未开启任何有效站点，无法搜索资源')
logger.info(f'未开启有效站点，插件资源源返回 {len(plugin_results)} 条资源')
yield {
"type": "done",
"stage": "searching",
"value": 100,
"text": "未开启任何有效站点，无法搜索资源",
"text": f"搜索完成，共 {len(plugin_results)} 条资源",
"items": [],
"finished": 0,
"total": 0
"total": 0,
"total_items": len(plugin_results),
}
return
@@ -2604,12 +2637,12 @@ class SearchChain(ChainBase):
"""
async with semaphore:
if area == "imdbid":
site_result = await self.async_search_torrents(site=site,
site_result = await self.async_search_site_torrents(site=site,
keyword=mediainfo.imdb_id if mediainfo else None,
mtype=mediainfo.type if mediainfo else mtype,
page=search_page)
else:
site_result = await self.async_search_torrents(site=site,
site_result = await self.async_search_site_torrents(site=site,
keyword=keyword,
mtype=mediainfo.type if mediainfo else mtype,
page=search_page)
@@ -2629,7 +2662,7 @@ class SearchChain(ChainBase):
for site in indexer_sites:
submit_site_page(site=site, page_index=0)
results_count = 0
results_count = len(plugin_results)
try:
while tasks:
if global_vars.is_system_stopped:
app/modules/indexer/__init__.py
+27 -2
View File
@@ -634,12 +634,14 @@ class IndexerModule(_ModuleBase):
:return: 用户数据
"""
def __get_site_obj() -> Optional[SiteParserBase]:
def __get_site_obj(schema_value: Optional[str] = None) -> Optional[SiteParserBase]:
"""
获取站点解析器
:param schema_value: 指定 schema, 默认取站点声明的 schema
"""
schema_value = schema_value or site.get("schema")
for site_schema in self._site_schemas:
if site_schema.schema and site_schema.schema.value == site.get("schema"):
if site_schema.schema and site_schema.schema.value == schema_value:
return site_schema(
site_name=site.get("name"),
url=site.get("url"),
@@ -651,6 +653,7 @@ class IndexerModule(_ModuleBase):
api_url=site.get("api_url"))
return None
# 按站点声明的 schema 获取解析器
site_obj = __get_site_obj()
if not site_obj:
if not site.get("public"):
@@ -662,6 +665,28 @@ class IndexerModule(_ModuleBase):
logger.info(f"站点 {site.get('name')} 开始以 {site.get('schema')} 模型解析数据...")
site_obj.parse()
logger.debug(f"站点 {site.get('name')} 数据解析完成")
# 站点声明的 schema 解析失败(userid 为空)时, 自动尝试其他解析器,
# 兼容资源文件 schema 标注错误/变种站点的场景
if not site_obj.userid and not site.get("public"):
tried = {site.get("schema")}
for site_schema in self._site_schemas:
if not site_schema.schema or site_schema.schema.value in tried:
continue
tried.add(site_schema.schema.value)
logger.info(f"站点 {site.get('name')} schema {site.get('schema')} 解析失败, "
f"尝试 {site_schema.schema.value} 模型...")
alt_obj = __get_site_obj(site_schema.schema.value)
if not alt_obj:
continue
try:
alt_obj.parse()
except Exception as e:
logger.error(f"站点 {site.get('name')} 以 {site_schema.schema.value} 解析失败: {str(e)}")
continue
if alt_obj.userid:
site_obj = alt_obj
logger.info(f"站点 {site.get('name')} 改用 {site_schema.schema.value} 模型解析成功")
break
return SiteUserData(
domain=site_rules.extract_domain(site.get("url")),
userid=site_obj.userid,
app/modules/indexer/parser/gazelle.py
+21
View File
@@ -93,6 +93,27 @@ class GazelleSiteUserInfo(SiteParserBase):
'//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()')
if join_at_text:
self.join_at = time_tools.normalize_datetime(join_at_text[0].strip())
# 兼容部分 Gazelle 站点(如 JPopsuki)以文本形式展示上传/下载:
# <li>Uploaded: 77.44 GB</li> / <li>Downloaded: 8.51 GB</li>
if not self.upload:
upload_text = html.xpath(
'//li[starts-with(normalize-space(text()), "Uploaded:")]')
if upload_text:
size_match = re.search(
r"([\d.,]+\s*[GMKT]?i?B)", upload_text[0].xpath("string(.)"), re.I)
if size_match:
self.upload = size_tools.parse_size(size_match.group(1))
if not self.download:
download_text = html.xpath(
'//li[starts-with(normalize-space(text()), "Downloaded:")]')
if download_text:
size_match = re.search(
r"([\d.,]+\s*[GMKT]?i?B)", download_text[0].xpath("string(.)"), re.I)
if size_match:
self.download = size_tools.parse_size(size_match.group(1))
if not self.ratio and self.upload and self.download:
self.ratio = round(self.upload / self.download, 3)
finally:
if html is not None:
del html
app/modules/indexer/parser/nexus_project.py
+39 -1
View File
@@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-
import re
from lxml import etree
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
@@ -11,9 +13,45 @@ class NexusProjectSiteUserInfo(NexusPhpSiteUserInfo):
def _parse_site_page(self, html_text: str):
html_text = self._prepare_html_text(html_text)
user_detail = re.search(r"userdetails.php\?id=(\d+)", html_text)
user_detail = re.search(r"userdetails\.php\?id=(\d+)", html_text)
if user_detail and user_detail.group().strip():
self._user_detail_page = user_detail.group().strip().lstrip('/')
self.userid = user_detail.group(1)
else:
# 兼容部分 NexusProject 变种站点(如 star-space.net)的
# p_user/user_detail.php?uid= 用户定位格式
user_detail = re.search(r"user_detail\.php\?uid=(\d+)", html_text)
if user_detail and user_detail.group().strip():
self._user_detail_page = user_detail.group().strip().lstrip('/')
self.userid = user_detail.group(1)
uname = re.search(
r"user_detail\.php\?uid=\d+[^>]*?>\s*<[^>]*>([^<]+)<", html_text)
if uname:
self.username = uname.group(1).strip()
self._torrent_seeding_page = f"viewusertorrents.php?id={self.userid}&show=seeding"
def _parse_user_traffic_info(self, html_text):
# 兼容部分 NexusProject 变种站点(如 star-space.net)以
# span#user_info / span#user_info_no_hover 文本展示流量:
# "上传：445.11 G" / "下载：29.61 G"(单位无 B 后缀)
try:
html = etree.HTML(html_text)
if html is not None:
body_text = " ".join(html.xpath(
'//span[@id="user_info"]//text() | //span[@id="user_info_no_hover"]//text()'))
size_match = re.search(r"上传[：:]\s*([\d.,]+)\s*([GMKT]?i?B?)", body_text, re.I)
if size_match:
self.upload = self.num_filesize(
f"{size_match.group(1).replace(',', '')} {size_match.group(2).upper()}B")
size_match = re.search(r"下载[：:]\s*([\d.,]+)\s*([GMKT]?i?B?)", body_text, re.I)
if size_match:
self.download = self.num_filesize(
f"{size_match.group(1).replace(',', '')} {size_match.group(2).upper()}B")
if self.upload and self.download:
self.ratio = round(self.upload / self.download, 3)
if self.upload or self.download:
return
except Exception:
pass
super()._parse_user_traffic_info(html_text)