mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-06 07:56:52 +08:00
refactor: 收敛 factory 模块级副作用并统一 async 路径进度为异步后端
- 将 configure_token_codec/configure_plugin_routes 移入 create_app(),消除 import 期副作用 - runtime 层新增 AsyncCacheProxy/AsyncTTLCache/AsyncProgressHelper,共享 progress region - search/system/history/scheduler/dashboard 的事件循环路径切换异步进度后端 - 同步进度回调经事件循环提交或线程池执行,避免阻塞事件循环
This commit is contained in:
@@ -2,6 +2,7 @@ from pathlib import Path
|
|||||||
from typing import Any, List, Optional, Annotated
|
from typing import Any, List, Optional, Annotated
|
||||||
|
|
||||||
from fastapi import Depends
|
from fastapi import Depends
|
||||||
|
from fastapi.concurrency import run_in_threadpool
|
||||||
|
|
||||||
from app.schemas.dashboard import DashboardMemoryInfo as _SchemaDashboardMemoryInfo
|
from app.schemas.dashboard import DashboardMemoryInfo as _SchemaDashboardMemoryInfo
|
||||||
from app.schemas.dashboard import DashboardSystemInfo as _SchemaDashboardSystemInfo
|
from app.schemas.dashboard import DashboardSystemInfo as _SchemaDashboardSystemInfo
|
||||||
@@ -160,7 +161,8 @@ async def schedule(_: Any = Depends(get_current_active_superuser)) -> Any:
|
|||||||
"""
|
"""
|
||||||
查询后台服务信息
|
查询后台服务信息
|
||||||
"""
|
"""
|
||||||
return Scheduler().list()
|
# 同步 list() 内含同步进度读取,放到线程池执行避免阻塞事件循环
|
||||||
|
return await run_in_threadpool(Scheduler().list)
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
@@ -174,7 +176,8 @@ async def schedule_progress(
|
|||||||
"""
|
"""
|
||||||
查询指定后台服务的执行进度。
|
查询指定后台服务的执行进度。
|
||||||
"""
|
"""
|
||||||
progress = Scheduler().get_progress(job_id)
|
# 异步进度后端读取,避免同步 Redis 调用阻塞事件循环
|
||||||
|
progress = await Scheduler().aget_progress(job_id)
|
||||||
if not progress:
|
if not progress:
|
||||||
return _SchemaResponse(success=False, message="后台服务不存在")
|
return _SchemaResponse(success=False, message="后台服务不存在")
|
||||||
return _SchemaResponse(success=True, data=progress.model_dump())
|
return _SchemaResponse(success=True, data=progress.model_dump())
|
||||||
@@ -189,7 +192,8 @@ async def schedule2(_: Annotated[str, Depends(verify_apitoken)]) -> Any:
|
|||||||
"""
|
"""
|
||||||
查询下载器信息 API_TOKEN认证(?token=xxx)
|
查询下载器信息 API_TOKEN认证(?token=xxx)
|
||||||
"""
|
"""
|
||||||
return Scheduler().list()
|
# 同步 list() 内含同步进度读取,放到线程池执行避免阻塞事件循环
|
||||||
|
return await run_in_threadpool(Scheduler().list)
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
@@ -203,7 +207,8 @@ async def schedule_progress2(
|
|||||||
"""
|
"""
|
||||||
查询指定后台服务的执行进度 API_TOKEN认证(?token=xxx)
|
查询指定后台服务的执行进度 API_TOKEN认证(?token=xxx)
|
||||||
"""
|
"""
|
||||||
progress = Scheduler().get_progress(job_id)
|
# 异步进度后端读取,避免同步 Redis 调用阻塞事件循环
|
||||||
|
progress = await Scheduler().aget_progress(job_id)
|
||||||
if not progress:
|
if not progress:
|
||||||
return _SchemaResponse(success=False, message="后台服务不存在")
|
return _SchemaResponse(success=False, message="后台服务不存在")
|
||||||
return _SchemaResponse(success=True, data=progress.model_dump())
|
return _SchemaResponse(success=True, data=progress.model_dump())
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ from app.api.deps import (
|
|||||||
get_history_query_service,
|
get_history_query_service,
|
||||||
get_transfer_history_mutation_command,
|
get_transfer_history_mutation_command,
|
||||||
)
|
)
|
||||||
from app.runtime.progress import ProgressHelper
|
from app.runtime.progress import AsyncProgressHelper
|
||||||
from app.application.history import (
|
from app.application.history import (
|
||||||
DownloadHistoryMutationCommand,
|
DownloadHistoryMutationCommand,
|
||||||
HistoryQueryService,
|
HistoryQueryService,
|
||||||
@@ -49,19 +49,24 @@ def normalize_history_ids(history_ids: list[int]) -> list[int]:
|
|||||||
|
|
||||||
|
|
||||||
def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str):
|
def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str):
|
||||||
"""在后台线程中启动单条 AI 重新整理任务,并通过 ProgressHelper 实时更新进度。"""
|
"""在后台任务中启动单条 AI 重新整理任务,并通过异步进度辅助类实时更新进度。"""
|
||||||
progress = ProgressHelper(progress_key)
|
progress = AsyncProgressHelper(progress_key)
|
||||||
progress.start()
|
|
||||||
progress.update(
|
|
||||||
text=f"智能助手正在准备整理记录 #{history_id} ...",
|
|
||||||
data={"history_id": history_id, "success": True},
|
|
||||||
)
|
|
||||||
|
|
||||||
def update_output(text: str):
|
def update_output(text: str):
|
||||||
progress.update(text=text, data={"history_id": history_id})
|
# 输出回调由 agent 在事件循环上同步调用,不能直接 await;
|
||||||
|
# 提交到全局事件循环非阻塞执行,避免同步缓存后端阻塞事件循环。
|
||||||
|
asyncio.run_coroutine_threadsafe(
|
||||||
|
progress.update(text=text, data={"history_id": history_id}),
|
||||||
|
global_vars.loop,
|
||||||
|
)
|
||||||
|
|
||||||
async def runner():
|
async def runner():
|
||||||
try:
|
try:
|
||||||
|
await progress.start()
|
||||||
|
await progress.update(
|
||||||
|
text=f"智能助手正在准备整理记录 #{history_id} ...",
|
||||||
|
data={"history_id": history_id, "success": True},
|
||||||
|
)
|
||||||
manager = get_running_agent_manager()
|
manager = get_running_agent_manager()
|
||||||
if manager is None:
|
if manager is None:
|
||||||
logger.warning("智能助手服务未运行,跳过单条整理历史 AI 重做")
|
logger.warning("智能助手服务未运行,跳过单条整理历史 AI 重做")
|
||||||
@@ -73,12 +78,12 @@ def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str):
|
|||||||
reply_mode=ReplyMode.CAPTURE_ONLY,
|
reply_mode=ReplyMode.CAPTURE_ONLY,
|
||||||
allow_message_tools=False,
|
allow_message_tools=False,
|
||||||
)
|
)
|
||||||
progress.update(
|
await progress.update(
|
||||||
text="智能助手整理完成",
|
text="智能助手整理完成",
|
||||||
data={"history_id": history_id, "success": True, "completed": True},
|
data={"history_id": history_id, "success": True, "completed": True},
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
progress.update(
|
await progress.update(
|
||||||
text=f"智能助手整理失败:{str(e)}",
|
text=f"智能助手整理失败:{str(e)}",
|
||||||
data={
|
data={
|
||||||
"history_id": history_id,
|
"history_id": history_id,
|
||||||
@@ -88,7 +93,7 @@ def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str):
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
progress.end()
|
await progress.end()
|
||||||
|
|
||||||
asyncio.run_coroutine_threadsafe(runner(), global_vars.loop)
|
asyncio.run_coroutine_threadsafe(runner(), global_vars.loop)
|
||||||
|
|
||||||
@@ -98,19 +103,24 @@ def _start_batch_ai_redo_task(
|
|||||||
prompt: str,
|
prompt: str,
|
||||||
progress_key: str,
|
progress_key: str,
|
||||||
):
|
):
|
||||||
"""在后台线程中启动批量 AI 重新整理任务,并通过 ProgressHelper 实时更新进度。"""
|
"""在后台任务中启动批量 AI 重新整理任务,并通过异步进度辅助类实时更新进度。"""
|
||||||
progress = ProgressHelper(progress_key)
|
progress = AsyncProgressHelper(progress_key)
|
||||||
progress.start()
|
|
||||||
progress.update(
|
|
||||||
text=f"智能助手正在准备批量整理 {len(history_ids)} 条记录 ...",
|
|
||||||
data={"history_ids": history_ids, "success": True},
|
|
||||||
)
|
|
||||||
|
|
||||||
def update_output(text: str):
|
def update_output(text: str):
|
||||||
progress.update(text=text, data={"history_ids": history_ids})
|
# 输出回调由 agent 在事件循环上同步调用,不能直接 await;
|
||||||
|
# 提交到全局事件循环非阻塞执行,避免同步缓存后端阻塞事件循环。
|
||||||
|
asyncio.run_coroutine_threadsafe(
|
||||||
|
progress.update(text=text, data={"history_ids": history_ids}),
|
||||||
|
global_vars.loop,
|
||||||
|
)
|
||||||
|
|
||||||
async def runner():
|
async def runner():
|
||||||
try:
|
try:
|
||||||
|
await progress.start()
|
||||||
|
await progress.update(
|
||||||
|
text=f"智能助手正在准备批量整理 {len(history_ids)} 条记录 ...",
|
||||||
|
data={"history_ids": history_ids, "success": True},
|
||||||
|
)
|
||||||
manager = get_running_agent_manager()
|
manager = get_running_agent_manager()
|
||||||
if manager is None:
|
if manager is None:
|
||||||
logger.warning("智能助手服务未运行,跳过批量整理历史 AI 重做")
|
logger.warning("智能助手服务未运行,跳过批量整理历史 AI 重做")
|
||||||
@@ -122,12 +132,12 @@ def _start_batch_ai_redo_task(
|
|||||||
reply_mode=ReplyMode.CAPTURE_ONLY,
|
reply_mode=ReplyMode.CAPTURE_ONLY,
|
||||||
allow_message_tools=False,
|
allow_message_tools=False,
|
||||||
)
|
)
|
||||||
progress.update(
|
await progress.update(
|
||||||
text="智能助手批量整理完成",
|
text="智能助手批量整理完成",
|
||||||
data={"history_ids": history_ids, "success": True, "completed": True},
|
data={"history_ids": history_ids, "success": True, "completed": True},
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
progress.update(
|
await progress.update(
|
||||||
text=f"智能助手批量整理失败:{str(e)}",
|
text=f"智能助手批量整理失败:{str(e)}",
|
||||||
data={
|
data={
|
||||||
"history_ids": history_ids,
|
"history_ids": history_ids,
|
||||||
@@ -137,7 +147,7 @@ def _start_batch_ai_redo_task(
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
progress.end()
|
await progress.end()
|
||||||
|
|
||||||
asyncio.run_coroutine_threadsafe(runner(), global_vars.loop)
|
asyncio.run_coroutine_threadsafe(runner(), global_vars.loop)
|
||||||
|
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ from app.adapters.external.market import (
|
|||||||
split_plugin_market_repo_urls,
|
split_plugin_market_repo_urls,
|
||||||
)
|
)
|
||||||
from app.application.messaging.message import MessageHelper
|
from app.application.messaging.message import MessageHelper
|
||||||
from app.runtime.progress import ProgressHelper
|
from app.runtime.progress import AsyncProgressHelper
|
||||||
from app.application.rules import RuleHelper
|
from app.application.rules import RuleHelper
|
||||||
from app.adapters.external.server import MoviePilotServerHelper
|
from app.adapters.external.server import MoviePilotServerHelper
|
||||||
from app.runtime.state import SystemHelper
|
from app.runtime.state import SystemHelper
|
||||||
@@ -847,7 +847,7 @@ async def get_progress(
|
|||||||
"""
|
"""
|
||||||
实时获取处理进度,返回格式为SSE
|
实时获取处理进度,返回格式为SSE
|
||||||
"""
|
"""
|
||||||
progress = ProgressHelper(process_type)
|
progress = AsyncProgressHelper(process_type)
|
||||||
locale = LocaleHelper.get_current_locale()
|
locale = LocaleHelper.get_current_locale()
|
||||||
|
|
||||||
async def event_generator():
|
async def event_generator():
|
||||||
@@ -855,7 +855,7 @@ async def get_progress(
|
|||||||
while not global_vars.is_system_stopped:
|
while not global_vars.is_system_stopped:
|
||||||
if await request.is_disconnected():
|
if await request.is_disconnected():
|
||||||
break
|
break
|
||||||
detail = progress.get(locale=locale)
|
detail = await progress.get(locale=locale)
|
||||||
yield f"data: {json.dumps(detail)}\n\n"
|
yield f"data: {json.dumps(detail)}\n\n"
|
||||||
await asyncio.sleep(0.5)
|
await asyncio.sleep(0.5)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
|
|||||||
+39
-36
@@ -22,7 +22,7 @@ from app.domain.meta.metamusic import MetaMusic
|
|||||||
from app.domain.metainfo import MetaInfo
|
from app.domain.metainfo import MetaInfo
|
||||||
from app.domain.context import MusicInfo
|
from app.domain.context import MusicInfo
|
||||||
from app.application.configuration import get_configured_system_config
|
from app.application.configuration import get_configured_system_config
|
||||||
from app.runtime.progress import ProgressHelper
|
from app.runtime.progress import AsyncProgressHelper, ProgressHelper
|
||||||
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
|
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
|
||||||
from app.application.search.state import (
|
from app.application.search.state import (
|
||||||
SearchStateService,
|
SearchStateService,
|
||||||
@@ -2396,9 +2396,9 @@ class SearchChain(ChainBase):
|
|||||||
logger.warn('未开启任何有效站点,无法搜索资源')
|
logger.warn('未开启任何有效站点,无法搜索资源')
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# 开始进度
|
# 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
|
||||||
progress = ProgressHelper(ProgressKey.Search)
|
progress = AsyncProgressHelper(ProgressKey.Search)
|
||||||
progress.start()
|
await progress.start()
|
||||||
# 开始计时
|
# 开始计时
|
||||||
start_time = datetime.now()
|
start_time = datetime.now()
|
||||||
search_pages = self._build_search_pages(page)
|
search_pages = self._build_search_pages(page)
|
||||||
@@ -2407,8 +2407,8 @@ class SearchChain(ChainBase):
|
|||||||
# 完成数
|
# 完成数
|
||||||
finish_count = 0
|
finish_count = 0
|
||||||
# 更新进度
|
# 更新进度
|
||||||
progress.update(value=0,
|
await progress.update(value=0,
|
||||||
text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
||||||
# 结果集
|
# 结果集
|
||||||
results = []
|
results = []
|
||||||
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
|
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
|
||||||
@@ -2470,8 +2470,8 @@ class SearchChain(ChainBase):
|
|||||||
f"{site.get('name')} 第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页"
|
f"{site.get('name')} 第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页"
|
||||||
)
|
)
|
||||||
logger.info(f"站点搜索进度:{finish_count} / {total_num}")
|
logger.info(f"站点搜索进度:{finish_count} / {total_num}")
|
||||||
progress.update(value=finish_count / total_num * 100,
|
await progress.update(value=finish_count / total_num * 100,
|
||||||
text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
|
text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
|
||||||
finally:
|
finally:
|
||||||
for task in pending_tasks:
|
for task in pending_tasks:
|
||||||
if not task.done():
|
if not task.done():
|
||||||
@@ -2482,11 +2482,11 @@ class SearchChain(ChainBase):
|
|||||||
# 计算耗时
|
# 计算耗时
|
||||||
end_time = datetime.now()
|
end_time = datetime.now()
|
||||||
# 更新进度
|
# 更新进度
|
||||||
progress.update(value=100,
|
await progress.update(value=100,
|
||||||
text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
logger.info(f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
logger.info(f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
# 结束进度
|
# 结束进度
|
||||||
progress.end()
|
await progress.end()
|
||||||
|
|
||||||
# 返回
|
# 返回
|
||||||
return results
|
return results
|
||||||
@@ -2527,14 +2527,15 @@ class SearchChain(ChainBase):
|
|||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
|
||||||
progress = ProgressHelper(ProgressKey.Search)
|
# 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
|
||||||
progress.start()
|
progress = AsyncProgressHelper(ProgressKey.Search)
|
||||||
|
await progress.start()
|
||||||
start_time = datetime.now()
|
start_time = datetime.now()
|
||||||
search_pages = self._build_search_pages(page)
|
search_pages = self._build_search_pages(page)
|
||||||
total_num = len(indexer_sites) * len(search_pages)
|
total_num = len(indexer_sites) * len(search_pages)
|
||||||
finish_count = 0
|
finish_count = 0
|
||||||
progress.update(value=0,
|
await progress.update(value=0,
|
||||||
text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
||||||
yield {
|
yield {
|
||||||
"type": "progress",
|
"type": "progress",
|
||||||
"stage": "searching",
|
"stage": "searching",
|
||||||
@@ -2606,7 +2607,7 @@ class SearchChain(ChainBase):
|
|||||||
logger.info(f"站点搜索进度:{finish_count} / {total_num}")
|
logger.info(f"站点搜索进度:{finish_count} / {total_num}")
|
||||||
progress_value = finish_count / total_num * 100
|
progress_value = finish_count / total_num * 100
|
||||||
progress_text = f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
|
progress_text = f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
|
||||||
progress.update(value=progress_value, text=progress_text)
|
await progress.update(value=progress_value, text=progress_text)
|
||||||
yield {
|
yield {
|
||||||
"type": "append",
|
"type": "append",
|
||||||
"stage": "searching",
|
"stage": "searching",
|
||||||
@@ -2628,10 +2629,10 @@ class SearchChain(ChainBase):
|
|||||||
await asyncio.gather(*tasks.keys(), return_exceptions=True)
|
await asyncio.gather(*tasks.keys(), return_exceptions=True)
|
||||||
|
|
||||||
end_time = datetime.now()
|
end_time = datetime.now()
|
||||||
progress.update(value=100,
|
await progress.update(value=100,
|
||||||
text=f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
text=f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
logger.info(f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
logger.info(f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
progress.end()
|
await progress.end()
|
||||||
|
|
||||||
async def __async_search_subtitles_all_sites(self, keyword: str,
|
async def __async_search_subtitles_all_sites(self, keyword: str,
|
||||||
sites: List[int] = None,
|
sites: List[int] = None,
|
||||||
@@ -2657,14 +2658,15 @@ class SearchChain(ChainBase):
|
|||||||
logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕')
|
logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕')
|
||||||
return []
|
return []
|
||||||
|
|
||||||
progress = ProgressHelper(ProgressKey.Search)
|
# 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
|
||||||
progress.start()
|
progress = AsyncProgressHelper(ProgressKey.Search)
|
||||||
|
await progress.start()
|
||||||
start_time = datetime.now()
|
start_time = datetime.now()
|
||||||
search_pages = self._build_search_pages(page)
|
search_pages = self._build_search_pages(page)
|
||||||
total_num = len(indexer_sites) * len(search_pages)
|
total_num = len(indexer_sites) * len(search_pages)
|
||||||
finish_count = 0
|
finish_count = 0
|
||||||
progress.update(value=0,
|
await progress.update(value=0,
|
||||||
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
||||||
results = []
|
results = []
|
||||||
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
|
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
|
||||||
|
|
||||||
@@ -2714,8 +2716,8 @@ class SearchChain(ChainBase):
|
|||||||
f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页"
|
f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页"
|
||||||
)
|
)
|
||||||
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
|
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
|
||||||
progress.update(value=finish_count / total_num * 100,
|
await progress.update(value=finish_count / total_num * 100,
|
||||||
text=f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
|
text=f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
|
||||||
finally:
|
finally:
|
||||||
for task in pending_tasks:
|
for task in pending_tasks:
|
||||||
if not task.done():
|
if not task.done():
|
||||||
@@ -2724,10 +2726,10 @@ class SearchChain(ChainBase):
|
|||||||
await asyncio.gather(*pending_tasks.keys(), return_exceptions=True)
|
await asyncio.gather(*pending_tasks.keys(), return_exceptions=True)
|
||||||
|
|
||||||
end_time = datetime.now()
|
end_time = datetime.now()
|
||||||
progress.update(value=100,
|
await progress.update(value=100,
|
||||||
text=f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
text=f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
logger.info(f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
logger.info(f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
progress.end()
|
await progress.end()
|
||||||
return results
|
return results
|
||||||
|
|
||||||
async def __async_search_subtitles_all_sites_stream(self, keyword: str,
|
async def __async_search_subtitles_all_sites_stream(self, keyword: str,
|
||||||
@@ -2762,14 +2764,15 @@ class SearchChain(ChainBase):
|
|||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
|
||||||
progress = ProgressHelper(ProgressKey.Search)
|
# 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
|
||||||
progress.start()
|
progress = AsyncProgressHelper(ProgressKey.Search)
|
||||||
|
await progress.start()
|
||||||
start_time = datetime.now()
|
start_time = datetime.now()
|
||||||
search_pages = self._build_search_pages(page)
|
search_pages = self._build_search_pages(page)
|
||||||
total_num = len(indexer_sites) * len(search_pages)
|
total_num = len(indexer_sites) * len(search_pages)
|
||||||
finish_count = 0
|
finish_count = 0
|
||||||
progress.update(value=0,
|
await progress.update(value=0,
|
||||||
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
|
||||||
yield {
|
yield {
|
||||||
"type": "progress",
|
"type": "progress",
|
||||||
"stage": "searching",
|
"stage": "searching",
|
||||||
@@ -2831,7 +2834,7 @@ class SearchChain(ChainBase):
|
|||||||
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
|
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
|
||||||
progress_value = finish_count / total_num * 100
|
progress_value = finish_count / total_num * 100
|
||||||
progress_text = f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
|
progress_text = f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
|
||||||
progress.update(value=progress_value, text=progress_text)
|
await progress.update(value=progress_value, text=progress_text)
|
||||||
yield {
|
yield {
|
||||||
"type": "append",
|
"type": "append",
|
||||||
"stage": "searching",
|
"stage": "searching",
|
||||||
@@ -2853,10 +2856,10 @@ class SearchChain(ChainBase):
|
|||||||
await asyncio.gather(*tasks.keys(), return_exceptions=True)
|
await asyncio.gather(*tasks.keys(), return_exceptions=True)
|
||||||
|
|
||||||
end_time = datetime.now()
|
end_time = datetime.now()
|
||||||
progress.update(value=100,
|
await progress.update(value=100,
|
||||||
text=f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
text=f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
logger.info(f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
logger.info(f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds} 秒")
|
||||||
progress.end()
|
await progress.end()
|
||||||
|
|
||||||
@eventmanager.register(EventType.SiteDeleted)
|
@eventmanager.register(EventType.SiteDeleted)
|
||||||
def remove_site(self, event: Event):
|
def remove_site(self, event: Event):
|
||||||
|
|||||||
+23
-22
@@ -314,6 +314,7 @@ def create_app() -> FastAPI:
|
|||||||
allow_methods=["*"],
|
allow_methods=["*"],
|
||||||
allow_headers=["*"],
|
allow_headers=["*"],
|
||||||
)
|
)
|
||||||
|
|
||||||
@_app.middleware("http")
|
@_app.middleware("http")
|
||||||
async def locale_context_middleware(
|
async def locale_context_middleware(
|
||||||
request: Request,
|
request: Request,
|
||||||
@@ -330,29 +331,29 @@ def create_app() -> FastAPI:
|
|||||||
finally:
|
finally:
|
||||||
LocaleHelper.reset_current_locale(token)
|
LocaleHelper.reset_current_locale(token)
|
||||||
|
|
||||||
|
# HTTP 适配器只持有令牌编解码端口,具体实现由组合根在创建应用时连接。
|
||||||
|
configure_token_codec(create_access_token, decode_access_token)
|
||||||
|
|
||||||
|
# 向 application 层插件路由服务注入应用实例,插件 API 的动态注册/移除
|
||||||
|
# 统一经服务完成,避免 api.endpoints 反向依赖本模块。
|
||||||
|
configure_plugin_routes(FastAPIDynamicRouteRegistry(
|
||||||
|
app=_app,
|
||||||
|
plugin_ids=lambda: PluginManager().get_running_plugin_ids(),
|
||||||
|
plugin_apis=lambda plugin_id: PluginManager().get_plugin_apis(plugin_id),
|
||||||
|
verify_token=verify_token,
|
||||||
|
verify_apikey=verify_apikey,
|
||||||
|
prefix=f"{settings.API_V1_STR}/plugin",
|
||||||
|
protected_routes={
|
||||||
|
f"{settings.API_V1_STR}/openapi.json",
|
||||||
|
"/docs",
|
||||||
|
"/docs/oauth2-redirect",
|
||||||
|
"/redoc",
|
||||||
|
},
|
||||||
|
log=logger,
|
||||||
|
))
|
||||||
|
|
||||||
return _app
|
return _app
|
||||||
|
|
||||||
|
|
||||||
# HTTP 适配器只持有令牌编解码端口,具体实现由组合根连接。
|
# 创建 FastAPI 应用实例;所有组合根装配副作用都在 create_app() 内部完成
|
||||||
configure_token_codec(create_access_token, decode_access_token)
|
|
||||||
|
|
||||||
# 创建 FastAPI 应用实例
|
|
||||||
app = create_app()
|
app = create_app()
|
||||||
|
|
||||||
# 向 application 层插件路由服务注入应用实例,插件 API 的动态注册/移除
|
|
||||||
# 统一经服务完成,避免 api.endpoints 反向依赖本模块。
|
|
||||||
configure_plugin_routes(FastAPIDynamicRouteRegistry(
|
|
||||||
app=app,
|
|
||||||
plugin_ids=lambda: PluginManager().get_running_plugin_ids(),
|
|
||||||
plugin_apis=lambda plugin_id: PluginManager().get_plugin_apis(plugin_id),
|
|
||||||
verify_token=verify_token,
|
|
||||||
verify_apikey=verify_apikey,
|
|
||||||
prefix=f"{settings.API_V1_STR}/plugin",
|
|
||||||
protected_routes={
|
|
||||||
f"{settings.API_V1_STR}/openapi.json",
|
|
||||||
"/docs",
|
|
||||||
"/docs/oauth2-redirect",
|
|
||||||
"/redoc",
|
|
||||||
},
|
|
||||||
log=logger,
|
|
||||||
))
|
|
||||||
|
|||||||
@@ -1158,6 +1158,143 @@ class CacheProxy:
|
|||||||
self._cache_backend.close()
|
self._cache_backend.close()
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncCacheProxy:
|
||||||
|
"""
|
||||||
|
异步缓存代理类,将异步缓存后端的方法直接代理到实例上
|
||||||
|
|
||||||
|
与同步 CacheProxy 的唯一差异是方法均为 async,默认绑定构造时指定的 region。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, cache_backend: AsyncCacheBackend, region: str):
|
||||||
|
"""
|
||||||
|
初始化异步缓存代理
|
||||||
|
|
||||||
|
:param cache_backend: 异步缓存后端实例
|
||||||
|
:param region: 缓存区域
|
||||||
|
"""
|
||||||
|
self._cache_backend = cache_backend
|
||||||
|
self._region = region
|
||||||
|
|
||||||
|
def is_redis(self) -> bool:
|
||||||
|
"""
|
||||||
|
检查当前缓存后端是否为 Redis(纯状态判断,无需 await)
|
||||||
|
"""
|
||||||
|
return self._cache_backend.is_redis()
|
||||||
|
|
||||||
|
async def get(self, key: str, **kwargs) -> Any:
|
||||||
|
"""
|
||||||
|
获取缓存值
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
return await self._cache_backend.get(key, **kwargs)
|
||||||
|
|
||||||
|
async def set(self, key: str, value: Any, **kwargs) -> None:
|
||||||
|
"""
|
||||||
|
设置缓存值
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
await self._cache_backend.set(key, value, **kwargs)
|
||||||
|
|
||||||
|
async def delete(self, key: str, **kwargs) -> None:
|
||||||
|
"""
|
||||||
|
删除缓存值
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
await self._cache_backend.delete(key, **kwargs)
|
||||||
|
|
||||||
|
async def exists(self, key: str, **kwargs) -> bool:
|
||||||
|
"""
|
||||||
|
检查缓存键是否存在
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
return await self._cache_backend.exists(key, **kwargs)
|
||||||
|
|
||||||
|
async def clear(self, **kwargs) -> None:
|
||||||
|
"""
|
||||||
|
清除缓存
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
await self._cache_backend.clear(**kwargs)
|
||||||
|
|
||||||
|
async def items(self, **kwargs):
|
||||||
|
"""
|
||||||
|
获取所有缓存项
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
async for item in self._cache_backend.items(**kwargs):
|
||||||
|
yield item
|
||||||
|
|
||||||
|
async def keys(self, **kwargs):
|
||||||
|
"""
|
||||||
|
获取所有缓存键
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
async for key in self._cache_backend.keys(**kwargs):
|
||||||
|
yield key
|
||||||
|
|
||||||
|
async def values(self, **kwargs):
|
||||||
|
"""
|
||||||
|
获取所有缓存值
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
async for value in self._cache_backend.values(**kwargs):
|
||||||
|
yield value
|
||||||
|
|
||||||
|
async def update(self, other: Dict[str, Any], **kwargs) -> None:
|
||||||
|
"""
|
||||||
|
更新缓存
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
await self._cache_backend.update(other, **kwargs)
|
||||||
|
|
||||||
|
async def pop(self, key: str, default: Any = None, **kwargs) -> Any:
|
||||||
|
"""
|
||||||
|
弹出缓存项
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
return await self._cache_backend.pop(key, default, **kwargs)
|
||||||
|
|
||||||
|
async def popitem(self, **kwargs) -> Tuple[str, Any]:
|
||||||
|
"""
|
||||||
|
弹出最后一个缓存项
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
return await self._cache_backend.popitem(**kwargs)
|
||||||
|
|
||||||
|
async def setdefault(self, key: str, default: Any = None, **kwargs) -> Any:
|
||||||
|
"""
|
||||||
|
设置默认值
|
||||||
|
"""
|
||||||
|
kwargs.setdefault('region', self._region)
|
||||||
|
return await self._cache_backend.setdefault(key, default, **kwargs)
|
||||||
|
|
||||||
|
async def close(self) -> None:
|
||||||
|
"""
|
||||||
|
关闭缓存连接
|
||||||
|
"""
|
||||||
|
await self._cache_backend.close()
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncTTLCache(AsyncCacheProxy):
|
||||||
|
"""
|
||||||
|
基于 TTL 的异步缓存类,与同步 TTLCache 使用同一 region 语义,
|
||||||
|
内存后端共享进程内存储,Redis 后端共享同一键空间
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||||
|
maxsize: Optional[int] = DEFAULT_CACHE_SIZE,
|
||||||
|
ttl: Optional[int] = DEFAULT_CACHE_TTL):
|
||||||
|
"""
|
||||||
|
初始化异步 TTL 缓存
|
||||||
|
|
||||||
|
:param maxsize: 缓存的最大条目数
|
||||||
|
:param ttl: 缓存的存活时间,单位秒
|
||||||
|
:param region: 缓存的区,为 None 时使用默认区
|
||||||
|
"""
|
||||||
|
super().__init__(AsyncCache(cache_type='ttl', maxsize=maxsize, ttl=ttl), region)
|
||||||
|
|
||||||
|
|
||||||
class TTLCache(CacheProxy):
|
class TTLCache(CacheProxy):
|
||||||
"""
|
"""
|
||||||
基于 TTL 的缓存类,兼容 cachetools.TTLCache 接口
|
基于 TTL 的缓存类,兼容 cachetools.TTLCache 接口
|
||||||
|
|||||||
+116
-1
@@ -1,7 +1,7 @@
|
|||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional, Union
|
from typing import Optional, Union
|
||||||
|
|
||||||
from app.runtime.cache import TTLCache
|
from app.runtime.cache import AsyncTTLCache, TTLCache
|
||||||
from app.runtime.localization import LocaleHelper
|
from app.runtime.localization import LocaleHelper
|
||||||
from app.schemas.types import ProgressKey
|
from app.schemas.types import ProgressKey
|
||||||
|
|
||||||
@@ -115,3 +115,118 @@ class ProgressHelper:
|
|||||||
)
|
)
|
||||||
detail["data"] = localized_data
|
detail["data"] = localized_data
|
||||||
return detail
|
return detail
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncProgressHelper:
|
||||||
|
"""
|
||||||
|
处理进度辅助类(异步)
|
||||||
|
|
||||||
|
与 ProgressHelper 共用同一个进度 region:内存后端共享进程内存储,
|
||||||
|
Redis 后端共享同一键空间,因此同步写入、异步读取(或反之)均互通。
|
||||||
|
供事件循环上的异步调用方使用,避免同步缓存后端阻塞事件循环。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, key: Union[ProgressKey, str]) -> None:
|
||||||
|
"""为指定业务键绑定独立的异步进度缓存区域。"""
|
||||||
|
if isinstance(key, Enum):
|
||||||
|
key = key.value
|
||||||
|
self._key = key
|
||||||
|
self._progress = AsyncTTLCache(region="progress", maxsize=1024, ttl=24 * 60 * 60)
|
||||||
|
|
||||||
|
async def __reset(self) -> None:
|
||||||
|
"""
|
||||||
|
重置进度
|
||||||
|
"""
|
||||||
|
await self._progress.set(self._key, {
|
||||||
|
"enable": False,
|
||||||
|
"value": 0,
|
||||||
|
"text": "请稍候...",
|
||||||
|
"data": {}
|
||||||
|
})
|
||||||
|
|
||||||
|
async def start(self) -> None:
|
||||||
|
"""
|
||||||
|
开始进度
|
||||||
|
"""
|
||||||
|
await self.__reset()
|
||||||
|
current = await self._progress.get(self._key)
|
||||||
|
if not current:
|
||||||
|
return
|
||||||
|
current['enable'] = True
|
||||||
|
await self._progress.set(self._key, current)
|
||||||
|
|
||||||
|
async def end(
|
||||||
|
self,
|
||||||
|
text: Optional[str] = "",
|
||||||
|
data: Optional[dict] = None,
|
||||||
|
value: Optional[Union[float, int]] = 100,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
结束进度
|
||||||
|
"""
|
||||||
|
current = await self._progress.get(self._key)
|
||||||
|
if not current:
|
||||||
|
return
|
||||||
|
if data is not None:
|
||||||
|
if not current.get('data'):
|
||||||
|
current['data'] = {}
|
||||||
|
current['data'].update(data)
|
||||||
|
current["enable"] = False
|
||||||
|
if value is not None:
|
||||||
|
current["value"] = max(min(float(value), 100), 0)
|
||||||
|
current["text"] = text or ""
|
||||||
|
await self._progress.set(self._key, current)
|
||||||
|
|
||||||
|
async def update(
|
||||||
|
self,
|
||||||
|
value: Optional[Union[float, int]] = None,
|
||||||
|
text: Optional[str] = None,
|
||||||
|
data: Optional[dict] = None,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
更新进度
|
||||||
|
"""
|
||||||
|
current = await self._progress.get(self._key)
|
||||||
|
if not current or not current.get('enable'):
|
||||||
|
return
|
||||||
|
if value is not None:
|
||||||
|
current['value'] = max(min(float(value), 100), 0)
|
||||||
|
if text is not None:
|
||||||
|
current['text'] = text
|
||||||
|
if data is not None:
|
||||||
|
if not current.get('data'):
|
||||||
|
current['data'] = {}
|
||||||
|
current['data'].update(data)
|
||||||
|
await self._progress.set(self._key, current)
|
||||||
|
|
||||||
|
async def get(self, locale: Optional[str] = None) -> Optional[dict]:
|
||||||
|
"""
|
||||||
|
获取当前进度,并按语言补充前端展示字段。
|
||||||
|
|
||||||
|
:param locale: 目标语言,未传入时使用当前请求上下文语言
|
||||||
|
:return: 当前进度字典
|
||||||
|
"""
|
||||||
|
current = await self._progress.get(self._key)
|
||||||
|
if not current:
|
||||||
|
return current
|
||||||
|
|
||||||
|
detail = current.copy()
|
||||||
|
text = detail.get("text")
|
||||||
|
if isinstance(text, str):
|
||||||
|
detail["text_i18n"] = LocaleHelper.translate_text(text, locale=locale)
|
||||||
|
|
||||||
|
data = detail.get("data")
|
||||||
|
if isinstance(data, dict):
|
||||||
|
localized_data = data.copy()
|
||||||
|
error = localized_data.get("error")
|
||||||
|
message = localized_data.get("message")
|
||||||
|
if isinstance(error, str):
|
||||||
|
localized_data["error_i18n"] = LocaleHelper.translate_text(
|
||||||
|
error, locale=locale
|
||||||
|
)
|
||||||
|
if isinstance(message, str):
|
||||||
|
localized_data["message_i18n"] = LocaleHelper.translate_text(
|
||||||
|
message, locale=locale
|
||||||
|
)
|
||||||
|
detail["data"] = localized_data
|
||||||
|
return detail
|
||||||
|
|||||||
+83
-13
@@ -31,7 +31,7 @@ from app.db.oper.systemconfig import SystemConfigOper
|
|||||||
from app.application.maintenance import build_cleanup_service
|
from app.application.maintenance import build_cleanup_service
|
||||||
from app.application.image import WallpaperHelper
|
from app.application.image import WallpaperHelper
|
||||||
from app.application.messaging.message import MessageHelper
|
from app.application.messaging.message import MessageHelper
|
||||||
from app.runtime.progress import ProgressHelper
|
from app.runtime.progress import AsyncProgressHelper, ProgressHelper
|
||||||
from app.adapters.external.server import MoviePilotServerHelper
|
from app.adapters.external.server import MoviePilotServerHelper
|
||||||
from app.runtime.extensions.service_config import ServiceConfigHelper
|
from app.runtime.extensions.service_config import ServiceConfigHelper
|
||||||
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
|
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
|
||||||
@@ -609,7 +609,7 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
)
|
)
|
||||||
return job
|
return job
|
||||||
|
|
||||||
def __finish_job(
|
async def __finish_job(
|
||||||
self,
|
self,
|
||||||
job_id: str,
|
job_id: str,
|
||||||
success: bool = True,
|
success: bool = True,
|
||||||
@@ -627,10 +627,11 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
job["last_finished_at"] = finished_at
|
job["last_finished_at"] = finished_at
|
||||||
job["last_error"] = error
|
job["last_error"] = error
|
||||||
job_name = job.get("name") if job else job_id
|
job_name = job.get("name") if job else job_id
|
||||||
progress = ProgressHelper(self._get_progress_key(job_id))
|
# 收尾可能发生在事件循环上(__run_coro_job),使用异步进度后端避免阻塞
|
||||||
current_progress = progress.get() or {}
|
progress = AsyncProgressHelper(self._get_progress_key(job_id))
|
||||||
|
current_progress = await progress.get() or {}
|
||||||
progress_value = 100 if success else current_progress.get("value", 0)
|
progress_value = 100 if success else current_progress.get("value", 0)
|
||||||
progress.end(
|
await progress.end(
|
||||||
text=f"{job_name} {'执行完成' if success else '执行失败'}",
|
text=f"{job_name} {'执行完成' if success else '执行失败'}",
|
||||||
data={
|
data={
|
||||||
"id": job_id,
|
"id": job_id,
|
||||||
@@ -682,6 +683,45 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
data=data,
|
data=data,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def aget_progress(self, job_id: str) -> Optional[_SchemaScheduleProgress]:
|
||||||
|
"""
|
||||||
|
查询指定定时服务的执行进度(异步版本,供事件循环上的端点使用)。
|
||||||
|
"""
|
||||||
|
if not job_id:
|
||||||
|
return None
|
||||||
|
with self._lock:
|
||||||
|
job = self._jobs.get(job_id)
|
||||||
|
job_name = job.get("name") if job else job_id
|
||||||
|
provider_name = job.get("provider_name", "[系统]") if job else None
|
||||||
|
running = bool(job.get("running")) if job else False
|
||||||
|
last_started_at = job.get("last_started_at") if job else None
|
||||||
|
last_finished_at = job.get("last_finished_at") if job else None
|
||||||
|
last_error = job.get("last_error") if job else None
|
||||||
|
# 异步后端读取,避免在事件循环上阻塞
|
||||||
|
detail = await AsyncProgressHelper(self._get_progress_key(job_id)).get() or {}
|
||||||
|
if not job and not detail:
|
||||||
|
return None
|
||||||
|
data = detail.get("data") or {}
|
||||||
|
value = detail.get("value", 0)
|
||||||
|
try:
|
||||||
|
value = float(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
value = 0.0
|
||||||
|
return _SchemaScheduleProgress(
|
||||||
|
id=job_id,
|
||||||
|
name=data.get("name") or job_name,
|
||||||
|
provider=data.get("provider") or provider_name,
|
||||||
|
enable=bool(detail.get("enable", running)),
|
||||||
|
value=max(min(value, 100), 0),
|
||||||
|
text=detail.get("text"),
|
||||||
|
status=data.get("status") or ("running" if running else "waiting"),
|
||||||
|
success=data.get("success"),
|
||||||
|
started_at=data.get("started_at") or last_started_at,
|
||||||
|
finished_at=data.get("finished_at") or last_finished_at,
|
||||||
|
error=data.get("error") or last_error,
|
||||||
|
data=data,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def __handle_job_error(job_id: str, job: dict, error: Exception) -> None:
|
def __handle_job_error(job_id: str, job: dict, error: Exception) -> None:
|
||||||
"""
|
"""
|
||||||
@@ -726,11 +766,19 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
}
|
}
|
||||||
if data:
|
if data:
|
||||||
progress_data.update(data)
|
progress_data.update(data)
|
||||||
ProgressHelper(self._get_progress_key(job_id)).update(
|
key = self._get_progress_key(job_id)
|
||||||
value=value,
|
|
||||||
text=text,
|
async def _update() -> None:
|
||||||
data=progress_data,
|
# 异步后端更新,避免任务函数在事件循环内调用回调时阻塞
|
||||||
)
|
await AsyncProgressHelper(key).update(
|
||||||
|
value=value,
|
||||||
|
text=text,
|
||||||
|
data=progress_data,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 回调可能在事件循环内(async 任务)或线程池中(sync 任务)被调用,
|
||||||
|
# 统一经事件循环提交;无运行中循环时同步执行兜底
|
||||||
|
self._submit_to_loop(_update())
|
||||||
|
|
||||||
return update_progress
|
return update_progress
|
||||||
|
|
||||||
@@ -778,7 +826,8 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
error = str(err)
|
error = str(err)
|
||||||
self.__handle_job_error(job_id=job_id, job=job, error=err)
|
self.__handle_job_error(job_id=job_id, job=job, error=err)
|
||||||
finally:
|
finally:
|
||||||
self.__finish_job(job_id=job_id, success=success, error=error)
|
# 协程收尾在事件循环上完成,同步路径(线程池/调用线程)提交到事件循环执行
|
||||||
|
await self.__finish_job(job_id=job_id, success=success, error=error)
|
||||||
|
|
||||||
def start(self, job_id: str, *args, **kwargs) -> None:
|
def start(self, job_id: str, *args, **kwargs) -> None:
|
||||||
"""
|
"""
|
||||||
@@ -845,8 +894,29 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
self.__handle_job_error(job_id=job_id, job=job, error=e)
|
self.__handle_job_error(job_id=job_id, job=job, error=e)
|
||||||
finally:
|
finally:
|
||||||
if not deferred_finish:
|
if not deferred_finish:
|
||||||
# 运行结束
|
# 同步上下文执行异步收尾:优先提交到当前/全局事件循环,无循环时新建循环
|
||||||
self.__finish_job(job_id=job_id, success=success, error=error)
|
self._submit_to_loop(self.__finish_job(
|
||||||
|
job_id=job_id, success=success, error=error
|
||||||
|
))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _submit_to_loop(coro: Any) -> None:
|
||||||
|
"""
|
||||||
|
把协程提交到事件循环执行,兼容以下调用环境:
|
||||||
|
- 已在事件循环内(async 任务内部):排队为独立任务,避免阻塞
|
||||||
|
- 外部线程且全局循环在运行:跨线程提交,非阻塞
|
||||||
|
- 无运行中循环(测试/CLI):新建循环同步执行,确保进度不丢失
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
running_loop = asyncio.get_running_loop()
|
||||||
|
except RuntimeError:
|
||||||
|
running_loop = None
|
||||||
|
if running_loop:
|
||||||
|
asyncio.create_task(coro)
|
||||||
|
elif global_vars.loop and global_vars.loop.is_running():
|
||||||
|
asyncio.run_coroutine_threadsafe(coro, global_vars.loop)
|
||||||
|
else:
|
||||||
|
asyncio.run(coro)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_agent_task_job_id(task_id: int) -> str:
|
def _get_agent_task_job_id(task_id: int) -> str:
|
||||||
|
|||||||
@@ -1652,7 +1652,7 @@
|
|||||||
"consumers": [
|
"consumers": [
|
||||||
{
|
{
|
||||||
"caller": "app.scheduler",
|
"caller": "app.scheduler",
|
||||||
"line": 1046
|
"line": 1116
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"producers": []
|
"producers": []
|
||||||
@@ -1665,7 +1665,7 @@
|
|||||||
"consumers": [
|
"consumers": [
|
||||||
{
|
{
|
||||||
"caller": "app.chain.search",
|
"caller": "app.chain.search",
|
||||||
"line": 2861
|
"line": 2864
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"caller": "app.chain.subscribe",
|
"caller": "app.chain.subscribe",
|
||||||
@@ -1813,7 +1813,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"caller": "app.scheduler",
|
"caller": "app.scheduler",
|
||||||
"line": 696
|
"line": 736
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in New Issue
Block a user