refactor: 收敛 factory 模块级副作用并统一 async 路径进度为异步后端

- 将 configure_token_codec/configure_plugin_routes 移入 create_app(),消除 import 期副作用
- runtime 层新增 AsyncCacheProxy/AsyncTTLCache/AsyncProgressHelper,共享 progress region
- search/system/history/scheduler/dashboard 的事件循环路径切换异步进度后端
- 同步进度回调经事件循环提交或线程池执行,避免阻塞事件循环
This commit is contained in:
jxxghp
2026-08-19 09:11:42 +08:00
parent 91d339a8dd
commit 03ada7eb01
9 changed files with 446 additions and 105 deletions
+9 -4
View File
@@ -2,6 +2,7 @@ from pathlib import Path
from typing import Any, List, Optional, Annotated from typing import Any, List, Optional, Annotated
from fastapi import Depends from fastapi import Depends
from fastapi.concurrency import run_in_threadpool
from app.schemas.dashboard import DashboardMemoryInfo as _SchemaDashboardMemoryInfo from app.schemas.dashboard import DashboardMemoryInfo as _SchemaDashboardMemoryInfo
from app.schemas.dashboard import DashboardSystemInfo as _SchemaDashboardSystemInfo from app.schemas.dashboard import DashboardSystemInfo as _SchemaDashboardSystemInfo
@@ -160,7 +161,8 @@ async def schedule(_: Any = Depends(get_current_active_superuser)) -> Any:
""" """
查询后台服务信息 查询后台服务信息
""" """
return Scheduler().list() # 同步 list() 内含同步进度读取,放到线程池执行避免阻塞事件循环
return await run_in_threadpool(Scheduler().list)
@router.get( @router.get(
@@ -174,7 +176,8 @@ async def schedule_progress(
""" """
查询指定后台服务的执行进度。 查询指定后台服务的执行进度。
""" """
progress = Scheduler().get_progress(job_id) # 异步进度后端读取,避免同步 Redis 调用阻塞事件循环
progress = await Scheduler().aget_progress(job_id)
if not progress: if not progress:
return _SchemaResponse(success=False, message="后台服务不存在") return _SchemaResponse(success=False, message="后台服务不存在")
return _SchemaResponse(success=True, data=progress.model_dump()) return _SchemaResponse(success=True, data=progress.model_dump())
@@ -189,7 +192,8 @@ async def schedule2(_: Annotated[str, Depends(verify_apitoken)]) -> Any:
""" """
查询下载器信息 API_TOKEN认证(?token=xxx 查询下载器信息 API_TOKEN认证(?token=xxx
""" """
return Scheduler().list() # 同步 list() 内含同步进度读取,放到线程池执行避免阻塞事件循环
return await run_in_threadpool(Scheduler().list)
@router.get( @router.get(
@@ -203,7 +207,8 @@ async def schedule_progress2(
""" """
查询指定后台服务的执行进度 API_TOKEN认证(?token=xxx 查询指定后台服务的执行进度 API_TOKEN认证(?token=xxx
""" """
progress = Scheduler().get_progress(job_id) # 异步进度后端读取,避免同步 Redis 调用阻塞事件循环
progress = await Scheduler().aget_progress(job_id)
if not progress: if not progress:
return _SchemaResponse(success=False, message="后台服务不存在") return _SchemaResponse(success=False, message="后台服务不存在")
return _SchemaResponse(success=True, data=progress.model_dump()) return _SchemaResponse(success=True, data=progress.model_dump())
+33 -23
View File
@@ -28,7 +28,7 @@ from app.api.deps import (
get_history_query_service, get_history_query_service,
get_transfer_history_mutation_command, get_transfer_history_mutation_command,
) )
from app.runtime.progress import ProgressHelper from app.runtime.progress import AsyncProgressHelper
from app.application.history import ( from app.application.history import (
DownloadHistoryMutationCommand, DownloadHistoryMutationCommand,
HistoryQueryService, HistoryQueryService,
@@ -49,19 +49,24 @@ def normalize_history_ids(history_ids: list[int]) -> list[int]:
def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str): def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str):
"""在后台线程中启动单条 AI 重新整理任务,并通过 ProgressHelper 实时更新进度。""" """在后台任务中启动单条 AI 重新整理任务,并通过异步进度辅助类实时更新进度。"""
progress = ProgressHelper(progress_key) progress = AsyncProgressHelper(progress_key)
progress.start()
progress.update(
text=f"智能助手正在准备整理记录 #{history_id} ...",
data={"history_id": history_id, "success": True},
)
def update_output(text: str): def update_output(text: str):
progress.update(text=text, data={"history_id": history_id}) # 输出回调由 agent 在事件循环上同步调用,不能直接 await;
# 提交到全局事件循环非阻塞执行,避免同步缓存后端阻塞事件循环。
asyncio.run_coroutine_threadsafe(
progress.update(text=text, data={"history_id": history_id}),
global_vars.loop,
)
async def runner(): async def runner():
try: try:
await progress.start()
await progress.update(
text=f"智能助手正在准备整理记录 #{history_id} ...",
data={"history_id": history_id, "success": True},
)
manager = get_running_agent_manager() manager = get_running_agent_manager()
if manager is None: if manager is None:
logger.warning("智能助手服务未运行,跳过单条整理历史 AI 重做") logger.warning("智能助手服务未运行,跳过单条整理历史 AI 重做")
@@ -73,12 +78,12 @@ def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str):
reply_mode=ReplyMode.CAPTURE_ONLY, reply_mode=ReplyMode.CAPTURE_ONLY,
allow_message_tools=False, allow_message_tools=False,
) )
progress.update( await progress.update(
text="智能助手整理完成", text="智能助手整理完成",
data={"history_id": history_id, "success": True, "completed": True}, data={"history_id": history_id, "success": True, "completed": True},
) )
except Exception as e: except Exception as e:
progress.update( await progress.update(
text=f"智能助手整理失败:{str(e)}", text=f"智能助手整理失败:{str(e)}",
data={ data={
"history_id": history_id, "history_id": history_id,
@@ -88,7 +93,7 @@ def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str):
}, },
) )
finally: finally:
progress.end() await progress.end()
asyncio.run_coroutine_threadsafe(runner(), global_vars.loop) asyncio.run_coroutine_threadsafe(runner(), global_vars.loop)
@@ -98,19 +103,24 @@ def _start_batch_ai_redo_task(
prompt: str, prompt: str,
progress_key: str, progress_key: str,
): ):
"""在后台线程中启动批量 AI 重新整理任务,并通过 ProgressHelper 实时更新进度。""" """在后台任务中启动批量 AI 重新整理任务,并通过异步进度辅助类实时更新进度。"""
progress = ProgressHelper(progress_key) progress = AsyncProgressHelper(progress_key)
progress.start()
progress.update(
text=f"智能助手正在准备批量整理 {len(history_ids)} 条记录 ...",
data={"history_ids": history_ids, "success": True},
)
def update_output(text: str): def update_output(text: str):
progress.update(text=text, data={"history_ids": history_ids}) # 输出回调由 agent 在事件循环上同步调用,不能直接 await;
# 提交到全局事件循环非阻塞执行,避免同步缓存后端阻塞事件循环。
asyncio.run_coroutine_threadsafe(
progress.update(text=text, data={"history_ids": history_ids}),
global_vars.loop,
)
async def runner(): async def runner():
try: try:
await progress.start()
await progress.update(
text=f"智能助手正在准备批量整理 {len(history_ids)} 条记录 ...",
data={"history_ids": history_ids, "success": True},
)
manager = get_running_agent_manager() manager = get_running_agent_manager()
if manager is None: if manager is None:
logger.warning("智能助手服务未运行,跳过批量整理历史 AI 重做") logger.warning("智能助手服务未运行,跳过批量整理历史 AI 重做")
@@ -122,12 +132,12 @@ def _start_batch_ai_redo_task(
reply_mode=ReplyMode.CAPTURE_ONLY, reply_mode=ReplyMode.CAPTURE_ONLY,
allow_message_tools=False, allow_message_tools=False,
) )
progress.update( await progress.update(
text="智能助手批量整理完成", text="智能助手批量整理完成",
data={"history_ids": history_ids, "success": True, "completed": True}, data={"history_ids": history_ids, "success": True, "completed": True},
) )
except Exception as e: except Exception as e:
progress.update( await progress.update(
text=f"智能助手批量整理失败:{str(e)}", text=f"智能助手批量整理失败:{str(e)}",
data={ data={
"history_ids": history_ids, "history_ids": history_ids,
@@ -137,7 +147,7 @@ def _start_batch_ai_redo_task(
}, },
) )
finally: finally:
progress.end() await progress.end()
asyncio.run_coroutine_threadsafe(runner(), global_vars.loop) asyncio.run_coroutine_threadsafe(runner(), global_vars.loop)
+3 -3
View File
@@ -52,7 +52,7 @@ from app.adapters.external.market import (
split_plugin_market_repo_urls, split_plugin_market_repo_urls,
) )
from app.application.messaging.message import MessageHelper from app.application.messaging.message import MessageHelper
from app.runtime.progress import ProgressHelper from app.runtime.progress import AsyncProgressHelper
from app.application.rules import RuleHelper from app.application.rules import RuleHelper
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.state import SystemHelper from app.runtime.state import SystemHelper
@@ -847,7 +847,7 @@ async def get_progress(
""" """
实时获取处理进度,返回格式为SSE 实时获取处理进度,返回格式为SSE
""" """
progress = ProgressHelper(process_type) progress = AsyncProgressHelper(process_type)
locale = LocaleHelper.get_current_locale() locale = LocaleHelper.get_current_locale()
async def event_generator(): async def event_generator():
@@ -855,7 +855,7 @@ async def get_progress(
while not global_vars.is_system_stopped: while not global_vars.is_system_stopped:
if await request.is_disconnected(): if await request.is_disconnected():
break break
detail = progress.get(locale=locale) detail = await progress.get(locale=locale)
yield f"data: {json.dumps(detail)}\n\n" yield f"data: {json.dumps(detail)}\n\n"
await asyncio.sleep(0.5) await asyncio.sleep(0.5)
except asyncio.CancelledError: except asyncio.CancelledError:
+39 -36
View File
@@ -22,7 +22,7 @@ from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.domain.context import MusicInfo from app.domain.context import MusicInfo
from app.application.configuration import get_configured_system_config from app.application.configuration import get_configured_system_config
from app.runtime.progress import ProgressHelper from app.runtime.progress import AsyncProgressHelper, ProgressHelper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.application.search.state import ( from app.application.search.state import (
SearchStateService, SearchStateService,
@@ -2396,9 +2396,9 @@ class SearchChain(ChainBase):
logger.warn('未开启任何有效站点,无法搜索资源') logger.warn('未开启任何有效站点,无法搜索资源')
return [] return []
# 开始进度 # 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
progress = ProgressHelper(ProgressKey.Search) progress = AsyncProgressHelper(ProgressKey.Search)
progress.start() await progress.start()
# 开始计时 # 开始计时
start_time = datetime.now() start_time = datetime.now()
search_pages = self._build_search_pages(page) search_pages = self._build_search_pages(page)
@@ -2407,8 +2407,8 @@ class SearchChain(ChainBase):
# 完成数 # 完成数
finish_count = 0 finish_count = 0
# 更新进度 # 更新进度
progress.update(value=0, await progress.update(value=0,
text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...") text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
# 结果集 # 结果集
results = [] results = []
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num) semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
@@ -2470,8 +2470,8 @@ class SearchChain(ChainBase):
f"{site.get('name')}{search_page} 页返回 {len(result or [])} 条,停止继续翻页" f"{site.get('name')}{search_page} 页返回 {len(result or [])} 条,停止继续翻页"
) )
logger.info(f"站点搜索进度:{finish_count} / {total_num}") logger.info(f"站点搜索进度:{finish_count} / {total_num}")
progress.update(value=finish_count / total_num * 100, await progress.update(value=finish_count / total_num * 100,
text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...") text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
finally: finally:
for task in pending_tasks: for task in pending_tasks:
if not task.done(): if not task.done():
@@ -2482,11 +2482,11 @@ class SearchChain(ChainBase):
# 计算耗时 # 计算耗时
end_time = datetime.now() end_time = datetime.now()
# 更新进度 # 更新进度
progress.update(value=100, await progress.update(value=100,
text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds}") text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds}")
logger.info(f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds}") logger.info(f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds}")
# 结束进度 # 结束进度
progress.end() await progress.end()
# 返回 # 返回
return results return results
@@ -2527,14 +2527,15 @@ class SearchChain(ChainBase):
} }
return return
progress = ProgressHelper(ProgressKey.Search) # 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
progress.start() progress = AsyncProgressHelper(ProgressKey.Search)
await progress.start()
start_time = datetime.now() start_time = datetime.now()
search_pages = self._build_search_pages(page) search_pages = self._build_search_pages(page)
total_num = len(indexer_sites) * len(search_pages) total_num = len(indexer_sites) * len(search_pages)
finish_count = 0 finish_count = 0
progress.update(value=0, await progress.update(value=0,
text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...") text=f"开始搜索,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
yield { yield {
"type": "progress", "type": "progress",
"stage": "searching", "stage": "searching",
@@ -2606,7 +2607,7 @@ class SearchChain(ChainBase):
logger.info(f"站点搜索进度:{finish_count} / {total_num}") logger.info(f"站点搜索进度:{finish_count} / {total_num}")
progress_value = finish_count / total_num * 100 progress_value = finish_count / total_num * 100
progress_text = f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..." progress_text = f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
progress.update(value=progress_value, text=progress_text) await progress.update(value=progress_value, text=progress_text)
yield { yield {
"type": "append", "type": "append",
"stage": "searching", "stage": "searching",
@@ -2628,10 +2629,10 @@ class SearchChain(ChainBase):
await asyncio.gather(*tasks.keys(), return_exceptions=True) await asyncio.gather(*tasks.keys(), return_exceptions=True)
end_time = datetime.now() end_time = datetime.now()
progress.update(value=100, await progress.update(value=100,
text=f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds}") text=f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds}")
logger.info(f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds}") logger.info(f"站点搜索完成,有效资源数:{results_count},总耗时 {(end_time - start_time).seconds}")
progress.end() await progress.end()
async def __async_search_subtitles_all_sites(self, keyword: str, async def __async_search_subtitles_all_sites(self, keyword: str,
sites: List[int] = None, sites: List[int] = None,
@@ -2657,14 +2658,15 @@ class SearchChain(ChainBase):
logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕') logger.warn('未开启任何支持字幕搜索的有效站点,无法搜索字幕')
return [] return []
progress = ProgressHelper(ProgressKey.Search) # 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
progress.start() progress = AsyncProgressHelper(ProgressKey.Search)
await progress.start()
start_time = datetime.now() start_time = datetime.now()
search_pages = self._build_search_pages(page) search_pages = self._build_search_pages(page)
total_num = len(indexer_sites) * len(search_pages) total_num = len(indexer_sites) * len(search_pages)
finish_count = 0 finish_count = 0
progress.update(value=0, await progress.update(value=0,
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...") text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
results = [] results = []
semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num) semaphore = asyncio.Semaphore(settings.CONF.threadpool or total_num)
@@ -2714,8 +2716,8 @@ class SearchChain(ChainBase):
f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页" f"{site.get('name')} 字幕第 {search_page} 页返回 {len(result or [])} 条,停止继续翻页"
) )
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}") logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
progress.update(value=finish_count / total_num * 100, await progress.update(value=finish_count / total_num * 100,
text=f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...") text=f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ...")
finally: finally:
for task in pending_tasks: for task in pending_tasks:
if not task.done(): if not task.done():
@@ -2724,10 +2726,10 @@ class SearchChain(ChainBase):
await asyncio.gather(*pending_tasks.keys(), return_exceptions=True) await asyncio.gather(*pending_tasks.keys(), return_exceptions=True)
end_time = datetime.now() end_time = datetime.now()
progress.update(value=100, await progress.update(value=100,
text=f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds}") text=f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds}")
logger.info(f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds}") logger.info(f"站点字幕搜索完成,有效字幕数:{len(results)},总耗时 {(end_time - start_time).seconds}")
progress.end() await progress.end()
return results return results
async def __async_search_subtitles_all_sites_stream(self, keyword: str, async def __async_search_subtitles_all_sites_stream(self, keyword: str,
@@ -2762,14 +2764,15 @@ class SearchChain(ChainBase):
} }
return return
progress = ProgressHelper(ProgressKey.Search) # 开始进度(异步后端,避免同步 Redis 在事件循环上阻塞)
progress.start() progress = AsyncProgressHelper(ProgressKey.Search)
await progress.start()
start_time = datetime.now() start_time = datetime.now()
search_pages = self._build_search_pages(page) search_pages = self._build_search_pages(page)
total_num = len(indexer_sites) * len(search_pages) total_num = len(indexer_sites) * len(search_pages)
finish_count = 0 finish_count = 0
progress.update(value=0, await progress.update(value=0,
text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...") text=f"开始搜索字幕,共 {len(indexer_sites)} 个站点,{len(search_pages)} 页 ...")
yield { yield {
"type": "progress", "type": "progress",
"stage": "searching", "stage": "searching",
@@ -2831,7 +2834,7 @@ class SearchChain(ChainBase):
logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}") logger.info(f"站点字幕搜索进度:{finish_count} / {total_num}")
progress_value = finish_count / total_num * 100 progress_value = finish_count / total_num * 100
progress_text = f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..." progress_text = f"正在搜索字幕{keyword or ''},已完成 {finish_count} / {total_num} 个请求 ..."
progress.update(value=progress_value, text=progress_text) await progress.update(value=progress_value, text=progress_text)
yield { yield {
"type": "append", "type": "append",
"stage": "searching", "stage": "searching",
@@ -2853,10 +2856,10 @@ class SearchChain(ChainBase):
await asyncio.gather(*tasks.keys(), return_exceptions=True) await asyncio.gather(*tasks.keys(), return_exceptions=True)
end_time = datetime.now() end_time = datetime.now()
progress.update(value=100, await progress.update(value=100,
text=f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds}") text=f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds}")
logger.info(f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds}") logger.info(f"站点字幕搜索完成,有效字幕数:{results_count},总耗时 {(end_time - start_time).seconds}")
progress.end() await progress.end()
@eventmanager.register(EventType.SiteDeleted) @eventmanager.register(EventType.SiteDeleted)
def remove_site(self, event: Event): def remove_site(self, event: Event):
+23 -22
View File
@@ -314,6 +314,7 @@ def create_app() -> FastAPI:
allow_methods=["*"], allow_methods=["*"],
allow_headers=["*"], allow_headers=["*"],
) )
@_app.middleware("http") @_app.middleware("http")
async def locale_context_middleware( async def locale_context_middleware(
request: Request, request: Request,
@@ -330,29 +331,29 @@ def create_app() -> FastAPI:
finally: finally:
LocaleHelper.reset_current_locale(token) LocaleHelper.reset_current_locale(token)
# HTTP 适配器只持有令牌编解码端口,具体实现由组合根在创建应用时连接。
configure_token_codec(create_access_token, decode_access_token)
# 向 application 层插件路由服务注入应用实例,插件 API 的动态注册/移除
# 统一经服务完成,避免 api.endpoints 反向依赖本模块。
configure_plugin_routes(FastAPIDynamicRouteRegistry(
app=_app,
plugin_ids=lambda: PluginManager().get_running_plugin_ids(),
plugin_apis=lambda plugin_id: PluginManager().get_plugin_apis(plugin_id),
verify_token=verify_token,
verify_apikey=verify_apikey,
prefix=f"{settings.API_V1_STR}/plugin",
protected_routes={
f"{settings.API_V1_STR}/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
},
log=logger,
))
return _app return _app
# HTTP 适配器只持有令牌编解码端口,具体实现由组合根连接。 # 创建 FastAPI 应用实例;所有组合根装配副作用都在 create_app() 内部完成
configure_token_codec(create_access_token, decode_access_token)
# 创建 FastAPI 应用实例
app = create_app() app = create_app()
# 向 application 层插件路由服务注入应用实例,插件 API 的动态注册/移除
# 统一经服务完成,避免 api.endpoints 反向依赖本模块。
configure_plugin_routes(FastAPIDynamicRouteRegistry(
app=app,
plugin_ids=lambda: PluginManager().get_running_plugin_ids(),
plugin_apis=lambda plugin_id: PluginManager().get_plugin_apis(plugin_id),
verify_token=verify_token,
verify_apikey=verify_apikey,
prefix=f"{settings.API_V1_STR}/plugin",
protected_routes={
f"{settings.API_V1_STR}/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
},
log=logger,
))
+137
View File
@@ -1158,6 +1158,143 @@ class CacheProxy:
self._cache_backend.close() self._cache_backend.close()
class AsyncCacheProxy:
"""
异步缓存代理类,将异步缓存后端的方法直接代理到实例上
与同步 CacheProxy 的唯一差异是方法均为 async,默认绑定构造时指定的 region。
"""
def __init__(self, cache_backend: AsyncCacheBackend, region: str):
"""
初始化异步缓存代理
:param cache_backend: 异步缓存后端实例
:param region: 缓存区域
"""
self._cache_backend = cache_backend
self._region = region
def is_redis(self) -> bool:
"""
检查当前缓存后端是否为 Redis(纯状态判断,无需 await)
"""
return self._cache_backend.is_redis()
async def get(self, key: str, **kwargs) -> Any:
"""
获取缓存值
"""
kwargs.setdefault('region', self._region)
return await self._cache_backend.get(key, **kwargs)
async def set(self, key: str, value: Any, **kwargs) -> None:
"""
设置缓存值
"""
kwargs.setdefault('region', self._region)
await self._cache_backend.set(key, value, **kwargs)
async def delete(self, key: str, **kwargs) -> None:
"""
删除缓存值
"""
kwargs.setdefault('region', self._region)
await self._cache_backend.delete(key, **kwargs)
async def exists(self, key: str, **kwargs) -> bool:
"""
检查缓存键是否存在
"""
kwargs.setdefault('region', self._region)
return await self._cache_backend.exists(key, **kwargs)
async def clear(self, **kwargs) -> None:
"""
清除缓存
"""
kwargs.setdefault('region', self._region)
await self._cache_backend.clear(**kwargs)
async def items(self, **kwargs):
"""
获取所有缓存项
"""
kwargs.setdefault('region', self._region)
async for item in self._cache_backend.items(**kwargs):
yield item
async def keys(self, **kwargs):
"""
获取所有缓存键
"""
kwargs.setdefault('region', self._region)
async for key in self._cache_backend.keys(**kwargs):
yield key
async def values(self, **kwargs):
"""
获取所有缓存值
"""
kwargs.setdefault('region', self._region)
async for value in self._cache_backend.values(**kwargs):
yield value
async def update(self, other: Dict[str, Any], **kwargs) -> None:
"""
更新缓存
"""
kwargs.setdefault('region', self._region)
await self._cache_backend.update(other, **kwargs)
async def pop(self, key: str, default: Any = None, **kwargs) -> Any:
"""
弹出缓存项
"""
kwargs.setdefault('region', self._region)
return await self._cache_backend.pop(key, default, **kwargs)
async def popitem(self, **kwargs) -> Tuple[str, Any]:
"""
弹出最后一个缓存项
"""
kwargs.setdefault('region', self._region)
return await self._cache_backend.popitem(**kwargs)
async def setdefault(self, key: str, default: Any = None, **kwargs) -> Any:
"""
设置默认值
"""
kwargs.setdefault('region', self._region)
return await self._cache_backend.setdefault(key, default, **kwargs)
async def close(self) -> None:
"""
关闭缓存连接
"""
await self._cache_backend.close()
class AsyncTTLCache(AsyncCacheProxy):
"""
基于 TTL 的异步缓存类,与同步 TTLCache 使用同一 region 语义,
内存后端共享进程内存储,Redis 后端共享同一键空间
"""
def __init__(self,
region: Optional[str] = DEFAULT_CACHE_REGION,
maxsize: Optional[int] = DEFAULT_CACHE_SIZE,
ttl: Optional[int] = DEFAULT_CACHE_TTL):
"""
初始化异步 TTL 缓存
:param maxsize: 缓存的最大条目数
:param ttl: 缓存的存活时间,单位秒
:param region: 缓存的区,为 None 时使用默认区
"""
super().__init__(AsyncCache(cache_type='ttl', maxsize=maxsize, ttl=ttl), region)
class TTLCache(CacheProxy): class TTLCache(CacheProxy):
""" """
基于 TTL 的缓存类,兼容 cachetools.TTLCache 接口 基于 TTL 的缓存类,兼容 cachetools.TTLCache 接口
+116 -1
View File
@@ -1,7 +1,7 @@
from enum import Enum from enum import Enum
from typing import Optional, Union from typing import Optional, Union
from app.runtime.cache import TTLCache from app.runtime.cache import AsyncTTLCache, TTLCache
from app.runtime.localization import LocaleHelper from app.runtime.localization import LocaleHelper
from app.schemas.types import ProgressKey from app.schemas.types import ProgressKey
@@ -115,3 +115,118 @@ class ProgressHelper:
) )
detail["data"] = localized_data detail["data"] = localized_data
return detail return detail
class AsyncProgressHelper:
"""
处理进度辅助类(异步)
与 ProgressHelper 共用同一个进度 region:内存后端共享进程内存储,
Redis 后端共享同一键空间,因此同步写入、异步读取(或反之)均互通。
供事件循环上的异步调用方使用,避免同步缓存后端阻塞事件循环。
"""
def __init__(self, key: Union[ProgressKey, str]) -> None:
"""为指定业务键绑定独立的异步进度缓存区域。"""
if isinstance(key, Enum):
key = key.value
self._key = key
self._progress = AsyncTTLCache(region="progress", maxsize=1024, ttl=24 * 60 * 60)
async def __reset(self) -> None:
"""
重置进度
"""
await self._progress.set(self._key, {
"enable": False,
"value": 0,
"text": "请稍候...",
"data": {}
})
async def start(self) -> None:
"""
开始进度
"""
await self.__reset()
current = await self._progress.get(self._key)
if not current:
return
current['enable'] = True
await self._progress.set(self._key, current)
async def end(
self,
text: Optional[str] = "",
data: Optional[dict] = None,
value: Optional[Union[float, int]] = 100,
) -> None:
"""
结束进度
"""
current = await self._progress.get(self._key)
if not current:
return
if data is not None:
if not current.get('data'):
current['data'] = {}
current['data'].update(data)
current["enable"] = False
if value is not None:
current["value"] = max(min(float(value), 100), 0)
current["text"] = text or ""
await self._progress.set(self._key, current)
async def update(
self,
value: Optional[Union[float, int]] = None,
text: Optional[str] = None,
data: Optional[dict] = None,
) -> None:
"""
更新进度
"""
current = await self._progress.get(self._key)
if not current or not current.get('enable'):
return
if value is not None:
current['value'] = max(min(float(value), 100), 0)
if text is not None:
current['text'] = text
if data is not None:
if not current.get('data'):
current['data'] = {}
current['data'].update(data)
await self._progress.set(self._key, current)
async def get(self, locale: Optional[str] = None) -> Optional[dict]:
"""
获取当前进度,并按语言补充前端展示字段。
:param locale: 目标语言,未传入时使用当前请求上下文语言
:return: 当前进度字典
"""
current = await self._progress.get(self._key)
if not current:
return current
detail = current.copy()
text = detail.get("text")
if isinstance(text, str):
detail["text_i18n"] = LocaleHelper.translate_text(text, locale=locale)
data = detail.get("data")
if isinstance(data, dict):
localized_data = data.copy()
error = localized_data.get("error")
message = localized_data.get("message")
if isinstance(error, str):
localized_data["error_i18n"] = LocaleHelper.translate_text(
error, locale=locale
)
if isinstance(message, str):
localized_data["message_i18n"] = LocaleHelper.translate_text(
message, locale=locale
)
detail["data"] = localized_data
return detail
+83 -13
View File
@@ -31,7 +31,7 @@ from app.db.oper.systemconfig import SystemConfigOper
from app.application.maintenance import build_cleanup_service from app.application.maintenance import build_cleanup_service
from app.application.image import WallpaperHelper from app.application.image import WallpaperHelper
from app.application.messaging.message import MessageHelper from app.application.messaging.message import MessageHelper
from app.runtime.progress import ProgressHelper from app.runtime.progress import AsyncProgressHelper, ProgressHelper
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.extensions.service_config import ServiceConfigHelper from app.runtime.extensions.service_config import ServiceConfigHelper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
@@ -609,7 +609,7 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
) )
return job return job
def __finish_job( async def __finish_job(
self, self,
job_id: str, job_id: str,
success: bool = True, success: bool = True,
@@ -627,10 +627,11 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
job["last_finished_at"] = finished_at job["last_finished_at"] = finished_at
job["last_error"] = error job["last_error"] = error
job_name = job.get("name") if job else job_id job_name = job.get("name") if job else job_id
progress = ProgressHelper(self._get_progress_key(job_id)) # 收尾可能发生在事件循环上(__run_coro_job),使用异步进度后端避免阻塞
current_progress = progress.get() or {} progress = AsyncProgressHelper(self._get_progress_key(job_id))
current_progress = await progress.get() or {}
progress_value = 100 if success else current_progress.get("value", 0) progress_value = 100 if success else current_progress.get("value", 0)
progress.end( await progress.end(
text=f"{job_name} {'执行完成' if success else '执行失败'}", text=f"{job_name} {'执行完成' if success else '执行失败'}",
data={ data={
"id": job_id, "id": job_id,
@@ -682,6 +683,45 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
data=data, data=data,
) )
async def aget_progress(self, job_id: str) -> Optional[_SchemaScheduleProgress]:
"""
查询指定定时服务的执行进度(异步版本,供事件循环上的端点使用)。
"""
if not job_id:
return None
with self._lock:
job = self._jobs.get(job_id)
job_name = job.get("name") if job else job_id
provider_name = job.get("provider_name", "[系统]") if job else None
running = bool(job.get("running")) if job else False
last_started_at = job.get("last_started_at") if job else None
last_finished_at = job.get("last_finished_at") if job else None
last_error = job.get("last_error") if job else None
# 异步后端读取,避免在事件循环上阻塞
detail = await AsyncProgressHelper(self._get_progress_key(job_id)).get() or {}
if not job and not detail:
return None
data = detail.get("data") or {}
value = detail.get("value", 0)
try:
value = float(value)
except (TypeError, ValueError):
value = 0.0
return _SchemaScheduleProgress(
id=job_id,
name=data.get("name") or job_name,
provider=data.get("provider") or provider_name,
enable=bool(detail.get("enable", running)),
value=max(min(value, 100), 0),
text=detail.get("text"),
status=data.get("status") or ("running" if running else "waiting"),
success=data.get("success"),
started_at=data.get("started_at") or last_started_at,
finished_at=data.get("finished_at") or last_finished_at,
error=data.get("error") or last_error,
data=data,
)
@staticmethod @staticmethod
def __handle_job_error(job_id: str, job: dict, error: Exception) -> None: def __handle_job_error(job_id: str, job: dict, error: Exception) -> None:
""" """
@@ -726,11 +766,19 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
} }
if data: if data:
progress_data.update(data) progress_data.update(data)
ProgressHelper(self._get_progress_key(job_id)).update( key = self._get_progress_key(job_id)
value=value,
text=text, async def _update() -> None:
data=progress_data, # 异步后端更新,避免任务函数在事件循环内调用回调时阻塞
) await AsyncProgressHelper(key).update(
value=value,
text=text,
data=progress_data,
)
# 回调可能在事件循环内(async 任务)或线程池中(sync 任务)被调用,
# 统一经事件循环提交;无运行中循环时同步执行兜底
self._submit_to_loop(_update())
return update_progress return update_progress
@@ -778,7 +826,8 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
error = str(err) error = str(err)
self.__handle_job_error(job_id=job_id, job=job, error=err) self.__handle_job_error(job_id=job_id, job=job, error=err)
finally: finally:
self.__finish_job(job_id=job_id, success=success, error=error) # 协程收尾在事件循环上完成,同步路径(线程池/调用线程)提交到事件循环执行
await self.__finish_job(job_id=job_id, success=success, error=error)
def start(self, job_id: str, *args, **kwargs) -> None: def start(self, job_id: str, *args, **kwargs) -> None:
""" """
@@ -845,8 +894,29 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
self.__handle_job_error(job_id=job_id, job=job, error=e) self.__handle_job_error(job_id=job_id, job=job, error=e)
finally: finally:
if not deferred_finish: if not deferred_finish:
# 运行结束 # 同步上下文执行异步收尾:优先提交到当前/全局事件循环,无循环时新建循环
self.__finish_job(job_id=job_id, success=success, error=error) self._submit_to_loop(self.__finish_job(
job_id=job_id, success=success, error=error
))
@staticmethod
def _submit_to_loop(coro: Any) -> None:
"""
把协程提交到事件循环执行,兼容以下调用环境:
- 已在事件循环内(async 任务内部):排队为独立任务,避免阻塞
- 外部线程且全局循环在运行:跨线程提交,非阻塞
- 无运行中循环(测试/CLI):新建循环同步执行,确保进度不丢失
"""
try:
running_loop = asyncio.get_running_loop()
except RuntimeError:
running_loop = None
if running_loop:
asyncio.create_task(coro)
elif global_vars.loop and global_vars.loop.is_running():
asyncio.run_coroutine_threadsafe(coro, global_vars.loop)
else:
asyncio.run(coro)
@staticmethod @staticmethod
def _get_agent_task_job_id(task_id: int) -> str: def _get_agent_task_job_id(task_id: int) -> str:
+3 -3
View File
@@ -1652,7 +1652,7 @@
"consumers": [ "consumers": [
{ {
"caller": "app.scheduler", "caller": "app.scheduler",
"line": 1046 "line": 1116
} }
], ],
"producers": [] "producers": []
@@ -1665,7 +1665,7 @@
"consumers": [ "consumers": [
{ {
"caller": "app.chain.search", "caller": "app.chain.search",
"line": 2861 "line": 2864
}, },
{ {
"caller": "app.chain.subscribe", "caller": "app.chain.subscribe",
@@ -1813,7 +1813,7 @@
}, },
{ {
"caller": "app.scheduler", "caller": "app.scheduler",
"line": 696 "line": 736
} }
] ]
}, },