From 3784e15670217f946aaf8b8907fac7aeed772378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=80=9D=E8=AF=BA=E7=89=B9?= Date: Tue, 15 Apr 2025 12:19:14 +0800 Subject: [PATCH] =?UTF-8?q?feat(core):=20=E5=AE=9E=E7=8E=B0=20Celery?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=BC=82=E6=AD=A5=E7=94=9F=E6=88=90=E7=AC=94?= =?UTF-8?q?=E8=AE=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 Celery 配置文件 celery_app.py - 创建 note_tasks.py 文件，定义生成笔记的 Celery 任务 - 修改 note_router，使用 Celery 任务异步处理笔记生成 - 重构 bili_downloader 和 youtube_downloader，支持多质量选择和错误处理 - 更新 .env.example，添加 Celery 配置项 --- .env.example | 10 +++ backend/app/core/celery_app.py | 28 +++++++++ .../app/downloaders/bilibili_downloader.py | 60 ++++++++++++------ backend/app/downloaders/youtube_downloader.py | 62 ++++++++++++------- backend/app/routers/note.py | 16 +++-- backend/app/tasks/__init__.py | 0 backend/app/tasks/note_tasks.py | 27 ++++++++ backend/app/utils/note_helper.py | 9 +++ 8 files changed, 165 insertions(+), 47 deletions(-) create mode 100644 backend/app/core/celery_app.py create mode 100644 backend/app/tasks/__init__.py create mode 100644 backend/app/tasks/note_tasks.py diff --git a/.env.example b/.env.example index 4a22083..73128fe 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,17 @@ +### + # @Author: 思诺特 jefferyhcool@gmail.com + # @Date: 2025-04-14 08:49:59 + # @LastEditors: 思诺特 jefferyhcool@gmail.com + # @LastEditTime: 2025-04-15 12:18:39 + # @FilePath: \BiliNote\.env.example + # @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE +### # 通用端口配置 BACKEND_PORT=8001 FRONTEND_PORT=3015 BACKEND_HOST=0.0.0.0 +CELERY_BROKER_URL= #redis 地址 +CELERY_RESULT_BACKEND= #redis 地址 # 前端访问后端用（生产环境建议写公网或宿主机 IP） VITE_API_BASE_URL=http://127.0.0.1:8001 diff --git a/backend/app/core/celery_app.py b/backend/app/core/celery_app.py new file mode 100644 index 0000000..8b8f834 --- /dev/null +++ b/backend/app/core/celery_app.py @@ -0,0 +1,28 @@ +from celery import Celery +from dotenv import load_dotenv +import os + +# 加载 .env 文件中的环境变量 +load_dotenv() + +# 从环境变量中读取配置（可适配不同环境） +BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") +BACKEND_URL = os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") + +# 初始化 Celery 实例 +celery_app = Celery( + "bilinote", + broker=BROKER_URL, + backend=BACKEND_URL, +) + +# 基础配置 +celery_app.conf.update( + task_track_started=True, # 任务启动时即可记录状态 + task_time_limit=600, # 每个任务最大运行时间（秒） + task_serializer="json", + result_serializer="json", + accept_content=["json"], + timezone="Asia/Shanghai", # 设置时区 + enable_utc=False, +) diff --git a/backend/app/downloaders/bilibili_downloader.py b/backend/app/downloaders/bilibili_downloader.py index 532efa1..af4fcb8 100644 --- a/backend/app/downloaders/bilibili_downloader.py +++ b/backend/app/downloaders/bilibili_downloader.py @@ -6,8 +6,9 @@ import yt_dlp from app.downloaders.base import Downloader, DownloadQuality, QUALITY_MAP from app.models.notes_model import AudioDownloadResult +from app.utils.logger import get_logger from app.utils.path_helper import get_data_dir - +logger=get_logger(__name__) class BilibiliDownloader(Downloader, ABC): def __init__(self): @@ -55,36 +56,55 @@ class BilibiliDownloader(Downloader, ABC): ) def download_video( - self, - video_url: str, - output_dir: Union[str, None] = None, + self, + video_url: str, + output_dir: Union[str, None] = None, + quality: DownloadQuality = "medium", ) -> str: - """ - 下载视频，返回视频文件路径 - """ if output_dir is None: output_dir = get_data_dir() os.makedirs(output_dir, exist_ok=True) output_path = os.path.join(output_dir, "%(id)s.%(ext)s") - ydl_opts = { - 'format': 'best[height<=480][ext=mp4]/best[height<=480]/best', - 'outtmpl': output_path, - 'noplaylist': True, - 'quiet': False, - 'merge_output_format': 'mp4', # 确保合并成 mp4 + format_map = { + "fast": "best[height<=480]", + "medium": "best[height<=720]", + "slow": "bestvideo+bestaudio/best" } + preferred_format = format_map.get(quality, "best[height<=720]") - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - info = ydl.extract_info(video_url, download=True) - video_id = info.get("id") - video_path = os.path.join(output_dir, f"{video_id}.mp4") + # ⛑️ 多级格式容错 fallback + fallback_formats = [ + preferred_format, + "best[ext=mp4]", + "bestvideo+bestaudio", + "best" + ] - if not os.path.exists(video_path): - raise FileNotFoundError(f"视频文件未找到: {video_path}") + last_error = None + for fmt in fallback_formats: + ydl_opts = { + 'format': fmt, + 'outtmpl': output_path, + 'noplaylist': True, + 'quiet': False, + 'merge_output_format': 'mp4', + } - return video_path + try: + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(video_url, download=True) + video_id = info.get("id") + video_path = os.path.join(output_dir, f"{video_id}.mp4") + if os.path.exists(video_path): + return video_path + except yt_dlp.utils.DownloadError as e: + logger.warning(f"⚠️ 尝试格式失败：{fmt}") + last_error = e + continue + + raise last_error or Exception("未能成功下载视频") def delete_video(self, video_path: str) -> str: """ diff --git a/backend/app/downloaders/youtube_downloader.py b/backend/app/downloaders/youtube_downloader.py index f915171..cec3624 100644 --- a/backend/app/downloaders/youtube_downloader.py +++ b/backend/app/downloaders/youtube_downloader.py @@ -6,9 +6,10 @@ import yt_dlp from app.downloaders.base import Downloader, DownloadQuality from app.models.notes_model import AudioDownloadResult +from app.utils.logger import get_logger from app.utils.path_helper import get_data_dir - +logger=get_logger(__name__) class YoutubeDownloader(Downloader, ABC): def __init__(self): @@ -30,7 +31,7 @@ class YoutubeDownloader(Downloader, ABC): output_path = os.path.join(output_dir, "%(id)s.%(ext)s") ydl_opts = { - 'format': 'best[height<=480][ext=mp4]/best[height<=480]/best', + 'format': 'best[ext=mp4][height<=720]/best[height<=720]/best', 'outtmpl': output_path, 'noplaylist': True, 'quiet': False, @@ -56,33 +57,52 @@ class YoutubeDownloader(Downloader, ABC): ) def download_video( - self, - video_url: str, - output_dir: Union[str, None] = None, + self, + video_url: str, + output_dir: Union[str, None] = None, + quality: DownloadQuality = "medium", ) -> str: - """ - 下载视频，返回视频文件路径 - """ if output_dir is None: output_dir = get_data_dir() os.makedirs(output_dir, exist_ok=True) output_path = os.path.join(output_dir, "%(id)s.%(ext)s") - ydl_opts = { - 'format': 'worst[ext=mp4]/worst', - 'outtmpl': output_path, - 'noplaylist': True, - 'quiet': False, - 'merge_output_format': 'mp4', # 确保合并成 mp4 + format_map = { + "fast": "best[height<=480]", + "medium": "best[height<=720]", + "slow": "bestvideo+bestaudio/best" } + preferred_format = format_map.get(quality, "best[height<=720]") - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - info = ydl.extract_info(video_url, download=True) - video_id = info.get("id") - video_path = os.path.join(output_dir, f"{video_id}.mp4") + # ⛑️ 多级格式容错 fallback + fallback_formats = [ + preferred_format, + "best[ext=mp4]", + "bestvideo+bestaudio", + "best" + ] - if not os.path.exists(video_path): - raise FileNotFoundError(f"视频文件未找到: {video_path}") + last_error = None + for fmt in fallback_formats: + ydl_opts = { + 'format': fmt, + 'outtmpl': output_path, + 'noplaylist': True, + 'quiet': False, + 'merge_output_format': 'mp4', + } - return video_path + try: + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(video_url, download=True) + video_id = info.get("id") + video_path = os.path.join(output_dir, f"{video_id}.mp4") + if os.path.exists(video_path): + return video_path + except yt_dlp.utils.DownloadError as e: + logger.warning(f"⚠️ 尝试格式失败：{fmt}") + last_error = e + continue + + raise last_error or Exception("未能成功下载视频") diff --git a/backend/app/routers/note.py b/backend/app/routers/note.py index e8d05b8..cb7941b 100644 --- a/backend/app/routers/note.py +++ b/backend/app/routers/note.py @@ -20,7 +20,7 @@ import httpx # from app.services.downloader import download_raw_audio # from app.services.whisperer import transcribe_audio - +from app.tasks.note_tasks import generate_note_task router = APIRouter() @@ -48,10 +48,7 @@ class VideoRequest(BaseModel): NOTE_OUTPUT_DIR = "note_results" -def save_note_to_file(task_id: str, note): - os.makedirs(NOTE_OUTPUT_DIR, exist_ok=True) - with open(os.path.join(NOTE_OUTPUT_DIR, f"{task_id}.json"), "w", encoding="utf-8") as f: - json.dump(asdict(note), f, ensure_ascii=False, indent=2) + def run_note_task(task_id: str, video_url: str, platform: str, quality: DownloadQuality, link: bool = False,screenshot: bool = False): @@ -96,7 +93,14 @@ def generate_note(data: VideoRequest, background_tasks: BackgroundTasks): task_id = str(uuid.uuid4()) - background_tasks.add_task(run_note_task, task_id, data.video_url, data.platform, data.quality,data.link ,data.screenshot) + generate_note_task.delay( + task_id=task_id, + video_url=data.video_url, + platform=data.platform, + quality=data.quality.value, + link=data.link, + screenshot=data.screenshot + ) return R.success({"task_id": task_id}) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/tasks/note_tasks.py b/backend/app/tasks/note_tasks.py new file mode 100644 index 0000000..c70aa18 --- /dev/null +++ b/backend/app/tasks/note_tasks.py @@ -0,0 +1,27 @@ +# app/tasks/note_tasks.py +import os +import json + + +from app.services.note import NoteGenerator +from app.core.celery_app import celery_app +from dataclasses import asdict +from app.enmus.note_enums import DownloadQuality +from app.utils.note_helper import save_note_to_file + +NOTE_OUTPUT_DIR = "note_results" + +@celery_app.task(name="generate_note_task") +def generate_note_task(task_id: str, video_url: str, platform: str, quality: str, link: bool = False, screenshot: bool = False): + try: + note = NoteGenerator().generate( + video_url=video_url, + platform=platform, + quality=DownloadQuality(quality), + task_id=task_id, + link=link, + screenshot=screenshot + ) + save_note_to_file(task_id, note) + except Exception as e: + save_note_to_file(task_id, {"error": str(e)}) diff --git a/backend/app/utils/note_helper.py b/backend/app/utils/note_helper.py index 430c3a0..baef6d9 100644 --- a/backend/app/utils/note_helper.py +++ b/backend/app/utils/note_helper.py @@ -1,10 +1,14 @@ +import json +import os import re import re import re +from dataclasses import asdict +NOTE_OUTPUT_DIR = "note_results" def replace_content_markers(markdown: str, video_id: str, platform: str = 'bilibili') -> str: """ 替换 *Content-04:16*、Content-04:16 或 Content-[04:16] 为超链接，跳转到对应平台视频的时间位置 @@ -30,3 +34,8 @@ def replace_content_markers(markdown: str, video_id: str, platform: str = 'bilib return f"[原片 @ {mm}:{ss}]({url})" return re.sub(pattern, replacer, markdown) + +def save_note_to_file(task_id: str, note): + os.makedirs(NOTE_OUTPUT_DIR, exist_ok=True) + with open(os.path.join(NOTE_OUTPUT_DIR, f"{task_id}.json"), "w", encoding="utf-8") as f: + json.dump(asdict(note), f, ensure_ascii=False, indent=2) \ No newline at end of file