GithubBackup/BiliNote
1
0
Fork 0
mirror of https://github.com/JefferyHcool/BiliNote.git synced 2026-05-06 20:42:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): 实现 Celery任务异步生成笔记

Browse Source 
- 新增 Celery 配置文件 celery_app.py
- 创建 note_tasks.py 文件，定义生成笔记的 Celery 任务
- 修改 note_router，使用 Celery 任务异步处理笔记生成
- 重构 bili_downloader 和 youtube_downloader，支持多质量选择和错误处理
- 更新 .env.example，添加 Celery 配置项
This commit is contained in:
思诺特
2025-04-15 12:19:14 +08:00
parent a395f8e1c1
commit 3784e15670
8 changed files with 165 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.env.example
View File

@@ -1,7 +1,17 @@
###
# @Author: 思诺特 jefferyhcool@gmail.com
# @Date: 2025-04-14 08:49:59
# @LastEditors: 思诺特 jefferyhcool@gmail.com
# @LastEditTime: 2025-04-15 12:18:39
# @FilePath: \BiliNote\.env.example
# @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE
###
# 通用端口配置
BACKEND_PORT=8001
FRONTEND_PORT=3015
BACKEND_HOST=0.0.0.0
CELERY_BROKER_URL= #redis 地址
CELERY_RESULT_BACKEND= #redis 地址
# 前端访问后端用（生产环境建议写公网或宿主机 IP）
VITE_API_BASE_URL=http://127.0.0.1:8001

28
backend/app/core/celery_app.py Normal file
View File

@@ -0,0 +1,28 @@
from celery import Celery
from dotenv import load_dotenv
import os
# 加载 .env 文件中的环境变量
load_dotenv()
# 从环境变量中读取配置（可适配不同环境）
BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
BACKEND_URL = os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0")
# 初始化 Celery 实例
celery_app = Celery(
"bilinote",
broker=BROKER_URL,
backend=BACKEND_URL,
)
# 基础配置
celery_app.conf.update(
task_track_started=True, # 任务启动时即可记录状态
task_time_limit=600, # 每个任务最大运行时间（秒）
task_serializer="json",
result_serializer="json",
accept_content=["json"],
timezone="Asia/Shanghai", # 设置时区
enable_utc=False,
)

60
backend/app/downloaders/bilibili_downloader.py
View File

@@ -6,8 +6,9 @@ import yt_dlp
from app.downloaders.base import Downloader, DownloadQuality, QUALITY_MAP
from app.models.notes_model import AudioDownloadResult
from app.utils.logger import get_logger
from app.utils.path_helper import get_data_dir
logger=get_logger(__name__)
class BilibiliDownloader(Downloader, ABC):
def __init__(self):
@@ -55,36 +56,55 @@ class BilibiliDownloader(Downloader, ABC):
)
def download_video(
self,
video_url: str,
output_dir: Union[str, None] = None,
self,
video_url: str,
output_dir: Union[str, None] = None,
quality: DownloadQuality = "medium",
) -> str:
"""
下载视频，返回视频文件路径
"""
if output_dir is None:
output_dir = get_data_dir()
os.makedirs(output_dir, exist_ok=True)
output_path = os.path.join(output_dir, "%(id)s.%(ext)s")
ydl_opts = {
'format': 'best[height<=480][ext=mp4]/best[height<=480]/best',
'outtmpl': output_path,
'noplaylist': True,
'quiet': False,
'merge_output_format': 'mp4', # 确保合并成 mp4
format_map = {
"fast": "best[height<=480]",
"medium": "best[height<=720]",
"slow": "bestvideo+bestaudio/best"
}
preferred_format = format_map.get(quality, "best[height<=720]")
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(video_url, download=True)
video_id = info.get("id")
video_path = os.path.join(output_dir, f"{video_id}.mp4")
# ⛑️ 多级格式容错 fallback
fallback_formats = [
preferred_format,
"best[ext=mp4]",
"bestvideo+bestaudio",
"best"
]
if not os.path.exists(video_path):
raise FileNotFoundError(f"视频文件未找到: {video_path}")
last_error = None
for fmt in fallback_formats:
ydl_opts = {
'format': fmt,
'outtmpl': output_path,
'noplaylist': True,
'quiet': False,
'merge_output_format': 'mp4',
}
return video_path
try:
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(video_url, download=True)
video_id = info.get("id")
video_path = os.path.join(output_dir, f"{video_id}.mp4")
if os.path.exists(video_path):
return video_path
except yt_dlp.utils.DownloadError as e:
logger.warning(f"⚠️ 尝试格式失败：{fmt}")
last_error = e
continue
raise last_error or Exception("未能成功下载视频")
def delete_video(self, video_path: str) -> str:
"""

62
backend/app/downloaders/youtube_downloader.py
View File

@@ -6,9 +6,10 @@ import yt_dlp
from app.downloaders.base import Downloader, DownloadQuality
from app.models.notes_model import AudioDownloadResult
from app.utils.logger import get_logger
from app.utils.path_helper import get_data_dir
logger=get_logger(__name__)
class YoutubeDownloader(Downloader, ABC):
def __init__(self):
@@ -30,7 +31,7 @@ class YoutubeDownloader(Downloader, ABC):
output_path = os.path.join(output_dir, "%(id)s.%(ext)s")
ydl_opts = {
'format': 'best[height<=480][ext=mp4]/best[height<=480]/best',
'format': 'best[ext=mp4][height<=720]/best[height<=720]/best',
'outtmpl': output_path,
'noplaylist': True,
'quiet': False,
@@ -56,33 +57,52 @@ class YoutubeDownloader(Downloader, ABC):
)
def download_video(
self,
video_url: str,
output_dir: Union[str, None] = None,
self,
video_url: str,
output_dir: Union[str, None] = None,
quality: DownloadQuality = "medium",
) -> str:
"""
下载视频，返回视频文件路径
"""
if output_dir is None:
output_dir = get_data_dir()
os.makedirs(output_dir, exist_ok=True)
output_path = os.path.join(output_dir, "%(id)s.%(ext)s")
ydl_opts = {
'format': 'worst[ext=mp4]/worst',
'outtmpl': output_path,
'noplaylist': True,
'quiet': False,
'merge_output_format': 'mp4', # 确保合并成 mp4
format_map = {
"fast": "best[height<=480]",
"medium": "best[height<=720]",
"slow": "bestvideo+bestaudio/best"
}
preferred_format = format_map.get(quality, "best[height<=720]")
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(video_url, download=True)
video_id = info.get("id")
video_path = os.path.join(output_dir, f"{video_id}.mp4")
# ⛑️ 多级格式容错 fallback
fallback_formats = [
preferred_format,
"best[ext=mp4]",
"bestvideo+bestaudio",
"best"
]
if not os.path.exists(video_path):
raise FileNotFoundError(f"视频文件未找到: {video_path}")
last_error = None
for fmt in fallback_formats:
ydl_opts = {
'format': fmt,
'outtmpl': output_path,
'noplaylist': True,
'quiet': False,
'merge_output_format': 'mp4',
}
return video_path
try:
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(video_url, download=True)
video_id = info.get("id")
video_path = os.path.join(output_dir, f"{video_id}.mp4")
if os.path.exists(video_path):
return video_path
except yt_dlp.utils.DownloadError as e:
logger.warning(f"⚠️ 尝试格式失败：{fmt}")
last_error = e
continue
raise last_error or Exception("未能成功下载视频")

16
backend/app/routers/note.py
View File

@@ -20,7 +20,7 @@ import httpx
# from app.services.downloader import download_raw_audio
# from app.services.whisperer import transcribe_audio
from app.tasks.note_tasks import generate_note_task
router = APIRouter()
@@ -48,10 +48,7 @@ class VideoRequest(BaseModel):
NOTE_OUTPUT_DIR = "note_results"
def save_note_to_file(task_id: str, note):
os.makedirs(NOTE_OUTPUT_DIR, exist_ok=True)
with open(os.path.join(NOTE_OUTPUT_DIR, f"{task_id}.json"), "w", encoding="utf-8") as f:
json.dump(asdict(note), f, ensure_ascii=False, indent=2)
def run_note_task(task_id: str, video_url: str, platform: str, quality: DownloadQuality, link: bool = False,screenshot: bool = False):
@@ -96,7 +93,14 @@ def generate_note(data: VideoRequest, background_tasks: BackgroundTasks):
task_id = str(uuid.uuid4())
background_tasks.add_task(run_note_task, task_id, data.video_url, data.platform, data.quality,data.link ,data.screenshot)
generate_note_task.delay(
task_id=task_id,
video_url=data.video_url,
platform=data.platform,
quality=data.quality.value,
link=data.link,
screenshot=data.screenshot
)
return R.success({"task_id": task_id})
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))

0
backend/app/tasks/__init__.py Normal file
View File

27
backend/app/tasks/note_tasks.py Normal file
View File

@@ -0,0 +1,27 @@
# app/tasks/note_tasks.py
import os
import json
from app.services.note import NoteGenerator
from app.core.celery_app import celery_app
from dataclasses import asdict
from app.enmus.note_enums import DownloadQuality
from app.utils.note_helper import save_note_to_file
NOTE_OUTPUT_DIR = "note_results"
@celery_app.task(name="generate_note_task")
def generate_note_task(task_id: str, video_url: str, platform: str, quality: str, link: bool = False, screenshot: bool = False):
try:
note = NoteGenerator().generate(
video_url=video_url,
platform=platform,
quality=DownloadQuality(quality),
task_id=task_id,
link=link,
screenshot=screenshot
)
save_note_to_file(task_id, note)
except Exception as e:
save_note_to_file(task_id, {"error": str(e)})

9
backend/app/utils/note_helper.py
View File

@@ -1,10 +1,14 @@
import json
import os
import re
import re
import re
from dataclasses import asdict
NOTE_OUTPUT_DIR = "note_results"
def replace_content_markers(markdown: str, video_id: str, platform: str = 'bilibili') -> str:
"""
替换 *Content-04:16*、Content-04:16 或 Content-[04:16] 为超链接，跳转到对应平台视频的时间位置
@@ -30,3 +34,8 @@ def replace_content_markers(markdown: str, video_id: str, platform: str = 'bilib
return f"[原片 @ {mm}:{ss}]({url})"
return re.sub(pattern, replacer, markdown)
def save_note_to_file(task_id: str, note):
os.makedirs(NOTE_OUTPUT_DIR, exist_ok=True)
with open(os.path.join(NOTE_OUTPUT_DIR, f"{task_id}.json"), "w", encoding="utf-8") as f:
json.dump(asdict(note), f, ensure_ascii=False, indent=2)