From f1b091b846caf7ddd8ac64270e940bc862043a9a Mon Sep 17 00:00:00 2001 From: huangjianwu Date: Thu, 14 May 2026 19:01:55 +0800 Subject: [PATCH] =?UTF-8?q?chore(deploy):=20docker=20=E9=95=9C=E5=83=8F?= =?UTF-8?q?=E6=BA=90/restart=20=E7=AD=96=E7=95=A5=20+=20.env=20=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=20+=20=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 所有 Dockerfile 加 BASE_REGISTRY build-arg，国内拉不到 docker.io 时可换 daocloud 等镜像源；compose 透传该 arg - docker-compose: restart 从 on-failure:3 改 unless-stopped（避免短暂 崩溃后永久打死）；gpu compose 补齐 healthcheck/restart/mem_limit - Dockerfile.complete: supervisord 用 %(ENV_*)s 透传环境变量给 backend 子进程（之前只白名单 2 个，docker run -e 配的变量后端看不到） - .env.example: 修正 VITE_API_BASE_URL 端口（8000→8483）、 WHISPER_MODEL_SIZE medium→tiny（首次启动不被大模型下载卡住）、 补 Docker 部署说明注释 - README: 新增 Docker 部署常见问题 FAQ（镜像源/restart/数据持久化等） - CLAUDE.md: 勘误（移除不存在的 messaging/i18n/worker_registry 描述， 修正 events 路径），补 pytest/typecheck 命令 Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 35 ++++++++++++++---- BillNote_frontend/Dockerfile | 9 +++-- CLAUDE.md | 16 +++++---- Dockerfile.complete | 23 +++++++++--- README.md | 69 +++++++++++++++++++++++++++++++++--- backend/Dockerfile | 6 +++- backend/Dockerfile.gpu | 4 ++- docker-compose.gpu.yml | 25 +++++++++++-- docker-compose.yml | 19 ++++++++-- 9 files changed, 176 insertions(+), 30 deletions(-) diff --git a/.env.example b/.env.example index e2ca898..3589634 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,28 @@ +# ============================================================================= +# BiliNote 环境变量示例 +# Docker 部署：cp .env.example .env，按需修改，然后 docker-compose up --build -d +# +# 注意区分两类变量： +# 1) VITE_* 是【构建时】变量，会被烘进前端 JS bundle。改完必须 +# docker-compose build frontend && docker-compose up -d 才会生效， +# 只 docker-compose restart 不行。 +# 2) 其他后端变量是【运行时】变量，改完 docker-compose up -d 即可。 +# +# 提醒：LLM API key 不要写在这里！请部署完成后，从前端「模型供应商」页面录入， +# 这些 key 会保存到 SQLite 数据库（./backend/bili_note.db）并随容器持久化。 +# ============================================================================= + # 通用端口配置 BACKEND_PORT=8483 # 后端端口 FRONTEND_PORT=3015 BACKEND_HOST=0.0.0.0 # 默认为 0.0.0.0，表示监听所有 IP 地址 不建议动 -APP_PORT= 3015 # docker 部署时用 -# 前端访问后端用 (开发环境使用) -VITE_API_BASE_URL=http://127.0.0.1:8000 +APP_PORT=3015 # docker 部署时对外暴露端口（浏览器访问的端口） + +# 前端访问后端用（开发环境直连；Docker 部署下走 nginx 代理，此值仅作回退） +VITE_API_BASE_URL=http://127.0.0.1:8483 VITE_SCREENSHOT_BASE_URL=http://127.0.0.1:8483/static/screenshots VITE_FRONTEND_PORT=3015 + # 生产环境配置 ENV=production STATIC=/static @@ -14,11 +30,16 @@ OUT_DIR=./static/screenshots NOTE_OUTPUT_DIR=note_results IMAGE_BASE_URL=/static/screenshots DATA_DIR=data -# FFMPEG 配置 + +# FFMPEG 配置（Docker 镜像已内置 ffmpeg，留空即可；自建/桌面端可填绝对路径） FFMPEG_BIN_PATH= -# transcriber 相关配置 -TRANSCRIBER_TYPE=fast-whisper # fast-whisper/bcut/kuaishou/mlx-whisper(仅Apple平台)/groq -WHISPER_MODEL_SIZE=medium +# 转写器配置 +# TRANSCRIBER_TYPE 可选：fast-whisper / bcut / kuaishou / mlx-whisper(仅 Apple Silicon) / groq +TRANSCRIBER_TYPE=fast-whisper +# WHISPER_MODEL_SIZE 默认 tiny (~75MB)，首次启动快；想要更高识别质量可在前端 +# 「音频转写配置」页切到 base/small/medium/large。直接在这里改大尺寸会触发 +# 首次启动下载 ~1.5GB 文件，慢网络或 4GB 内存的容器容易 OOM。 +WHISPER_MODEL_SIZE=tiny GROQ_TRANSCRIBER_MODEL=whisper-large-v3-turbo # groq提供的faster-whisper 默认为 whisper-large-v3-turbo diff --git a/BillNote_frontend/Dockerfile b/BillNote_frontend/Dockerfile index 16e62a0..9e54ab9 100644 --- a/BillNote_frontend/Dockerfile +++ b/BillNote_frontend/Dockerfile @@ -1,6 +1,9 @@ # === 前端构建阶段 === # Tailwind v4 / Vite 6 需要 Node 20+，alpine + pnpm 会按 lockfile 拉 musl native binary。 -FROM node:20-alpine AS builder +# BASE_REGISTRY 默认 docker.io，国内拉不到可换 daocloud / 阿里云镜像： +# docker-compose build --build-arg BASE_REGISTRY=docker.m.daocloud.io +ARG BASE_REGISTRY=docker.io +FROM ${BASE_REGISTRY}/library/node:20-alpine AS builder # pnpm pin 到 9.x：lockfile 是 v9 生成；pnpm 11 要求 Node 22+ 与 node:20 不兼容 RUN corepack enable && corepack prepare pnpm@9.15.0 --activate @@ -16,7 +19,9 @@ COPY ./BillNote_frontend/ ./ RUN pnpm run build # --- 阶段2：使用 nginx 作为静态服务器 --- -FROM nginx:1.25-alpine +# 重新声明 ARG —— buildkit 跨阶段不自动继承 +ARG BASE_REGISTRY=docker.io +FROM ${BASE_REGISTRY}/library/nginx:1.25-alpine RUN rm -rf /etc/nginx/conf.d/default.conf COPY ./BillNote_frontend/deploy/default.conf /etc/nginx/conf.d/default.conf diff --git a/CLAUDE.md b/CLAUDE.md index f9cf392..dc7d6f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,8 @@ BiliNote is an AI video note generation tool. It extracts content from video lin cd backend pip install -r requirements.txt python main.py # Starts on 0.0.0.0:8483 +pytest # Run tests in backend/tests/ +pytest tests/test_request_chunker.py::test_name # Run a single test ``` ### Frontend (React 19 + Vite + TypeScript) @@ -43,6 +45,8 @@ pnpm install pnpm dev # watch mode → ./extension/ pnpm build # production build → ./extension/ pnpm typecheck +pnpm test # Vitest unit tests +pnpm test:e2e # Playwright e2e ``` Load unpacked at `chrome://extensions/` → select `BillNote_extension/extension/`. Talks to the same backend at `http://localhost:8483` (configurable in the options page). CORS in `backend/main.py` already accepts `chrome-extension://` and `moz-extension://` via regex. @@ -56,15 +60,15 @@ Load unpacked at `chrome://extensions/` → select `BillNote_extension/extension - `chat_service.py` + `chat_tools.py` + `vector_store.py` — RAG-based AI Q&A with Function Calling, indexing transcripts and video metadata - `cookie_manager.py` — per-platform cookie storage; injected into yt-dlp by downloaders (e.g. Bilibili) - `transcriber_config_manager.py` — persisted transcriber settings - - `worker_registry.py` — **optional** Nacos registration + heartbeat for distributed worker mode (no-op when `NACOS_SERVER_ADDR` unset) -- `app/messaging/` — **optional** RabbitMQ producer/consumer publishing task progress/results to `bilinote.task.feedback` exchange. Silently degrades when `RABBITMQ_URL` is unset; always import-safe. - `app/downloaders/` — Platform adapters (bilibili, youtube, douyin, kuaishou, local) with shared `base.py` interface - `app/transcriber/` — Speech-to-text engines (fast-whisper, groq, bcut, kuaishou, mlx-whisper) with factory in `transcriber_provider.py`. YouTube path prefers existing subtitles and skips audio download when available. - `app/gpt/` — LLM integration with factory pattern (`gpt_factory.py`), prompt templates (`prompt.py`, `prompt_builder.py`), and `request_chunker.py` for long transcripts - `app/db/` — SQLite + SQLAlchemy: DAO pattern (`provider_dao.py`, `model_dao.py`, `video_task_dao.py`), models in `models/` - `app/utils/` — `response.py` (ResponseWrapper for consistent JSON), `video_helper.py` (screenshots via FFmpeg), `export.py` (PDF/DOCX), `ppt_generator.py`, `minio_client.py` -- `app/i18n/` — backend localization -- `events/` (root level) — Blinker signal system for post-processing (e.g., temp file cleanup after transcription) +- `app/validators/video_url_validator.py` — URL → platform detection (mirrored client-side in the extension) +- `app/exceptions/` — `BizException` + handlers wired in `main.py` via `register_exception_handlers` +- `backend/events/` — Blinker signal system for post-processing (e.g., temp file cleanup after transcription); registered in `lifespan` startup +- `backend/ffmpeg_helper.py` — `ensure_ffmpeg_or_raise` is called at startup; respects `FFMPEG_BIN_PATH` **Frontend** (`BillNote_frontend/src/`) — React 19 + Vite + Tailwind + shadcn/ui: - `pages/HomePage/` — Main note generation UI: `NoteForm.tsx` (input), `MarkdownViewer.tsx` (preview), `MarkmapComponent.tsx` (mind map) @@ -94,8 +98,8 @@ Load unpacked at `chrome://extensions/` → select `BillNote_extension/extension - **Environment**: Root `.env` (copy from `.env.example`). LLM API keys are configured through the UI, not env vars. - **Database**: SQLite at `backend/app/db/bili_note.db`, auto-initialized on first run - **FFmpeg**: Required system dependency for video/audio processing -- **Vite proxy**: Dev server proxies `/api` and `/static` to backend (configured in `vite.config.ts`, reads env from parent dir) -- **Distributed mode (optional)**: Setting `NACOS_SERVER_ADDR` enables Nacos worker registration; setting `RABBITMQ_URL` enables MQ feedback. Both are no-ops when unset — single-node deployment works without either. Other knobs: `WORKER_ID`, `WORKER_SELF_URL`, `WORKER_MAX_CONCURRENT`, `TASK_MAX_WORKERS`. +- **Vite proxy**: Dev server proxies `/api` and `/static` to backend (configured in `vite.config.ts`, reads env from parent dir; falls back to current dir when `DOCKER_BUILD` is set) +- **CORS**: `backend/main.py` uses a regex (`CORS_ORIGIN_REGEX`) that allows localhost, `tauri.localhost`, and `chrome-extension://` / `moz-extension://` origins — required for the desktop app and the browser extension. ## Code Style diff --git a/Dockerfile.complete b/Dockerfile.complete index d7f6e29..4ad647e 100644 --- a/Dockerfile.complete +++ b/Dockerfile.complete @@ -1,5 +1,9 @@ +# BASE_REGISTRY 默认 docker.io；国内拉不到可换镜像源： +# docker build --build-arg BASE_REGISTRY=docker.m.daocloud.io -f Dockerfile.complete . +ARG BASE_REGISTRY=docker.io + # === 阶段1：构建 Backend === -FROM python:3.11-slim AS backend-builder +FROM ${BASE_REGISTRY}/library/python:3.11-slim AS backend-builder ARG APT_MIRROR=mirrors.tuna.tsinghua.edu.cn ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple @@ -28,7 +32,8 @@ COPY ./backend /tmp/backend # === 阶段2：构建 Frontend === # Node 18-alpine 跑不动 Tailwind v4 / Vite 6（前者要求 Node 20+，后者推荐 Node 20+）， # 升到 node:20-alpine。alpine 走 musl，pnpm 会按 lockfile 拉 *-linux-x64-musl native binary。 -FROM node:20-alpine AS frontend-builder +ARG BASE_REGISTRY=docker.io +FROM ${BASE_REGISTRY}/library/node:20-alpine AS frontend-builder # pnpm 版本 pin 到 9 系列： # - lockfile (BillNote_frontend/pnpm-lock.yaml) 是 lockfileVersion '9.0'，由 pnpm 9 生成 @@ -50,7 +55,8 @@ ENV DOCKER_BUILD=1 RUN pnpm run build # === 阶段3：完整应用镜像 === -FROM python:3.11-slim +ARG BASE_REGISTRY=docker.io +FROM ${BASE_REGISTRY}/library/python:3.11-slim ARG APT_MIRROR=mirrors.tuna.tsinghua.edu.cn @@ -85,6 +91,10 @@ RUN rm -rf /etc/nginx/conf.d/default.conf COPY ./nginx/default.conf /etc/nginx/conf.d/default.conf # 创建 supervisor 配置 +# 关键点：supervisord 默认 *不* 把自己的环境变量传给子进程。 +# 在 [supervisord] 块用 environment= 设兜底默认值；在 [program:backend] 用 +# %(ENV_*)s 显式引用，等价于「把 host 通过 docker run -e 或 env_file 传进来的 +# 变量再透传给 python main.py」。漏掉这一步就是用户「改 .env 没反应」的根因。 RUN mkdir -p /var/log/supervisor COPY <> .env`。 + +注意：Chinese 公共 docker 镜像源时常被关停，2025-2026 之间可用的列表会变；如果 `docker.m.daocloud.io` 不通，搜一下"Docker 镜像加速 可用"找最新可用源即可。 + +**1. 容器一直 restart / unhealthy** + +先看后端日志： +```bash +docker logs -f bilinote-backend +``` +后端启动会按顺序打印 `[startup 1/5] ... [startup 5/5] 启动完成`。若日志卡在某一步或出现 `[startup FAILED]`，就是那一步的问题，常见： +- **卡在 `[startup 3/5]`**：转写器配置读不到。检查 `.env` 里 `TRANSCRIBER_TYPE` 是否写错，`mlx-whisper` 只能在 Apple Silicon 用，Linux/Docker 请用 `fast-whisper` 或 `groq`。 +- **首次跑视频时容器被 kill**：whisper 模型下载触发 OOM。先把 `.env` 里 `WHISPER_MODEL_SIZE` 改成 `tiny`，跑通后再去前端「音频转写配置」里逐档升。 + +**2. 改了 `.env` 没生效** + +区分两类变量： +- `VITE_*` 是**构建时**变量（前端 bundle 里硬编码），改完必须 `docker-compose build frontend && docker-compose up -d`。只 `restart` 不会重新打包。 +- 其他后端变量（`TRANSCRIBER_TYPE`、`WHISPER_MODEL_SIZE`、`FFMPEG_BIN_PATH` 等）是**运行时**变量，改完 `docker-compose up -d` 即可。 + +注意：**LLM API key 不要写 `.env`**，从前端「模型供应商」页面录入，会保存到 SQLite 数据库并持久化。 + +**3. 数据存在哪？删容器会丢吗？** + +`docker-compose` 用的是 `./backend:/app` 绑挂，下面这些文件都在宿主机的 `./backend/` 目录里、删容器不会丢： +- `./backend/bili_note.db` —— SQLite 库（含 LLM 供应商配置、笔记历史） +- `./backend/config/transcriber.json` —— 转写器运行时配置 +- `./backend/static/screenshots/` —— 视频截图 +- `./backend/uploads/` —— 上传的本地视频 + +要彻底重置就 `docker-compose down && rm backend/bili_note.db backend/config/transcriber.json`。 + +**4. 前端打开是空白页 / 报 502** + +通常是 nginx 起来了但 backend 还没 healthy。`docker ps` 看 backend 容器 STATUS 是不是 `(healthy)`；若长期 `(unhealthy)`，按问题 1 排查后端日志。 + +**5. 不要用 `restart: on-failure:N`** + +如果你 fork 后改过 compose 文件、把 restart 策略改成了 `on-failure:3`：任何 3 次连续崩溃都会让容器永远不再启动，之后改 `.env` 也没用。本项目自带的 compose 已经统一用 `unless-stopped`。 + ### 方式二：源码部署 #### 1. 克隆仓库 diff --git a/backend/Dockerfile b/backend/Dockerfile index 9e899bf..d08fb37 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,8 @@ -FROM python:3.11-slim +# BASE_REGISTRY 默认走 docker.io；国内拉不到 docker.io 时可换 daocloud / 阿里云 / 自建镜像源： +# docker-compose build --build-arg BASE_REGISTRY=docker.m.daocloud.io +# 或写到 docker-compose.yml 的 build.args / 环境变量里 +ARG BASE_REGISTRY=docker.io +FROM ${BASE_REGISTRY}/library/python:3.11-slim ARG APT_MIRROR=mirrors.tuna.tsinghua.edu.cn ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple diff --git a/backend/Dockerfile.gpu b/backend/Dockerfile.gpu index d74a3cb..aff0073 100644 --- a/backend/Dockerfile.gpu +++ b/backend/Dockerfile.gpu @@ -1,4 +1,6 @@ -FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04 +# BASE_REGISTRY 默认走 docker.io；国内可换 daocloud / 阿里云镜像（注意所选镜像需支持 nvidia/cuda 命名空间） +ARG BASE_REGISTRY=docker.io +FROM ${BASE_REGISTRY}/nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04 ARG APT_MIRROR=mirrors.tuna.tsinghua.edu.cn ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple diff --git a/docker-compose.gpu.yml b/docker-compose.gpu.yml index 1b1fdbc..327bed2 100644 --- a/docker-compose.gpu.yml +++ b/docker-compose.gpu.yml @@ -6,6 +6,8 @@ services: context: . dockerfile: backend/Dockerfile.gpu args: + # 国内拉不到 docker.io 时设置 BASE_REGISTRY；注意所选镜像需要支持 nvidia/cuda 命名空间 + BASE_REGISTRY: ${BASE_REGISTRY:-docker.io} APT_MIRROR: ${APT_MIRROR:-mirrors.tuna.tsinghua.edu.cn} PIP_INDEX: ${PIP_INDEX:-https://pypi.tuna.tsinghua.edu.cn/simple} env_file: @@ -14,9 +16,20 @@ services: - BACKEND_PORT=${BACKEND_PORT} - BACKEND_HOST=${BACKEND_HOST} volumes: + # 同 docker-compose.yml：./backend 绑到 /app，DB / 转写器配置 / 截图 / 上传都持久化 - ./backend:/app expose: - "${BACKEND_PORT}" # 不再对外暴露，用于 nginx 内部通信 + # 用 unless-stopped 避免短暂崩溃把容器永久打死后再也读不到 .env 修改 + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:${BACKEND_PORT}/api/sys_health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s # GPU 镜像首次加载 CUDA 比 CPU 慢，给久一点 + # GPU 部署默认跑较大模型，把内存限制提到 8g 避免 host OOM + mem_limit: 8g deploy: resources: reservations: @@ -30,10 +43,14 @@ services: build: context: . dockerfile: BillNote_frontend/Dockerfile + args: + BASE_REGISTRY: ${BASE_REGISTRY:-docker.io} env_file: - .env expose: - "80" # 不暴露给宿主机，只供 nginx 访问 + restart: unless-stopped + mem_limit: 512m nginx: container_name: bilinote-nginx @@ -43,5 +60,9 @@ services: volumes: - ./nginx/default.conf:/etc/nginx/conf.d/default.conf depends_on: - - backend - - frontend + backend: + condition: service_healthy + frontend: + condition: service_started + restart: unless-stopped + mem_limit: 256m diff --git a/docker-compose.yml b/docker-compose.yml index 4f5383a..bcb022d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ services: context: . dockerfile: backend/Dockerfile args: + # 国内拉不到 docker.io 时设置 BASE_REGISTRY=docker.m.daocloud.io（或其他可用镜像） + BASE_REGISTRY: ${BASE_REGISTRY:-docker.io} APT_MIRROR: ${APT_MIRROR:-mirrors.tuna.tsinghua.edu.cn} PIP_INDEX: ${PIP_INDEX:-https://pypi.tuna.tsinghua.edu.cn/simple} env_file: @@ -14,16 +16,25 @@ services: - BACKEND_PORT=${BACKEND_PORT} - BACKEND_HOST=${BACKEND_HOST} volumes: + # 把整个 backend/ 目录绑到 /app，意味着这些都持久化到宿主机、删容器不丢： + # ./backend/bili_note.db — SQLite 数据库（含 LLM 供应商配置、笔记历史） + # ./backend/config/transcriber.json — 转写器运行时配置 + # ./backend/static/screenshots/ — 视频截图 + # ./backend/uploads/ — 上传的本地视频 - ./backend:/app expose: - "${BACKEND_PORT}" # 不再对外暴露，用于 nginx 内部通信 - restart: on-failure:3 + # 用 unless-stopped 而非 on-failure:N，避免任何短暂崩溃把容器永久打死后 + # 再也接收不到用户修过的 .env。手动 docker-compose stop 仍可正常停下。 + restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:${BACKEND_PORT}/api/sys_health"] interval: 30s timeout: 10s retries: 3 start_period: 15s + # WHISPER_MODEL_SIZE 选 medium 及以上请把这里调到 8g+， + # 否则首次模型加载时容易被 host OOM-killer 干掉。 mem_limit: 4g frontend: @@ -31,11 +42,13 @@ services: build: context: . dockerfile: BillNote_frontend/Dockerfile + args: + BASE_REGISTRY: ${BASE_REGISTRY:-docker.io} env_file: - .env expose: - "80" # 不暴露给宿主机，只供 nginx 访问 - restart: on-failure:3 + restart: unless-stopped mem_limit: 512m nginx: @@ -50,5 +63,5 @@ services: condition: service_healthy frontend: condition: service_started - restart: on-failure:3 + restart: unless-stopped mem_limit: 256m