fix(transcriber): whisper 模型下载/加载统一走 HF cache 布局

此前用 modelscope 下到自定义目录 whisper-{size}/ 再把该路径传给
WhisperModel。但 faster-whisper 1.1.1 只要 path 含 '/' 就当成 HF
repo_id 处理,没有「本地目录直接返回」分支 → 在线请求失败后 fallback
local_files_only,又因 modelscope 布局命不中 HF cache → LocalEntryNotFound,
误导用户以为是「离线模式」。

改为下载与加载路径对齐:
- 下载:huggingface_hub.snapshot_download(cache_dir=model_dir),落到 HF
  cache 布局 models--Systran--faster-whisper-{size}/snapshots/<hash>/
- 加载:WhisperModel(model_size_or_path=size, download_root=model_dir),
  让 faster-whisper 自己映射到 Systran/faster-whisper-* 并命中同一 cache
- 完整性检测 / 损坏自愈(_purge_cache) 同步按 HF cache 布局,并兼容老
  modelscope 目录(向后兼容已下载的老用户)

HF_ENDPOINT 已在 Dockerfile 指向 hf-mirror.com,国内可用。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
huangjianwu
2026-05-22 11:27:03 +08:00
parent 1cc7f38e14
commit 261c95cf12
2 changed files with 78 additions and 65 deletions

View File

@@ -119,12 +119,21 @@ _downloading: dict[str, str] = {} # model_size -> status ("downloading" | "done
def _check_whisper_model_exists(model_size: str, subdir: str = "whisper") -> bool:
"""检查指定 whisper 模型是否已下载完整到本地。
必须 model.bin 落盘才算完成,仅有空目录或半成品不能算「已下载」——
否则监控页会显示绿勾但加载时报「Unable to open file 'model.bin'」。
faster-whisper 把模型缓存在 HF cache 布局下:
<model_dir>/models--Systran--faster-whisper-{size}/snapshots/<hash>/model.bin
必须能在某个 snapshot 目录里找到 model.bin 才算完成。
(历史 modelscope 布局 <model_dir>/whisper-{size}/model.bin 也兼容识别。)
"""
model_dir = get_model_dir(subdir)
model_path = os.path.join(model_dir, f"whisper-{model_size}")
return (Path(model_path) / "model.bin").exists()
model_dir = Path(get_model_dir(subdir))
# HF cache 布局
hf_repo_dir = model_dir / f"models--Systran--faster-whisper-{model_size}" / "snapshots"
if hf_repo_dir.exists():
for snapshot in hf_repo_dir.iterdir():
if (snapshot / "model.bin").exists():
return True
# 历史 modelscope 布局(向后兼容老用户)
legacy = model_dir / f"whisper-{model_size}" / "model.bin"
return legacy.exists()
def _check_mlx_whisper_model_exists(model_size: str) -> bool:
@@ -189,24 +198,37 @@ class ModelDownloadRequest(BaseModel):
def _do_download_whisper(model_size: str):
"""后台下载 faster-whisper 模型。"""
from app.transcriber.whisper import MODEL_MAP
from modelscope import snapshot_download
"""后台下载 faster-whisper 模型。
直接走 huggingface_hub.snapshot_download把模型放到 HF cache 布局里——
这样 faster-whisper 加载时WhisperModel(model_size_or_path=size_name,
download_root=model_dir))能直接命中缓存,跟加载路径完全对齐。
"""
from huggingface_hub import snapshot_download
try:
_downloading[model_size] = "downloading"
model_dir = get_model_dir("whisper")
model_path = os.path.join(model_dir, f"whisper-{model_size}")
# 用 model.bin 判定而非目录存在:半成品目录不能算「已下载」
if (Path(model_path) / "model.bin").exists():
# 已经下好就不重复下
if _check_whisper_model_exists(model_size, "whisper"):
_downloading[model_size] = "done"
return
repo_id = MODEL_MAP.get(model_size)
if not repo_id:
_downloading[model_size] = "failed"
return
logger.info(f"开始下载 whisper 模型: {model_size}")
snapshot_download(repo_id, local_dir=model_path)
repo_id = f"Systran/faster-whisper-{model_size}"
logger.info(f"开始下载 whisper 模型: {repo_id}")
# 跟 faster-whisper utils.py 用同样的 allow_patterns避免多下无关文件
# 不传 local_dir 让它走 HF 默认 cache 布局(与加载逻辑对齐)
snapshot_download(
repo_id,
cache_dir=model_dir,
allow_patterns=[
"config.json",
"preprocessor_config.json",
"model.bin",
"tokenizer.json",
"vocabulary.*",
],
)
logger.info(f"whisper 模型下载完成: {model_size}")
_downloading[model_size] = "done"
except Exception as e: