feat: add raw stream upload functionality

This commit is contained in:
shiyu
2026-05-16 18:14:52 +08:00
parent 2af2a8756f
commit e3a5317f6f
4 changed files with 106 additions and 15 deletions
+13
View File
@@ -172,6 +172,19 @@ async def upload_stream(
return success(result) return success(result)
@router.put("/upload-raw/{full_path:path}")
@audit(action=AuditAction.UPLOAD, description="原始流上传文件")
@require_path_permission(PathAction.WRITE, "full_path")
async def upload_raw_stream(
request: Request,
current_user: Annotated[User, Depends(get_current_active_user)],
full_path: str,
overwrite: bool = Query(True, description="是否覆盖已存在文件"),
):
result = await VirtualFSService.upload_raw_stream(full_path, request, overwrite)
return success(result)
@router.get("/{full_path:path}") @router.get("/{full_path:path}")
@audit(action=AuditAction.READ, description="浏览目录") @audit(action=AuditAction.READ, description="浏览目录")
@require_path_permission(PathAction.READ, "full_path") @require_path_permission(PathAction.READ, "full_path")
+19 -1
View File
@@ -2,7 +2,7 @@ import mimetypes
import re import re
from urllib.parse import quote from urllib.parse import quote
from fastapi import HTTPException, UploadFile from fastapi import HTTPException, Request, UploadFile
from fastapi.responses import Response from fastapi.responses import Response
from domain.config import ConfigService from domain.config import ConfigService
@@ -271,6 +271,24 @@ class VirtualFSRouteMixin(VirtualFSTempLinkMixin):
size = int(result or 0) size = int(result or 0)
return {"uploaded": True, "path": path, "size": size, "overwrite": overwrite} return {"uploaded": True, "path": path, "size": size, "overwrite": overwrite}
@classmethod
async def upload_raw_stream(cls, full_path: str, request: Request, overwrite: bool):
full_path = cls._normalize_path(full_path)
if full_path.endswith("/"):
raise HTTPException(400, detail="Path must be a file")
result = await cls.write_file_stream(full_path, request.stream(), overwrite=overwrite)
path = full_path
size = 0
if isinstance(result, dict):
path = result.get("path") or path
size_val = result.get("size")
if isinstance(size_val, int):
size = size_val
else:
size = int(result or 0)
return {"uploaded": True, "path": path, "size": size, "overwrite": overwrite}
@classmethod @classmethod
async def list_directory(cls, full_path: str, page_num: int, page_size: int, sort_by: str, sort_order: str): async def list_directory(cls, full_path: str, page_num: int, page_size: int, sort_by: str, sort_order: str):
full_path = cls._normalize_path(full_path) full_path = cls._normalize_path(full_path)
+34
View File
@@ -100,6 +100,40 @@ export const vfsApi = {
getTempLinkToken: (path: string, expiresIn: number = 3600) => getTempLinkToken: (path: string, expiresIn: number = 3600) =>
request<{token: string, path: string, url: string}>(`/fs/temp-link/${encodeURI(path.replace(/^\/+/, ''))}?expires_in=${expiresIn}`), request<{token: string, path: string, url: string}>(`/fs/temp-link/${encodeURI(path.replace(/^\/+/, ''))}?expires_in=${expiresIn}`),
getTempPublicUrl: (token: string) => `${API_BASE_URL}/fs/public/${token}`, getTempPublicUrl: (token: string) => `${API_BASE_URL}/fs/public/${token}`,
uploadRaw: (fullPath: string, file: File, overwrite: boolean = true, onProgress?: (loaded: number, total: number) => void) => {
const enc = encodeURI(fullPath.replace(/^\/+/, ''));
return new Promise<any>((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('PUT', `${API_BASE_URL}/fs/upload-raw/${enc}?overwrite=${overwrite}`);
const token = localStorage.getItem('token');
if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`);
xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream');
xhr.upload.onprogress = (ev) => {
if (ev.lengthComputable && onProgress) onProgress(ev.loaded, ev.total);
};
xhr.onreadystatechange = () => {
if (xhr.readyState === 4) {
if (xhr.status >= 200 && xhr.status < 300) {
try {
const json = JSON.parse(xhr.responseText);
if (json.code === 0) return resolve(json.data);
return reject(new Error(json.msg || json.message || 'Upload failed'));
} catch {
return reject(new Error('Invalid response'));
}
} else {
let err = 'Upload failed';
try {
const json = JSON.parse(xhr.responseText);
err = json.detail || json.msg || json.message || err;
} catch { void 0; }
reject(new Error(err));
}
}
};
xhr.send(file);
});
},
uploadStream: (fullPath: string, file: File, overwrite: boolean = true, onProgress?: (loaded: number, total: number) => void) => { uploadStream: (fullPath: string, file: File, overwrite: boolean = true, onProgress?: (loaded: number, total: number) => void) => {
const enc = encodeURI(fullPath.replace(/^\/+/, '')); const enc = encodeURI(fullPath.replace(/^\/+/, ''));
return new Promise<any>((resolve, reject) => { return new Promise<any>((resolve, reject) => {
@@ -43,6 +43,8 @@ interface RawUploadDirectory {
type RawUploadItem = RawUploadFile | RawUploadDirectory; type RawUploadItem = RawUploadFile | RawUploadDirectory;
const MAX_UPLOAD_CONCURRENCY = 3;
const generateId = (() => { const generateId = (() => {
const cryptoApi = typeof crypto !== 'undefined' ? crypto : undefined; const cryptoApi = typeof crypto !== 'undefined' ? crypto : undefined;
return () => { return () => {
@@ -457,15 +459,15 @@ export function useUploader(path: string, onUploadComplete: () => void) {
} }
}, [ensureDirectory, updateFile, t]); }, [ensureDirectory, updateFile, t]);
const processFileTask = useCallback(async (task: UploadFile) => { const prepareFileTask = useCallback(async (task: UploadFile): Promise<{ task: UploadFile; overwrite: boolean } | null> => {
if (!task.file) { if (!task.file) {
updateFile(task.id, { status: 'error', error: t('Missing file content') }); updateFile(task.id, { status: 'error', error: t('Missing file content') });
return; return null;
} }
if (skipAllRef.current) { if (skipAllRef.current) {
updateFile(task.id, { status: 'skipped', progress: 0 }); updateFile(task.id, { status: 'skipped', progress: 0 });
return; return null;
} }
let shouldOverwrite = overwriteAllRef.current; let shouldOverwrite = overwriteAllRef.current;
@@ -475,19 +477,28 @@ export function useUploader(path: string, onUploadComplete: () => void) {
const decision = await awaitConflictDecision(task); const decision = await awaitConflictDecision(task);
if (decision === 'skip') { if (decision === 'skip') {
updateFile(task.id, { status: 'skipped', progress: 0 }); updateFile(task.id, { status: 'skipped', progress: 0 });
return; return null;
} }
shouldOverwrite = true; shouldOverwrite = true;
} }
} }
setConflict(null); setConflict(null);
updateFile(task.id, { status: 'uploading', progress: 0, loadedBytes: 0 });
const parentDir = task.targetPath.replace(/\/[^/]+$/, '') || '/'; const parentDir = task.targetPath.replace(/\/[^/]+$/, '') || '/';
await ensureDirectoryTree(parentDir);
updateFile(task.id, { status: 'pending', progress: 0, loadedBytes: 0 });
return { task, overwrite: shouldOverwrite };
}, [ensureDirectoryTree, awaitConflictDecision, updateFile, t]);
const uploadPreparedFile = useCallback(async (task: UploadFile, shouldOverwrite: boolean) => {
if (!task.file) {
updateFile(task.id, { status: 'error', error: t('Missing file content') });
return;
}
updateFile(task.id, { status: 'uploading', progress: 0, loadedBytes: 0 });
try { try {
await ensureDirectoryTree(parentDir); const uploadResult = await vfsApi.uploadRaw(task.targetPath, task.file, shouldOverwrite, (loaded, total) => {
const uploadResult = await vfsApi.uploadStream(task.targetPath, task.file, shouldOverwrite, (loaded, total) => {
mutateFiles((prev) => prev.map((f) => { mutateFiles((prev) => prev.map((f) => {
if (f.id !== task.id) return f; if (f.id !== task.id) return f;
const effectiveTotal = total > 0 ? total : f.size; const effectiveTotal = total > 0 ? total : f.size;
@@ -506,22 +517,34 @@ export function useUploader(path: string, onUploadComplete: () => void) {
const finalSize = typeof uploadResult?.size === 'number' && uploadResult.size > 0 const finalSize = typeof uploadResult?.size === 'number' && uploadResult.size > 0
? uploadResult.size ? uploadResult.size
: task.size; : task.size;
const link = await vfsApi.getTempLinkToken(actualPath, 60 * 60 * 24 * 365 * 10);
const permanentLink = vfsApi.getTempPublicUrl(link.token);
updateFile(task.id, { updateFile(task.id, {
status: 'success', status: 'success',
progress: 100, progress: 100,
loadedBytes: finalSize, loadedBytes: finalSize,
size: finalSize, size: finalSize,
targetPath: actualPath, targetPath: actualPath,
permanentLink,
}); });
} catch (err: unknown) { } catch (err: unknown) {
const error = err instanceof Error ? err.message : t('Upload failed'); const error = err instanceof Error ? err.message : t('Upload failed');
updateFile(task.id, { status: 'error', error, progress: 0 }); updateFile(task.id, { status: 'error', error, progress: 0 });
message.error(`${task.relativePath}: ${error}`); message.error(`${task.relativePath}: ${error}`);
} }
}, [ensureDirectoryTree, awaitConflictDecision, mutateFiles, updateFile, t]); }, [mutateFiles, updateFile, t]);
const uploadPreparedFiles = useCallback(async (preparedFiles: Array<{ task: UploadFile; overwrite: boolean }>) => {
let nextIndex = 0;
const workerCount = Math.min(MAX_UPLOAD_CONCURRENCY, preparedFiles.length);
const runWorker = async () => {
while (nextIndex < preparedFiles.length) {
const current = preparedFiles[nextIndex];
nextIndex += 1;
await uploadPreparedFile(current.task, current.overwrite);
}
};
await Promise.all(Array.from({ length: workerCount }, runWorker));
}, [uploadPreparedFile]);
const startUpload = useCallback(async () => { const startUpload = useCallback(async () => {
if (isUploadingRef.current) return; if (isUploadingRef.current) return;
@@ -530,6 +553,7 @@ export function useUploader(path: string, onUploadComplete: () => void) {
isUploadingRef.current = true; isUploadingRef.current = true;
setIsUploading(true); setIsUploading(true);
try { try {
const preparedFiles: Array<{ task: UploadFile; overwrite: boolean }> = [];
for (const task of filesRef.current) { for (const task of filesRef.current) {
if (task.status !== 'pending' && task.status !== 'waiting') { if (task.status !== 'pending' && task.status !== 'waiting') {
continue; continue;
@@ -537,15 +561,17 @@ export function useUploader(path: string, onUploadComplete: () => void) {
if (task.type === 'directory') { if (task.type === 'directory') {
await processDirectoryTask(task); await processDirectoryTask(task);
} else { } else {
await processFileTask(task); const prepared = await prepareFileTask(task);
if (prepared) preparedFiles.push(prepared);
} }
} }
await uploadPreparedFiles(preparedFiles);
onUploadComplete(); onUploadComplete();
} finally { } finally {
isUploadingRef.current = false; isUploadingRef.current = false;
setIsUploading(false); setIsUploading(false);
} }
}, [onUploadComplete, processDirectoryTask, processFileTask]); }, [onUploadComplete, processDirectoryTask, prepareFileTask, uploadPreparedFiles]);
const totalFileBytes = useMemo( const totalFileBytes = useMemo(
() => files.reduce((acc, f) => acc + (f.type === 'file' ? f.size : 0), 0), () => files.reduce((acc, f) => acc + (f.type === 'file' ? f.size : 0), 0),