mirror of
https://github.com/DrizzleTime/Foxel.git
synced 2026-09-08 09:06:50 +08:00
feat(Processors): add process directory endpoint and UI support
This commit is contained in:
+135
-1
@@ -3,6 +3,8 @@ from fastapi import APIRouter, Depends, Body, HTTPException
|
|||||||
from fastapi.concurrency import run_in_threadpool
|
from fastapi.concurrency import run_in_threadpool
|
||||||
from typing import Annotated
|
from typing import Annotated
|
||||||
from services.processors.registry import (
|
from services.processors.registry import (
|
||||||
|
get,
|
||||||
|
get_config_schema,
|
||||||
get_config_schemas,
|
get_config_schemas,
|
||||||
get_module_path,
|
get_module_path,
|
||||||
reload_processors,
|
reload_processors,
|
||||||
@@ -11,7 +13,8 @@ from services.task_queue import task_queue_service
|
|||||||
from services.auth import get_current_active_user, User
|
from services.auth import get_current_active_user, User
|
||||||
from api.response import success
|
from api.response import success
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from services.virtual_fs import path_is_directory
|
from services.virtual_fs import path_is_directory, resolve_adapter_and_rel
|
||||||
|
from typing import List, Optional, Tuple
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/processors", tags=["processors"])
|
router = APIRouter(prefix="/api/processors", tags=["processors"])
|
||||||
|
|
||||||
@@ -42,6 +45,15 @@ class ProcessRequest(BaseModel):
|
|||||||
overwrite: bool = False
|
overwrite: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessDirectoryRequest(BaseModel):
|
||||||
|
path: str
|
||||||
|
processor_type: str
|
||||||
|
config: dict
|
||||||
|
overwrite: bool = True
|
||||||
|
max_depth: Optional[int] = None
|
||||||
|
suffix: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
class UpdateSourceRequest(BaseModel):
|
class UpdateSourceRequest(BaseModel):
|
||||||
source: str
|
source: str
|
||||||
|
|
||||||
@@ -69,6 +81,128 @@ async def process_file_with_processor(
|
|||||||
return success({"task_id": task.id})
|
return success({"task_id": task.id})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/process-directory")
|
||||||
|
async def process_directory_with_processor(
|
||||||
|
current_user: Annotated[User, Depends(get_current_active_user)],
|
||||||
|
req: ProcessDirectoryRequest = Body(...)
|
||||||
|
):
|
||||||
|
if req.max_depth is not None and req.max_depth < 0:
|
||||||
|
raise HTTPException(400, detail="max_depth must be >= 0")
|
||||||
|
|
||||||
|
is_dir = await path_is_directory(req.path)
|
||||||
|
if not is_dir:
|
||||||
|
raise HTTPException(400, detail="Path must be a directory")
|
||||||
|
|
||||||
|
schema = get_config_schema(req.processor_type)
|
||||||
|
_processor = get(req.processor_type)
|
||||||
|
if not schema or not _processor:
|
||||||
|
raise HTTPException(404, detail="Processor not found")
|
||||||
|
|
||||||
|
produces_file = bool(schema.get("produces_file"))
|
||||||
|
raw_suffix = req.suffix if req.suffix is not None else None
|
||||||
|
if raw_suffix is not None and raw_suffix.strip() == "":
|
||||||
|
raw_suffix = None
|
||||||
|
suffix = raw_suffix
|
||||||
|
overwrite = req.overwrite
|
||||||
|
|
||||||
|
if produces_file:
|
||||||
|
if not overwrite and not suffix:
|
||||||
|
raise HTTPException(400, detail="Suffix is required when not overwriting files")
|
||||||
|
else:
|
||||||
|
overwrite = False
|
||||||
|
suffix = None
|
||||||
|
|
||||||
|
supported_exts = schema.get("supported_exts") or []
|
||||||
|
allowed_exts = {
|
||||||
|
ext.lower().lstrip('.')
|
||||||
|
for ext in supported_exts
|
||||||
|
if isinstance(ext, str)
|
||||||
|
}
|
||||||
|
|
||||||
|
def matches_extension(file_rel: str) -> bool:
|
||||||
|
if not allowed_exts:
|
||||||
|
return True
|
||||||
|
if '.' not in file_rel:
|
||||||
|
return '' in allowed_exts
|
||||||
|
ext = file_rel.rsplit('.', 1)[-1].lower()
|
||||||
|
return ext in allowed_exts or f'.{ext}' in allowed_exts
|
||||||
|
|
||||||
|
adapter_instance, adapter_model, root, rel = await resolve_adapter_and_rel(req.path)
|
||||||
|
rel = rel.rstrip('/')
|
||||||
|
|
||||||
|
list_dir = getattr(adapter_instance, "list_dir", None)
|
||||||
|
if not callable(list_dir):
|
||||||
|
raise HTTPException(501, detail="Adapter does not implement list_dir")
|
||||||
|
|
||||||
|
def build_absolute_path(mount_path: str, rel_path: str) -> str:
|
||||||
|
rel_norm = rel_path.lstrip('/')
|
||||||
|
mount_norm = mount_path.rstrip('/')
|
||||||
|
if not mount_norm:
|
||||||
|
return '/' + rel_norm if rel_norm else '/'
|
||||||
|
return f"{mount_norm}/{rel_norm}" if rel_norm else mount_norm
|
||||||
|
|
||||||
|
def apply_suffix(path_str: str, suffix_str: str) -> str:
|
||||||
|
path_obj = Path(path_str)
|
||||||
|
name = path_obj.name
|
||||||
|
if not name:
|
||||||
|
return path_str
|
||||||
|
if '.' in name:
|
||||||
|
base, ext = name.rsplit('.', 1)
|
||||||
|
new_name = f"{base}{suffix_str}.{ext}"
|
||||||
|
else:
|
||||||
|
new_name = f"{name}{suffix_str}"
|
||||||
|
return str(path_obj.with_name(new_name))
|
||||||
|
|
||||||
|
scheduled_tasks: List[str] = []
|
||||||
|
stack: List[Tuple[str, int]] = [(rel, 0)]
|
||||||
|
page_size = 200
|
||||||
|
|
||||||
|
while stack:
|
||||||
|
current_rel, depth = stack.pop()
|
||||||
|
page = 1
|
||||||
|
while True:
|
||||||
|
entries, total = await list_dir(root, current_rel, page, page_size, "name", "asc")
|
||||||
|
entries = entries or []
|
||||||
|
if not entries and (total or 0) == 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
for entry in entries:
|
||||||
|
name = entry.get("name")
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
child_rel = f"{current_rel}/{name}" if current_rel else name
|
||||||
|
if entry.get("is_dir"):
|
||||||
|
if req.max_depth is None or depth < req.max_depth:
|
||||||
|
stack.append((child_rel.rstrip('/'), depth + 1))
|
||||||
|
continue
|
||||||
|
if not matches_extension(child_rel):
|
||||||
|
continue
|
||||||
|
absolute_path = build_absolute_path(adapter_model.path, child_rel)
|
||||||
|
save_to = None
|
||||||
|
if produces_file and not overwrite and suffix:
|
||||||
|
save_to = apply_suffix(absolute_path, suffix)
|
||||||
|
task = await task_queue_service.add_task(
|
||||||
|
"process_file",
|
||||||
|
{
|
||||||
|
"path": absolute_path,
|
||||||
|
"processor_type": req.processor_type,
|
||||||
|
"config": req.config,
|
||||||
|
"save_to": save_to,
|
||||||
|
"overwrite": overwrite,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
scheduled_tasks.append(task.id)
|
||||||
|
|
||||||
|
if total is None or page * page_size >= total:
|
||||||
|
break
|
||||||
|
page += 1
|
||||||
|
|
||||||
|
return success({
|
||||||
|
"task_ids": scheduled_tasks,
|
||||||
|
"scheduled": len(scheduled_tasks),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
@router.get("/source/{processor_type}")
|
@router.get("/source/{processor_type}")
|
||||||
async def get_processor_source(
|
async def get_processor_source(
|
||||||
processor_type: str,
|
processor_type: str,
|
||||||
|
|||||||
@@ -34,6 +34,18 @@ export const processorsApi = {
|
|||||||
method: 'POST',
|
method: 'POST',
|
||||||
json: params,
|
json: params,
|
||||||
}),
|
}),
|
||||||
|
processDirectory: (params: {
|
||||||
|
path: string;
|
||||||
|
processor_type: string;
|
||||||
|
config: any;
|
||||||
|
overwrite: boolean;
|
||||||
|
max_depth?: number | null;
|
||||||
|
suffix?: string | null;
|
||||||
|
}) =>
|
||||||
|
request<{ task_ids: string[]; scheduled: number }>('/processors/process-directory', {
|
||||||
|
method: 'POST',
|
||||||
|
json: params,
|
||||||
|
}),
|
||||||
getSource: (type: string) =>
|
getSource: (type: string) =>
|
||||||
request<{ source: string; module_path: string }>('/processors/source/' + encodeURIComponent(type), {
|
request<{ source: string; module_path: string }>('/processors/source/' + encodeURIComponent(type), {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
|
|||||||
@@ -418,6 +418,17 @@ export const zh = {
|
|||||||
'Source Editor': '源码编辑',
|
'Source Editor': '源码编辑',
|
||||||
'Module Path': '模块路径',
|
'Module Path': '模块路径',
|
||||||
'Directory processing always overwrites original files': '选择目录时会强制覆盖原文件',
|
'Directory processing always overwrites original files': '选择目录时会强制覆盖原文件',
|
||||||
|
'Directory execution will enqueue one task per file': '目录模式会为每个文件单独创建任务',
|
||||||
|
'Directory scope': '目录范围',
|
||||||
|
'Current level only': '仅当前层级',
|
||||||
|
'Include subdirectories': '包含子目录',
|
||||||
|
'Max depth': '最大层级',
|
||||||
|
'Leave empty to traverse all subdirectories': '留空表示遍历所有子目录',
|
||||||
|
'Depth must be greater or equal to 0': '层级必须大于或等于 0',
|
||||||
|
'Output suffix': '输出后缀',
|
||||||
|
'Suffix will be inserted before the file extension, e.g. demo_processed.mp4': '后缀会插入到文件扩展名前,例如 demo_processed.mp4',
|
||||||
|
'Suffix such as _processed': '例如 _processed 的后缀',
|
||||||
|
'Suffix cannot be empty': '后缀不能为空',
|
||||||
'No data': '暂无数据',
|
'No data': '暂无数据',
|
||||||
|
|
||||||
// Path selector
|
// Path selector
|
||||||
|
|||||||
@@ -1,13 +1,16 @@
|
|||||||
import { memo, useCallback, useEffect, useMemo, useState } from 'react';
|
import { memo, useCallback, useEffect, useMemo, useState } from 'react';
|
||||||
import {
|
import {
|
||||||
Button,
|
Button,
|
||||||
|
Alert,
|
||||||
Card,
|
Card,
|
||||||
Empty,
|
Empty,
|
||||||
Flex,
|
Flex,
|
||||||
Form,
|
Form,
|
||||||
Input,
|
Input,
|
||||||
|
InputNumber,
|
||||||
message,
|
message,
|
||||||
Modal,
|
Modal,
|
||||||
|
Segmented,
|
||||||
Space,
|
Space,
|
||||||
Spin,
|
Spin,
|
||||||
Switch,
|
Switch,
|
||||||
@@ -134,23 +137,36 @@ const ProcessorsPage = memo(function ProcessorsPage() {
|
|||||||
overwrite: !!selectedProcessorMeta.produces_file,
|
overwrite: !!selectedProcessorMeta.produces_file,
|
||||||
save_to: undefined,
|
save_to: undefined,
|
||||||
config: defaults,
|
config: defaults,
|
||||||
|
directory_scope: 'current',
|
||||||
|
max_depth: undefined,
|
||||||
|
suffix: undefined,
|
||||||
});
|
});
|
||||||
setIsDirectory(false);
|
setIsDirectory(false);
|
||||||
}, [selectedProcessorMeta, form]);
|
}, [selectedProcessorMeta, form]);
|
||||||
|
|
||||||
const overwriteValue = Form.useWatch('overwrite', form) ?? false;
|
const producesFile = selectedProcessorMeta?.produces_file ?? false;
|
||||||
|
const overwriteWatch = Form.useWatch('overwrite', form);
|
||||||
|
const overwriteValue = producesFile ? !!overwriteWatch : false;
|
||||||
|
const directoryScope = Form.useWatch('directory_scope', form) ?? 'current';
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (overwriteValue) {
|
if (overwriteValue) {
|
||||||
form.setFieldsValue({ save_to: undefined });
|
form.setFieldsValue({ save_to: undefined, suffix: undefined });
|
||||||
}
|
}
|
||||||
}, [overwriteValue, form]);
|
}, [overwriteValue, form]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isDirectory) {
|
if (isDirectory) {
|
||||||
form.setFieldsValue({ overwrite: true, save_to: undefined });
|
form.setFieldsValue({
|
||||||
|
overwrite: producesFile ? true : false,
|
||||||
|
save_to: undefined,
|
||||||
|
directory_scope: 'current',
|
||||||
|
max_depth: undefined,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
form.setFieldsValue({ suffix: undefined });
|
||||||
}
|
}
|
||||||
}, [isDirectory, form]);
|
}, [isDirectory, form, producesFile]);
|
||||||
|
|
||||||
const handleSelectProcessor = useCallback((type: string) => {
|
const handleSelectProcessor = useCallback((type: string) => {
|
||||||
if (type === selectedType) return;
|
if (type === selectedType) return;
|
||||||
@@ -232,17 +248,38 @@ const ProcessorsPage = memo(function ProcessorsPage() {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
setRunning(true);
|
setRunning(true);
|
||||||
const payload: any = {
|
const overwriteFlag = producesFile ? !!values.overwrite : false;
|
||||||
path: values.path,
|
if (isDirectory) {
|
||||||
processor_type: selectedType,
|
const scope: 'current' | 'recursive' = values.directory_scope || 'current';
|
||||||
config: finalConfig,
|
let maxDepth: number | null = scope === 'current' ? 0 : null;
|
||||||
overwrite: !!values.overwrite,
|
if (scope === 'recursive' && typeof values.max_depth === 'number') {
|
||||||
};
|
maxDepth = values.max_depth;
|
||||||
if (values.save_to && !values.overwrite) {
|
}
|
||||||
payload.save_to = values.save_to;
|
const suffixValue = producesFile && !overwriteFlag && typeof values.suffix === 'string'
|
||||||
|
? values.suffix.trim() || null
|
||||||
|
: null;
|
||||||
|
const resp = await processorsApi.processDirectory({
|
||||||
|
path: values.path,
|
||||||
|
processor_type: selectedType,
|
||||||
|
config: finalConfig,
|
||||||
|
overwrite: overwriteFlag,
|
||||||
|
max_depth: maxDepth,
|
||||||
|
suffix: suffixValue,
|
||||||
|
});
|
||||||
|
messageApi.success(`${t('Task submitted')}: ${resp.scheduled}`);
|
||||||
|
} else {
|
||||||
|
const payload: any = {
|
||||||
|
path: values.path,
|
||||||
|
processor_type: selectedType,
|
||||||
|
config: finalConfig,
|
||||||
|
overwrite: overwriteFlag,
|
||||||
|
};
|
||||||
|
if (values.save_to && !overwriteFlag) {
|
||||||
|
payload.save_to = values.save_to;
|
||||||
|
}
|
||||||
|
const resp = await processorsApi.process(payload);
|
||||||
|
messageApi.success(`${t('Task submitted')}: ${resp.task_id}`);
|
||||||
}
|
}
|
||||||
const resp = await processorsApi.process(payload);
|
|
||||||
messageApi.success(`${t('Task submitted')}: ${resp.task_id}`);
|
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
if (err?.errorFields) {
|
if (err?.errorFields) {
|
||||||
return;
|
return;
|
||||||
@@ -251,7 +288,7 @@ const ProcessorsPage = memo(function ProcessorsPage() {
|
|||||||
} finally {
|
} finally {
|
||||||
setRunning(false);
|
setRunning(false);
|
||||||
}
|
}
|
||||||
}, [form, messageApi, selectedProcessorMeta, selectedType, t]);
|
}, [form, isDirectory, messageApi, producesFile, selectedProcessorMeta, selectedType, t]);
|
||||||
|
|
||||||
const selectedConfigPath = pathModalField === 'path'
|
const selectedConfigPath = pathModalField === 'path'
|
||||||
? (selectedType ? form.getFieldValue('path') : undefined) || '/'
|
? (selectedType ? form.getFieldValue('path') : undefined) || '/'
|
||||||
@@ -379,11 +416,6 @@ const ProcessorsPage = memo(function ProcessorsPage() {
|
|||||||
<Form form={form} layout="vertical" disabled={!selectedType} style={{ padding: '12px 0' }}>
|
<Form form={form} layout="vertical" disabled={!selectedType} style={{ padding: '12px 0' }}>
|
||||||
{selectedType ? (
|
{selectedType ? (
|
||||||
<>
|
<>
|
||||||
{isDirectory && (
|
|
||||||
<Text type="secondary" style={{ display: 'block', marginBottom: 12 }}>
|
|
||||||
{t('Directory processing always overwrites original files')}
|
|
||||||
</Text>
|
|
||||||
)}
|
|
||||||
<Form.Item
|
<Form.Item
|
||||||
label={t('Target Path')}
|
label={t('Target Path')}
|
||||||
required
|
required
|
||||||
@@ -402,16 +434,71 @@ const ProcessorsPage = memo(function ProcessorsPage() {
|
|||||||
<Button onClick={() => openPathSelector('path', 'directory')}>{t('Select Directory')}</Button>
|
<Button onClick={() => openPathSelector('path', 'directory')}>{t('Select Directory')}</Button>
|
||||||
</Flex>
|
</Flex>
|
||||||
</Form.Item>
|
</Form.Item>
|
||||||
|
{isDirectory && (
|
||||||
|
<Space direction="vertical" size={12} style={{ width: '100%', marginBottom: 12 }}>
|
||||||
|
<Alert
|
||||||
|
type="info"
|
||||||
|
showIcon
|
||||||
|
message={t('Directory execution will enqueue one task per file')}
|
||||||
|
/>
|
||||||
|
<Form.Item name="directory_scope" label={t('Directory scope')} initialValue="current">
|
||||||
|
<Segmented
|
||||||
|
options={[
|
||||||
|
{ label: t('Current level only'), value: 'current' },
|
||||||
|
{ label: t('Include subdirectories'), value: 'recursive' },
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</Form.Item>
|
||||||
|
{directoryScope === 'recursive' && (
|
||||||
|
<Form.Item
|
||||||
|
name="max_depth"
|
||||||
|
label={t('Max depth')}
|
||||||
|
extra={t('Leave empty to traverse all subdirectories')}
|
||||||
|
rules={[{
|
||||||
|
validator: async (_: any, value: number | null) => {
|
||||||
|
if (value === undefined || value === null) return;
|
||||||
|
if (value < 0) throw new Error(t('Depth must be greater or equal to 0'));
|
||||||
|
},
|
||||||
|
}]}
|
||||||
|
>
|
||||||
|
<InputNumber min={0} placeholder={t('Unlimited')} style={{ width: '100%' }} />
|
||||||
|
</Form.Item>
|
||||||
|
)}
|
||||||
|
</Space>
|
||||||
|
)}
|
||||||
|
|
||||||
<Form.Item
|
{producesFile && (
|
||||||
name="overwrite"
|
<Form.Item
|
||||||
label={t('Overwrite original')}
|
name="overwrite"
|
||||||
valuePropName="checked"
|
label={t('Overwrite original')}
|
||||||
>
|
valuePropName="checked"
|
||||||
<Switch disabled={isDirectory} />
|
>
|
||||||
</Form.Item>
|
<Switch />
|
||||||
|
</Form.Item>
|
||||||
|
)}
|
||||||
|
|
||||||
{selectedProcessorMeta?.produces_file && !overwriteValue && (
|
{isDirectory && producesFile && !overwriteValue && (
|
||||||
|
<Form.Item
|
||||||
|
name="suffix"
|
||||||
|
label={t('Output suffix')}
|
||||||
|
rules={[
|
||||||
|
{ required: true, message: t('Please input a suffix') },
|
||||||
|
{
|
||||||
|
validator: async (_: any, value: string) => {
|
||||||
|
if (typeof value !== 'string') return;
|
||||||
|
if (!value.trim()) {
|
||||||
|
throw new Error(t('Suffix cannot be empty'));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
extra={t('Suffix will be inserted before the file extension, e.g. demo_processed.mp4')}
|
||||||
|
>
|
||||||
|
<Input placeholder={t('Suffix such as _processed')} />
|
||||||
|
</Form.Item>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!isDirectory && producesFile && !overwriteValue && (
|
||||||
<Form.Item label={t('Save To')}>
|
<Form.Item label={t('Save To')}>
|
||||||
<Flex gap={8} align="center">
|
<Flex gap={8} align="center">
|
||||||
<div style={{ flex: 1 }}>
|
<div style={{ flex: 1 }}>
|
||||||
|
|||||||
Reference in New Issue
Block a user