mirror of
https://github.com/DrizzleTime/Foxel.git
synced 2026-09-06 08:07:22 +08:00
Initial commit
This commit is contained in:
shiyu
@@ -0,0 +1,35 @@
|
||||
from .base import BaseProcessor
|
||||
from .registry import (
|
||||
CONFIG_SCHEMAS,
|
||||
TYPE_MAP,
|
||||
get_config_schema,
|
||||
get_config_schemas,
|
||||
get_last_discovery_errors,
|
||||
get_module_path,
|
||||
reload_processors,
|
||||
)
|
||||
from .service import (
|
||||
ProcessorService,
|
||||
get_processor,
|
||||
list_processors,
|
||||
reload_processor_modules,
|
||||
)
|
||||
from .types import ProcessDirectoryRequest, ProcessRequest, UpdateSourceRequest
|
||||
|
||||
__all__ = [
|
||||
"BaseProcessor",
|
||||
"CONFIG_SCHEMAS",
|
||||
"TYPE_MAP",
|
||||
"get_config_schema",
|
||||
"get_config_schemas",
|
||||
"get_last_discovery_errors",
|
||||
"get_module_path",
|
||||
"reload_processors",
|
||||
"ProcessorService",
|
||||
"get_processor",
|
||||
"list_processors",
|
||||
"reload_processor_modules",
|
||||
"ProcessDirectoryRequest",
|
||||
"ProcessRequest",
|
||||
"UpdateSourceRequest",
|
||||
]
|
||||
@@ -0,0 +1,108 @@
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Request
|
||||
|
||||
from api.response import success
|
||||
from domain.audit import AuditAction, audit
|
||||
from domain.auth import User, get_current_active_user
|
||||
from domain.permission import require_path_permission
|
||||
from domain.permission import require_system_permission
|
||||
from domain.permission.service import PermissionService
|
||||
from domain.permission.types import PathAction
|
||||
from domain.permission.types import SystemPermission
|
||||
from domain.processors.registry import get_config_schema
|
||||
from .service import ProcessorService
|
||||
from .types import (
|
||||
ProcessDirectoryRequest,
|
||||
ProcessRequest,
|
||||
UpdateSourceRequest,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/processors", tags=["processors"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
@audit(action=AuditAction.READ, description="获取处理器列表")
|
||||
async def list_processors(
|
||||
request: Request,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)],
|
||||
):
|
||||
data = ProcessorService.list_processors()
|
||||
return success(data)
|
||||
|
||||
|
||||
@router.post("/process")
|
||||
@audit(
|
||||
action=AuditAction.CREATE,
|
||||
description="处理单个文件",
|
||||
body_fields=["path", "processor_type", "save_to", "overwrite"],
|
||||
)
|
||||
@require_path_permission(PathAction.READ, "req.path")
|
||||
async def process_file_with_processor(
|
||||
request: Request,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)],
|
||||
req: ProcessRequest = Body(...),
|
||||
):
|
||||
meta = get_config_schema(req.processor_type) or {}
|
||||
if meta.get("produces_file"):
|
||||
if req.overwrite:
|
||||
await PermissionService.require_path_permission(current_user.id, req.path, PathAction.WRITE)
|
||||
elif req.save_to:
|
||||
await PermissionService.require_path_permission(current_user.id, req.save_to, PathAction.WRITE)
|
||||
data = await ProcessorService.process_file(req)
|
||||
return success(data)
|
||||
|
||||
|
||||
@router.post("/process-directory")
|
||||
@audit(
|
||||
action=AuditAction.CREATE,
|
||||
description="批量处理目录",
|
||||
body_fields=["path", "processor_type", "overwrite", "max_depth", "suffix"],
|
||||
)
|
||||
@require_path_permission(PathAction.READ, "req.path")
|
||||
async def process_directory_with_processor(
|
||||
request: Request,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)],
|
||||
req: ProcessDirectoryRequest = Body(...),
|
||||
):
|
||||
meta = get_config_schema(req.processor_type) or {}
|
||||
if meta.get("produces_file"):
|
||||
await PermissionService.require_path_permission(current_user.id, req.path, PathAction.WRITE)
|
||||
data = await ProcessorService.process_directory(req)
|
||||
return success(data)
|
||||
|
||||
|
||||
@router.get("/source/{processor_type}")
|
||||
@audit(action=AuditAction.READ, description="获取处理器源码")
|
||||
@require_system_permission(SystemPermission.ROLE_MANAGE)
|
||||
async def get_processor_source(
|
||||
request: Request,
|
||||
processor_type: str,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)],
|
||||
):
|
||||
data = await ProcessorService.get_source(processor_type)
|
||||
return success(data)
|
||||
|
||||
|
||||
@router.put("/source/{processor_type}")
|
||||
@audit(action=AuditAction.UPDATE, description="更新处理器源码")
|
||||
@require_system_permission(SystemPermission.ROLE_MANAGE)
|
||||
async def update_processor_source(
|
||||
request: Request,
|
||||
processor_type: str,
|
||||
req: UpdateSourceRequest,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)],
|
||||
):
|
||||
data = await ProcessorService.update_source(processor_type, req)
|
||||
return success(data)
|
||||
|
||||
|
||||
@router.post("/reload")
|
||||
@audit(action=AuditAction.UPDATE, description="重载处理器模块")
|
||||
@require_system_permission(SystemPermission.ROLE_MANAGE)
|
||||
async def reload_processor_modules(
|
||||
request: Request,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)],
|
||||
):
|
||||
data = ProcessorService.reload()
|
||||
return success(data)
|
||||
@@ -0,0 +1,19 @@
|
||||
from typing import Protocol, Dict, Any
|
||||
|
||||
|
||||
class BaseProcessor(Protocol):
|
||||
name: str
|
||||
supported_exts: list
|
||||
config_schema: list
|
||||
produces_file: bool
|
||||
supports_directory: bool
|
||||
requires_input_bytes: bool
|
||||
|
||||
async def process(self, input_bytes: bytes, path: str, config: Dict[str, Any]) -> Any:
|
||||
"""处理文件内容/路径并返回结果。produces_file=True 时应返回 bytes/Response。"""
|
||||
...
|
||||
|
||||
# 约定：每个处理器需定义
|
||||
# PROCESSOR_TYPE: str
|
||||
# CONFIG_SCHEMA: list
|
||||
# PROCESSOR_FACTORY: Callable[[], BaseProcessor]
|
||||
@@ -0,0 +1 @@
|
||||
# 内置处理器包
|
||||
@@ -0,0 +1,70 @@
|
||||
from typing import Dict, Any
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from fastapi.responses import Response
|
||||
|
||||
from ..base import BaseProcessor
|
||||
|
||||
|
||||
class ImageWatermarkProcessor:
|
||||
name = "图片水印"
|
||||
supported_exts = ["jpg", "jpeg", "png", "bmp"]
|
||||
config_schema = [
|
||||
{"key": "text", "label": "水印文字", "type": "string", "required": True},
|
||||
{
|
||||
"key": "position",
|
||||
"label": "位置",
|
||||
"type": "select",
|
||||
"required": False,
|
||||
"default": "bottom-right",
|
||||
"options": [
|
||||
{"value": "top-left", "label": "左上"},
|
||||
{"value": "center", "label": "居中"},
|
||||
{"value": "bottom-right", "label": "右下"},
|
||||
],
|
||||
},
|
||||
{"key": "font_size", "label": "字体大小", "type": "number", "required": False, "default": 24},
|
||||
]
|
||||
produces_file = True
|
||||
|
||||
async def process(self, input_bytes: bytes, path: str, config: Dict[str, Any]) -> Response:
|
||||
text = config.get("text", "")
|
||||
position = config.get("position", "bottom-right")
|
||||
font_size = int(config.get("font_size", 24))
|
||||
|
||||
img = Image.open(BytesIO(input_bytes)).convert("RGBA")
|
||||
watermark = Image.new("RGBA", img.size)
|
||||
draw = ImageDraw.Draw(watermark)
|
||||
try:
|
||||
font = ImageFont.truetype("arial.ttf", font_size)
|
||||
except Exception:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
w, h = img.size
|
||||
try:
|
||||
text_w, text_h = font.getsize(text)
|
||||
except AttributeError:
|
||||
bbox = draw.textbbox((0, 0), text, font=font)
|
||||
text_w, text_h = bbox[2] - bbox[0], bbox[3] - bbox[1]
|
||||
|
||||
if position == "bottom-right":
|
||||
xy = (w - text_w - 10, h - text_h - 10)
|
||||
elif position == "top-left":
|
||||
xy = (10, 10)
|
||||
else:
|
||||
xy = (w // 2 - text_w // 2, h // 2 - text_h // 2)
|
||||
|
||||
draw.text(xy, text, font=font, fill=(255, 255, 255, 128))
|
||||
out = Image.alpha_composite(img, watermark)
|
||||
buf = BytesIO()
|
||||
out.convert("RGB").save(buf, format="JPEG")
|
||||
|
||||
return Response(content=buf.getvalue(), media_type="image/jpeg")
|
||||
|
||||
|
||||
PROCESSOR_TYPE = "image_watermark"
|
||||
PROCESSOR_NAME = ImageWatermarkProcessor.name
|
||||
SUPPORTED_EXTS = ImageWatermarkProcessor.supported_exts
|
||||
CONFIG_SCHEMA = ImageWatermarkProcessor.config_schema
|
||||
PROCESSOR_FACTORY = lambda: ImageWatermarkProcessor()
|
||||
@@ -0,0 +1,251 @@
|
||||
import base64
|
||||
import mimetypes
|
||||
import os
|
||||
from io import BytesIO
|
||||
from typing import Dict, Any, List, Tuple
|
||||
|
||||
from fastapi.responses import Response
|
||||
from PIL import Image
|
||||
|
||||
from ..base import BaseProcessor
|
||||
from domain.ai import (
|
||||
DEFAULT_VECTOR_DIMENSION,
|
||||
FILE_COLLECTION_NAME,
|
||||
VECTOR_COLLECTION_NAME,
|
||||
VectorDBService,
|
||||
describe_image_base64,
|
||||
get_text_embedding,
|
||||
provider_service,
|
||||
)
|
||||
|
||||
|
||||
CHUNK_SIZE = 800
|
||||
CHUNK_OVERLAP = 200
|
||||
MAX_IMAGE_EDGE = 1600
|
||||
JPEG_QUALITY = 85
|
||||
|
||||
|
||||
def _chunk_text(content: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> List[Tuple[int, str, int, int]]:
|
||||
"""按固定窗口拆分文本，返回(chunk_id, chunk_text, start, end)。"""
|
||||
if chunk_size <= 0:
|
||||
chunk_size = CHUNK_SIZE
|
||||
if overlap >= chunk_size:
|
||||
overlap = max(chunk_size // 4, 1)
|
||||
|
||||
chunks: List[Tuple[int, str, int, int]] = []
|
||||
step = chunk_size - overlap
|
||||
idx = 0
|
||||
start = 0
|
||||
length = len(content)
|
||||
|
||||
while start < length:
|
||||
end = min(length, start + chunk_size)
|
||||
chunk = content[start:end].strip()
|
||||
if chunk:
|
||||
chunks.append((idx, chunk, start, end))
|
||||
idx += 1
|
||||
if end >= length:
|
||||
break
|
||||
start += step
|
||||
return chunks
|
||||
|
||||
|
||||
def _guess_mime(path: str) -> str:
|
||||
mime, _ = mimetypes.guess_type(path)
|
||||
return mime or "application/octet-stream"
|
||||
|
||||
|
||||
def _chunk_key(path: str, chunk_id: str) -> str:
|
||||
return f"{path}#chunk={chunk_id}"
|
||||
|
||||
|
||||
def _compress_image_for_embedding(input_bytes: bytes) -> Tuple[bytes, Dict[str, Any] | None]:
|
||||
"""压缩图片，降低发送到视觉模型的体积。"""
|
||||
if Image is None:
|
||||
return input_bytes, None
|
||||
|
||||
try:
|
||||
with Image.open(BytesIO(input_bytes)) as img:
|
||||
img = img.convert("RGB")
|
||||
width, height = img.size
|
||||
longest_edge = max(width, height)
|
||||
scale = 1.0
|
||||
if longest_edge > MAX_IMAGE_EDGE:
|
||||
scale = MAX_IMAGE_EDGE / float(longest_edge)
|
||||
new_size = (max(int(width * scale), 1), max(int(height * scale), 1))
|
||||
resample_mode = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
|
||||
img = img.resize(new_size, resample=resample_mode)
|
||||
|
||||
buffer = BytesIO()
|
||||
img.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
compressed = buffer.getvalue()
|
||||
|
||||
if len(compressed) < len(input_bytes):
|
||||
return compressed, {
|
||||
"original_bytes": len(input_bytes),
|
||||
"compressed_bytes": len(compressed),
|
||||
"scaled": scale < 1.0,
|
||||
"width": img.width,
|
||||
"height": img.height,
|
||||
}
|
||||
except Exception: # pragma: no cover - 任意图像处理异常时回退
|
||||
return input_bytes, None
|
||||
|
||||
return input_bytes, None
|
||||
|
||||
|
||||
class VectorIndexProcessor:
|
||||
name = "向量索引"
|
||||
supported_exts: List[str] = [] # 留空表示不限扩展名
|
||||
config_schema = [
|
||||
{
|
||||
"key": "action", "label": "操作", "type": "select", "required": True, "default": "create",
|
||||
"options": [
|
||||
{"value": "create", "label": "创建索引"},
|
||||
{"value": "destroy", "label": "销毁索引"},
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "index_type", "label": "索引类型", "type": "select", "required": True, "default": "vector",
|
||||
"options": [
|
||||
{"value": "vector", "label": "向量索引"},
|
||||
{"value": "simple", "label": "普通索引"},
|
||||
]
|
||||
}
|
||||
]
|
||||
produces_file = False
|
||||
requires_input_bytes = False
|
||||
|
||||
async def process(self, input_bytes: bytes, path: str, config: Dict[str, Any]) -> Response:
|
||||
async def ensure_input_bytes() -> bytes:
|
||||
if input_bytes:
|
||||
return input_bytes
|
||||
from domain.virtual_fs import VirtualFSService
|
||||
return await VirtualFSService.read_file(path)
|
||||
|
||||
action = config.get("action", "create")
|
||||
index_type = config.get("index_type", "vector")
|
||||
vector_db = VectorDBService()
|
||||
vector_collection = VECTOR_COLLECTION_NAME
|
||||
file_collection = FILE_COLLECTION_NAME
|
||||
|
||||
if action == "destroy":
|
||||
target_collection = file_collection if index_type == "simple" else vector_collection
|
||||
await vector_db.delete_vector(target_collection, path)
|
||||
return Response(content=f"文件 {path} 的 {index_type} 索引已销毁", media_type="text/plain")
|
||||
|
||||
mime_type = _guess_mime(path)
|
||||
|
||||
if index_type == "simple":
|
||||
await vector_db.ensure_collection(file_collection, vector=False)
|
||||
await vector_db.delete_vector(file_collection, path)
|
||||
await vector_db.upsert_vector(file_collection, {
|
||||
"path": path,
|
||||
"source_path": path,
|
||||
"chunk_id": "filename",
|
||||
"mime": mime_type,
|
||||
"type": "filename",
|
||||
"name": os.path.basename(path),
|
||||
})
|
||||
return Response(content=f"文件 {path} 的普通索引已创建", media_type="text/plain")
|
||||
|
||||
file_ext = path.split('.')[-1].lower()
|
||||
details: Dict[str, Any] = {"path": path, "action": "create", "index_type": "vector"}
|
||||
|
||||
embedding_model = await provider_service.get_default_model("embedding")
|
||||
vector_dim = DEFAULT_VECTOR_DIMENSION
|
||||
if embedding_model and getattr(embedding_model, "embedding_dimensions", None):
|
||||
try:
|
||||
vector_dim = int(embedding_model.embedding_dimensions)
|
||||
except (TypeError, ValueError):
|
||||
vector_dim = DEFAULT_VECTOR_DIMENSION
|
||||
if vector_dim <= 0:
|
||||
vector_dim = DEFAULT_VECTOR_DIMENSION
|
||||
|
||||
await vector_db.ensure_collection(vector_collection, vector=True, dim=vector_dim)
|
||||
await vector_db.delete_vector(vector_collection, path)
|
||||
|
||||
if file_ext in ["jpg", "jpeg", "png", "bmp"]:
|
||||
file_bytes = await ensure_input_bytes()
|
||||
processed_bytes, compression = _compress_image_for_embedding(file_bytes)
|
||||
base64_image = base64.b64encode(processed_bytes).decode("utf-8")
|
||||
description = await describe_image_base64(base64_image)
|
||||
embedding = await get_text_embedding(description)
|
||||
image_mime = "image/jpeg" if compression else mime_type
|
||||
await vector_db.upsert_vector(vector_collection, {
|
||||
"path": _chunk_key(path, "image"),
|
||||
"source_path": path,
|
||||
"chunk_id": "image",
|
||||
"embedding": embedding,
|
||||
"text": description,
|
||||
"mime": image_mime,
|
||||
"type": "image",
|
||||
})
|
||||
details["description"] = description
|
||||
if compression:
|
||||
details["image_compression"] = compression
|
||||
return Response(content=f"图片已索引，描述：{description}", media_type="text/plain")
|
||||
|
||||
if file_ext in ["txt", "md"]:
|
||||
try:
|
||||
file_bytes = await ensure_input_bytes()
|
||||
text = file_bytes.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return Response(content="文本文件解码失败", status_code=400)
|
||||
|
||||
chunks = _chunk_text(text)
|
||||
if not chunks:
|
||||
await vector_db.upsert_vector(vector_collection, {
|
||||
"path": _chunk_key(path, "0"),
|
||||
"source_path": path,
|
||||
"chunk_id": "0",
|
||||
"embedding": await get_text_embedding(text or path),
|
||||
"text": text,
|
||||
"mime": mime_type,
|
||||
"type": "text",
|
||||
"start_offset": 0,
|
||||
"end_offset": len(text),
|
||||
})
|
||||
details["chunks"] = 1
|
||||
return Response(content="文本文件已索引", media_type="text/plain")
|
||||
|
||||
chunk_count = 0
|
||||
for chunk_id, chunk_text, start, end in chunks:
|
||||
embedding = await get_text_embedding(chunk_text)
|
||||
await vector_db.upsert_vector(vector_collection, {
|
||||
"path": _chunk_key(path, str(chunk_id)),
|
||||
"source_path": path,
|
||||
"chunk_id": str(chunk_id),
|
||||
"embedding": embedding,
|
||||
"text": chunk_text,
|
||||
"mime": mime_type,
|
||||
"type": "text",
|
||||
"start_offset": start,
|
||||
"end_offset": end,
|
||||
})
|
||||
chunk_count += 1
|
||||
|
||||
details["chunks"] = chunk_count
|
||||
sample = chunks[0][1]
|
||||
details["sample"] = sample[:120]
|
||||
return Response(content="文本文件已索引", media_type="text/plain")
|
||||
|
||||
# 其他类型暂未支持向量索引，回退为文件名索引
|
||||
await vector_db.ensure_collection(file_collection, vector=False)
|
||||
await vector_db.delete_vector(file_collection, path)
|
||||
await vector_db.upsert_vector(file_collection, {
|
||||
"path": path,
|
||||
"source_path": path,
|
||||
"chunk_id": "filename",
|
||||
"mime": mime_type,
|
||||
"type": "filename",
|
||||
"name": os.path.basename(path),
|
||||
})
|
||||
return Response(content="暂不支持该类型的向量索引，已创建文件名索引", media_type="text/plain")
|
||||
|
||||
|
||||
PROCESSOR_TYPE = "vector_index"
|
||||
PROCESSOR_NAME = VectorIndexProcessor.name
|
||||
SUPPORTED_EXTS = VectorIndexProcessor.supported_exts
|
||||
CONFIG_SCHEMA = VectorIndexProcessor.config_schema
|
||||
def PROCESSOR_FACTORY(): return VectorIndexProcessor()
|
||||
@@ -0,0 +1,145 @@
|
||||
import inspect
|
||||
import pkgutil
|
||||
from importlib import import_module, reload
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Callable, Dict, Optional
|
||||
|
||||
from .base import BaseProcessor
|
||||
|
||||
ProcessorFactory = Callable[[], BaseProcessor]
|
||||
TYPE_MAP: Dict[str, ProcessorFactory] = {}
|
||||
CONFIG_SCHEMAS: Dict[str, dict] = {}
|
||||
MODULE_MAP: Dict[str, ModuleType] = {}
|
||||
LAST_DISCOVERY_ERRORS: list[str] = []
|
||||
|
||||
|
||||
def discover_processors(force_reload: bool = False) -> list[str]:
|
||||
"""扫描并缓存可用的处理器模块。"""
|
||||
from . import builtin as processors_pkg
|
||||
|
||||
TYPE_MAP.clear()
|
||||
CONFIG_SCHEMAS.clear()
|
||||
MODULE_MAP.clear()
|
||||
|
||||
global LAST_DISCOVERY_ERRORS
|
||||
LAST_DISCOVERY_ERRORS = []
|
||||
|
||||
for modinfo in pkgutil.iter_modules(processors_pkg.__path__):
|
||||
if modinfo.name.startswith("_"):
|
||||
continue
|
||||
|
||||
full_name = f"{processors_pkg.__name__}.{modinfo.name}"
|
||||
try:
|
||||
module = import_module(full_name)
|
||||
if force_reload:
|
||||
module = reload(module)
|
||||
except Exception as exc:
|
||||
LAST_DISCOVERY_ERRORS.append(f"Failed to import {full_name}: {exc}")
|
||||
continue
|
||||
|
||||
processor_type = getattr(module, "PROCESSOR_TYPE", None)
|
||||
processor_name = getattr(module, "PROCESSOR_NAME", None)
|
||||
supported_exts = getattr(module, "SUPPORTED_EXTS", None)
|
||||
schema = getattr(module, "CONFIG_SCHEMA", None)
|
||||
factory = getattr(module, "PROCESSOR_FACTORY", None)
|
||||
|
||||
if not processor_type:
|
||||
continue
|
||||
|
||||
if factory is None:
|
||||
for attr in module.__dict__.values():
|
||||
if inspect.isclass(attr) and attr.__name__.endswith("Processor"):
|
||||
|
||||
def _mk(cls=attr):
|
||||
return lambda: cls()
|
||||
|
||||
factory = _mk()
|
||||
break
|
||||
|
||||
if not callable(factory):
|
||||
LAST_DISCOVERY_ERRORS.append(f"Processor {full_name} missing factory")
|
||||
continue
|
||||
|
||||
try:
|
||||
sample = factory()
|
||||
except Exception as exc:
|
||||
LAST_DISCOVERY_ERRORS.append(f"Failed to instantiate processor {processor_type}: {exc}")
|
||||
continue
|
||||
|
||||
TYPE_MAP[processor_type] = factory
|
||||
MODULE_MAP[processor_type] = module
|
||||
|
||||
produces_file = getattr(module, "produces_file", None)
|
||||
if produces_file is None and hasattr(sample, "produces_file"):
|
||||
produces_file = getattr(sample, "produces_file")
|
||||
|
||||
supports_directory = getattr(module, "supports_directory", None)
|
||||
if supports_directory is None and hasattr(sample, "supports_directory"):
|
||||
supports_directory = getattr(sample, "supports_directory")
|
||||
|
||||
module_file = getattr(module, "__file__", None)
|
||||
module_path: Optional[str] = None
|
||||
if module_file:
|
||||
try:
|
||||
module_path = str(Path(module_file).resolve())
|
||||
except Exception:
|
||||
module_path = module_file
|
||||
|
||||
if isinstance(supported_exts, list):
|
||||
normalized_exts = [str(ext) for ext in supported_exts]
|
||||
elif supported_exts:
|
||||
normalized_exts = [str(supported_exts)]
|
||||
else:
|
||||
normalized_exts = []
|
||||
|
||||
if not normalized_exts and hasattr(sample, "supported_exts"):
|
||||
sample_exts = getattr(sample, "supported_exts") or []
|
||||
if isinstance(sample_exts, list):
|
||||
normalized_exts = [str(ext) for ext in sample_exts]
|
||||
|
||||
if isinstance(schema, list):
|
||||
CONFIG_SCHEMAS[processor_type] = {
|
||||
"type": processor_type,
|
||||
"name": processor_name or processor_type,
|
||||
"supported_exts": normalized_exts,
|
||||
"config_schema": schema,
|
||||
"produces_file": produces_file if produces_file is not None else False,
|
||||
"supports_directory": supports_directory if supports_directory is not None else False,
|
||||
"module_path": module_path,
|
||||
}
|
||||
|
||||
return LAST_DISCOVERY_ERRORS
|
||||
|
||||
|
||||
def get_config_schemas() -> Dict[str, dict]:
|
||||
return CONFIG_SCHEMAS
|
||||
|
||||
|
||||
def get_config_schema(processor_type: str):
|
||||
return CONFIG_SCHEMAS.get(processor_type)
|
||||
|
||||
|
||||
def get(processor_type: str) -> BaseProcessor | None:
|
||||
factory = TYPE_MAP.get(processor_type)
|
||||
if factory:
|
||||
return factory()
|
||||
return None
|
||||
|
||||
|
||||
def get_module_path(processor_type: str) -> Optional[str]:
|
||||
meta = CONFIG_SCHEMAS.get(processor_type)
|
||||
if not meta:
|
||||
return None
|
||||
return meta.get("module_path")
|
||||
|
||||
|
||||
def get_last_discovery_errors() -> list[str]:
|
||||
return LAST_DISCOVERY_ERRORS
|
||||
|
||||
|
||||
def reload_processors() -> list[str]:
|
||||
return discover_processors(force_reload=True)
|
||||
|
||||
|
||||
discover_processors()
|
||||
@@ -0,0 +1,258 @@
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from domain.tasks import task_queue_service
|
||||
from domain.virtual_fs import VirtualFSService
|
||||
from .registry import (
|
||||
get,
|
||||
get_config_schema,
|
||||
get_config_schemas,
|
||||
get_module_path,
|
||||
reload_processors,
|
||||
)
|
||||
from .types import (
|
||||
ProcessDirectoryRequest,
|
||||
ProcessRequest,
|
||||
UpdateSourceRequest,
|
||||
)
|
||||
|
||||
|
||||
class ProcessorService:
|
||||
@classmethod
|
||||
def get_processor(cls, processor_type: str):
|
||||
return get(processor_type)
|
||||
|
||||
@classmethod
|
||||
def list_processors(cls):
|
||||
schemas = get_config_schemas()
|
||||
out = []
|
||||
for t, meta in schemas.items():
|
||||
out.append({
|
||||
"type": meta["type"],
|
||||
"name": meta["name"],
|
||||
"supported_exts": meta.get("supported_exts", []),
|
||||
"config_schema": meta["config_schema"],
|
||||
"produces_file": meta.get("produces_file", False),
|
||||
"supports_directory": meta.get("supports_directory", False),
|
||||
"module_path": meta.get("module_path"),
|
||||
})
|
||||
return out
|
||||
|
||||
@classmethod
|
||||
async def process_file(cls, req: ProcessRequest):
|
||||
processor = cls.get_processor(req.processor_type)
|
||||
if not processor:
|
||||
raise HTTPException(404, detail="Processor not found")
|
||||
|
||||
is_dir = await VirtualFSService.path_is_directory(req.path)
|
||||
supports_directory = bool(getattr(processor, "supports_directory", False))
|
||||
if is_dir and not supports_directory and not req.overwrite:
|
||||
raise HTTPException(400, detail="Directory processing requires overwrite")
|
||||
|
||||
save_to = None if is_dir else (req.path if req.overwrite else req.save_to)
|
||||
task = await task_queue_service.add_task(
|
||||
"process_file",
|
||||
{
|
||||
"path": req.path,
|
||||
"processor_type": req.processor_type,
|
||||
"config": req.config,
|
||||
"save_to": save_to,
|
||||
"overwrite": req.overwrite,
|
||||
},
|
||||
)
|
||||
return {"task_id": task.id}
|
||||
|
||||
@classmethod
|
||||
async def process_directory(cls, req: ProcessDirectoryRequest):
|
||||
if req.max_depth is not None and req.max_depth < 0:
|
||||
raise HTTPException(400, detail="max_depth must be >= 0")
|
||||
|
||||
is_dir = await VirtualFSService.path_is_directory(req.path)
|
||||
if not is_dir:
|
||||
raise HTTPException(400, detail="Path must be a directory")
|
||||
|
||||
schema = get_config_schema(req.processor_type)
|
||||
_processor = get(req.processor_type)
|
||||
if not schema or not _processor:
|
||||
raise HTTPException(404, detail="Processor not found")
|
||||
|
||||
produces_file = bool(schema.get("produces_file"))
|
||||
raw_suffix = req.suffix if req.suffix is not None else None
|
||||
if raw_suffix is not None and raw_suffix.strip() == "":
|
||||
raw_suffix = None
|
||||
suffix = raw_suffix
|
||||
overwrite = req.overwrite
|
||||
|
||||
if produces_file:
|
||||
if not overwrite and not suffix:
|
||||
raise HTTPException(400, detail="Suffix is required when not overwriting files")
|
||||
else:
|
||||
overwrite = False
|
||||
suffix = None
|
||||
payload = {
|
||||
"path": req.path,
|
||||
"processor_type": req.processor_type,
|
||||
"config": req.config,
|
||||
"overwrite": overwrite,
|
||||
"max_depth": req.max_depth,
|
||||
"suffix": suffix,
|
||||
}
|
||||
task = await task_queue_service.add_task("process_directory_scan", payload)
|
||||
return {"task_id": task.id}
|
||||
|
||||
@classmethod
|
||||
async def scan_directory(cls, req: ProcessDirectoryRequest):
|
||||
if req.max_depth is not None and req.max_depth < 0:
|
||||
raise HTTPException(400, detail="max_depth must be >= 0")
|
||||
|
||||
is_dir = await VirtualFSService.path_is_directory(req.path)
|
||||
if not is_dir:
|
||||
raise HTTPException(400, detail="Path must be a directory")
|
||||
|
||||
schema = get_config_schema(req.processor_type)
|
||||
_processor = get(req.processor_type)
|
||||
if not schema or not _processor:
|
||||
raise HTTPException(404, detail="Processor not found")
|
||||
|
||||
produces_file = bool(schema.get("produces_file"))
|
||||
raw_suffix = req.suffix if req.suffix is not None else None
|
||||
if raw_suffix is not None and raw_suffix.strip() == "":
|
||||
raw_suffix = None
|
||||
suffix = raw_suffix
|
||||
overwrite = req.overwrite
|
||||
|
||||
if produces_file:
|
||||
if not overwrite and not suffix:
|
||||
raise HTTPException(400, detail="Suffix is required when not overwriting files")
|
||||
else:
|
||||
overwrite = False
|
||||
suffix = None
|
||||
|
||||
supported_exts = schema.get("supported_exts") or []
|
||||
allowed_exts = {
|
||||
ext.lower().lstrip('.')
|
||||
for ext in supported_exts
|
||||
if isinstance(ext, str)
|
||||
}
|
||||
|
||||
def matches_extension(file_rel: str) -> bool:
|
||||
if not allowed_exts:
|
||||
return True
|
||||
if '.' not in file_rel:
|
||||
return '' in allowed_exts
|
||||
ext = file_rel.rsplit('.', 1)[-1].lower()
|
||||
return ext in allowed_exts or f'.{ext}' in allowed_exts
|
||||
|
||||
adapter_instance, adapter_model, root, rel = await VirtualFSService.resolve_adapter_and_rel(req.path)
|
||||
rel = rel.rstrip('/')
|
||||
|
||||
list_dir = getattr(adapter_instance, "list_dir", None)
|
||||
if not callable(list_dir):
|
||||
raise HTTPException(501, detail="Adapter does not implement list_dir")
|
||||
|
||||
def build_absolute_path(mount_path: str, rel_path: str) -> str:
|
||||
rel_norm = rel_path.lstrip('/')
|
||||
mount_norm = mount_path.rstrip('/')
|
||||
if not mount_norm:
|
||||
return '/' + rel_norm if rel_norm else '/'
|
||||
return f"{mount_norm}/{rel_norm}" if rel_norm else mount_norm
|
||||
|
||||
def apply_suffix(path_str: str, suffix_str: str) -> str:
|
||||
path_obj = Path(path_str)
|
||||
name = path_obj.name
|
||||
if not name:
|
||||
return path_str
|
||||
if '.' in name:
|
||||
base, ext = name.rsplit('.', 1)
|
||||
new_name = f"{base}{suffix_str}.{ext}"
|
||||
else:
|
||||
new_name = f"{name}{suffix_str}"
|
||||
return str(path_obj.with_name(new_name))
|
||||
|
||||
scheduled_count = 0
|
||||
stack: List[Tuple[str, int]] = [(rel, 0)]
|
||||
page_size = 200
|
||||
|
||||
while stack:
|
||||
current_rel, depth = stack.pop()
|
||||
page = 1
|
||||
while True:
|
||||
entries, total = await list_dir(root, current_rel, page, page_size, "name", "asc")
|
||||
entries = entries or []
|
||||
if not entries and (total or 0) == 0:
|
||||
break
|
||||
|
||||
for entry in entries:
|
||||
name = entry.get("name")
|
||||
if not name:
|
||||
continue
|
||||
child_rel = f"{current_rel}/{name}" if current_rel else name
|
||||
if entry.get("is_dir"):
|
||||
if req.max_depth is None or depth < req.max_depth:
|
||||
stack.append((child_rel.rstrip('/'), depth + 1))
|
||||
continue
|
||||
if not matches_extension(child_rel):
|
||||
continue
|
||||
absolute_path = build_absolute_path(adapter_model.path, child_rel)
|
||||
save_to = None
|
||||
if produces_file and not overwrite and suffix:
|
||||
save_to = apply_suffix(absolute_path, suffix)
|
||||
await task_queue_service.add_task(
|
||||
"process_file",
|
||||
{
|
||||
"path": absolute_path,
|
||||
"processor_type": req.processor_type,
|
||||
"config": req.config,
|
||||
"save_to": save_to,
|
||||
"overwrite": overwrite,
|
||||
},
|
||||
)
|
||||
scheduled_count += 1
|
||||
|
||||
if total is None or page * page_size >= total:
|
||||
break
|
||||
page += 1
|
||||
|
||||
return {"scheduled": scheduled_count}
|
||||
|
||||
@classmethod
|
||||
async def get_source(cls, processor_type: str):
|
||||
module_path = get_module_path(processor_type)
|
||||
if not module_path:
|
||||
raise HTTPException(404, detail="Processor not found")
|
||||
path_obj = Path(module_path)
|
||||
if not path_obj.exists():
|
||||
raise HTTPException(404, detail="Processor source not found")
|
||||
try:
|
||||
content = await run_in_threadpool(path_obj.read_text, encoding='utf-8')
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, detail=f"Failed to read source: {exc}")
|
||||
return {"source": content, "module_path": str(path_obj)}
|
||||
|
||||
@classmethod
|
||||
async def update_source(cls, processor_type: str, req: UpdateSourceRequest):
|
||||
module_path = get_module_path(processor_type)
|
||||
if not module_path:
|
||||
raise HTTPException(404, detail="Processor not found")
|
||||
path_obj = Path(module_path)
|
||||
if not path_obj.exists():
|
||||
raise HTTPException(404, detail="Processor source not found")
|
||||
try:
|
||||
await run_in_threadpool(path_obj.write_text, req.source, encoding='utf-8')
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, detail=f"Failed to write source: {exc}")
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def reload(cls):
|
||||
errors = reload_processors()
|
||||
if errors:
|
||||
raise HTTPException(500, detail="; ".join(errors))
|
||||
return True
|
||||
|
||||
|
||||
get_processor = ProcessorService.get_processor
|
||||
list_processors = ProcessorService.list_processors
|
||||
reload_processor_modules = ProcessorService.reload
|
||||
@@ -0,0 +1,24 @@
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ProcessRequest(BaseModel):
|
||||
path: str
|
||||
processor_type: str
|
||||
config: Dict[str, Any]
|
||||
save_to: Optional[str] = None
|
||||
overwrite: bool = False
|
||||
|
||||
|
||||
class ProcessDirectoryRequest(BaseModel):
|
||||
path: str
|
||||
processor_type: str
|
||||
config: Dict[str, Any]
|
||||
overwrite: bool = True
|
||||
max_depth: Optional[int] = None
|
||||
suffix: Optional[str] = None
|
||||
|
||||
|
||||
class UpdateSourceRequest(BaseModel):
|
||||
source: str
|
||||
Reference in New Issue
Block a user