feat(vector_db): Implement Vector Database Service with multiple providers

This commit is contained in:
shiyu
2025-09-19 13:45:48 +08:00
parent 0a06f4d02c
commit fbeb673126
19 changed files with 1496 additions and 142 deletions

View File

@@ -40,7 +40,7 @@ async def set_config(
if key == "AI_EMBED_DIM" and str(original_value) != value_to_save:
try:
service = VectorDBService()
service.clear_all_data()
await service.clear_all_data()
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to clear vector database: {exc}")

View File

@@ -9,7 +9,7 @@ router = APIRouter(prefix="/api/search", tags=["search"])
async def search_files_by_vector(q: str, top_k: int):
embedding = await get_text_embedding(q)
vector_db = VectorDBService()
results = vector_db.search_vectors("vector_collection", embedding, top_k)
results = await vector_db.search_vectors("vector_collection", embedding, top_k)
items = [
SearchResultItem(id=res["id"], path=res["entity"]["path"], score=res["distance"])
for res in results[0]
@@ -18,7 +18,7 @@ async def search_files_by_vector(q: str, top_k: int):
async def search_files_by_name(q: str, top_k: int):
vector_db = VectorDBService()
results = vector_db.search_by_path("vector_collection", q, top_k)
results = await vector_db.search_by_path("vector_collection", q, top_k)
items = [
SearchResultItem(id=idx, path=res["entity"]["path"], score=res["distance"])
for idx, res in enumerate(results[0])
@@ -38,4 +38,4 @@ async def search_files(
elif mode == "filename":
return await search_files_by_name(q, top_k)
else:
return {"items": [], "query": q, "error": "Invalid search mode"}
return {"items": [], "query": q, "error": "Invalid search mode"}

View File

@@ -1,19 +1,100 @@
from typing import Any, Dict
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from services.auth import get_current_active_user
from models.database import UserAccount
from services.vector_db import VectorDBService
from services.vector_db import (
VectorDBService,
VectorDBConfigManager,
list_providers,
get_provider_entry,
)
from services.vector_db.providers import get_provider_class
from api.response import success
router = APIRouter(prefix="/api/vector-db", tags=["vector-db"])
class VectorDBConfigPayload(BaseModel):
type: str = Field(..., description="向量数据库提供者类型")
config: Dict[str, Any] = Field(default_factory=dict, description="提供者配置参数")
@router.post("/clear-all", summary="清空向量数据库")
async def clear_vector_db(user: UserAccount = Depends(get_current_active_user)):
if user.username != 'admin':
raise HTTPException(status_code=403, detail="仅管理员可操作")
try:
service = VectorDBService()
service.clear_all_data()
await service.clear_all_data()
return success(msg="向量数据库已清空")
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
raise HTTPException(status_code=500, detail=str(e))
@router.get("/stats", summary="获取向量数据库统计")
async def get_vector_db_stats(user: UserAccount = Depends(get_current_active_user)):
if user.username != 'admin':
raise HTTPException(status_code=403, detail="仅管理员可操作")
try:
service = VectorDBService()
data = await service.get_all_stats()
return success(data=data)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/providers", summary="列出可用向量数据库提供者")
async def list_vector_providers(user: UserAccount = Depends(get_current_active_user)):
if user.username != 'admin':
raise HTTPException(status_code=403, detail="仅管理员可操作")
return success(list_providers())
@router.get("/config", summary="获取当前向量数据库配置")
async def get_vector_db_config(user: UserAccount = Depends(get_current_active_user)):
if user.username != 'admin':
raise HTTPException(status_code=403, detail="仅管理员可操作")
service = VectorDBService()
data = await service.current_provider()
return success(data)
@router.post("/config", summary="更新向量数据库配置")
async def update_vector_db_config(payload: VectorDBConfigPayload, user: UserAccount = Depends(get_current_active_user)):
if user.username != 'admin':
raise HTTPException(status_code=403, detail="仅管理员可操作")
entry = get_provider_entry(payload.type)
if not entry:
raise HTTPException(status_code=400, detail=f"未知的向量数据库类型: {payload.type}")
if not entry.get("enabled", True):
raise HTTPException(status_code=400, detail="该向量数据库类型暂不可用")
provider_cls = get_provider_class(payload.type)
if not provider_cls:
raise HTTPException(status_code=400, detail=f"未找到类型 {payload.type} 对应的实现")
# 先尝试建立连接,确保配置有效
test_provider = provider_cls(payload.config)
try:
await test_provider.initialize()
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc))
finally:
client = getattr(test_provider, "client", None)
close_fn = getattr(client, "close", None)
if callable(close_fn):
try:
close_fn()
except Exception:
pass
await VectorDBConfigManager.save_config(payload.type, payload.config)
service = VectorDBService()
await service.reload()
config_data = await service.current_provider()
stats = await service.get_all_stats()
return success({"config": config_data, "stats": stats})

View File

@@ -64,6 +64,7 @@ dependencies = [
"python-multipart==0.0.20",
"pytz==2025.2",
"pyyaml==6.0.2",
"qdrant-client==1.15.1",
"rawpy==0.25.1",
"rich==14.1.0",
"rich-toolkit==0.15.0",

View File

@@ -34,7 +34,7 @@ class VectorIndexProcessor:
vector_db = VectorDBService()
collection_name = "vector_collection"
if action == "destroy":
vector_db.delete_vector(collection_name, path)
await vector_db.delete_vector(collection_name, path)
await LogService.info(
"processor:vector_index",
f"Destroyed {index_type} index for {path}",
@@ -43,8 +43,8 @@ class VectorIndexProcessor:
return Response(content=f"文件 {path}{index_type} 索引已销毁", media_type="text/plain")
if index_type == 'simple':
vector_db.ensure_collection(collection_name, vector=False)
vector_db.upsert_vector(collection_name, {'path': path})
await vector_db.ensure_collection(collection_name, vector=False)
await vector_db.upsert_vector(collection_name, {'path': path})
await LogService.info(
"processor:vector_index",
f"Created simple index for {path}",
@@ -80,8 +80,8 @@ class VectorIndexProcessor:
if vector_dim <= 0:
vector_dim = DEFAULT_VECTOR_DIMENSION
vector_db.ensure_collection(collection_name, vector=True, dim=vector_dim)
vector_db.upsert_vector(
await vector_db.ensure_collection(collection_name, vector=True, dim=vector_dim)
await vector_db.upsert_vector(
collection_name, {'path': path, 'embedding': embedding})
await LogService.info(

View File

@@ -1,92 +0,0 @@
from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusClient
DEFAULT_VECTOR_DIMENSION = 4096
class VectorDBService:
_instance = None
def __new__(cls, *args, **kwargs):
if not cls._instance:
cls._instance = super(VectorDBService, cls).__new__(cls)
return cls._instance
def __init__(self):
if not hasattr(self, 'client'):
self.client = MilvusClient("data/db/milvus.db")
def ensure_collection(self, collection_name, vector: bool = True, dim: int = DEFAULT_VECTOR_DIMENSION):
if self.client.has_collection(collection_name):
return
if vector:
try:
vector_dim = int(dim)
except (TypeError, ValueError):
vector_dim = DEFAULT_VECTOR_DIMENSION
if vector_dim <= 0:
vector_dim = DEFAULT_VECTOR_DIMENSION
fields = [
FieldSchema(name="path", dtype=DataType.VARCHAR,
max_length=512, is_primary=True, auto_id=False),
FieldSchema(name="embedding",
dtype=DataType.FLOAT_VECTOR, dim=vector_dim)
]
schema = CollectionSchema(
fields, description="Image vector collection")
self.client.create_collection(collection_name, schema=schema)
index_params = MilvusClient.prepare_index_params()
index_params.add_index(
field_name="embedding",
index_type="IVF_FLAT",
index_name="vector_index",
metric_type="COSINE",
params={
"nlist": 64,
}
)
self.client.create_index(
collection_name,
index_params=index_params
)
else:
fields = [
FieldSchema(name="path", dtype=DataType.VARCHAR,
max_length=512, is_primary=True, auto_id=False),
]
schema = CollectionSchema(fields, description="Simple file index")
self.client.create_collection(collection_name, schema=schema)
def upsert_vector(self, collection_name, data):
self.client.upsert(collection_name, data)
def delete_vector(self, collection_name, path: str):
self.client.delete(collection_name, ids=[path])
def search_vectors(self, collection_name, query_embedding, top_k=5):
search_params = {"metric_type": "COSINE"}
results = self.client.search(
collection_name,
data=[query_embedding],
anns_field="embedding",
search_params=search_params,
limit=top_k,
output_fields=["path"]
)
print(results)
return results
def search_by_path(self, collection_name, query_path, top_k=20):
results = self.client.query(
collection_name,
filter=f"path like '%{query_path}%'",
limit=top_k,
output_fields=["path"]
)
return [[{'id': r['path'], 'distance': 1.0, 'entity': {'path': r['path']}} for r in results]]
def clear_all_data(self):
"""清空所有集合的内容"""
collections = self.client.list_collections()
for collection_name in collections:
self.client.drop_collection(collection_name)

View File

@@ -0,0 +1,11 @@
from .service import VectorDBService, DEFAULT_VECTOR_DIMENSION
from .providers import list_providers, get_provider_entry
from .config_manager import VectorDBConfigManager
__all__ = [
"VectorDBService",
"DEFAULT_VECTOR_DIMENSION",
"list_providers",
"get_provider_entry",
"VectorDBConfigManager",
]

View File

@@ -0,0 +1,43 @@
from __future__ import annotations
import json
from typing import Any, Dict, Tuple
from services.config import ConfigCenter
class VectorDBConfigManager:
TYPE_KEY = "VECTOR_DB_TYPE"
CONFIG_KEY = "VECTOR_DB_CONFIG"
DEFAULT_TYPE = "milvus_lite"
@classmethod
async def load_config(cls) -> Tuple[str, Dict[str, Any]]:
raw_type = await ConfigCenter.get(cls.TYPE_KEY, cls.DEFAULT_TYPE)
provider_type = str(raw_type or cls.DEFAULT_TYPE)
raw_config = await ConfigCenter.get(cls.CONFIG_KEY)
config_dict: Dict[str, Any] = {}
if isinstance(raw_config, str) and raw_config:
try:
config_dict = json.loads(raw_config)
except json.JSONDecodeError:
config_dict = {}
elif isinstance(raw_config, dict):
config_dict = raw_config
return provider_type, config_dict
@classmethod
async def save_config(cls, provider_type: str, config: Dict[str, Any]) -> None:
await ConfigCenter.set(cls.TYPE_KEY, provider_type)
await ConfigCenter.set(cls.CONFIG_KEY, json.dumps(config or {}))
@classmethod
async def get_type(cls) -> str:
provider_type, _ = await cls.load_config()
return provider_type
@classmethod
async def get_config(cls) -> Dict[str, Any]:
_, config = await cls.load_config()
return config

View File

@@ -0,0 +1,56 @@
from __future__ import annotations
from typing import Dict, List, Type
from .base import BaseVectorProvider
from .milvus_lite import MilvusLiteProvider
from .milvus_server import MilvusServerProvider
from .qdrant import QdrantProvider
_PROVIDER_REGISTRY: Dict[str, Dict[str, object]] = {
MilvusLiteProvider.type: {
"class": MilvusLiteProvider,
"label": MilvusLiteProvider.label,
"description": MilvusLiteProvider.description,
"enabled": MilvusLiteProvider.enabled,
"config_schema": MilvusLiteProvider.config_schema,
},
MilvusServerProvider.type: {
"class": MilvusServerProvider,
"label": MilvusServerProvider.label,
"description": MilvusServerProvider.description,
"enabled": MilvusServerProvider.enabled,
"config_schema": MilvusServerProvider.config_schema,
},
QdrantProvider.type: {
"class": QdrantProvider,
"label": QdrantProvider.label,
"description": QdrantProvider.description,
"enabled": QdrantProvider.enabled,
"config_schema": QdrantProvider.config_schema,
},
}
def list_providers() -> List[Dict[str, object]]:
return [
{
"type": type_key,
"label": meta["label"],
"description": meta.get("description"),
"enabled": meta.get("enabled", True),
"config_schema": meta.get("config_schema", []),
}
for type_key, meta in _PROVIDER_REGISTRY.items()
]
def get_provider_entry(provider_type: str) -> Dict[str, object] | None:
return _PROVIDER_REGISTRY.get(provider_type)
def get_provider_class(provider_type: str) -> Type[BaseVectorProvider] | None:
entry = get_provider_entry(provider_type)
if not entry:
return None
return entry.get("class") # type: ignore[return-value]

View File

@@ -0,0 +1,41 @@
from __future__ import annotations
from typing import Any, Dict, List
class BaseVectorProvider:
"""向量数据库提供者基础类,所有实际实现需继承该类"""
type: str = ""
label: str = ""
description: str | None = None
enabled: bool = True
config_schema: List[Dict[str, Any]] = []
def __init__(self, config: Dict[str, Any] | None = None):
self.config = config or {}
async def initialize(self) -> None:
"""执行初始化逻辑,例如建立连接"""
raise NotImplementedError
def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None:
raise NotImplementedError
def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None:
raise NotImplementedError
def delete_vector(self, collection_name: str, path: str) -> None:
raise NotImplementedError
def search_vectors(self, collection_name: str, query_embedding, top_k: int):
raise NotImplementedError
def search_by_path(self, collection_name: str, query_path: str, top_k: int):
raise NotImplementedError
def get_all_stats(self) -> Dict[str, Any]:
raise NotImplementedError
def clear_all_data(self) -> None:
raise NotImplementedError

View File

@@ -0,0 +1,196 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, List, Optional
from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusClient
from .base import BaseVectorProvider
class MilvusLiteProvider(BaseVectorProvider):
type = "milvus_lite"
label = "Milvus Lite"
description = "Embedded Milvus Lite (local file storage)."
enabled = True
config_schema: List[Dict[str, Any]] = [
{
"key": "db_path",
"label": "Database file path",
"type": "text",
"default": "data/db/milvus.db",
"required": False,
}
]
def __init__(self, config: Dict[str, Any] | None = None):
super().__init__(config)
self.db_path = Path(self.config.get("db_path") or "data/db/milvus.db")
self.client: MilvusClient | None = None
async def initialize(self) -> None:
try:
self.client = MilvusClient(str(self.db_path))
except Exception as exc: # pragma: no cover - depends on local environment
raise RuntimeError(f"Failed to open Milvus Lite at {self.db_path}: {exc}") from exc
def _get_client(self) -> MilvusClient:
if not self.client:
raise RuntimeError("Milvus Lite client is not initialized")
return self.client
@staticmethod
def _to_int(value: Any) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None:
client = self._get_client()
if client.has_collection(collection_name):
return
if vector:
vector_dim = dim if isinstance(dim, int) and dim > 0 else 0
if vector_dim <= 0:
vector_dim = 4096
fields = [
FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False),
FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=vector_dim),
]
schema = CollectionSchema(fields, description="Image vector collection")
client.create_collection(collection_name, schema=schema)
index_params = MilvusClient.prepare_index_params()
index_params.add_index(
field_name="embedding",
index_type="IVF_FLAT",
index_name="vector_index",
metric_type="COSINE",
params={"nlist": 64},
)
client.create_index(collection_name, index_params=index_params)
else:
fields = [
FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False),
]
schema = CollectionSchema(fields, description="Simple file index")
client.create_collection(collection_name, schema=schema)
def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None:
self._get_client().upsert(collection_name, data)
def delete_vector(self, collection_name: str, path: str) -> None:
self._get_client().delete(collection_name, ids=[path])
def search_vectors(self, collection_name: str, query_embedding, top_k: int):
search_params = {"metric_type": "COSINE"}
return self._get_client().search(
collection_name,
data=[query_embedding],
anns_field="embedding",
search_params=search_params,
limit=top_k,
output_fields=["path"],
)
def search_by_path(self, collection_name: str, query_path: str, top_k: int):
filter_expr = f"path like '%{query_path}%'" if query_path else "path like '%%'"
results = self._get_client().query(
collection_name,
filter=filter_expr,
limit=top_k,
output_fields=["path"],
)
return [[{"id": r["path"], "distance": 1.0, "entity": {"path": r["path"]}} for r in results]]
def get_all_stats(self) -> Dict[str, Any]:
client = self._get_client()
try:
collection_names = client.list_collections()
except Exception as exc:
raise RuntimeError(f"Failed to list collections: {exc}") from exc
collections: List[Dict[str, Any]] = []
total_vectors = 0
total_estimated_memory = 0
for name in collection_names:
try:
stats = client.get_collection_stats(name) or {}
except Exception:
stats = {}
row_count = self._to_int(stats.get("row_count"))
total_vectors += row_count
dimension: Optional[int] = None
is_vector_collection = False
try:
description = client.describe_collection(name)
except Exception:
description = None
if description:
for field in description.get("fields", []):
if field.get("type") == DataType.FLOAT_VECTOR:
params = field.get("params") or {}
dimension = self._to_int(params.get("dim")) or 4096
is_vector_collection = True
break
estimated_memory = 0
if is_vector_collection and dimension:
estimated_memory = row_count * dimension * 4
total_estimated_memory += estimated_memory
indexes: List[Dict[str, Any]] = []
try:
index_names = client.list_indexes(name) or []
except Exception:
index_names = []
for index_name in index_names:
try:
detail = client.describe_index(name, index_name) or {}
except Exception:
detail = {}
indexes.append(
{
"index_name": index_name,
"index_type": detail.get("index_type"),
"metric_type": detail.get("metric_type"),
"indexed_rows": self._to_int(detail.get("indexed_rows")),
"pending_index_rows": self._to_int(detail.get("pending_index_rows")),
"state": detail.get("state"),
}
)
collections.append(
{
"name": name,
"row_count": row_count,
"dimension": dimension if is_vector_collection else None,
"estimated_memory_bytes": estimated_memory,
"is_vector_collection": is_vector_collection,
"indexes": indexes,
}
)
db_file_size = None
try:
if self.db_path.exists():
db_file_size = self.db_path.stat().st_size
except OSError:
db_file_size = None
return {
"collections": collections,
"collection_count": len(collections),
"total_vectors": total_vectors,
"estimated_total_memory_bytes": total_estimated_memory,
"db_file_size_bytes": db_file_size,
}
def clear_all_data(self) -> None:
client = self._get_client()
for collection_name in client.list_collections():
client.drop_collection(collection_name)

View File

@@ -0,0 +1,197 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional
from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusClient
from .base import BaseVectorProvider
class MilvusServerProvider(BaseVectorProvider):
type = "milvus_server"
label = "Milvus Server"
description = "Remote Milvus instance accessed via URI."
enabled = True
config_schema: List[Dict[str, Any]] = [
{
"key": "uri",
"label": "Server URI",
"type": "text",
"required": True,
"placeholder": "http://localhost:19530",
},
{
"key": "token",
"label": "Token",
"type": "password",
"required": False,
"placeholder": "user:password",
},
]
def __init__(self, config: Dict[str, Any] | None = None):
super().__init__(config)
self.client: MilvusClient | None = None
async def initialize(self) -> None:
uri = self.config.get("uri")
if not uri:
raise RuntimeError("Milvus Server URI is required")
try:
self.client = MilvusClient(uri=uri, token=self.config.get("token"))
except Exception as exc: # pragma: no cover - depends on remote availability
raise RuntimeError(f"Failed to connect to Milvus Server {uri}: {exc}") from exc
def _get_client(self) -> MilvusClient:
if not self.client:
raise RuntimeError("Milvus Server client is not initialized")
return self.client
@staticmethod
def _to_int(value: Any) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None:
client = self._get_client()
if client.has_collection(collection_name):
return
if vector:
vector_dim = dim if isinstance(dim, int) and dim > 0 else 0
if vector_dim <= 0:
vector_dim = 4096
fields = [
FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False),
FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=vector_dim),
]
schema = CollectionSchema(fields, description="Image vector collection")
client.create_collection(collection_name, schema=schema)
index_params = MilvusClient.prepare_index_params()
index_params.add_index(
field_name="embedding",
index_type="IVF_FLAT",
index_name="vector_index",
metric_type="COSINE",
params={"nlist": 64},
)
client.create_index(collection_name, index_params=index_params)
else:
fields = [
FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False),
]
schema = CollectionSchema(fields, description="Simple file index")
client.create_collection(collection_name, schema=schema)
def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None:
self._get_client().upsert(collection_name, data)
def delete_vector(self, collection_name: str, path: str) -> None:
self._get_client().delete(collection_name, ids=[path])
def search_vectors(self, collection_name: str, query_embedding, top_k: int):
search_params = {"metric_type": "COSINE"}
return self._get_client().search(
collection_name,
data=[query_embedding],
anns_field="embedding",
search_params=search_params,
limit=top_k,
output_fields=["path"],
)
def search_by_path(self, collection_name: str, query_path: str, top_k: int):
filter_expr = f"path like '%{query_path}%'" if query_path else "path like '%%'"
results = self._get_client().query(
collection_name,
filter=filter_expr,
limit=top_k,
output_fields=["path"],
)
return [[{"id": r["path"], "distance": 1.0, "entity": {"path": r["path"]}} for r in results]]
def get_all_stats(self) -> Dict[str, Any]:
client = self._get_client()
try:
collection_names = client.list_collections()
except Exception as exc:
raise RuntimeError(f"Failed to list collections: {exc}") from exc
collections: List[Dict[str, Any]] = []
total_vectors = 0
total_estimated_memory = 0
for name in collection_names:
try:
stats = client.get_collection_stats(name) or {}
except Exception:
stats = {}
row_count = self._to_int(stats.get("row_count"))
total_vectors += row_count
dimension: Optional[int] = None
is_vector_collection = False
try:
description = client.describe_collection(name)
except Exception:
description = None
if description:
for field in description.get("fields", []):
if field.get("type") == DataType.FLOAT_VECTOR:
params = field.get("params") or {}
dimension = self._to_int(params.get("dim")) or 4096
is_vector_collection = True
break
estimated_memory = 0
if is_vector_collection and dimension:
estimated_memory = row_count * dimension * 4
total_estimated_memory += estimated_memory
indexes: List[Dict[str, Any]] = []
try:
index_names = client.list_indexes(name) or []
except Exception:
index_names = []
for index_name in index_names:
try:
detail = client.describe_index(name, index_name) or {}
except Exception:
detail = {}
indexes.append(
{
"index_name": index_name,
"index_type": detail.get("index_type"),
"metric_type": detail.get("metric_type"),
"indexed_rows": self._to_int(detail.get("indexed_rows")),
"pending_index_rows": self._to_int(detail.get("pending_index_rows")),
"state": detail.get("state"),
}
)
collections.append(
{
"name": name,
"row_count": row_count,
"dimension": dimension if is_vector_collection else None,
"estimated_memory_bytes": estimated_memory,
"is_vector_collection": is_vector_collection,
"indexes": indexes,
}
)
return {
"collections": collections,
"collection_count": len(collections),
"total_vectors": total_vectors,
"estimated_total_memory_bytes": total_estimated_memory,
"db_file_size_bytes": None,
}
def clear_all_data(self) -> None:
client = self._get_client()
for collection_name in client.list_collections():
client.drop_collection(collection_name)

View File

@@ -0,0 +1,237 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional, Sequence
from uuid import NAMESPACE_URL, uuid5
from qdrant_client import QdrantClient
from qdrant_client.http import models as qmodels
from .base import BaseVectorProvider
class QdrantProvider(BaseVectorProvider):
type = "qdrant"
label = "Qdrant"
description = "Qdrant vector database (HTTP API)."
enabled = True
config_schema: List[Dict[str, Any]] = [
{
"key": "url",
"label": "Server URL",
"type": "text",
"required": True,
"placeholder": "http://localhost:6333",
},
{
"key": "api_key",
"label": "API Key",
"type": "password",
"required": False,
},
]
def __init__(self, config: Dict[str, Any] | None = None):
super().__init__(config)
self.client: Optional[QdrantClient] = None
async def initialize(self) -> None:
url = (self.config.get("url") or "").strip()
if not url:
raise RuntimeError("Qdrant URL is required")
api_key = (self.config.get("api_key") or None) or None
try:
client = QdrantClient(url=url, api_key=api_key)
# 简单连通性校验
client.get_collections()
self.client = client
except Exception as exc: # pragma: no cover - 依赖外部服务
raise RuntimeError(f"Failed to connect to Qdrant at {url}: {exc}") from exc
def _get_client(self) -> QdrantClient:
if not self.client:
raise RuntimeError("Qdrant client is not initialized")
return self.client
@staticmethod
def _vector_params(vector: bool, dim: int) -> qmodels.VectorParams:
size = dim if vector and isinstance(dim, int) and dim > 0 else 1
return qmodels.VectorParams(size=size, distance=qmodels.Distance.COSINE)
def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None:
client = self._get_client()
try:
if client.collection_exists(collection_name):
return
except Exception as exc: # pragma: no cover - 依赖外部服务
raise RuntimeError(f"Failed to check Qdrant collection '{collection_name}': {exc}") from exc
vectors_config = self._vector_params(vector, dim)
try:
client.create_collection(collection_name=collection_name, vectors_config=vectors_config)
except Exception as exc: # pragma: no cover
if "already exists" in str(exc).lower():
return
raise RuntimeError(f"Failed to create Qdrant collection '{collection_name}': {exc}") from exc
@staticmethod
def _point_id(path: str) -> str:
return str(uuid5(NAMESPACE_URL, path))
def _prepare_point(self, data: Dict[str, Any]) -> qmodels.PointStruct:
path = data.get("path")
if not path:
raise ValueError("Qdrant upsert requires 'path' in data")
embedding = data.get("embedding")
if embedding is None:
vector = [0.0]
else:
vector = [float(x) for x in embedding]
payload = {"path": path}
return qmodels.PointStruct(id=self._point_id(path), vector=vector, payload=payload)
def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None:
client = self._get_client()
point = self._prepare_point(data)
client.upsert(collection_name=collection_name, wait=True, points=[point])
def delete_vector(self, collection_name: str, path: str) -> None:
client = self._get_client()
selector = qmodels.PointIdsList(points=[self._point_id(path)])
client.delete(collection_name=collection_name, points_selector=selector, wait=True)
def _format_search_results(self, points: Sequence[qmodels.ScoredPoint]):
return [
{
"id": point.id,
"distance": point.score,
"entity": {"path": (point.payload or {}).get("path")},
}
for point in points
]
def search_vectors(self, collection_name: str, query_embedding, top_k: int):
client = self._get_client()
vector = [float(x) for x in query_embedding]
points = client.search(
collection_name=collection_name,
query_vector=vector,
limit=top_k,
with_payload=True,
)
return [self._format_search_results(points)]
def search_by_path(self, collection_name: str, query_path: str, top_k: int):
client = self._get_client()
results: List[Dict[str, Any]] = []
offset: Optional[str | int] = None
remaining = max(top_k, 1)
while len(results) < top_k:
batch_size = min(max(remaining * 2, 10), 200)
records, next_offset = client.scroll(
collection_name=collection_name,
limit=batch_size,
offset=offset,
with_payload=True,
)
if not records:
break
for record in records:
path = (record.payload or {}).get("path")
if query_path and path:
if query_path not in path:
continue
results.append({"id": record.id, "distance": 1.0, "entity": {"path": path}})
if len(results) >= top_k:
break
if next_offset is None or len(results) >= top_k:
break
offset = next_offset
remaining = top_k - len(results)
return [results]
def _extract_vector_config(self, vectors) -> Optional[qmodels.VectorParams]:
if isinstance(vectors, qmodels.VectorParams):
return vectors
if isinstance(vectors, dict):
for value in vectors.values():
if isinstance(value, qmodels.VectorParams):
return value
return None
def get_all_stats(self) -> Dict[str, Any]:
client = self._get_client()
try:
response = client.get_collections()
except Exception as exc: # pragma: no cover
raise RuntimeError(f"Failed to list Qdrant collections: {exc}") from exc
collections: List[Dict[str, Any]] = []
total_vectors = 0
total_estimated_memory = 0
for description in response.collections or []:
name = description.name
try:
info = client.get_collection(name)
except Exception:
continue
row_count = int(info.points_count or 0)
total_vectors += row_count
vector_params = self._extract_vector_config(info.config.params.vectors if info.config and info.config.params else None)
dimension = int(vector_params.size) if vector_params and vector_params.size else None
estimated_memory = row_count * dimension * 4 if dimension else 0
total_estimated_memory += estimated_memory
distance = str(vector_params.distance) if vector_params and vector_params.distance else None
indexed_rows = int(info.indexed_vectors_count or 0)
pending_rows = max(row_count - indexed_rows, 0)
collections.append(
{
"name": name,
"row_count": row_count,
"dimension": dimension,
"estimated_memory_bytes": estimated_memory,
"is_vector_collection": dimension is not None and dimension > 1,
"indexes": [
{
"index_name": "hnsw",
"index_type": "HNSW",
"metric_type": distance,
"indexed_rows": indexed_rows,
"pending_index_rows": pending_rows,
"state": info.status,
}
],
}
)
return {
"collections": collections,
"collection_count": len(collections),
"total_vectors": total_vectors,
"estimated_total_memory_bytes": total_estimated_memory,
"db_file_size_bytes": None,
}
def clear_all_data(self) -> None:
client = self._get_client()
try:
response = client.get_collections()
except Exception as exc: # pragma: no cover
raise RuntimeError(f"Failed to list Qdrant collections: {exc}") from exc
for description in response.collections or []:
try:
client.delete_collection(description.name)
except Exception:
continue

View File

@@ -0,0 +1,99 @@
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
from .config_manager import VectorDBConfigManager
from .providers import get_provider_class, get_provider_entry
from .providers.base import BaseVectorProvider
DEFAULT_VECTOR_DIMENSION = 4096
class VectorDBService:
_instance: "VectorDBService" | None = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self):
if not hasattr(self, "_provider"):
self._provider: Optional[BaseVectorProvider] = None
self._provider_type: Optional[str] = None
self._provider_config: Dict[str, Any] | None = None
self._lock = asyncio.Lock()
async def _ensure_provider(self) -> BaseVectorProvider:
if self._provider is None:
await self.reload()
assert self._provider is not None # for type checker
return self._provider
async def reload(self) -> BaseVectorProvider:
async with self._lock:
provider_type, provider_config = await VectorDBConfigManager.load_config()
normalized_config = dict(provider_config or {})
if (
self._provider
and self._provider_type == provider_type
and self._provider_config == normalized_config
):
return self._provider
entry = get_provider_entry(provider_type)
if not entry:
raise RuntimeError(f"Unknown vector database provider: {provider_type}")
if not entry.get("enabled", True):
raise RuntimeError(f"Vector database provider '{provider_type}' is disabled")
provider_cls = get_provider_class(provider_type)
if not provider_cls:
raise RuntimeError(f"Provider class not found for '{provider_type}'")
provider = provider_cls(provider_config)
await provider.initialize()
self._provider = provider
self._provider_type = provider_type
self._provider_config = normalized_config
return provider
async def ensure_collection(self, collection_name: str, vector: bool = True, dim: int = DEFAULT_VECTOR_DIMENSION) -> None:
provider = await self._ensure_provider()
provider.ensure_collection(collection_name, vector, dim)
async def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None:
provider = await self._ensure_provider()
provider.upsert_vector(collection_name, data)
async def delete_vector(self, collection_name: str, path: str) -> None:
provider = await self._ensure_provider()
provider.delete_vector(collection_name, path)
async def search_vectors(self, collection_name: str, query_embedding, top_k: int = 5):
provider = await self._ensure_provider()
return provider.search_vectors(collection_name, query_embedding, top_k)
async def search_by_path(self, collection_name: str, query_path: str, top_k: int = 20):
provider = await self._ensure_provider()
return provider.search_by_path(collection_name, query_path, top_k)
async def get_all_stats(self) -> Dict[str, Any]:
provider = await self._ensure_provider()
return provider.get_all_stats()
async def clear_all_data(self) -> None:
provider = await self._ensure_provider()
provider.clear_all_data()
async def current_provider(self) -> Dict[str, Any]:
provider_type, provider_config = await VectorDBConfigManager.load_config()
entry = get_provider_entry(provider_type) or {}
return {
"type": provider_type,
"config": provider_config,
"label": entry.get("label"),
"enabled": entry.get("enabled", True),
}

81
uv.lock generated
View File

@@ -415,6 +415,7 @@ dependencies = [
{ name = "python-multipart" },
{ name = "pytz" },
{ name = "pyyaml" },
{ name = "qdrant-client" },
{ name = "rawpy" },
{ name = "rich" },
{ name = "rich-toolkit" },
@@ -505,6 +506,7 @@ requires-dist = [
{ name = "python-multipart", specifier = "==0.0.20" },
{ name = "pytz", specifier = "==2025.2" },
{ name = "pyyaml", specifier = "==6.0.2" },
{ name = "qdrant-client", specifier = "==1.15.1" },
{ name = "rawpy", specifier = "==0.25.1" },
{ name = "rich", specifier = "==14.1.0" },
{ name = "rich-toolkit", specifier = "==0.15.0" },
@@ -604,6 +606,28 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
]
[[package]]
name = "h2"
version = "4.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "hpack" },
{ name = "hyperframe" },
]
sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" },
]
[[package]]
name = "hpack"
version = "4.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" },
]
[[package]]
name = "httpcore"
version = "1.0.9"
@@ -647,6 +671,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
]
[package.optional-dependencies]
http2 = [
{ name = "h2" },
]
[[package]]
name = "hyperframe"
version = "6.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
]
[[package]]
name = "idna"
version = "3.10"
@@ -950,6 +988,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/89/c7/5572fa4a3f45740eaab6ae86fcdf7195b55beac1371ac8c619d880cfe948/pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa", size = 2512835, upload-time = "2025-07-01T09:15:50.399Z" },
]
[[package]]
name = "portalocker"
version = "3.2.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pywin32", marker = "sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" },
]
[[package]]
name = "propcache"
version = "0.3.2"
@@ -1161,6 +1211,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" },
]
[[package]]
name = "pywin32"
version = "311"
source = { registry = "https://pypi.org/simple" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700, upload-time = "2025-07-14T20:13:26.471Z" },
{ url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700, upload-time = "2025-07-14T20:13:28.243Z" },
{ url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" },
{ url = "https://files.pythonhosted.org/packages/c9/31/097f2e132c4f16d99a22bfb777e0fd88bd8e1c634304e102f313af69ace5/pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee", size = 8840714, upload-time = "2025-07-14T20:13:32.449Z" },
{ url = "https://files.pythonhosted.org/packages/90/4b/07c77d8ba0e01349358082713400435347df8426208171ce297da32c313d/pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87", size = 9656800, upload-time = "2025-07-14T20:13:34.312Z" },
{ url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" },
]
[[package]]
name = "pyyaml"
version = "6.0.2"
@@ -1178,6 +1241,24 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },
]
[[package]]
name = "qdrant-client"
version = "1.15.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "grpcio" },
{ name = "httpx", extra = ["http2"] },
{ name = "numpy" },
{ name = "portalocker" },
{ name = "protobuf" },
{ name = "pydantic" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/79/8b/76c7d325e11d97cb8eb5e261c3759e9ed6664735afbf32fdded5b580690c/qdrant_client-1.15.1.tar.gz", hash = "sha256:631f1f3caebfad0fd0c1fba98f41be81d9962b7bf3ca653bed3b727c0e0cbe0e", size = 295297, upload-time = "2025-07-31T19:35:19.627Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/33/d8df6a2b214ffbe4138db9a1efe3248f67dc3c671f82308bea1582ecbbb7/qdrant_client-1.15.1-py3-none-any.whl", hash = "sha256:2b975099b378382f6ca1cfb43f0d59e541be6e16a5892f282a4b8de7eff5cb63", size = 337331, upload-time = "2025-07-31T19:35:17.539Z" },
]
[[package]]
name = "rawpy"
version = "0.25.1"

View File

@@ -1,5 +1,65 @@
import client from './client';
export interface VectorDBIndexInfo {
index_name: string;
index_type?: string;
metric_type?: string;
indexed_rows: number;
pending_index_rows: number;
state?: string;
}
export interface VectorDBCollectionStats {
name: string;
row_count: number;
dimension: number | null;
estimated_memory_bytes: number;
is_vector_collection: boolean;
indexes: VectorDBIndexInfo[];
}
export interface VectorDBStats {
collections: VectorDBCollectionStats[];
collection_count: number;
total_vectors: number;
estimated_total_memory_bytes: number;
db_file_size_bytes: number | null;
}
export interface VectorDBProviderField {
key: string;
label: string;
type: 'text' | 'password';
required?: boolean;
default?: string;
placeholder?: string;
}
export interface VectorDBProviderMeta {
type: string;
label: string;
description?: string;
enabled: boolean;
config_schema: VectorDBProviderField[];
}
export interface VectorDBCurrentConfig {
type: string;
config: Record<string, string>;
label?: string;
enabled?: boolean;
}
export interface UpdateVectorDBConfigResponse {
config: VectorDBCurrentConfig;
stats: VectorDBStats;
}
export const vectorDBApi = {
getProviders: () => client<VectorDBProviderMeta[]>('/vector-db/providers', { method: 'GET' }),
getConfig: () => client<VectorDBCurrentConfig>('/vector-db/config', { method: 'GET' }),
getStats: () => client<VectorDBStats>('/vector-db/stats', { method: 'GET' }),
updateConfig: (payload: { type: string; config: Record<string, string> }) =>
client<UpdateVectorDBConfigResponse>('/vector-db/config', { method: 'POST', json: payload }),
clearAll: () => client('/vector-db/clear-all', { method: 'POST' }),
};
};

View File

@@ -205,6 +205,32 @@ export const en = {
'Embedding Dimension': 'Embedding Dimension',
'Vector Database': 'Vector Database',
'Vector Database Settings': 'Vector Database Settings',
'Current Statistics': 'Current Statistics',
'Collections': 'Collections',
'Vectors': 'Vectors',
'Database Size': 'Database Size',
'Estimated Memory': 'Estimated Memory',
'No collections': 'No collections',
'Dimension': 'Dimension',
'Non-vector collection': 'Non-vector collection',
'Estimated memory': 'Estimated memory',
'Indexes': 'Indexes',
'Unnamed index': 'Unnamed index',
'Indexed rows': 'Indexed rows',
'Pending rows': 'Pending rows',
'Estimated memory is calculated as vectors x dimension x 4 bytes (float32).': 'Estimated memory is calculated as vectors x dimension x 4 bytes (float32).',
'Database Provider': 'Database Provider',
'Please select a provider': 'Please select a provider',
'Coming soon': 'Coming soon',
'This provider is not available yet': 'This provider is not available yet',
'Database file path': 'Database file path',
'Server URI': 'Server URI',
'Token': 'Token',
'Server URL': 'Server URL',
'API Key': 'API Key',
'Embedded Milvus Lite (local file storage).': 'Embedded Milvus Lite (local file storage).',
'Remote Milvus instance accessed via URI.': 'Remote Milvus instance accessed via URI.',
'Qdrant vector database (HTTP API).': 'Qdrant vector database (HTTP API).',
'Database Type': 'Database Type',
'Confirm embedding dimension change': 'Confirm embedding dimension change',
'Changing the embedding dimension will clear the vector database automatically. You will need to rebuild indexes afterwards. Continue?': 'Changing the embedding dimension will clear the vector database automatically. You will need to rebuild indexes afterwards. Continue?',

View File

@@ -207,6 +207,32 @@ export const zh = {
'Embedding Dimension': '向量维度',
'Vector Database': '向量数据库',
'Vector Database Settings': '向量数据库设置',
'Current Statistics': '当前统计',
'Collections': '集合',
'Vectors': '向量',
'Database Size': '数据库大小',
'Estimated Memory': '估算内存',
'No collections': '暂无集合',
'Dimension': '维度',
'Non-vector collection': '非向量集合',
'Estimated memory': '估算内存',
'Indexes': '索引',
'Unnamed index': '未命名索引',
'Indexed rows': '已索引行数',
'Pending rows': '待索引行数',
'Estimated memory is calculated as vectors x dimension x 4 bytes (float32).': '估算内存 = 向量数量 x 维度 x 4 字节float32。',
'Database Provider': '数据库提供者',
'Please select a provider': '请选择提供者',
'Coming soon': '敬请期待',
'This provider is not available yet': '该提供者暂不可用',
'Database file path': '数据库文件路径',
'Server URI': '服务器 URI',
'Token': '令牌',
'Server URL': '服务器地址',
'API Key': 'API Key',
'Embedded Milvus Lite (local file storage).': '嵌入式 Milvus Lite本地文件存储。',
'Remote Milvus instance accessed via URI.': '通过 URI 访问的远程 Milvus 实例。',
'Qdrant vector database (HTTP API).': 'Qdrant 向量数据库HTTP API。',
'Database Type': '数据库类型',
'Confirm embedding dimension change': '确认修改向量维度',
'Changing the embedding dimension will clear the vector database automatically. You will need to rebuild indexes afterwards. Continue?': '修改向量维度会自动清空向量数据库,之后需要重建索引,是否继续?',

View File

@@ -1,8 +1,8 @@
import { Form, Input, Button, message, Tabs, Space, Card, Select, Modal, Radio, InputNumber } from 'antd';
import { useEffect, useState } from 'react';
import { Form, Input, Button, message, Tabs, Space, Card, Select, Modal, Radio, InputNumber, Spin, Empty, Alert } from 'antd';
import { useEffect, useState, useCallback } from 'react';
import PageCard from '../../components/PageCard';
import { getAllConfig, setConfig } from '../../api/config';
import { vectorDBApi } from '../../api/vectorDB';
import { vectorDBApi, type VectorDBStats, type VectorDBProviderMeta, type VectorDBCurrentConfig } from '../../api/vectorDB';
import { AppstoreOutlined, RobotOutlined, DatabaseOutlined, SkinOutlined } from '@ant-design/icons';
import { useTheme } from '../../contexts/ThemeContext';
import '../../styles/settings-tabs.css';
@@ -32,6 +32,20 @@ const EMBED_CONFIG_KEYS = [
const ALL_AI_KEYS = [...VISION_CONFIG_KEYS, ...EMBED_CONFIG_KEYS, { key: EMBED_DIM_KEY, default: DEFAULT_EMBED_DIMENSION }];
const formatBytes = (bytes?: number | null) => {
if (bytes === null || bytes === undefined) return '-';
if (bytes === 0) return '0 B';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
let value = bytes;
let unitIndex = 0;
while (value >= 1024 && unitIndex < units.length - 1) {
value /= 1024;
unitIndex += 1;
}
const precision = value >= 10 || unitIndex === 0 ? 0 : 1;
return `${value.toFixed(precision)} ${units[unitIndex]}`;
};
// Theme related config keys
const THEME_KEYS = {
MODE: 'THEME_MODE',
@@ -42,9 +56,19 @@ const THEME_KEYS = {
};
export default function SystemSettingsPage() {
const [vectorConfigForm] = Form.useForm();
const [loading, setLoading] = useState(false);
const [config, setConfigState] = useState<Record<string, string> | null>(null);
const [activeTab, setActiveTab] = useState('appearance');
const [vectorStats, setVectorStats] = useState<VectorDBStats | null>(null);
const [vectorStatsLoading, setVectorStatsLoading] = useState(false);
const [vectorStatsError, setVectorStatsError] = useState<string | null>(null);
const [vectorProviders, setVectorProviders] = useState<VectorDBProviderMeta[]>([]);
const [vectorConfig, setVectorConfig] = useState<VectorDBCurrentConfig | null>(null);
const [vectorConfigLoading, setVectorConfigLoading] = useState(false);
const [vectorConfigSaving, setVectorConfigSaving] = useState(false);
const [vectorMetaError, setVectorMetaError] = useState<string | null>(null);
const [selectedProviderType, setSelectedProviderType] = useState<string | null>(null);
const { refreshTheme, previewTheme } = useTheme();
const { t } = useI18n();
@@ -52,6 +76,72 @@ export default function SystemSettingsPage() {
getAllConfig().then((data) => setConfigState(data as Record<string, string>));
}, []);
const fetchVectorStats = useCallback(async () => {
setVectorStatsLoading(true);
setVectorStatsError(null);
try {
const data = await vectorDBApi.getStats();
setVectorStats(data);
} catch (e: any) {
const msg = e?.message || t('Load failed');
setVectorStatsError(msg);
message.error(msg);
} finally {
setVectorStatsLoading(false);
}
}, [t]);
const buildProviderConfigValues = useCallback((provider: VectorDBProviderMeta | undefined, existing?: Record<string, string>) => {
if (!provider) return {};
const values: Record<string, string> = {};
const schema = provider.config_schema || [];
schema.forEach((field) => {
const current = existing && existing[field.key] !== undefined && existing[field.key] !== null
? String(existing[field.key])
: undefined;
if (current !== undefined) {
values[field.key] = current;
} else if (field.default !== undefined && field.default !== null) {
values[field.key] = String(field.default);
} else {
values[field.key] = '';
}
});
return values;
}, []);
const fetchVectorMeta = useCallback(async () => {
setVectorConfigLoading(true);
setVectorMetaError(null);
try {
const [providers, current] = await Promise.all([
vectorDBApi.getProviders(),
vectorDBApi.getConfig(),
]);
setVectorProviders(providers);
setVectorConfig(current);
const enabled = providers.filter((item) => item.enabled);
let nextType: string | null = current?.type ?? null;
if (nextType && !providers.some((item) => item.type === nextType)) {
nextType = null;
}
if (!nextType) {
nextType = enabled[0]?.type ?? providers[0]?.type ?? null;
}
setSelectedProviderType(nextType);
const provider = providers.find((item) => item.type === nextType);
const configValues = buildProviderConfigValues(provider, nextType === current?.type ? current?.config : undefined);
vectorConfigForm.setFieldsValue({ type: nextType || undefined, config: configValues });
} catch (e: any) {
const msg = e?.message || t('Load failed');
setVectorMetaError(msg);
message.error(msg);
} finally {
setVectorConfigLoading(false);
}
}, [buildProviderConfigValues, message, t, vectorConfigForm]);
const handleSave = async (values: any) => {
setLoading(true);
try {
@@ -70,6 +160,40 @@ export default function SystemSettingsPage() {
setLoading(false);
};
const handleProviderChange = useCallback((value: string) => {
setSelectedProviderType(value);
const provider = vectorProviders.find((item) => item.type === value);
const existing = value === vectorConfig?.type ? vectorConfig?.config : undefined;
const configValues = buildProviderConfigValues(provider, existing);
vectorConfigForm.setFieldsValue({ type: value, config: configValues });
}, [vectorProviders, vectorConfig, buildProviderConfigValues, vectorConfigForm]);
const handleVectorConfigSave = useCallback(async (values: { type: string; config?: Record<string, string> }) => {
if (!values?.type) {
return;
}
setVectorConfigSaving(true);
try {
const configPayload = Object.fromEntries(
Object.entries(values.config || {}).filter(([, val]) => val !== undefined && val !== null && String(val).trim() !== '')
.map(([key, val]) => [key, String(val)])
);
const response = await vectorDBApi.updateConfig({ type: values.type, config: configPayload });
setVectorConfig(response.config);
setVectorStats(response.stats);
setVectorStatsError(null);
setSelectedProviderType(response.config.type);
const provider = vectorProviders.find((item) => item.type === response.config.type);
const mergedValues = buildProviderConfigValues(provider, response.config.config);
vectorConfigForm.setFieldsValue({ type: response.config.type, config: mergedValues });
message.success(t('Saved successfully'));
} catch (e: any) {
message.error(e?.message || t('Save failed'));
} finally {
setVectorConfigSaving(false);
}
}, [buildProviderConfigValues, message, t, vectorConfigForm, vectorProviders]);
// 离开“外观设置”时,恢复后端持久化配置(取消未保存的预览)
useEffect(() => {
if (activeTab !== 'appearance') {
@@ -77,6 +201,27 @@ export default function SystemSettingsPage() {
}
}, [activeTab]);
useEffect(() => {
if (activeTab === 'vector-db') {
if (!vectorProviders.length && !vectorConfigLoading) {
fetchVectorMeta();
}
if (!vectorStats && !vectorStatsLoading) {
fetchVectorStats();
}
}
}, [
activeTab,
fetchVectorMeta,
fetchVectorStats,
vectorProviders.length,
vectorConfigLoading,
vectorStats,
vectorStatsLoading,
]);
const selectedProvider = vectorProviders.find((item) => item.type === selectedProviderType || (!selectedProviderType && item.enabled));
if (!config) {
return <PageCard title={t('System Settings')}><div>{t('Loading...')}</div></PageCard>;
}
@@ -275,41 +420,187 @@ export default function SystemSettingsPage() {
),
children: (
<Card title={t('Vector Database Settings')} style={{ marginTop: 24 }}>
<Form layout="vertical">
<Form.Item label={t('Database Type')}>
<Select
size="large"
value={'Milvus Lite'}
disabled
options={[{ value: 'Milvus Lite', label: 'Milvus Lite' }]}
/>
</Form.Item>
<Form.Item>
<Button
danger
block
onClick={() => {
Modal.confirm({
title: t('Confirm clear vector database?'),
content: t('This will delete all collections irreversibly.'),
okText: t('Confirm Clear'),
okType: 'danger',
cancelText: t('Cancel'),
onOk: async () => {
try {
await vectorDBApi.clearAll();
message.success(t('Vector database cleared'));
} catch (e: any) {
message.error(e.message || t('Clear failed'));
}
},
});
}}
<Space direction="vertical" size={24} style={{ width: '100%' }}>
<Space direction="vertical" size={16} style={{ width: '100%' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', flexWrap: 'wrap', gap: 12 }}>
<strong>{t('Current Statistics')}</strong>
<Button onClick={() => { fetchVectorMeta(); fetchVectorStats(); }} loading={vectorStatsLoading || vectorConfigLoading} disabled={(vectorStatsLoading || vectorConfigLoading) && !vectorStats}>
{t('Refresh')}
</Button>
</div>
{vectorMetaError ? (
<Alert type="error" showIcon message={vectorMetaError} />
) : null}
{vectorStatsLoading && !vectorStats ? (
<Spin />
) : vectorStats ? (
<Space direction="vertical" size={16} style={{ width: '100%' }}>
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 24 }}>
<div>
<div style={{ color: '#888' }}>{t('Collections')}</div>
<div style={{ fontSize: 20, fontWeight: 600 }}>{vectorStats.collection_count}</div>
</div>
<div>
<div style={{ color: '#888' }}>{t('Vectors')}</div>
<div style={{ fontSize: 20, fontWeight: 600 }}>{vectorStats.total_vectors}</div>
</div>
<div>
<div style={{ color: '#888' }}>{t('Database Size')}</div>
<div style={{ fontSize: 20, fontWeight: 600 }}>{formatBytes(vectorStats.db_file_size_bytes)}</div>
</div>
<div>
<div style={{ color: '#888' }}>{t('Estimated Memory')}</div>
<div style={{ fontSize: 20, fontWeight: 600 }}>{formatBytes(vectorStats.estimated_total_memory_bytes)}</div>
</div>
</div>
{vectorStats.collections.length ? (
<Space direction="vertical" style={{ width: '100%' }} size={16}>
{vectorStats.collections.map((collection) => (
<div key={collection.name} style={{ border: '1px solid #f0f0f0', borderRadius: 8, padding: 16 }}>
<Space direction="vertical" size={12} style={{ width: '100%' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', flexWrap: 'wrap', gap: 12 }}>
<strong>{collection.name}</strong>
<span style={{ color: '#888' }}>
{collection.is_vector_collection && collection.dimension
? `${t('Dimension')}: ${collection.dimension}`
: t('Non-vector collection')}
</span>
</div>
<div>{t('Vectors')}: {collection.row_count}</div>
{collection.is_vector_collection ? (
<div>{t('Estimated memory')}: {formatBytes(collection.estimated_memory_bytes)}</div>
) : null}
{collection.indexes.length ? (
<Space direction="vertical" size={4} style={{ width: '100%' }}>
<span>{t('Indexes')}:</span>
<ul style={{ paddingLeft: 20, margin: 0 }}>
{collection.indexes.map((index) => (
<li key={`${collection.name}-${index.index_name || 'default'}`}>
<span>{index.index_name || t('Unnamed index')}</span>
<span>{' · '}{index.index_type || '-'}</span>
<span>{' · '}{index.metric_type || '-'}</span>
<span>{' · '}{t('Indexed rows')}: {index.indexed_rows}</span>
<span>{' · '}{t('Pending rows')}: {index.pending_index_rows}</span>
<span>{' · '}{t('Status')}: {index.state || '-'}</span>
</li>
))}
</ul>
</Space>
) : null}
</Space>
</div>
))}
</Space>
) : (
<Empty description={t('No collections')} />
)}
<div style={{ color: '#888' }}>
{t('Estimated memory is calculated as vectors x dimension x 4 bytes (float32).')}
</div>
</Space>
) : vectorStatsError ? (
<div style={{ color: '#ff4d4f' }}>{vectorStatsError}</div>
) : (
<Empty description={t('No collections')} />
)}
</Space>
{vectorConfigLoading && !vectorProviders.length ? (
<Spin />
) : (
<Form
layout="vertical"
form={vectorConfigForm}
onFinish={handleVectorConfigSave}
initialValues={{ type: selectedProviderType || undefined, config: {} }}
>
{t('Clear Vector DB')}
</Button>
</Form.Item>
</Form>
<Form.Item
name="type"
label={t('Database Provider')}
rules={[{ required: true, message: t('Please select a provider') }]}
>
<Select
size="large"
options={vectorProviders.map((provider) => ({
value: provider.type,
label: provider.enabled ? provider.label : `${provider.label} (${t('Coming soon')})`,
disabled: !provider.enabled,
}))}
onChange={handleProviderChange}
loading={vectorConfigLoading && !vectorProviders.length}
/>
</Form.Item>
{selectedProvider?.description ? (
<Alert
type="info"
showIcon
message={t(selectedProvider.description)}
style={{ marginBottom: 16 }}
/>
) : null}
{selectedProvider?.config_schema?.map((field) => (
<Form.Item
key={field.key}
name={['config', field.key]}
label={t(field.label)}
rules={field.required ? [{ required: true, message: t('Please input {label}', { label: t(field.label) }) }] : []}
>
{field.type === 'password' ? (
<Input.Password size="large" placeholder={field.placeholder ? t(field.placeholder) : undefined} />
) : (
<Input size="large" placeholder={field.placeholder ? t(field.placeholder) : undefined} />
)}
</Form.Item>
))}
{selectedProvider && !selectedProvider.enabled ? (
<Alert
type="warning"
showIcon
message={t('This provider is not available yet')}
style={{ marginBottom: 16 }}
/>
) : null}
<Form.Item>
<Space direction="vertical" style={{ width: '100%' }}>
<Button
type="primary"
htmlType="submit"
loading={vectorConfigSaving}
block
disabled={!selectedProvider?.enabled}
>
{t('Save')}
</Button>
<Button
danger
htmlType="button"
block
onClick={() => {
Modal.confirm({
title: t('Confirm clear vector database?'),
content: t('This will delete all collections irreversibly.'),
okText: t('Confirm Clear'),
okType: 'danger',
cancelText: t('Cancel'),
onOk: async () => {
try {
await vectorDBApi.clearAll();
message.success(t('Vector database cleared'));
await fetchVectorStats();
await fetchVectorMeta();
} catch (e: any) {
message.error(e.message || t('Clear failed'));
}
},
});
}}
>
{t('Clear Vector DB')}
</Button>
</Space>
</Form.Item>
</Form>
)}
</Space>
</Card>
),
},