From fbeb67312606709b7d618ba3e46ae40ceb769151 Mon Sep 17 00:00:00 2001 From: shiyu Date: Fri, 19 Sep 2025 13:45:48 +0800 Subject: [PATCH] feat(vector_db): Implement Vector Database Service with multiple providers --- api/routes/config.py | 2 +- api/routes/search.py | 6 +- api/routes/vector_db.py | 87 ++++- pyproject.toml | 1 + services/processors/vector_index.py | 10 +- services/vector_db.py | 92 ----- services/vector_db/__init__.py | 11 + services/vector_db/config_manager.py | 43 +++ services/vector_db/providers/__init__.py | 56 +++ services/vector_db/providers/base.py | 41 ++ services/vector_db/providers/milvus_lite.py | 196 ++++++++++ services/vector_db/providers/milvus_server.py | 197 ++++++++++ services/vector_db/providers/qdrant.py | 237 ++++++++++++ services/vector_db/service.py | 99 +++++ uv.lock | 81 ++++ web/src/api/vectorDB.ts | 62 ++- web/src/i18n/locales/en.ts | 26 ++ web/src/i18n/locales/zh.ts | 26 ++ .../SystemSettingsPage/SystemSettingsPage.tsx | 365 ++++++++++++++++-- 19 files changed, 1496 insertions(+), 142 deletions(-) delete mode 100644 services/vector_db.py create mode 100644 services/vector_db/__init__.py create mode 100644 services/vector_db/config_manager.py create mode 100644 services/vector_db/providers/__init__.py create mode 100644 services/vector_db/providers/base.py create mode 100644 services/vector_db/providers/milvus_lite.py create mode 100644 services/vector_db/providers/milvus_server.py create mode 100644 services/vector_db/providers/qdrant.py create mode 100644 services/vector_db/service.py diff --git a/api/routes/config.py b/api/routes/config.py index d1b9819..b8650da 100644 --- a/api/routes/config.py +++ b/api/routes/config.py @@ -40,7 +40,7 @@ async def set_config( if key == "AI_EMBED_DIM" and str(original_value) != value_to_save: try: service = VectorDBService() - service.clear_all_data() + await service.clear_all_data() except Exception as exc: raise HTTPException(status_code=500, detail=f"Failed to clear vector database: {exc}") diff --git a/api/routes/search.py b/api/routes/search.py index 664bfbb..1544939 100644 --- a/api/routes/search.py +++ b/api/routes/search.py @@ -9,7 +9,7 @@ router = APIRouter(prefix="/api/search", tags=["search"]) async def search_files_by_vector(q: str, top_k: int): embedding = await get_text_embedding(q) vector_db = VectorDBService() - results = vector_db.search_vectors("vector_collection", embedding, top_k) + results = await vector_db.search_vectors("vector_collection", embedding, top_k) items = [ SearchResultItem(id=res["id"], path=res["entity"]["path"], score=res["distance"]) for res in results[0] @@ -18,7 +18,7 @@ async def search_files_by_vector(q: str, top_k: int): async def search_files_by_name(q: str, top_k: int): vector_db = VectorDBService() - results = vector_db.search_by_path("vector_collection", q, top_k) + results = await vector_db.search_by_path("vector_collection", q, top_k) items = [ SearchResultItem(id=idx, path=res["entity"]["path"], score=res["distance"]) for idx, res in enumerate(results[0]) @@ -38,4 +38,4 @@ async def search_files( elif mode == "filename": return await search_files_by_name(q, top_k) else: - return {"items": [], "query": q, "error": "Invalid search mode"} \ No newline at end of file + return {"items": [], "query": q, "error": "Invalid search mode"} diff --git a/api/routes/vector_db.py b/api/routes/vector_db.py index 9d41e3e..935ee69 100644 --- a/api/routes/vector_db.py +++ b/api/routes/vector_db.py @@ -1,19 +1,100 @@ +from typing import Any, Dict + from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + from services.auth import get_current_active_user from models.database import UserAccount -from services.vector_db import VectorDBService +from services.vector_db import ( + VectorDBService, + VectorDBConfigManager, + list_providers, + get_provider_entry, +) +from services.vector_db.providers import get_provider_class from api.response import success router = APIRouter(prefix="/api/vector-db", tags=["vector-db"]) +class VectorDBConfigPayload(BaseModel): + type: str = Field(..., description="向量数据库提供者类型") + config: Dict[str, Any] = Field(default_factory=dict, description="提供者配置参数") + + @router.post("/clear-all", summary="清空向量数据库") async def clear_vector_db(user: UserAccount = Depends(get_current_active_user)): if user.username != 'admin': raise HTTPException(status_code=403, detail="仅管理员可操作") try: service = VectorDBService() - service.clear_all_data() + await service.clear_all_data() return success(msg="向量数据库已清空") except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) \ No newline at end of file + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/stats", summary="获取向量数据库统计") +async def get_vector_db_stats(user: UserAccount = Depends(get_current_active_user)): + if user.username != 'admin': + raise HTTPException(status_code=403, detail="仅管理员可操作") + try: + service = VectorDBService() + data = await service.get_all_stats() + return success(data=data) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/providers", summary="列出可用向量数据库提供者") +async def list_vector_providers(user: UserAccount = Depends(get_current_active_user)): + if user.username != 'admin': + raise HTTPException(status_code=403, detail="仅管理员可操作") + return success(list_providers()) + + +@router.get("/config", summary="获取当前向量数据库配置") +async def get_vector_db_config(user: UserAccount = Depends(get_current_active_user)): + if user.username != 'admin': + raise HTTPException(status_code=403, detail="仅管理员可操作") + service = VectorDBService() + data = await service.current_provider() + return success(data) + + +@router.post("/config", summary="更新向量数据库配置") +async def update_vector_db_config(payload: VectorDBConfigPayload, user: UserAccount = Depends(get_current_active_user)): + if user.username != 'admin': + raise HTTPException(status_code=403, detail="仅管理员可操作") + + entry = get_provider_entry(payload.type) + if not entry: + raise HTTPException(status_code=400, detail=f"未知的向量数据库类型: {payload.type}") + if not entry.get("enabled", True): + raise HTTPException(status_code=400, detail="该向量数据库类型暂不可用") + + provider_cls = get_provider_class(payload.type) + if not provider_cls: + raise HTTPException(status_code=400, detail=f"未找到类型 {payload.type} 对应的实现") + + # 先尝试建立连接，确保配置有效 + test_provider = provider_cls(payload.config) + try: + await test_provider.initialize() + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) + finally: + client = getattr(test_provider, "client", None) + close_fn = getattr(client, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + pass + + await VectorDBConfigManager.save_config(payload.type, payload.config) + service = VectorDBService() + await service.reload() + config_data = await service.current_provider() + stats = await service.get_all_stats() + return success({"config": config_data, "stats": stats}) diff --git a/pyproject.toml b/pyproject.toml index f3ef306..b8df4c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ dependencies = [ "python-multipart==0.0.20", "pytz==2025.2", "pyyaml==6.0.2", + "qdrant-client==1.15.1", "rawpy==0.25.1", "rich==14.1.0", "rich-toolkit==0.15.0", diff --git a/services/processors/vector_index.py b/services/processors/vector_index.py index fb7f031..6e9afd8 100644 --- a/services/processors/vector_index.py +++ b/services/processors/vector_index.py @@ -34,7 +34,7 @@ class VectorIndexProcessor: vector_db = VectorDBService() collection_name = "vector_collection" if action == "destroy": - vector_db.delete_vector(collection_name, path) + await vector_db.delete_vector(collection_name, path) await LogService.info( "processor:vector_index", f"Destroyed {index_type} index for {path}", @@ -43,8 +43,8 @@ class VectorIndexProcessor: return Response(content=f"文件 {path} 的 {index_type} 索引已销毁", media_type="text/plain") if index_type == 'simple': - vector_db.ensure_collection(collection_name, vector=False) - vector_db.upsert_vector(collection_name, {'path': path}) + await vector_db.ensure_collection(collection_name, vector=False) + await vector_db.upsert_vector(collection_name, {'path': path}) await LogService.info( "processor:vector_index", f"Created simple index for {path}", @@ -80,8 +80,8 @@ class VectorIndexProcessor: if vector_dim <= 0: vector_dim = DEFAULT_VECTOR_DIMENSION - vector_db.ensure_collection(collection_name, vector=True, dim=vector_dim) - vector_db.upsert_vector( + await vector_db.ensure_collection(collection_name, vector=True, dim=vector_dim) + await vector_db.upsert_vector( collection_name, {'path': path, 'embedding': embedding}) await LogService.info( diff --git a/services/vector_db.py b/services/vector_db.py deleted file mode 100644 index 1ab47bf..0000000 --- a/services/vector_db.py +++ /dev/null @@ -1,92 +0,0 @@ -from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusClient - - -DEFAULT_VECTOR_DIMENSION = 4096 - - -class VectorDBService: - _instance = None - - def __new__(cls, *args, **kwargs): - if not cls._instance: - cls._instance = super(VectorDBService, cls).__new__(cls) - return cls._instance - - def __init__(self): - if not hasattr(self, 'client'): - self.client = MilvusClient("data/db/milvus.db") - - def ensure_collection(self, collection_name, vector: bool = True, dim: int = DEFAULT_VECTOR_DIMENSION): - if self.client.has_collection(collection_name): - return - if vector: - try: - vector_dim = int(dim) - except (TypeError, ValueError): - vector_dim = DEFAULT_VECTOR_DIMENSION - if vector_dim <= 0: - vector_dim = DEFAULT_VECTOR_DIMENSION - fields = [ - FieldSchema(name="path", dtype=DataType.VARCHAR, - max_length=512, is_primary=True, auto_id=False), - FieldSchema(name="embedding", - dtype=DataType.FLOAT_VECTOR, dim=vector_dim) - ] - schema = CollectionSchema( - fields, description="Image vector collection") - self.client.create_collection(collection_name, schema=schema) - index_params = MilvusClient.prepare_index_params() - index_params.add_index( - field_name="embedding", - index_type="IVF_FLAT", - index_name="vector_index", - metric_type="COSINE", - params={ - "nlist": 64, - } - ) - self.client.create_index( - collection_name, - index_params=index_params - ) - else: - fields = [ - FieldSchema(name="path", dtype=DataType.VARCHAR, - max_length=512, is_primary=True, auto_id=False), - ] - schema = CollectionSchema(fields, description="Simple file index") - self.client.create_collection(collection_name, schema=schema) - - def upsert_vector(self, collection_name, data): - self.client.upsert(collection_name, data) - - def delete_vector(self, collection_name, path: str): - self.client.delete(collection_name, ids=[path]) - - def search_vectors(self, collection_name, query_embedding, top_k=5): - search_params = {"metric_type": "COSINE"} - results = self.client.search( - collection_name, - data=[query_embedding], - anns_field="embedding", - search_params=search_params, - limit=top_k, - output_fields=["path"] - ) - print(results) - return results - - def search_by_path(self, collection_name, query_path, top_k=20): - results = self.client.query( - collection_name, - filter=f"path like '%{query_path}%'", - limit=top_k, - output_fields=["path"] - ) - return [[{'id': r['path'], 'distance': 1.0, 'entity': {'path': r['path']}} for r in results]] - - def clear_all_data(self): - """清空所有集合的内容""" - collections = self.client.list_collections() - for collection_name in collections: - self.client.drop_collection(collection_name) diff --git a/services/vector_db/__init__.py b/services/vector_db/__init__.py new file mode 100644 index 0000000..5f4c88b --- /dev/null +++ b/services/vector_db/__init__.py @@ -0,0 +1,11 @@ +from .service import VectorDBService, DEFAULT_VECTOR_DIMENSION +from .providers import list_providers, get_provider_entry +from .config_manager import VectorDBConfigManager + +__all__ = [ + "VectorDBService", + "DEFAULT_VECTOR_DIMENSION", + "list_providers", + "get_provider_entry", + "VectorDBConfigManager", +] diff --git a/services/vector_db/config_manager.py b/services/vector_db/config_manager.py new file mode 100644 index 0000000..c54f3cd --- /dev/null +++ b/services/vector_db/config_manager.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, Tuple + +from services.config import ConfigCenter + + +class VectorDBConfigManager: + TYPE_KEY = "VECTOR_DB_TYPE" + CONFIG_KEY = "VECTOR_DB_CONFIG" + DEFAULT_TYPE = "milvus_lite" + + @classmethod + async def load_config(cls) -> Tuple[str, Dict[str, Any]]: + raw_type = await ConfigCenter.get(cls.TYPE_KEY, cls.DEFAULT_TYPE) + provider_type = str(raw_type or cls.DEFAULT_TYPE) + + raw_config = await ConfigCenter.get(cls.CONFIG_KEY) + config_dict: Dict[str, Any] = {} + if isinstance(raw_config, str) and raw_config: + try: + config_dict = json.loads(raw_config) + except json.JSONDecodeError: + config_dict = {} + elif isinstance(raw_config, dict): + config_dict = raw_config + return provider_type, config_dict + + @classmethod + async def save_config(cls, provider_type: str, config: Dict[str, Any]) -> None: + await ConfigCenter.set(cls.TYPE_KEY, provider_type) + await ConfigCenter.set(cls.CONFIG_KEY, json.dumps(config or {})) + + @classmethod + async def get_type(cls) -> str: + provider_type, _ = await cls.load_config() + return provider_type + + @classmethod + async def get_config(cls) -> Dict[str, Any]: + _, config = await cls.load_config() + return config diff --git a/services/vector_db/providers/__init__.py b/services/vector_db/providers/__init__.py new file mode 100644 index 0000000..4b4d23b --- /dev/null +++ b/services/vector_db/providers/__init__.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from typing import Dict, List, Type + +from .base import BaseVectorProvider +from .milvus_lite import MilvusLiteProvider +from .milvus_server import MilvusServerProvider +from .qdrant import QdrantProvider + +_PROVIDER_REGISTRY: Dict[str, Dict[str, object]] = { + MilvusLiteProvider.type: { + "class": MilvusLiteProvider, + "label": MilvusLiteProvider.label, + "description": MilvusLiteProvider.description, + "enabled": MilvusLiteProvider.enabled, + "config_schema": MilvusLiteProvider.config_schema, + }, + MilvusServerProvider.type: { + "class": MilvusServerProvider, + "label": MilvusServerProvider.label, + "description": MilvusServerProvider.description, + "enabled": MilvusServerProvider.enabled, + "config_schema": MilvusServerProvider.config_schema, + }, + QdrantProvider.type: { + "class": QdrantProvider, + "label": QdrantProvider.label, + "description": QdrantProvider.description, + "enabled": QdrantProvider.enabled, + "config_schema": QdrantProvider.config_schema, + }, +} + + +def list_providers() -> List[Dict[str, object]]: + return [ + { + "type": type_key, + "label": meta["label"], + "description": meta.get("description"), + "enabled": meta.get("enabled", True), + "config_schema": meta.get("config_schema", []), + } + for type_key, meta in _PROVIDER_REGISTRY.items() + ] + + +def get_provider_entry(provider_type: str) -> Dict[str, object] | None: + return _PROVIDER_REGISTRY.get(provider_type) + + +def get_provider_class(provider_type: str) -> Type[BaseVectorProvider] | None: + entry = get_provider_entry(provider_type) + if not entry: + return None + return entry.get("class") # type: ignore[return-value] diff --git a/services/vector_db/providers/base.py b/services/vector_db/providers/base.py new file mode 100644 index 0000000..1e71266 --- /dev/null +++ b/services/vector_db/providers/base.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Any, Dict, List + + +class BaseVectorProvider: + """向量数据库提供者基础类，所有实际实现需继承该类""" + + type: str = "" + label: str = "" + description: str | None = None + enabled: bool = True + config_schema: List[Dict[str, Any]] = [] + + def __init__(self, config: Dict[str, Any] | None = None): + self.config = config or {} + + async def initialize(self) -> None: + """执行初始化逻辑，例如建立连接""" + raise NotImplementedError + + def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None: + raise NotImplementedError + + def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None: + raise NotImplementedError + + def delete_vector(self, collection_name: str, path: str) -> None: + raise NotImplementedError + + def search_vectors(self, collection_name: str, query_embedding, top_k: int): + raise NotImplementedError + + def search_by_path(self, collection_name: str, query_path: str, top_k: int): + raise NotImplementedError + + def get_all_stats(self) -> Dict[str, Any]: + raise NotImplementedError + + def clear_all_data(self) -> None: + raise NotImplementedError diff --git a/services/vector_db/providers/milvus_lite.py b/services/vector_db/providers/milvus_lite.py new file mode 100644 index 0000000..86759f2 --- /dev/null +++ b/services/vector_db/providers/milvus_lite.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Optional + +from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusClient + +from .base import BaseVectorProvider + + +class MilvusLiteProvider(BaseVectorProvider): + type = "milvus_lite" + label = "Milvus Lite" + description = "Embedded Milvus Lite (local file storage)." + enabled = True + config_schema: List[Dict[str, Any]] = [ + { + "key": "db_path", + "label": "Database file path", + "type": "text", + "default": "data/db/milvus.db", + "required": False, + } + ] + + def __init__(self, config: Dict[str, Any] | None = None): + super().__init__(config) + self.db_path = Path(self.config.get("db_path") or "data/db/milvus.db") + self.client: MilvusClient | None = None + + async def initialize(self) -> None: + try: + self.client = MilvusClient(str(self.db_path)) + except Exception as exc: # pragma: no cover - depends on local environment + raise RuntimeError(f"Failed to open Milvus Lite at {self.db_path}: {exc}") from exc + + def _get_client(self) -> MilvusClient: + if not self.client: + raise RuntimeError("Milvus Lite client is not initialized") + return self.client + + @staticmethod + def _to_int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None: + client = self._get_client() + if client.has_collection(collection_name): + return + if vector: + vector_dim = dim if isinstance(dim, int) and dim > 0 else 0 + if vector_dim <= 0: + vector_dim = 4096 + fields = [ + FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False), + FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=vector_dim), + ] + schema = CollectionSchema(fields, description="Image vector collection") + client.create_collection(collection_name, schema=schema) + index_params = MilvusClient.prepare_index_params() + index_params.add_index( + field_name="embedding", + index_type="IVF_FLAT", + index_name="vector_index", + metric_type="COSINE", + params={"nlist": 64}, + ) + client.create_index(collection_name, index_params=index_params) + else: + fields = [ + FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False), + ] + schema = CollectionSchema(fields, description="Simple file index") + client.create_collection(collection_name, schema=schema) + + def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None: + self._get_client().upsert(collection_name, data) + + def delete_vector(self, collection_name: str, path: str) -> None: + self._get_client().delete(collection_name, ids=[path]) + + def search_vectors(self, collection_name: str, query_embedding, top_k: int): + search_params = {"metric_type": "COSINE"} + return self._get_client().search( + collection_name, + data=[query_embedding], + anns_field="embedding", + search_params=search_params, + limit=top_k, + output_fields=["path"], + ) + + def search_by_path(self, collection_name: str, query_path: str, top_k: int): + filter_expr = f"path like '%{query_path}%'" if query_path else "path like '%%'" + results = self._get_client().query( + collection_name, + filter=filter_expr, + limit=top_k, + output_fields=["path"], + ) + return [[{"id": r["path"], "distance": 1.0, "entity": {"path": r["path"]}} for r in results]] + + def get_all_stats(self) -> Dict[str, Any]: + client = self._get_client() + try: + collection_names = client.list_collections() + except Exception as exc: + raise RuntimeError(f"Failed to list collections: {exc}") from exc + + collections: List[Dict[str, Any]] = [] + total_vectors = 0 + total_estimated_memory = 0 + + for name in collection_names: + try: + stats = client.get_collection_stats(name) or {} + except Exception: + stats = {} + row_count = self._to_int(stats.get("row_count")) + total_vectors += row_count + + dimension: Optional[int] = None + is_vector_collection = False + try: + description = client.describe_collection(name) + except Exception: + description = None + + if description: + for field in description.get("fields", []): + if field.get("type") == DataType.FLOAT_VECTOR: + params = field.get("params") or {} + dimension = self._to_int(params.get("dim")) or 4096 + is_vector_collection = True + break + + estimated_memory = 0 + if is_vector_collection and dimension: + estimated_memory = row_count * dimension * 4 + total_estimated_memory += estimated_memory + + indexes: List[Dict[str, Any]] = [] + try: + index_names = client.list_indexes(name) or [] + except Exception: + index_names = [] + + for index_name in index_names: + try: + detail = client.describe_index(name, index_name) or {} + except Exception: + detail = {} + indexes.append( + { + "index_name": index_name, + "index_type": detail.get("index_type"), + "metric_type": detail.get("metric_type"), + "indexed_rows": self._to_int(detail.get("indexed_rows")), + "pending_index_rows": self._to_int(detail.get("pending_index_rows")), + "state": detail.get("state"), + } + ) + + collections.append( + { + "name": name, + "row_count": row_count, + "dimension": dimension if is_vector_collection else None, + "estimated_memory_bytes": estimated_memory, + "is_vector_collection": is_vector_collection, + "indexes": indexes, + } + ) + + db_file_size = None + try: + if self.db_path.exists(): + db_file_size = self.db_path.stat().st_size + except OSError: + db_file_size = None + + return { + "collections": collections, + "collection_count": len(collections), + "total_vectors": total_vectors, + "estimated_total_memory_bytes": total_estimated_memory, + "db_file_size_bytes": db_file_size, + } + + def clear_all_data(self) -> None: + client = self._get_client() + for collection_name in client.list_collections(): + client.drop_collection(collection_name) diff --git a/services/vector_db/providers/milvus_server.py b/services/vector_db/providers/milvus_server.py new file mode 100644 index 0000000..d802734 --- /dev/null +++ b/services/vector_db/providers/milvus_server.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusClient + +from .base import BaseVectorProvider + + +class MilvusServerProvider(BaseVectorProvider): + type = "milvus_server" + label = "Milvus Server" + description = "Remote Milvus instance accessed via URI." + enabled = True + config_schema: List[Dict[str, Any]] = [ + { + "key": "uri", + "label": "Server URI", + "type": "text", + "required": True, + "placeholder": "http://localhost:19530", + }, + { + "key": "token", + "label": "Token", + "type": "password", + "required": False, + "placeholder": "user:password", + }, + ] + + def __init__(self, config: Dict[str, Any] | None = None): + super().__init__(config) + self.client: MilvusClient | None = None + + async def initialize(self) -> None: + uri = self.config.get("uri") + if not uri: + raise RuntimeError("Milvus Server URI is required") + try: + self.client = MilvusClient(uri=uri, token=self.config.get("token")) + except Exception as exc: # pragma: no cover - depends on remote availability + raise RuntimeError(f"Failed to connect to Milvus Server {uri}: {exc}") from exc + + def _get_client(self) -> MilvusClient: + if not self.client: + raise RuntimeError("Milvus Server client is not initialized") + return self.client + + @staticmethod + def _to_int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None: + client = self._get_client() + if client.has_collection(collection_name): + return + if vector: + vector_dim = dim if isinstance(dim, int) and dim > 0 else 0 + if vector_dim <= 0: + vector_dim = 4096 + fields = [ + FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False), + FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=vector_dim), + ] + schema = CollectionSchema(fields, description="Image vector collection") + client.create_collection(collection_name, schema=schema) + index_params = MilvusClient.prepare_index_params() + index_params.add_index( + field_name="embedding", + index_type="IVF_FLAT", + index_name="vector_index", + metric_type="COSINE", + params={"nlist": 64}, + ) + client.create_index(collection_name, index_params=index_params) + else: + fields = [ + FieldSchema(name="path", dtype=DataType.VARCHAR, max_length=512, is_primary=True, auto_id=False), + ] + schema = CollectionSchema(fields, description="Simple file index") + client.create_collection(collection_name, schema=schema) + + def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None: + self._get_client().upsert(collection_name, data) + + def delete_vector(self, collection_name: str, path: str) -> None: + self._get_client().delete(collection_name, ids=[path]) + + def search_vectors(self, collection_name: str, query_embedding, top_k: int): + search_params = {"metric_type": "COSINE"} + return self._get_client().search( + collection_name, + data=[query_embedding], + anns_field="embedding", + search_params=search_params, + limit=top_k, + output_fields=["path"], + ) + + def search_by_path(self, collection_name: str, query_path: str, top_k: int): + filter_expr = f"path like '%{query_path}%'" if query_path else "path like '%%'" + results = self._get_client().query( + collection_name, + filter=filter_expr, + limit=top_k, + output_fields=["path"], + ) + return [[{"id": r["path"], "distance": 1.0, "entity": {"path": r["path"]}} for r in results]] + + def get_all_stats(self) -> Dict[str, Any]: + client = self._get_client() + try: + collection_names = client.list_collections() + except Exception as exc: + raise RuntimeError(f"Failed to list collections: {exc}") from exc + + collections: List[Dict[str, Any]] = [] + total_vectors = 0 + total_estimated_memory = 0 + + for name in collection_names: + try: + stats = client.get_collection_stats(name) or {} + except Exception: + stats = {} + row_count = self._to_int(stats.get("row_count")) + total_vectors += row_count + + dimension: Optional[int] = None + is_vector_collection = False + try: + description = client.describe_collection(name) + except Exception: + description = None + + if description: + for field in description.get("fields", []): + if field.get("type") == DataType.FLOAT_VECTOR: + params = field.get("params") or {} + dimension = self._to_int(params.get("dim")) or 4096 + is_vector_collection = True + break + + estimated_memory = 0 + if is_vector_collection and dimension: + estimated_memory = row_count * dimension * 4 + total_estimated_memory += estimated_memory + + indexes: List[Dict[str, Any]] = [] + try: + index_names = client.list_indexes(name) or [] + except Exception: + index_names = [] + + for index_name in index_names: + try: + detail = client.describe_index(name, index_name) or {} + except Exception: + detail = {} + indexes.append( + { + "index_name": index_name, + "index_type": detail.get("index_type"), + "metric_type": detail.get("metric_type"), + "indexed_rows": self._to_int(detail.get("indexed_rows")), + "pending_index_rows": self._to_int(detail.get("pending_index_rows")), + "state": detail.get("state"), + } + ) + + collections.append( + { + "name": name, + "row_count": row_count, + "dimension": dimension if is_vector_collection else None, + "estimated_memory_bytes": estimated_memory, + "is_vector_collection": is_vector_collection, + "indexes": indexes, + } + ) + + return { + "collections": collections, + "collection_count": len(collections), + "total_vectors": total_vectors, + "estimated_total_memory_bytes": total_estimated_memory, + "db_file_size_bytes": None, + } + + def clear_all_data(self) -> None: + client = self._get_client() + for collection_name in client.list_collections(): + client.drop_collection(collection_name) diff --git a/services/vector_db/providers/qdrant.py b/services/vector_db/providers/qdrant.py new file mode 100644 index 0000000..161d26b --- /dev/null +++ b/services/vector_db/providers/qdrant.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence +from uuid import NAMESPACE_URL, uuid5 + +from qdrant_client import QdrantClient +from qdrant_client.http import models as qmodels + +from .base import BaseVectorProvider + + +class QdrantProvider(BaseVectorProvider): + type = "qdrant" + label = "Qdrant" + description = "Qdrant vector database (HTTP API)." + enabled = True + config_schema: List[Dict[str, Any]] = [ + { + "key": "url", + "label": "Server URL", + "type": "text", + "required": True, + "placeholder": "http://localhost:6333", + }, + { + "key": "api_key", + "label": "API Key", + "type": "password", + "required": False, + }, + ] + + def __init__(self, config: Dict[str, Any] | None = None): + super().__init__(config) + self.client: Optional[QdrantClient] = None + + async def initialize(self) -> None: + url = (self.config.get("url") or "").strip() + if not url: + raise RuntimeError("Qdrant URL is required") + + api_key = (self.config.get("api_key") or None) or None + try: + client = QdrantClient(url=url, api_key=api_key) + # 简单连通性校验 + client.get_collections() + self.client = client + except Exception as exc: # pragma: no cover - 依赖外部服务 + raise RuntimeError(f"Failed to connect to Qdrant at {url}: {exc}") from exc + + def _get_client(self) -> QdrantClient: + if not self.client: + raise RuntimeError("Qdrant client is not initialized") + return self.client + + @staticmethod + def _vector_params(vector: bool, dim: int) -> qmodels.VectorParams: + size = dim if vector and isinstance(dim, int) and dim > 0 else 1 + return qmodels.VectorParams(size=size, distance=qmodels.Distance.COSINE) + + def ensure_collection(self, collection_name: str, vector: bool, dim: int) -> None: + client = self._get_client() + try: + if client.collection_exists(collection_name): + return + except Exception as exc: # pragma: no cover - 依赖外部服务 + raise RuntimeError(f"Failed to check Qdrant collection '{collection_name}': {exc}") from exc + + vectors_config = self._vector_params(vector, dim) + try: + client.create_collection(collection_name=collection_name, vectors_config=vectors_config) + except Exception as exc: # pragma: no cover + if "already exists" in str(exc).lower(): + return + raise RuntimeError(f"Failed to create Qdrant collection '{collection_name}': {exc}") from exc + + @staticmethod + def _point_id(path: str) -> str: + return str(uuid5(NAMESPACE_URL, path)) + + def _prepare_point(self, data: Dict[str, Any]) -> qmodels.PointStruct: + path = data.get("path") + if not path: + raise ValueError("Qdrant upsert requires 'path' in data") + + embedding = data.get("embedding") + if embedding is None: + vector = [0.0] + else: + vector = [float(x) for x in embedding] + + payload = {"path": path} + return qmodels.PointStruct(id=self._point_id(path), vector=vector, payload=payload) + + def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None: + client = self._get_client() + point = self._prepare_point(data) + client.upsert(collection_name=collection_name, wait=True, points=[point]) + + def delete_vector(self, collection_name: str, path: str) -> None: + client = self._get_client() + selector = qmodels.PointIdsList(points=[self._point_id(path)]) + client.delete(collection_name=collection_name, points_selector=selector, wait=True) + + def _format_search_results(self, points: Sequence[qmodels.ScoredPoint]): + return [ + { + "id": point.id, + "distance": point.score, + "entity": {"path": (point.payload or {}).get("path")}, + } + for point in points + ] + + def search_vectors(self, collection_name: str, query_embedding, top_k: int): + client = self._get_client() + vector = [float(x) for x in query_embedding] + points = client.search( + collection_name=collection_name, + query_vector=vector, + limit=top_k, + with_payload=True, + ) + return [self._format_search_results(points)] + + def search_by_path(self, collection_name: str, query_path: str, top_k: int): + client = self._get_client() + results: List[Dict[str, Any]] = [] + offset: Optional[str | int] = None + remaining = max(top_k, 1) + + while len(results) < top_k: + batch_size = min(max(remaining * 2, 10), 200) + records, next_offset = client.scroll( + collection_name=collection_name, + limit=batch_size, + offset=offset, + with_payload=True, + ) + if not records: + break + + for record in records: + path = (record.payload or {}).get("path") + if query_path and path: + if query_path not in path: + continue + results.append({"id": record.id, "distance": 1.0, "entity": {"path": path}}) + if len(results) >= top_k: + break + + if next_offset is None or len(results) >= top_k: + break + offset = next_offset + remaining = top_k - len(results) + + return [results] + + def _extract_vector_config(self, vectors) -> Optional[qmodels.VectorParams]: + if isinstance(vectors, qmodels.VectorParams): + return vectors + if isinstance(vectors, dict): + for value in vectors.values(): + if isinstance(value, qmodels.VectorParams): + return value + return None + + def get_all_stats(self) -> Dict[str, Any]: + client = self._get_client() + try: + response = client.get_collections() + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Failed to list Qdrant collections: {exc}") from exc + + collections: List[Dict[str, Any]] = [] + total_vectors = 0 + total_estimated_memory = 0 + + for description in response.collections or []: + name = description.name + try: + info = client.get_collection(name) + except Exception: + continue + + row_count = int(info.points_count or 0) + total_vectors += row_count + + vector_params = self._extract_vector_config(info.config.params.vectors if info.config and info.config.params else None) + dimension = int(vector_params.size) if vector_params and vector_params.size else None + estimated_memory = row_count * dimension * 4 if dimension else 0 + total_estimated_memory += estimated_memory + distance = str(vector_params.distance) if vector_params and vector_params.distance else None + + indexed_rows = int(info.indexed_vectors_count or 0) + pending_rows = max(row_count - indexed_rows, 0) + + collections.append( + { + "name": name, + "row_count": row_count, + "dimension": dimension, + "estimated_memory_bytes": estimated_memory, + "is_vector_collection": dimension is not None and dimension > 1, + "indexes": [ + { + "index_name": "hnsw", + "index_type": "HNSW", + "metric_type": distance, + "indexed_rows": indexed_rows, + "pending_index_rows": pending_rows, + "state": info.status, + } + ], + } + ) + + return { + "collections": collections, + "collection_count": len(collections), + "total_vectors": total_vectors, + "estimated_total_memory_bytes": total_estimated_memory, + "db_file_size_bytes": None, + } + + def clear_all_data(self) -> None: + client = self._get_client() + try: + response = client.get_collections() + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Failed to list Qdrant collections: {exc}") from exc + + for description in response.collections or []: + try: + client.delete_collection(description.name) + except Exception: + continue diff --git a/services/vector_db/service.py b/services/vector_db/service.py new file mode 100644 index 0000000..2d5c260 --- /dev/null +++ b/services/vector_db/service.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import asyncio +from typing import Any, Dict, Optional + +from .config_manager import VectorDBConfigManager +from .providers import get_provider_class, get_provider_entry +from .providers.base import BaseVectorProvider + +DEFAULT_VECTOR_DIMENSION = 4096 + + +class VectorDBService: + _instance: "VectorDBService" | None = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if not hasattr(self, "_provider"): + self._provider: Optional[BaseVectorProvider] = None + self._provider_type: Optional[str] = None + self._provider_config: Dict[str, Any] | None = None + self._lock = asyncio.Lock() + + async def _ensure_provider(self) -> BaseVectorProvider: + if self._provider is None: + await self.reload() + assert self._provider is not None # for type checker + return self._provider + + async def reload(self) -> BaseVectorProvider: + async with self._lock: + provider_type, provider_config = await VectorDBConfigManager.load_config() + normalized_config = dict(provider_config or {}) + if ( + self._provider + and self._provider_type == provider_type + and self._provider_config == normalized_config + ): + return self._provider + + entry = get_provider_entry(provider_type) + if not entry: + raise RuntimeError(f"Unknown vector database provider: {provider_type}") + if not entry.get("enabled", True): + raise RuntimeError(f"Vector database provider '{provider_type}' is disabled") + + provider_cls = get_provider_class(provider_type) + if not provider_cls: + raise RuntimeError(f"Provider class not found for '{provider_type}'") + + provider = provider_cls(provider_config) + await provider.initialize() + + self._provider = provider + self._provider_type = provider_type + self._provider_config = normalized_config + return provider + + async def ensure_collection(self, collection_name: str, vector: bool = True, dim: int = DEFAULT_VECTOR_DIMENSION) -> None: + provider = await self._ensure_provider() + provider.ensure_collection(collection_name, vector, dim) + + async def upsert_vector(self, collection_name: str, data: Dict[str, Any]) -> None: + provider = await self._ensure_provider() + provider.upsert_vector(collection_name, data) + + async def delete_vector(self, collection_name: str, path: str) -> None: + provider = await self._ensure_provider() + provider.delete_vector(collection_name, path) + + async def search_vectors(self, collection_name: str, query_embedding, top_k: int = 5): + provider = await self._ensure_provider() + return provider.search_vectors(collection_name, query_embedding, top_k) + + async def search_by_path(self, collection_name: str, query_path: str, top_k: int = 20): + provider = await self._ensure_provider() + return provider.search_by_path(collection_name, query_path, top_k) + + async def get_all_stats(self) -> Dict[str, Any]: + provider = await self._ensure_provider() + return provider.get_all_stats() + + async def clear_all_data(self) -> None: + provider = await self._ensure_provider() + provider.clear_all_data() + + async def current_provider(self) -> Dict[str, Any]: + provider_type, provider_config = await VectorDBConfigManager.load_config() + entry = get_provider_entry(provider_type) or {} + return { + "type": provider_type, + "config": provider_config, + "label": entry.get("label"), + "enabled": entry.get("enabled", True), + } diff --git a/uv.lock b/uv.lock index 61a6cfc..4d9745b 100644 --- a/uv.lock +++ b/uv.lock @@ -415,6 +415,7 @@ dependencies = [ { name = "python-multipart" }, { name = "pytz" }, { name = "pyyaml" }, + { name = "qdrant-client" }, { name = "rawpy" }, { name = "rich" }, { name = "rich-toolkit" }, @@ -505,6 +506,7 @@ requires-dist = [ { name = "python-multipart", specifier = "==0.0.20" }, { name = "pytz", specifier = "==2025.2" }, { name = "pyyaml", specifier = "==6.0.2" }, + { name = "qdrant-client", specifier = "==1.15.1" }, { name = "rawpy", specifier = "==0.25.1" }, { name = "rich", specifier = "==14.1.0" }, { name = "rich-toolkit", specifier = "==0.15.0" }, @@ -604,6 +606,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, +] + +[[package]] +name = "hpack" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -647,6 +671,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "idna" version = "3.10" @@ -950,6 +988,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/c7/5572fa4a3f45740eaab6ae86fcdf7195b55beac1371ac8c619d880cfe948/pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa", size = 2512835, upload-time = "2025-07-01T09:15:50.399Z" }, ] +[[package]] +name = "portalocker" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" }, +] + [[package]] name = "propcache" version = "0.3.2" @@ -1161,6 +1211,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, ] +[[package]] +name = "pywin32" +version = "311" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700, upload-time = "2025-07-14T20:13:26.471Z" }, + { url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700, upload-time = "2025-07-14T20:13:28.243Z" }, + { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" }, + { url = "https://files.pythonhosted.org/packages/c9/31/097f2e132c4f16d99a22bfb777e0fd88bd8e1c634304e102f313af69ace5/pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee", size = 8840714, upload-time = "2025-07-14T20:13:32.449Z" }, + { url = "https://files.pythonhosted.org/packages/90/4b/07c77d8ba0e01349358082713400435347df8426208171ce297da32c313d/pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87", size = 9656800, upload-time = "2025-07-14T20:13:34.312Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, +] + [[package]] name = "pyyaml" version = "6.0.2" @@ -1178,6 +1241,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, ] +[[package]] +name = "qdrant-client" +version = "1.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "httpx", extra = ["http2"] }, + { name = "numpy" }, + { name = "portalocker" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/8b/76c7d325e11d97cb8eb5e261c3759e9ed6664735afbf32fdded5b580690c/qdrant_client-1.15.1.tar.gz", hash = "sha256:631f1f3caebfad0fd0c1fba98f41be81d9962b7bf3ca653bed3b727c0e0cbe0e", size = 295297, upload-time = "2025-07-31T19:35:19.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/33/d8df6a2b214ffbe4138db9a1efe3248f67dc3c671f82308bea1582ecbbb7/qdrant_client-1.15.1-py3-none-any.whl", hash = "sha256:2b975099b378382f6ca1cfb43f0d59e541be6e16a5892f282a4b8de7eff5cb63", size = 337331, upload-time = "2025-07-31T19:35:17.539Z" }, +] + [[package]] name = "rawpy" version = "0.25.1" diff --git a/web/src/api/vectorDB.ts b/web/src/api/vectorDB.ts index 57724d8..8a118ab 100644 --- a/web/src/api/vectorDB.ts +++ b/web/src/api/vectorDB.ts @@ -1,5 +1,65 @@ import client from './client'; +export interface VectorDBIndexInfo { + index_name: string; + index_type?: string; + metric_type?: string; + indexed_rows: number; + pending_index_rows: number; + state?: string; +} + +export interface VectorDBCollectionStats { + name: string; + row_count: number; + dimension: number | null; + estimated_memory_bytes: number; + is_vector_collection: boolean; + indexes: VectorDBIndexInfo[]; +} + +export interface VectorDBStats { + collections: VectorDBCollectionStats[]; + collection_count: number; + total_vectors: number; + estimated_total_memory_bytes: number; + db_file_size_bytes: number | null; +} + +export interface VectorDBProviderField { + key: string; + label: string; + type: 'text' | 'password'; + required?: boolean; + default?: string; + placeholder?: string; +} + +export interface VectorDBProviderMeta { + type: string; + label: string; + description?: string; + enabled: boolean; + config_schema: VectorDBProviderField[]; +} + +export interface VectorDBCurrentConfig { + type: string; + config: Record; + label?: string; + enabled?: boolean; +} + +export interface UpdateVectorDBConfigResponse { + config: VectorDBCurrentConfig; + stats: VectorDBStats; +} + export const vectorDBApi = { + getProviders: () => client('/vector-db/providers', { method: 'GET' }), + getConfig: () => client('/vector-db/config', { method: 'GET' }), + getStats: () => client('/vector-db/stats', { method: 'GET' }), + updateConfig: (payload: { type: string; config: Record }) => + client('/vector-db/config', { method: 'POST', json: payload }), clearAll: () => client('/vector-db/clear-all', { method: 'POST' }), -}; \ No newline at end of file +}; diff --git a/web/src/i18n/locales/en.ts b/web/src/i18n/locales/en.ts index 0d56069..86696ef 100644 --- a/web/src/i18n/locales/en.ts +++ b/web/src/i18n/locales/en.ts @@ -205,6 +205,32 @@ export const en = { 'Embedding Dimension': 'Embedding Dimension', 'Vector Database': 'Vector Database', 'Vector Database Settings': 'Vector Database Settings', + 'Current Statistics': 'Current Statistics', + 'Collections': 'Collections', + 'Vectors': 'Vectors', + 'Database Size': 'Database Size', + 'Estimated Memory': 'Estimated Memory', + 'No collections': 'No collections', + 'Dimension': 'Dimension', + 'Non-vector collection': 'Non-vector collection', + 'Estimated memory': 'Estimated memory', + 'Indexes': 'Indexes', + 'Unnamed index': 'Unnamed index', + 'Indexed rows': 'Indexed rows', + 'Pending rows': 'Pending rows', + 'Estimated memory is calculated as vectors x dimension x 4 bytes (float32).': 'Estimated memory is calculated as vectors x dimension x 4 bytes (float32).', + 'Database Provider': 'Database Provider', + 'Please select a provider': 'Please select a provider', + 'Coming soon': 'Coming soon', + 'This provider is not available yet': 'This provider is not available yet', + 'Database file path': 'Database file path', + 'Server URI': 'Server URI', + 'Token': 'Token', + 'Server URL': 'Server URL', + 'API Key': 'API Key', + 'Embedded Milvus Lite (local file storage).': 'Embedded Milvus Lite (local file storage).', + 'Remote Milvus instance accessed via URI.': 'Remote Milvus instance accessed via URI.', + 'Qdrant vector database (HTTP API).': 'Qdrant vector database (HTTP API).', 'Database Type': 'Database Type', 'Confirm embedding dimension change': 'Confirm embedding dimension change', 'Changing the embedding dimension will clear the vector database automatically. You will need to rebuild indexes afterwards. Continue?': 'Changing the embedding dimension will clear the vector database automatically. You will need to rebuild indexes afterwards. Continue?', diff --git a/web/src/i18n/locales/zh.ts b/web/src/i18n/locales/zh.ts index 3049168..370b62e 100644 --- a/web/src/i18n/locales/zh.ts +++ b/web/src/i18n/locales/zh.ts @@ -207,6 +207,32 @@ export const zh = { 'Embedding Dimension': '向量维度', 'Vector Database': '向量数据库', 'Vector Database Settings': '向量数据库设置', + 'Current Statistics': '当前统计', + 'Collections': '集合', + 'Vectors': '向量', + 'Database Size': '数据库大小', + 'Estimated Memory': '估算内存', + 'No collections': '暂无集合', + 'Dimension': '维度', + 'Non-vector collection': '非向量集合', + 'Estimated memory': '估算内存', + 'Indexes': '索引', + 'Unnamed index': '未命名索引', + 'Indexed rows': '已索引行数', + 'Pending rows': '待索引行数', + 'Estimated memory is calculated as vectors x dimension x 4 bytes (float32).': '估算内存 = 向量数量 x 维度 x 4 字节（float32）。', + 'Database Provider': '数据库提供者', + 'Please select a provider': '请选择提供者', + 'Coming soon': '敬请期待', + 'This provider is not available yet': '该提供者暂不可用', + 'Database file path': '数据库文件路径', + 'Server URI': '服务器 URI', + 'Token': '令牌', + 'Server URL': '服务器地址', + 'API Key': 'API Key', + 'Embedded Milvus Lite (local file storage).': '嵌入式 Milvus Lite，本地文件存储。', + 'Remote Milvus instance accessed via URI.': '通过 URI 访问的远程 Milvus 实例。', + 'Qdrant vector database (HTTP API).': 'Qdrant 向量数据库（HTTP API）。', 'Database Type': '数据库类型', 'Confirm embedding dimension change': '确认修改向量维度', 'Changing the embedding dimension will clear the vector database automatically. You will need to rebuild indexes afterwards. Continue?': '修改向量维度会自动清空向量数据库，之后需要重建索引，是否继续？', diff --git a/web/src/pages/SystemSettingsPage/SystemSettingsPage.tsx b/web/src/pages/SystemSettingsPage/SystemSettingsPage.tsx index b6ebbf9..061e38c 100644 --- a/web/src/pages/SystemSettingsPage/SystemSettingsPage.tsx +++ b/web/src/pages/SystemSettingsPage/SystemSettingsPage.tsx @@ -1,8 +1,8 @@ -import { Form, Input, Button, message, Tabs, Space, Card, Select, Modal, Radio, InputNumber } from 'antd'; -import { useEffect, useState } from 'react'; +import { Form, Input, Button, message, Tabs, Space, Card, Select, Modal, Radio, InputNumber, Spin, Empty, Alert } from 'antd'; +import { useEffect, useState, useCallback } from 'react'; import PageCard from '../../components/PageCard'; import { getAllConfig, setConfig } from '../../api/config'; -import { vectorDBApi } from '../../api/vectorDB'; +import { vectorDBApi, type VectorDBStats, type VectorDBProviderMeta, type VectorDBCurrentConfig } from '../../api/vectorDB'; import { AppstoreOutlined, RobotOutlined, DatabaseOutlined, SkinOutlined } from '@ant-design/icons'; import { useTheme } from '../../contexts/ThemeContext'; import '../../styles/settings-tabs.css'; @@ -32,6 +32,20 @@ const EMBED_CONFIG_KEYS = [ const ALL_AI_KEYS = [...VISION_CONFIG_KEYS, ...EMBED_CONFIG_KEYS, { key: EMBED_DIM_KEY, default: DEFAULT_EMBED_DIMENSION }]; +const formatBytes = (bytes?: number | null) => { + if (bytes === null || bytes === undefined) return '-'; + if (bytes === 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let value = bytes; + let unitIndex = 0; + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024; + unitIndex += 1; + } + const precision = value >= 10 || unitIndex === 0 ? 0 : 1; + return `${value.toFixed(precision)} ${units[unitIndex]}`; +}; + // Theme related config keys const THEME_KEYS = { MODE: 'THEME_MODE', @@ -42,9 +56,19 @@ const THEME_KEYS = { }; export default function SystemSettingsPage() { + const [vectorConfigForm] = Form.useForm(); const [loading, setLoading] = useState(false); const [config, setConfigState] = useState | null>(null); const [activeTab, setActiveTab] = useState('appearance'); + const [vectorStats, setVectorStats] = useState(null); + const [vectorStatsLoading, setVectorStatsLoading] = useState(false); + const [vectorStatsError, setVectorStatsError] = useState(null); + const [vectorProviders, setVectorProviders] = useState([]); + const [vectorConfig, setVectorConfig] = useState(null); + const [vectorConfigLoading, setVectorConfigLoading] = useState(false); + const [vectorConfigSaving, setVectorConfigSaving] = useState(false); + const [vectorMetaError, setVectorMetaError] = useState(null); + const [selectedProviderType, setSelectedProviderType] = useState(null); const { refreshTheme, previewTheme } = useTheme(); const { t } = useI18n(); @@ -52,6 +76,72 @@ export default function SystemSettingsPage() { getAllConfig().then((data) => setConfigState(data as Record)); }, []); + const fetchVectorStats = useCallback(async () => { + setVectorStatsLoading(true); + setVectorStatsError(null); + try { + const data = await vectorDBApi.getStats(); + setVectorStats(data); + } catch (e: any) { + const msg = e?.message || t('Load failed'); + setVectorStatsError(msg); + message.error(msg); + } finally { + setVectorStatsLoading(false); + } + }, [t]); + + const buildProviderConfigValues = useCallback((provider: VectorDBProviderMeta | undefined, existing?: Record) => { + if (!provider) return {}; + const values: Record = {}; + const schema = provider.config_schema || []; + schema.forEach((field) => { + const current = existing && existing[field.key] !== undefined && existing[field.key] !== null + ? String(existing[field.key]) + : undefined; + if (current !== undefined) { + values[field.key] = current; + } else if (field.default !== undefined && field.default !== null) { + values[field.key] = String(field.default); + } else { + values[field.key] = ''; + } + }); + return values; + }, []); + + const fetchVectorMeta = useCallback(async () => { + setVectorConfigLoading(true); + setVectorMetaError(null); + try { + const [providers, current] = await Promise.all([ + vectorDBApi.getProviders(), + vectorDBApi.getConfig(), + ]); + setVectorProviders(providers); + setVectorConfig(current); + + const enabled = providers.filter((item) => item.enabled); + let nextType: string | null = current?.type ?? null; + if (nextType && !providers.some((item) => item.type === nextType)) { + nextType = null; + } + if (!nextType) { + nextType = enabled[0]?.type ?? providers[0]?.type ?? null; + } + setSelectedProviderType(nextType); + const provider = providers.find((item) => item.type === nextType); + const configValues = buildProviderConfigValues(provider, nextType === current?.type ? current?.config : undefined); + vectorConfigForm.setFieldsValue({ type: nextType || undefined, config: configValues }); + } catch (e: any) { + const msg = e?.message || t('Load failed'); + setVectorMetaError(msg); + message.error(msg); + } finally { + setVectorConfigLoading(false); + } + }, [buildProviderConfigValues, message, t, vectorConfigForm]); + const handleSave = async (values: any) => { setLoading(true); try { @@ -70,6 +160,40 @@ export default function SystemSettingsPage() { setLoading(false); }; + const handleProviderChange = useCallback((value: string) => { + setSelectedProviderType(value); + const provider = vectorProviders.find((item) => item.type === value); + const existing = value === vectorConfig?.type ? vectorConfig?.config : undefined; + const configValues = buildProviderConfigValues(provider, existing); + vectorConfigForm.setFieldsValue({ type: value, config: configValues }); + }, [vectorProviders, vectorConfig, buildProviderConfigValues, vectorConfigForm]); + + const handleVectorConfigSave = useCallback(async (values: { type: string; config?: Record }) => { + if (!values?.type) { + return; + } + setVectorConfigSaving(true); + try { + const configPayload = Object.fromEntries( + Object.entries(values.config || {}).filter(([, val]) => val !== undefined && val !== null && String(val).trim() !== '') + .map(([key, val]) => [key, String(val)]) + ); + const response = await vectorDBApi.updateConfig({ type: values.type, config: configPayload }); + setVectorConfig(response.config); + setVectorStats(response.stats); + setVectorStatsError(null); + setSelectedProviderType(response.config.type); + const provider = vectorProviders.find((item) => item.type === response.config.type); + const mergedValues = buildProviderConfigValues(provider, response.config.config); + vectorConfigForm.setFieldsValue({ type: response.config.type, config: mergedValues }); + message.success(t('Saved successfully')); + } catch (e: any) { + message.error(e?.message || t('Save failed')); + } finally { + setVectorConfigSaving(false); + } + }, [buildProviderConfigValues, message, t, vectorConfigForm, vectorProviders]); + // 离开“外观设置”时，恢复后端持久化配置（取消未保存的预览） useEffect(() => { if (activeTab !== 'appearance') { @@ -77,6 +201,27 @@ export default function SystemSettingsPage() { } }, [activeTab]); + useEffect(() => { + if (activeTab === 'vector-db') { + if (!vectorProviders.length && !vectorConfigLoading) { + fetchVectorMeta(); + } + if (!vectorStats && !vectorStatsLoading) { + fetchVectorStats(); + } + } + }, [ + activeTab, + fetchVectorMeta, + fetchVectorStats, + vectorProviders.length, + vectorConfigLoading, + vectorStats, + vectorStatsLoading, + ]); + + const selectedProvider = vectorProviders.find((item) => item.type === selectedProviderType || (!selectedProviderType && item.enabled)); + if (!config) { return
{t('Loading...')}
; } @@ -275,41 +420,187 @@ export default function SystemSettingsPage() { ), children: ( -
- - ({ + value: provider.type, + label: provider.enabled ? provider.label : `${provider.label} (${t('Coming soon')})`, + disabled: !provider.enabled, + }))} + onChange={handleProviderChange} + loading={vectorConfigLoading && !vectorProviders.length} + /> + + {selectedProvider?.description ? ( + + ) : null} + {selectedProvider?.config_schema?.map((field) => ( + + {field.type === 'password' ? ( + + ) : ( + + )} + + ))} + {selectedProvider && !selectedProvider.enabled ? ( + + ) : null} + + + + + + + + )} +
), },