test: add unit tests for vector stores of Milvus, Qdrant and Weaviate (#3688)

api/tests/integration_tests/vdb/milvus/test_milvus.py

@@ -0,0 +1,38 @@
+import uuid
+
+from core.rag.datasource.vdb.milvus.milvus_vector import MilvusConfig, MilvusVector
+from models.dataset import Dataset
+from tests.integration_tests.vdb.test_vector_store import (
+    get_sample_document,
+    get_sample_embedding,
+    get_sample_query_vector,
+    setup_mock_redis,
+)
+
+
+def test_milvus_vector(setup_mock_redis) -> None:
+    dataset_id = str(uuid.uuid4())
+    vector = MilvusVector(
+        collection_name=Dataset.gen_collection_name_by_id(dataset_id),
+        config=MilvusConfig(
+            host='localhost',
+            port=19530,
+            user='root',
+            password='Milvus',
+        )
+    )
+
+    # create vector
+    vector.create(
+        texts=[get_sample_document(dataset_id)],
+        embeddings=[get_sample_embedding()],
+    )
+
+    # search by vector
+    hits_by_vector = vector.search_by_vector(query_vector=get_sample_query_vector())
+    assert len(hits_by_vector) >= 1
+
+    # milvus dos not support full text searching yet in < 2.3.x
+
+    # delete vector
+    vector.delete()

api/tests/integration_tests/vdb/qdrant/test_qdrant.py

@@ -0,0 +1,40 @@
+import uuid
+
+from core.rag.datasource.vdb.qdrant.qdrant_vector import QdrantConfig, QdrantVector
+from models.dataset import Dataset
+from tests.integration_tests.vdb.test_vector_store import (
+    get_sample_document,
+    get_sample_embedding,
+    get_sample_query_vector,
+    get_sample_text,
+    setup_mock_redis,
+)
+
+
+def test_qdrant_vector(setup_mock_redis)-> None:
+    dataset_id = str(uuid.uuid4())
+    vector = QdrantVector(
+        collection_name=Dataset.gen_collection_name_by_id(dataset_id),
+        group_id=dataset_id,
+        config=QdrantConfig(
+            endpoint='http://localhost:6333',
+            api_key='difyai123456',
+        )
+    )
+
+    # create vector
+    vector.create(
+        texts=[get_sample_document(dataset_id)],
+        embeddings=[get_sample_embedding()],
+    )
+
+    # search by vector
+    hits_by_vector = vector.search_by_vector(query_vector=get_sample_query_vector())
+    assert len(hits_by_vector) >= 1
+
+    # search by full text
+    hits_by_full_text = vector.search_by_full_text(query=get_sample_text())
+    assert len(hits_by_full_text) >= 1
+
+    # delete vector
+    vector.delete()

api/tests/integration_tests/vdb/test_vector_store.py

@@ -0,0 +1,46 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from core.rag.models.document import Document
+from extensions import ext_redis
+
+
+def get_sample_text() -> str:
+    return 'test_text'
+
+
+def get_sample_embedding() -> list[float]:
+    return [1.1, 2.2, 3.3]
+
+
+def get_sample_query_vector() -> list[float]:
+    return get_sample_embedding()
+
+
+def get_sample_document(sample_dataset_id: str) -> Document:
+    doc = Document(
+        page_content=get_sample_text(),
+        metadata={
+            "doc_id": sample_dataset_id,
+            "doc_hash": sample_dataset_id,
+            "document_id": sample_dataset_id,
+            "dataset_id": sample_dataset_id,
+        }
+    )
+    return doc
+
+
+@pytest.fixture
+def setup_mock_redis() -> None:
+    # get
+    ext_redis.redis_client.get = MagicMock(return_value=None)
+
+    # set
+    ext_redis.redis_client.set = MagicMock(return_value=None)
+
+    # lock
+    mock_redis_lock = MagicMock()
+    mock_redis_lock.__enter__ = MagicMock()
+    mock_redis_lock.__exit__ = MagicMock()
+    ext_redis.redis_client.lock = mock_redis_lock

api/tests/integration_tests/vdb/weaviate/test_weaviate.py

@@ -0,0 +1,41 @@
+import uuid
+
+from core.rag.datasource.vdb.weaviate.weaviate_vector import WeaviateConfig, WeaviateVector
+from models.dataset import Dataset
+from tests.integration_tests.vdb.test_vector_store import (
+    get_sample_document,
+    get_sample_embedding,
+    get_sample_query_vector,
+    get_sample_text,
+    setup_mock_redis,
+)
+
+
+def test_weaviate_vector(setup_mock_redis) -> None:
+    attributes = ['doc_id', 'dataset_id', 'document_id', 'doc_hash']
+    dataset_id = str(uuid.uuid4())
+    vector = WeaviateVector(
+        collection_name=Dataset.gen_collection_name_by_id(dataset_id),
+        config=WeaviateConfig(
+            endpoint='http://localhost:8080',
+            api_key='WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih',
+        ),
+        attributes=attributes
+    )
+
+    # create vector
+    vector.create(
+        texts=[get_sample_document(dataset_id)],
+        embeddings=[get_sample_embedding()],
+    )
+
+    # search by vector
+    hits_by_vector = vector.search_by_vector(query_vector=get_sample_query_vector())
+    assert len(hits_by_vector) >= 1
+
+    # search by full text
+    hits_by_full_text = vector.search_by_full_text(query=get_sample_text())
+    assert len(hits_by_full_text) >= 1
+
+    # delete vector
+    vector.delete()

api/tests/unit_tests/core/rag/datasource/vdb/milvus/test_milvus.py

@@ -18,7 +18,7 @@ def test_default_value():
         with pytest.raises(ValidationError) as e:
             MilvusConfig(**config)
         assert e.value.errors()[1]['msg'] == f'config MILVUS_{key.upper()} is required'
-    
+
     config = MilvusConfig(**valid_config)
     assert config.secure is False
     assert config.database == 'default'