feat: add voyage ai as a new model provider (#8747)

api/tests/integration_tests/model_runtime/voyage/test_provider.py

@@ -0,0 +1,25 @@
+import os
+from unittest.mock import Mock, patch
+
+import pytest
+
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.voyage.voyage import VoyageProvider
+
+
+def test_validate_provider_credentials():
+    provider = VoyageProvider()
+
+    with pytest.raises(CredentialsValidateFailedError):
+        provider.validate_provider_credentials(credentials={"api_key": "hahahaha"})
+    with patch("requests.post") as mock_post:
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "object": "list",
+            "data": [{"object": "embedding", "embedding": [0.23333 for _ in range(1024)], "index": 0}],
+            "model": "voyage-3",
+            "usage": {"total_tokens": 1},
+        }
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+        provider.validate_provider_credentials(credentials={"api_key": os.environ.get("VOYAGE_API_KEY")})

api/tests/integration_tests/model_runtime/voyage/test_rerank.py

@@ -0,0 +1,92 @@
+import os
+from unittest.mock import Mock, patch
+
+import pytest
+
+from core.model_runtime.entities.rerank_entities import RerankResult
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.voyage.rerank.rerank import VoyageRerankModel
+
+
+def test_validate_credentials():
+    model = VoyageRerankModel()
+
+    with pytest.raises(CredentialsValidateFailedError):
+        model.validate_credentials(
+            model="rerank-lite-1",
+            credentials={"api_key": "invalid_key"},
+        )
+    with patch("httpx.post") as mock_post:
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "object": "list",
+            "data": [
+                {
+                    "relevance_score": 0.546875,
+                    "index": 0,
+                    "document": "Carson City is the capital city of the American state of Nevada. At the 2010 United "
+                    "States Census, Carson City had a population of 55,274.",
+                },
+                {
+                    "relevance_score": 0.4765625,
+                    "index": 1,
+                    "document": "The Commonwealth of the Northern Mariana Islands is a group of islands in the "
+                    "Pacific Ocean that are a political division controlled by the United States. Its "
+                    "capital is Saipan.",
+                },
+            ],
+            "model": "rerank-lite-1",
+            "usage": {"total_tokens": 96},
+        }
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+        model.validate_credentials(
+            model="rerank-lite-1",
+            credentials={
+                "api_key": os.environ.get("VOYAGE_API_KEY"),
+            },
+        )
+
+
+def test_invoke_model():
+    model = VoyageRerankModel()
+    with patch("httpx.post") as mock_post:
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "object": "list",
+            "data": [
+                {
+                    "relevance_score": 0.84375,
+                    "index": 0,
+                    "document": "Kasumi is a girl name of Japanese origin meaning mist.",
+                },
+                {
+                    "relevance_score": 0.4765625,
+                    "index": 1,
+                    "document": "Her music is a kawaii bass, a mix of future bass, pop, and kawaii music and she "
+                    "leads a team named PopiParty.",
+                },
+            ],
+            "model": "rerank-lite-1",
+            "usage": {"total_tokens": 59},
+        }
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+        result = model.invoke(
+            model="rerank-lite-1",
+            credentials={
+                "api_key": os.environ.get("VOYAGE_API_KEY"),
+            },
+            query="Who is Kasumi?",
+            docs=[
+                "Kasumi is a girl name of Japanese origin meaning mist.",
+                "Her music is a kawaii bass, a mix of future bass, pop, and kawaii music and she leads a team named "
+                "PopiParty.",
+            ],
+            score_threshold=0.5,
+        )
+
+        assert isinstance(result, RerankResult)
+        assert len(result.docs) == 1
+        assert result.docs[0].index == 0
+        assert result.docs[0].score >= 0.5

api/tests/integration_tests/model_runtime/voyage/test_text_embedding.py

@@ -0,0 +1,70 @@
+import os
+from unittest.mock import Mock, patch
+
+import pytest
+
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.voyage.text_embedding.text_embedding import VoyageTextEmbeddingModel
+
+
+def test_validate_credentials():
+    model = VoyageTextEmbeddingModel()
+
+    with pytest.raises(CredentialsValidateFailedError):
+        model.validate_credentials(model="voyage-3", credentials={"api_key": "invalid_key"})
+    with patch("requests.post") as mock_post:
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "object": "list",
+            "data": [{"object": "embedding", "embedding": [0.23333 for _ in range(1024)], "index": 0}],
+            "model": "voyage-3",
+            "usage": {"total_tokens": 1},
+        }
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+        model.validate_credentials(model="voyage-3", credentials={"api_key": os.environ.get("VOYAGE_API_KEY")})
+
+
+def test_invoke_model():
+    model = VoyageTextEmbeddingModel()
+
+    with patch("requests.post") as mock_post:
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "object": "list",
+            "data": [
+                {"object": "embedding", "embedding": [0.23333 for _ in range(1024)], "index": 0},
+                {"object": "embedding", "embedding": [0.23333 for _ in range(1024)], "index": 1},
+            ],
+            "model": "voyage-3",
+            "usage": {"total_tokens": 2},
+        }
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+        result = model.invoke(
+            model="voyage-3",
+            credentials={
+                "api_key": os.environ.get("VOYAGE_API_KEY"),
+            },
+            texts=["hello", "world"],
+            user="abc-123",
+        )
+
+        assert isinstance(result, TextEmbeddingResult)
+        assert len(result.embeddings) == 2
+        assert result.usage.total_tokens == 2
+
+
+def test_get_num_tokens():
+    model = VoyageTextEmbeddingModel()
+
+    num_tokens = model.get_num_tokens(
+        model="voyage-3",
+        credentials={
+            "api_key": os.environ.get("VOYAGE_API_KEY"),
+        },
+        texts=["ping"],
+    )
+
+    assert num_tokens == 1