tts models support (#2033)

Co-authored-by: luowei <glpat-EjySCyNjWiLqAED-YmwM>
Co-authored-by: crazywoola <427733928@qq.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: Yeuoly <45712896+Yeuoly@users.noreply.github.com>

api/services/account_service.py

@@ -86,13 +86,13 @@ class AccountService:
                 db.session.commit()
 
         return account
-    
+
     @staticmethod
     def get_account_jwt_token(account):
         payload = {
             "user_id": account.id,
             "exp": datetime.utcnow() + timedelta(days=30),
-            "iss":  current_app.config['EDITION'],
+            "iss": current_app.config['EDITION'],
             "sub": 'Console API Passport',
         }
 
@@ -345,7 +345,7 @@ class TenantService:
         }
         if action not in ['add', 'remove', 'update']:
             raise InvalidActionError("Invalid action.")
-        
+
         if member:
             if operator.id == member.id:
                 raise CannotOperateSelfError("Cannot operate self.")
@@ -546,10 +546,10 @@ class RegisterService:
             return None
 
         return {
-                'account': account,
-                'data': invitation_data,
-                'tenant': tenant,
-                }
+            'account': account,
+            'data': invitation_data,
+            'tenant': tenant,
+        }
 
     @classmethod
     def _get_invitation_by_token(cls, token: str, workspace_id: str, email: str) -> Optional[Dict[str, str]]:

api/services/app_model_config_service.py

@@ -95,6 +95,21 @@ class AppModelConfigService:
         if not isinstance(config["speech_to_text"]["enabled"], bool):
             raise ValueError("enabled in speech_to_text must be of boolean type")
 
+        # text_to_speech
+        if 'text_to_speech' not in config or not config["text_to_speech"]:
+            config["text_to_speech"] = {
+                "enabled": False
+            }
+
+        if not isinstance(config["text_to_speech"], dict):
+            raise ValueError("text_to_speech must be of dict type")
+
+        if "enabled" not in config["text_to_speech"] or not config["text_to_speech"]["enabled"]:
+            config["text_to_speech"]["enabled"] = False
+
+        if not isinstance(config["text_to_speech"]["enabled"], bool):
+            raise ValueError("enabled in text_to_speech must be of boolean type")
+
         # return retriever resource
         if 'retriever_resource' not in config or not config["retriever_resource"]:
             config["retriever_resource"] = {
@@ -317,6 +332,7 @@ class AppModelConfigService:
             "suggested_questions": config["suggested_questions"],
             "suggested_questions_after_answer": config["suggested_questions_after_answer"],
             "speech_to_text": config["speech_to_text"],
+            "text_to_speech": config["text_to_speech"],
             "retriever_resource": config["retriever_resource"],
             "more_like_this": config["more_like_this"],
             "sensitive_word_avoidance": config["sensitive_word_avoidance"],

api/services/audio_service.py

@@ -1,22 +1,26 @@
 import io
+from typing import Optional
 
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
+from services.errors.audio import (AudioTooLargeServiceError,
+                                   NoAudioUploadedServiceError,
+                                   ProviderNotSupportTextToSpeechServiceError,
+                                   ProviderNotSupportSpeechToTextServiceError,
+                                   UnsupportedAudioTypeServiceError)
 from werkzeug.datastructures import FileStorage
 
 FILE_SIZE = 15
 FILE_SIZE_LIMIT = FILE_SIZE * 1024 * 1024
-ALLOWED_EXTENSIONS = ['mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav', 'webm']
+ALLOWED_EXTENSIONS = ['mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav', 'webm', 'amr']
 
 
 class AudioService:
     @classmethod
-    def transcript(cls, tenant_id: str, file: FileStorage):
+    def transcript_asr(cls, tenant_id: str, file: FileStorage, end_user: Optional[str] = None):
         if file is None:
             raise NoAudioUploadedServiceError()
-        
+
         extension = file.mimetype
         if extension not in [f'audio/{ext}' for ext in ALLOWED_EXTENSIONS]:
             raise UnsupportedAudioTypeServiceError()
@@ -33,8 +37,26 @@ class AudioService:
             tenant_id=tenant_id,
             model_type=ModelType.SPEECH2TEXT
         )
+        if model_instance is None:
+            raise ProviderNotSupportSpeechToTextServiceError()
 
         buffer = io.BytesIO(file_content)
         buffer.name = 'temp.mp3'
 
-        return {"text": model_instance.invoke_speech2text(buffer)}
+        return {"text": model_instance.invoke_speech2text(file=buffer, user=end_user)}
+
+    @classmethod
+    def transcript_tts(cls, tenant_id: str, text: str, streaming: bool, end_user: Optional[str] = None):
+        model_manager = ModelManager()
+        model_instance = model_manager.get_default_model_instance(
+            tenant_id=tenant_id,
+            model_type=ModelType.TTS
+        )
+        if model_instance is None:
+            raise ProviderNotSupportTextToSpeechServiceError()
+
+        try:
+            audio_response = model_instance.invoke_tts(content_text=text.strip(), user=end_user, streaming=streaming)
+            return audio_response
+        except Exception as e:
+            raise e

api/services/errors/audio.py

@@ -9,5 +9,10 @@ class AudioTooLargeServiceError(Exception):
 class UnsupportedAudioTypeServiceError(Exception):
     pass
 
+
 class ProviderNotSupportSpeechToTextServiceError(Exception):
-    pass
+    pass
+
+
+class ProviderNotSupportTextToSpeechServiceError(Exception):
+    pass