improve: introduce isort for linting Python imports (#1983)

api/core/agent/agent/agent_llm_callback.py

@@ -1,10 +1,10 @@
 import logging
-from typing import Optional, List
+from typing import List, Optional
 
 from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler
 from core.model_runtime.callbacks.base_callback import Callback
-from core.model_runtime.entities.llm_entities import LLMResultChunk, LLMResult
-from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
 logger = logging.getLogger(__name__)

api/core/agent/agent/calc_token_mixin.py

@@ -1,12 +1,11 @@
 from typing import List, cast
 
-from langchain.schema import BaseMessage
-
 from core.entities.application_entities import ModelConfigEntity
 from core.entities.message_entities import lc_messages_to_prompt_messages
 from core.model_runtime.entities.message_entities import PromptMessage
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from langchain.schema import BaseMessage
 
 
 class CalcTokenMixin:

api/core/agent/agent/multi_dataset_router_agent.py

@@ -1,20 +1,19 @@
-from typing import Tuple, List, Any, Union, Sequence, Optional, cast
+from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
-from langchain.agents import OpenAIFunctionsAgent, BaseSingleActionAgent
+from core.entities.application_entities import ModelConfigEntity
+from core.entities.message_entities import lc_messages_to_prompt_messages
+from core.model_manager import ModelInstance
+from core.model_runtime.entities.message_entities import PromptMessageTool
+from core.third_party.langchain.llms.fake import FakeLLM
+from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
 from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
 from langchain.callbacks.base import BaseCallbackManager
 from langchain.callbacks.manager import Callbacks
 from langchain.prompts.chat import BaseMessagePromptTemplate
-from langchain.schema import AgentAction, AgentFinish, SystemMessage, AIMessage
+from langchain.schema import AgentAction, AgentFinish, AIMessage, SystemMessage
 from langchain.tools import BaseTool
 from pydantic import root_validator
 
-from core.entities.application_entities import ModelConfigEntity
-from core.model_manager import ModelInstance
-from core.entities.message_entities import lc_messages_to_prompt_messages
-from core.model_runtime.entities.message_entities import PromptMessageTool
-from core.third_party.langchain.llms.fake import FakeLLM
-
 
 class MultiDatasetRouterAgent(OpenAIFunctionsAgent):
     """

api/core/agent/agent/openai_function_call.py

@@ -1,28 +1,26 @@
-from typing import List, Tuple, Any, Union, Sequence, Optional, cast
+from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
-from langchain.agents import OpenAIFunctionsAgent, BaseSingleActionAgent
-from langchain.agents.openai_functions_agent.base import _parse_ai_message, \
-    _format_intermediate_steps
+from core.agent.agent.agent_llm_callback import AgentLLMCallback
+from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
+from core.chain.llm_chain import LLMChain
+from core.entities.application_entities import ModelConfigEntity
+from core.entities.message_entities import lc_messages_to_prompt_messages
+from core.model_manager import ModelInstance
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.third_party.langchain.llms.fake import FakeLLM
+from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
+from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
 from langchain.callbacks.base import BaseCallbackManager
 from langchain.callbacks.manager import Callbacks
 from langchain.chat_models.openai import _convert_message_to_dict, _import_tiktoken
 from langchain.memory.prompt import SUMMARY_PROMPT
 from langchain.prompts.chat import BaseMessagePromptTemplate
-from langchain.schema import AgentAction, AgentFinish, SystemMessage, AIMessage, HumanMessage, BaseMessage, \
-    get_buffer_string
+from langchain.schema import (AgentAction, AgentFinish, AIMessage, BaseMessage, HumanMessage, SystemMessage,
+                              get_buffer_string)
 from langchain.tools import BaseTool
 from pydantic import root_validator
 
-from core.agent.agent.agent_llm_callback import AgentLLMCallback
-from core.agent.agent.calc_token_mixin import ExceededLLMTokensLimitError, CalcTokenMixin
-from core.chain.llm_chain import LLMChain
-from core.entities.application_entities import ModelConfigEntity
-from core.model_manager import ModelInstance
-from core.entities.message_entities import lc_messages_to_prompt_messages
-from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage
-from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.third_party.langchain.llms.fake import FakeLLM
-
 
 class AutoSummarizingOpenAIFunctionCallAgent(OpenAIFunctionsAgent, CalcTokenMixin):
     moving_summary_buffer: str = ""

api/core/agent/agent/output_parser/structured_chat.py

@@ -2,8 +2,8 @@ import json
 import re
 from typing import Union
 
-from langchain.agents.structured_chat.output_parser import StructuredChatOutputParser as LCStructuredChatOutputParser, \
-    logger
+from langchain.agents.structured_chat.output_parser import StructuredChatOutputParser as LCStructuredChatOutputParser
+from langchain.agents.structured_chat.output_parser import logger
 from langchain.schema import AgentAction, AgentFinish, OutputParserException
 
 

api/core/agent/agent/structed_multi_dataset_router_agent.py

@@ -1,18 +1,17 @@
 import re
-from typing import List, Tuple, Any, Union, Sequence, Optional, cast
-
-from langchain import BasePromptTemplate, PromptTemplate
-from langchain.agents import StructuredChatAgent, AgentOutputParser, Agent
-from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
-from langchain.callbacks.base import BaseCallbackManager
-from langchain.callbacks.manager import Callbacks
-from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, ChatPromptTemplate
-from langchain.schema import AgentAction, AgentFinish, OutputParserException
-from langchain.tools import BaseTool
-from langchain.agents.structured_chat.prompt import PREFIX, SUFFIX
+from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
 from core.chain.llm_chain import LLMChain
 from core.entities.application_entities import ModelConfigEntity
+from langchain import BasePromptTemplate, PromptTemplate
+from langchain.agents import Agent, AgentOutputParser, StructuredChatAgent
+from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
+from langchain.agents.structured_chat.prompt import PREFIX, SUFFIX
+from langchain.callbacks.base import BaseCallbackManager
+from langchain.callbacks.manager import Callbacks
+from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate
+from langchain.schema import AgentAction, AgentFinish, OutputParserException
+from langchain.tools import BaseTool
 
 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
 The nouns in the format of "Thought", "Action", "Action Input", "Final Answer" must be expressed in English.

api/core/agent/agent/structured_chat.py

@@ -1,23 +1,22 @@
 import re
-from typing import List, Tuple, Any, Union, Sequence, Optional, cast
-
-from langchain import BasePromptTemplate, PromptTemplate
-from langchain.agents import StructuredChatAgent, AgentOutputParser, Agent
-from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
-from langchain.callbacks.base import BaseCallbackManager
-from langchain.callbacks.manager import Callbacks
-from langchain.memory.prompt import SUMMARY_PROMPT
-from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, ChatPromptTemplate
-from langchain.schema import AgentAction, AgentFinish, AIMessage, HumanMessage, OutputParserException, BaseMessage, \
-    get_buffer_string
-from langchain.tools import BaseTool
-from langchain.agents.structured_chat.prompt import PREFIX, SUFFIX
+from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
 from core.chain.llm_chain import LLMChain
 from core.entities.application_entities import ModelConfigEntity
 from core.entities.message_entities import lc_messages_to_prompt_messages
+from langchain import BasePromptTemplate, PromptTemplate
+from langchain.agents import Agent, AgentOutputParser, StructuredChatAgent
+from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
+from langchain.agents.structured_chat.prompt import PREFIX, SUFFIX
+from langchain.callbacks.base import BaseCallbackManager
+from langchain.callbacks.manager import Callbacks
+from langchain.memory.prompt import SUMMARY_PROMPT
+from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate
+from langchain.schema import (AgentAction, AgentFinish, AIMessage, BaseMessage, HumanMessage, OutputParserException,
+                              get_buffer_string)
+from langchain.tools import BaseTool
 
 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
 The nouns in the format of "Thought", "Action", "Action Input", "Final Answer" must be expressed in English.

api/core/agent/agent_executor.py

@@ -1,11 +1,6 @@
 import enum
 import logging
-from typing import Union, Optional
-
-from langchain.agents import BaseSingleActionAgent, BaseMultiActionAgent
-from langchain.callbacks.manager import Callbacks
-from langchain.tools import BaseTool
-from pydantic import BaseModel, Extra
+from typing import Optional, Union
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.agent.agent.multi_dataset_router_agent import MultiDatasetRouterAgent
@@ -13,8 +8,6 @@ from core.agent.agent.openai_function_call import AutoSummarizingOpenAIFunctionC
 from core.agent.agent.output_parser.structured_chat import StructuredChatOutputParser
 from core.agent.agent.structed_multi_dataset_router_agent import StructuredMultiDatasetRouterAgent
 from core.agent.agent.structured_chat import AutoSummarizingStructuredChatAgent
-from langchain.agents import AgentExecutor as LCAgentExecutor
-
 from core.entities.application_entities import ModelConfigEntity
 from core.entities.message_entities import prompt_messages_to_lc_messages
 from core.helper import moderation
@@ -22,6 +15,11 @@ from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.errors.invoke import InvokeError
 from core.tool.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
 from core.tool.dataset_retriever_tool import DatasetRetrieverTool
+from langchain.agents import AgentExecutor as LCAgentExecutor
+from langchain.agents import BaseMultiActionAgent, BaseSingleActionAgent
+from langchain.callbacks.manager import Callbacks
+from langchain.tools import BaseTool
+from pydantic import BaseModel, Extra
 
 
 class PlanningStrategy(str, enum.Enum):

api/core/app_runner/agent_app_runner.py

@@ -4,16 +4,16 @@ from typing import cast
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.app_runner.app_runner import AppRunner
-from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler
-from core.entities.application_entities import ApplicationGenerateEntity, PromptTemplateEntity, ModelConfigEntity
 from core.application_queue_manager import ApplicationQueueManager
+from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler
+from core.entities.application_entities import ApplicationGenerateEntity, ModelConfigEntity, PromptTemplateEntity
 from core.features.agent_runner import AgentRunnerFeature
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.llm_entities import LLMUsage
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from extensions.ext_database import db
-from models.model import Conversation, Message, App, MessageChain, MessageAgentThought
+from models.model import App, Conversation, Message, MessageAgentThought, MessageChain
 
 logger = logging.getLogger(__name__)
 

api/core/app_runner/app_runner.py

@@ -1,12 +1,12 @@
 import time
-from typing import cast, Optional, List, Tuple, Generator, Union
+from typing import Generator, List, Optional, Tuple, Union, cast
 
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
-from core.entities.application_entities import ModelConfigEntity, PromptTemplateEntity, AppOrchestrationConfigEntity
+from core.entities.application_entities import AppOrchestrationConfigEntity, ModelConfigEntity, PromptTemplateEntity
 from core.file.file_obj import FileObj
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from core.model_runtime.entities.message_entities import PromptMessage, AssistantPromptMessage
+from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.errors.invoke import InvokeBadRequestError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel

api/core/app_runner/basic_app_runner.py

@@ -1,11 +1,11 @@
 import logging
-from typing import Tuple, Optional
+from typing import Optional, Tuple
 
 from core.app_runner.app_runner import AppRunner
-from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
-from core.entities.application_entities import ApplicationGenerateEntity, ModelConfigEntity, \
-    AppOrchestrationConfigEntity, InvokeFrom, ExternalDataVariableEntity, DatasetEntity
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
+from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
+from core.entities.application_entities import (ApplicationGenerateEntity, AppOrchestrationConfigEntity, DatasetEntity,
+                                                ExternalDataVariableEntity, InvokeFrom, ModelConfigEntity)
 from core.features.annotation_reply import AnnotationReplyFeature
 from core.features.dataset_retrieval import DatasetRetrievalFeature
 from core.features.external_data_fetch import ExternalDataFetchFeature
@@ -17,7 +17,7 @@ from core.model_runtime.entities.message_entities import PromptMessage
 from core.moderation.base import ModerationException
 from core.prompt.prompt_transform import AppMode
 from extensions.ext_database import db
-from models.model import Conversation, Message, App, MessageAnnotation
+from models.model import App, Conversation, Message, MessageAnnotation
 
 logger = logging.getLogger(__name__)
 

api/core/app_runner/generate_task_pipeline.py

@@ -1,25 +1,25 @@
 import json
 import logging
 import time
-from typing import Union, Generator, cast, Optional
+from typing import Generator, Optional, Union, cast
 
-from pydantic import BaseModel
-
-from core.app_runner.moderation_handler import OutputModerationHandler, ModerationRule
-from core.entities.application_entities import ApplicationGenerateEntity
+from core.app_runner.moderation_handler import ModerationRule, OutputModerationHandler
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
-from core.entities.queue_entities import QueueErrorEvent, QueueStopEvent, QueueMessageEndEvent, \
-    QueueRetrieverResourcesEvent, QueueAgentThoughtEvent, QueuePingEvent, QueueMessageEvent, QueueMessageReplaceEvent, \
-    AnnotationReplyEvent
-from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage, LLMResultChunk, LLMResultChunkDelta
-from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessageRole, \
-    TextPromptMessageContent, PromptMessageContentType, ImagePromptMessageContent, PromptMessage
-from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError
+from core.entities.application_entities import ApplicationGenerateEntity
+from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentThoughtEvent, QueueErrorEvent,
+                                          QueueMessageEndEvent, QueueMessageEvent, QueueMessageReplaceEvent,
+                                          QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent)
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent,
+                                                          PromptMessage, PromptMessageContentType, PromptMessageRole,
+                                                          TextPromptMessageContent)
+from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.prompt.prompt_template import PromptTemplateParser
 from events.message_event import message_was_created
 from extensions.ext_database import db
-from models.model import Message, Conversation, MessageAgentThought
+from models.model import Conversation, Message, MessageAgentThought
+from pydantic import BaseModel
 from services.annotation_service import AppAnnotationService
 
 logger = logging.getLogger(__name__)

api/core/app_runner/moderation_handler.py

@@ -1,14 +1,13 @@
 import logging
 import threading
 import time
-from typing import Any, Optional, Dict
-
-from flask import current_app, Flask
-from pydantic import BaseModel
+from typing import Any, Dict, Optional
 
 from core.application_queue_manager import PublishFrom
 from core.moderation.base import ModerationAction, ModerationOutputsResult
 from core.moderation.factory import ModerationFactory
+from flask import Flask, current_app
+from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
 

api/core/application_manager.py

@@ -2,31 +2,32 @@ import json
 import logging
 import threading
 import uuid
-from typing import cast, Optional, Any, Union, Generator, Tuple
-
-from flask import Flask, current_app
-from pydantic import ValidationError
+from typing import Any, Generator, Optional, Tuple, Union, cast
 
 from core.app_runner.agent_app_runner import AgentApplicationRunner
 from core.app_runner.basic_app_runner import BasicApplicationRunner
 from core.app_runner.generate_task_pipeline import GenerateTaskPipeline
-from core.entities.application_entities import ApplicationGenerateEntity, AppOrchestrationConfigEntity, \
-    ModelConfigEntity, PromptTemplateEntity, AdvancedChatPromptTemplateEntity, \
-    AdvancedCompletionPromptTemplateEntity, ExternalDataVariableEntity, DatasetEntity, DatasetRetrieveConfigEntity, \
-    AgentEntity, AgentToolEntity, FileUploadEntity, SensitiveWordAvoidanceEntity, InvokeFrom
+from core.application_queue_manager import ApplicationQueueManager, ConversationTaskStoppedException, PublishFrom
+from core.entities.application_entities import (AdvancedChatPromptTemplateEntity,
+                                                AdvancedCompletionPromptTemplateEntity, AgentEntity, AgentToolEntity,
+                                                ApplicationGenerateEntity, AppOrchestrationConfigEntity, DatasetEntity,
+                                                DatasetRetrieveConfigEntity, ExternalDataVariableEntity,
+                                                FileUploadEntity, InvokeFrom, ModelConfigEntity, PromptTemplateEntity,
+                                                SensitiveWordAvoidanceEntity)
 from core.entities.model_entities import ModelStatus
+from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.file.file_obj import FileObj
-from core.errors.error import QuotaExceededError, ProviderTokenNotInitError, ModelCurrentlyNotSupportError
 from core.model_runtime.entities.message_entities import PromptMessageRole
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.prompt.prompt_template import PromptTemplateParser
 from core.provider_manager import ProviderManager
-from core.application_queue_manager import ApplicationQueueManager, ConversationTaskStoppedException, PublishFrom
 from extensions.ext_database import db
+from flask import Flask, current_app
 from models.account import Account
-from models.model import EndUser, Conversation, Message, MessageFile, App
+from models.model import App, Conversation, EndUser, Message, MessageFile
+from pydantic import ValidationError
 
 logger = logging.getLogger(__name__)
 

api/core/application_queue_manager.py

@@ -1,17 +1,17 @@
 import queue
 import time
 from enum import Enum
-from typing import Generator, Any
-
-from sqlalchemy.orm import DeclarativeMeta
+from typing import Any, Generator
 
 from core.entities.application_entities import InvokeFrom
-from core.entities.queue_entities import QueueStopEvent, AppQueueEvent, QueuePingEvent, QueueErrorEvent, \
-    QueueAgentThoughtEvent, QueueMessageEndEvent, QueueRetrieverResourcesEvent, QueueMessageReplaceEvent, \
-    QueueMessageEvent, QueueMessage, AnnotationReplyEvent
+from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentThoughtEvent, QueueErrorEvent,
+                                          QueueMessage, QueueMessageEndEvent, QueueMessageEvent,
+                                          QueueMessageReplaceEvent, QueuePingEvent, QueueRetrieverResourcesEvent,
+                                          QueueStopEvent)
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
 from extensions.ext_redis import redis_client
 from models.model import MessageAgentThought
+from sqlalchemy.orm import DeclarativeMeta
 
 
 class PublishFrom(Enum):

api/core/callback_handler/agent_loop_gather_callback_handler.py

@@ -1,21 +1,19 @@
 import json
 import logging
 import time
-
-from typing import Any, Dict, List, Union, Optional, cast
-
-from langchain.agents import openai_functions_agent, openai_functions_multi_agent
-from langchain.callbacks.base import BaseCallbackHandler
-from langchain.schema import AgentAction, AgentFinish, LLMResult, ChatGeneration, BaseMessage
+from typing import Any, Dict, List, Optional, Union, cast
 
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.callback_handler.entity.agent_loop import AgentLoop
 from core.entities.application_entities import ModelConfigEntity
 from core.model_runtime.entities.llm_entities import LLMResult as RuntimeLLMResult
-from core.model_runtime.entities.message_entities import UserPromptMessage, AssistantPromptMessage, PromptMessage
+from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from extensions.ext_database import db
-from models.model import MessageChain, MessageAgentThought, Message
+from langchain.agents import openai_functions_agent, openai_functions_multi_agent
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.schema import AgentAction, AgentFinish, BaseMessage, ChatGeneration, LLMResult
+from models.model import Message, MessageAgentThought, MessageChain
 
 
 class AgentLoopGatherCallbackHandler(BaseCallbackHandler):

api/core/callback_handler/index_tool_callback_handler.py

@@ -1,11 +1,10 @@
 from typing import List, Union
 
-from langchain.schema import Document
-
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.entities.application_entities import InvokeFrom
 from extensions.ext_database import db
-from models.dataset import DocumentSegment, DatasetQuery
+from langchain.schema import Document
+from models.dataset import DatasetQuery, DocumentSegment
 from models.model import DatasetRetrieverResource
 
 

api/core/callback_handler/std_out_callback_handler.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Optional, Union
 
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain.input import print_text
-from langchain.schema import AgentAction, AgentFinish, LLMResult, BaseMessage
+from langchain.schema import AgentAction, AgentFinish, BaseMessage, LLMResult
 
 
 class DifyStdOutCallbackHandler(BaseCallbackHandler):

api/core/chain/llm_chain.py

@@ -1,15 +1,14 @@
-from typing import List, Dict, Any, Optional
-
-from langchain import LLMChain as LCLLMChain
-from langchain.callbacks.manager import CallbackManagerForChainRun
-from langchain.schema import LLMResult, Generation
-from langchain.schema.language_model import BaseLanguageModel
+from typing import Any, Dict, List, Optional
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.entities.application_entities import ModelConfigEntity
-from core.model_manager import ModelInstance
 from core.entities.message_entities import lc_messages_to_prompt_messages
+from core.model_manager import ModelInstance
 from core.third_party.langchain.llms.fake import FakeLLM
+from langchain import LLMChain as LCLLMChain
+from langchain.callbacks.manager import CallbackManagerForChainRun
+from langchain.schema import Generation, LLMResult
+from langchain.schema.language_model import BaseLanguageModel
 
 
 class LLMChain(LCLLMChain):

api/core/data_loader/file_extractor.py

@@ -1,12 +1,8 @@
 import tempfile
 from pathlib import Path
-from typing import List, Union, Optional
+from typing import List, Optional, Union
 
 import requests
-from flask import current_app
-from langchain.document_loaders import TextLoader, Docx2txtLoader
-from langchain.schema import Document
-
 from core.data_loader.loader.csv_loader import CSVLoader
 from core.data_loader.loader.excel import ExcelLoader
 from core.data_loader.loader.html import HTMLLoader
@@ -20,6 +16,9 @@ from core.data_loader.loader.unstructured.unstructured_pptx import UnstructuredP
 from core.data_loader.loader.unstructured.unstructured_text import UnstructuredTextLoader
 from core.data_loader.loader.unstructured.unstructured_xml import UnstructuredXmlLoader
 from extensions.ext_storage import storage
+from flask import current_app
+from langchain.document_loaders import Docx2txtLoader, TextLoader
+from langchain.schema import Document
 from models.model import UploadFile
 
 SUPPORT_URL_CONTENT_TYPES = ['application/pdf', 'text/plain']

api/core/data_loader/loader/csv_loader.py

@@ -1,6 +1,6 @@
-import logging
 import csv
-from typing import Optional, Dict, List
+import logging
+from typing import Dict, List, Optional
 
 from langchain.document_loaders import CSVLoader as LCCSVLoader
 from langchain.document_loaders.helpers import detect_file_encodings

api/core/data_loader/loader/markdown.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from typing import Optional, List, Tuple, cast
+from typing import List, Optional, Tuple, cast
 
 from langchain.document_loaders.base import BaseLoader
 from langchain.document_loaders.helpers import detect_file_encodings

api/core/data_loader/loader/notion.py

@@ -1,13 +1,12 @@
 import json
 import logging
-from typing import List, Dict, Any, Optional
+from typing import Any, Dict, List, Optional
 
 import requests
+from extensions.ext_database import db
 from flask import current_app
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document
-
-from extensions.ext_database import db
 from models.dataset import Document as DocumentModel
 from models.source import DataSourceBinding
 

api/core/data_loader/loader/pdf.py

@@ -1,11 +1,10 @@
 import logging
 from typing import List, Optional
 
+from extensions.ext_storage import storage
 from langchain.document_loaders import PyPDFium2Loader
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document
-
-from extensions.ext_storage import storage
 from models.model import UploadFile
 
 logger = logging.getLogger(__name__)

api/core/data_loader/loader/unstructured/unstructured_eml.py

@@ -1,6 +1,7 @@
-import logging
 import base64
+import logging
 from typing import List
+
 from bs4 import BeautifulSoup
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document

api/core/data_loader/loader/unstructured/unstructured_msg.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from typing import Optional, List, Tuple, cast
+from typing import List, Optional, Tuple, cast
 
 from langchain.document_loaders.base import BaseLoader
 from langchain.document_loaders.helpers import detect_file_encodings

api/core/data_loader/loader/unstructured/unstructured_ppt.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from typing import Optional, List, Tuple, cast
+from typing import List, Optional, Tuple, cast
 
 from langchain.document_loaders.base import BaseLoader
 from langchain.document_loaders.helpers import detect_file_encodings

api/core/data_loader/loader/unstructured/unstructured_pptx.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from typing import Optional, List, Tuple, cast
+from typing import List, Optional, Tuple, cast
 
 from langchain.document_loaders.base import BaseLoader
 from langchain.document_loaders.helpers import detect_file_encodings

api/core/data_loader/loader/unstructured/unstructured_text.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from typing import Optional, List, Tuple, cast
+from typing import List, Optional, Tuple, cast
 
 from langchain.document_loaders.base import BaseLoader
 from langchain.document_loaders.helpers import detect_file_encodings

api/core/data_loader/loader/unstructured/unstructured_xml.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from typing import Optional, List, Tuple, cast
+from typing import List, Optional, Tuple, cast
 
 from langchain.document_loaders.base import BaseLoader
 from langchain.document_loaders.helpers import detect_file_encodings

api/core/docstore/dataset_docstore.py

@@ -1,13 +1,12 @@
 from typing import Any, Dict, Optional, Sequence, cast
 
-from langchain.schema import Document
-from sqlalchemy import func
-
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
 from extensions.ext_database import db
+from langchain.schema import Document
 from models.dataset import Dataset, DocumentSegment
+from sqlalchemy import func
 
 
 class DatasetDocumentStore:

api/core/embedding/cached_embedding.py

@@ -2,13 +2,12 @@ import logging
 from typing import List, Optional
 
 import numpy as np
-from langchain.embeddings.base import Embeddings
-from sqlalchemy.exc import IntegrityError
-
 from core.model_manager import ModelInstance
 from extensions.ext_database import db
+from langchain.embeddings.base import Embeddings
 from libs import helper
 from models.dataset import Embedding
+from sqlalchemy.exc import IntegrityError
 
 logger = logging.getLogger(__name__)
 

api/core/entities/application_entities.py

@@ -1,12 +1,11 @@
 from enum import Enum
-from typing import Optional, Any, cast
-
-from pydantic import BaseModel
+from typing import Any, Optional, cast
 
 from core.entities.provider_configuration import ProviderModelBundle
 from core.file.file_obj import FileObj
 from core.model_runtime.entities.message_entities import PromptMessageRole
 from core.model_runtime.entities.model_entities import AIModelEntity
+from pydantic import BaseModel
 
 
 class ModelConfigEntity(BaseModel):

api/core/entities/message_entities.py

@@ -1,12 +1,12 @@
 import enum
 from typing import Any, cast
 
-from langchain.schema import HumanMessage, AIMessage, SystemMessage, BaseMessage, FunctionMessage
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent,
+                                                          PromptMessage, SystemPromptMessage, TextPromptMessageContent,
+                                                          ToolPromptMessage, UserPromptMessage)
+from langchain.schema import AIMessage, BaseMessage, FunctionMessage, HumanMessage, SystemMessage
 from pydantic import BaseModel
 
-from core.model_runtime.entities.message_entities import PromptMessage, UserPromptMessage, TextPromptMessageContent, \
-    ImagePromptMessageContent, AssistantPromptMessage, SystemPromptMessage, ToolPromptMessage
-
 
 class PromptMessageFileType(enum.Enum):
     IMAGE = 'image'

api/core/entities/model_entities.py

@@ -1,11 +1,10 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
-
 from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.model_entities import ProviderModel, ModelType
-from core.model_runtime.entities.provider_entities import SimpleProviderEntity, ProviderEntity
+from core.model_runtime.entities.model_entities import ModelType, ProviderModel
+from core.model_runtime.entities.provider_entities import ProviderEntity, SimpleProviderEntity
+from pydantic import BaseModel
 
 
 class ModelStatus(Enum):

api/core/entities/provider_configuration.py

@@ -1,25 +1,23 @@
 import datetime
 import json
 import logging
-
 from json import JSONDecodeError
-from typing import Optional, List, Dict, Tuple, Iterator
+from typing import Dict, Iterator, List, Optional, Tuple
 
-from pydantic import BaseModel
-
-from core.entities.model_entities import ModelWithProviderEntity, ModelStatus, SimpleModelProviderEntity
-from core.entities.provider_entities import SystemConfiguration, CustomConfiguration, SystemConfigurationStatus
+from core.entities.model_entities import ModelStatus, ModelWithProviderEntity, SimpleModelProviderEntity
+from core.entities.provider_entities import CustomConfiguration, SystemConfiguration, SystemConfigurationStatus
 from core.helper import encrypter
 from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType
-from core.model_runtime.entities.model_entities import ModelType, FetchFrom
-from core.model_runtime.entities.provider_entities import ProviderEntity, CredentialFormSchema, FormType, \
-    ConfigurateMethod
+from core.model_runtime.entities.model_entities import FetchFrom, ModelType
+from core.model_runtime.entities.provider_entities import (ConfigurateMethod, CredentialFormSchema, FormType,
+                                                           ProviderEntity)
 from core.model_runtime.model_providers import model_provider_factory
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 from core.model_runtime.model_providers.__base.model_provider import ModelProvider
 from core.model_runtime.utils import encoders
 from extensions.ext_database import db
-from models.provider import ProviderType, Provider, ProviderModel, TenantPreferredModelProvider
+from models.provider import Provider, ProviderModel, ProviderType, TenantPreferredModelProvider
+from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
 

api/core/entities/provider_entities.py

@@ -1,10 +1,9 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
-
 from core.model_runtime.entities.model_entities import ModelType
 from models.provider import ProviderQuotaType
+from pydantic import BaseModel
 
 
 class QuotaUnit(Enum):

api/core/entities/queue_entities.py

@@ -1,9 +1,8 @@
 from enum import Enum
 from typing import Any
 
-from pydantic import BaseModel
-
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
+from pydantic import BaseModel
 
 
 class QueueEvent(Enum):

api/core/extension/api_based_extension_requestor.py

@@ -1,7 +1,6 @@
 import os
 
 import requests
-
 from models.api_based_extension import APIBasedExtensionPoint
 
 

api/core/extension/extension.py

@@ -1,4 +1,4 @@
-from core.extension.extensible import ModuleExtension, ExtensionModule
+from core.extension.extensible import ExtensionModule, ModuleExtension
 from core.external_data_tool.base import ExternalDataTool
 from core.moderation.base import Moderation
 

api/core/external_data_tool/base.py

@@ -1,4 +1,4 @@
-from abc import abstractmethod, ABC
+from abc import ABC, abstractmethod
 from typing import Optional
 
 from core.extension.extensible import Extensible, ExtensionModule

api/core/features/agent_runner.py

@@ -1,19 +1,14 @@
 import logging
-from typing import cast, Optional, List
-
-from langchain import WikipediaAPIWrapper
-from langchain.callbacks.base import BaseCallbackHandler
-from langchain.tools import BaseTool, WikipediaQueryRun, Tool
-from pydantic import BaseModel, Field
+from typing import List, Optional, cast
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
-from core.agent.agent_executor import PlanningStrategy, AgentConfiguration, AgentExecutor
+from core.agent.agent_executor import AgentConfiguration, AgentExecutor, PlanningStrategy
 from core.application_queue_manager import ApplicationQueueManager
 from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
 from core.callback_handler.std_out_callback_handler import DifyStdOutCallbackHandler
-from core.entities.application_entities import ModelConfigEntity, InvokeFrom, \
-    AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity
+from core.entities.application_entities import (AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity, InvokeFrom,
+                                                ModelConfigEntity)
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.model_entities import ModelFeature, ModelType
 from core.model_runtime.model_providers import model_provider_factory
@@ -21,11 +16,15 @@ from core.model_runtime.model_providers.__base.large_language_model import Large
 from core.tool.current_datetime_tool import DatetimeTool
 from core.tool.dataset_retriever_tool import DatasetRetrieverTool
 from core.tool.provider.serpapi_provider import SerpAPIToolProvider
-from core.tool.serpapi_wrapper import OptimizedSerpAPIWrapper, OptimizedSerpAPIInput
+from core.tool.serpapi_wrapper import OptimizedSerpAPIInput, OptimizedSerpAPIWrapper
 from core.tool.web_reader_tool import WebReaderTool
 from extensions.ext_database import db
+from langchain import WikipediaAPIWrapper
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.tools import BaseTool, Tool, WikipediaQueryRun
 from models.dataset import Dataset
 from models.model import Message
+from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
 

api/core/features/annotation_reply.py

@@ -1,16 +1,15 @@
 import logging
 from typing import Optional
 
-from flask import current_app
-
 from core.embedding.cached_embedding import CacheEmbedding
 from core.entities.application_entities import InvokeFrom
 from core.index.vector_index.vector_index import VectorIndex
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
+from flask import current_app
 from models.dataset import Dataset
-from models.model import App, Message, AppAnnotationSetting, MessageAnnotation
+from models.model import App, AppAnnotationSetting, Message, MessageAnnotation
 from services.annotation_service import AppAnnotationService
 from services.dataset_service import DatasetCollectionBindingService
 

api/core/features/dataset_retrieval.py

@@ -1,16 +1,15 @@
-from typing import cast, Optional, List
+from typing import List, Optional, cast
 
-from langchain.tools import BaseTool
-
-from core.agent.agent_executor import PlanningStrategy, AgentConfiguration, AgentExecutor
+from core.agent.agent_executor import AgentConfiguration, AgentExecutor, PlanningStrategy
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
-from core.entities.application_entities import DatasetEntity, ModelConfigEntity, InvokeFrom, DatasetRetrieveConfigEntity
+from core.entities.application_entities import DatasetEntity, DatasetRetrieveConfigEntity, InvokeFrom, ModelConfigEntity
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.tool.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
 from core.tool.dataset_retriever_tool import DatasetRetrieverTool
 from extensions.ext_database import db
+from langchain.tools import BaseTool
 from models.dataset import Dataset
 
 

api/core/features/external_data_fetch.py

@@ -1,14 +1,12 @@
 import concurrent
 import json
 import logging
-
 from concurrent.futures import ThreadPoolExecutor
-from typing import Tuple, Optional
-
-from flask import current_app, Flask
+from typing import Optional, Tuple
 
 from core.entities.application_entities import ExternalDataVariableEntity
 from core.external_data_tool.factory import ExternalDataToolFactory
+from flask import Flask, current_app
 
 logger = logging.getLogger(__name__)
 

api/core/file/file_obj.py

@@ -1,12 +1,11 @@
 import enum
 from typing import Optional
 
-from pydantic import BaseModel
-
 from core.file.upload_file_parser import UploadFileParser
 from core.model_runtime.entities.message_entities import ImagePromptMessageContent
 from extensions.ext_database import db
 from models.model import UploadFile
+from pydantic import BaseModel
 
 
 class FileType(enum.Enum):

api/core/file/message_file_parser.py

@@ -1,12 +1,11 @@
-from typing import List, Union, Optional, Dict
+from typing import Dict, List, Optional, Union
 
 import requests
-
-from core.file.file_obj import FileObj, FileType, FileTransferMethod
+from core.file.file_obj import FileObj, FileTransferMethod, FileType
 from core.file.upload_file_parser import SUPPORT_EXTENSIONS
 from extensions.ext_database import db
 from models.account import Account
-from models.model import MessageFile, EndUser, AppModelConfig, UploadFile
+from models.model import AppModelConfig, EndUser, MessageFile, UploadFile
 
 
 class MessageFileParser:

api/core/file/upload_file_parser.py

@@ -6,9 +6,8 @@ import os
 import time
 from typing import Optional
 
-from flask import current_app
-
 from extensions.ext_storage import storage
+from flask import current_app
 
 SUPPORT_EXTENSIONS = ['jpg', 'jpeg', 'png', 'webp', 'gif', 'svg']
 

api/core/generator/llm_generator.py

@@ -1,17 +1,15 @@
 import json
 import logging
 
-from langchain.schema import OutputParserException
-
 from core.model_manager import ModelManager
-from core.model_runtime.entities.message_entities import UserPromptMessage, SystemPromptMessage
+from core.model_runtime.entities.message_entities import SystemPromptMessage, UserPromptMessage
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.prompt.output_parser.rule_config_generator import RuleConfigGeneratorOutputParser
-
 from core.prompt.output_parser.suggested_questions_after_answer import SuggestedQuestionsAfterAnswerOutputParser
 from core.prompt.prompt_template import PromptTemplateParser
 from core.prompt.prompts import CONVERSATION_TITLE_PROMPT, GENERATOR_QA_PROMPT
+from langchain.schema import OutputParserException
 
 
 class LLMGenerator:

api/core/helper/encrypter.py

@@ -2,7 +2,6 @@ import base64
 
 from extensions.ext_database import db
 from libs import rsa
-
 from models.account import Tenant
 
 

api/core/hosting_configuration.py

@@ -1,12 +1,11 @@
 import os
 from typing import Optional
 
-from flask import Flask
-from pydantic import BaseModel
-
 from core.entities.provider_entities import QuotaUnit, RestrictModel
 from core.model_runtime.entities.model_entities import ModelType
+from flask import Flask
 from models.provider import ProviderQuotaType
+from pydantic import BaseModel
 
 
 class HostingQuota(BaseModel):

api/core/index/base.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
-from abc import abstractmethod, ABC
-from typing import List, Any
 
-from langchain.schema import Document, BaseRetriever
+from abc import ABC, abstractmethod
+from typing import Any, List
 
+from langchain.schema import BaseRetriever, Document
 from models.dataset import Dataset
 
 

api/core/index/index.py

@@ -1,11 +1,10 @@
-from flask import current_app
-from langchain.embeddings import OpenAIEmbeddings
-
 from core.embedding.cached_embedding import CacheEmbedding
-from core.index.keyword_table_index.keyword_table_index import KeywordTableIndex, KeywordTableConfig
+from core.index.keyword_table_index.keyword_table_index import KeywordTableConfig, KeywordTableIndex
 from core.index.vector_index.vector_index import VectorIndex
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
+from flask import current_app
+from langchain.embeddings import OpenAIEmbeddings
 from models.dataset import Dataset
 
 

api/core/index/keyword_table_index/jieba_keyword_table_handler.py

@@ -2,9 +2,8 @@ import re
 from typing import Set
 
 import jieba
-from jieba.analyse import default_tfidf
-
 from core.index.keyword_table_index.stopwords import STOPWORDS
+from jieba.analyse import default_tfidf
 
 
 class JiebaKeywordTableHandler:

api/core/index/keyword_table_index/keyword_table_index.py

@@ -1,14 +1,13 @@
 import json
 from collections import defaultdict
-from typing import Any, List, Optional, Dict
-
-from langchain.schema import Document, BaseRetriever
-from pydantic import BaseModel, Field, Extra
+from typing import Any, Dict, List, Optional
 
 from core.index.base import BaseIndex
 from core.index.keyword_table_index.jieba_keyword_table_handler import JiebaKeywordTableHandler
 from extensions.ext_database import db
-from models.dataset import Dataset, DocumentSegment, DatasetKeywordTable
+from langchain.schema import BaseRetriever, Document
+from models.dataset import Dataset, DatasetKeywordTable, DocumentSegment
+from pydantic import BaseModel, Extra, Field
 
 
 class KeywordTableConfig(BaseModel):

api/core/index/vector_index/base.py

@@ -1,16 +1,16 @@
 import json
 import logging
 from abc import abstractmethod
-from typing import List, Any, cast
-
-from langchain.embeddings.base import Embeddings
-from langchain.schema import Document, BaseRetriever
-from langchain.vectorstores import VectorStore
+from typing import Any, List, cast
 
 from core.index.base import BaseIndex
 from extensions.ext_database import db
-from models.dataset import Dataset, DocumentSegment, DatasetCollectionBinding
+from langchain.embeddings.base import Embeddings
+from langchain.schema import BaseRetriever, Document
+from langchain.vectorstores import VectorStore
+from models.dataset import Dataset, DatasetCollectionBinding
 from models.dataset import Document as DatasetDocument
+from models.dataset import DocumentSegment
 
 
 class BaseVectorIndex(BaseIndex):

api/core/index/vector_index/milvus_vector_index.py

@@ -1,14 +1,13 @@
-from typing import cast, Any, List
-
-from langchain.embeddings.base import Embeddings
-from langchain.schema import Document
-from langchain.vectorstores import VectorStore
-from pydantic import BaseModel, root_validator
+from typing import Any, List, cast
 
 from core.index.base import BaseIndex
 from core.index.vector_index.base import BaseVectorIndex
 from core.vector_store.milvus_vector_store import MilvusVectorStore
+from langchain.embeddings.base import Embeddings
+from langchain.schema import Document
+from langchain.vectorstores import VectorStore
 from models.dataset import Dataset
+from pydantic import BaseModel, root_validator
 
 
 class MilvusConfig(BaseModel):

api/core/index/vector_index/qdrant_vector_index.py

@@ -1,18 +1,17 @@
 import os
-from typing import Optional, Any, List, cast
+from typing import Any, List, Optional, cast
 
 import qdrant_client
-from langchain.embeddings.base import Embeddings
-from langchain.schema import Document, BaseRetriever
-from langchain.vectorstores import VectorStore
-from pydantic import BaseModel
-from qdrant_client.http.models import HnswConfigDiff
-
 from core.index.base import BaseIndex
 from core.index.vector_index.base import BaseVectorIndex
 from core.vector_store.qdrant_vector_store import QdrantVectorStore
 from extensions.ext_database import db
+from langchain.embeddings.base import Embeddings
+from langchain.schema import BaseRetriever, Document
+from langchain.vectorstores import VectorStore
 from models.dataset import Dataset, DatasetCollectionBinding
+from pydantic import BaseModel
+from qdrant_client.http.models import HnswConfigDiff
 
 
 class QdrantConfig(BaseModel):

api/core/index/vector_index/vector_index.py

@@ -1,10 +1,9 @@
 import json
 
-from flask import current_app
-from langchain.embeddings.base import Embeddings
-
 from core.index.vector_index.base import BaseVectorIndex
 from extensions.ext_database import db
+from flask import current_app
+from langchain.embeddings.base import Embeddings
 from models.dataset import Dataset, Document
 
 
@@ -29,7 +28,7 @@ class VectorIndex:
             raise ValueError(f"Vector store must be specified.")
 
         if vector_type == "weaviate":
-            from core.index.vector_index.weaviate_vector_index import WeaviateVectorIndex, WeaviateConfig
+            from core.index.vector_index.weaviate_vector_index import WeaviateConfig, WeaviateVectorIndex
 
             return WeaviateVectorIndex(
                 dataset=dataset,
@@ -42,7 +41,7 @@ class VectorIndex:
                 attributes=attributes
             )
         elif vector_type == "qdrant":
-            from core.index.vector_index.qdrant_vector_index import QdrantVectorIndex, QdrantConfig
+            from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex
 
             return QdrantVectorIndex(
                 dataset=dataset,
@@ -55,7 +54,7 @@ class VectorIndex:
                 embeddings=embeddings
             )
         elif vector_type == "milvus":
-            from core.index.vector_index.milvus_vector_index import MilvusVectorIndex, MilvusConfig
+            from core.index.vector_index.milvus_vector_index import MilvusConfig, MilvusVectorIndex
 
             return MilvusVectorIndex(
                 dataset=dataset,

api/core/index/vector_index/weaviate_vector_index.py

@@ -1,16 +1,15 @@
-from typing import Optional, cast, Any, List
+from typing import Any, List, Optional, cast
 
 import requests
 import weaviate
-from langchain.embeddings.base import Embeddings
-from langchain.schema import Document, BaseRetriever
-from langchain.vectorstores import VectorStore
-from pydantic import BaseModel, root_validator
-
 from core.index.base import BaseIndex
 from core.index.vector_index.base import BaseVectorIndex
 from core.vector_store.weaviate_vector_store import WeaviateVectorStore
+from langchain.embeddings.base import Embeddings
+from langchain.schema import BaseRetriever, Document
+from langchain.vectorstores import VectorStore
 from models.dataset import Dataset
+from pydantic import BaseModel, root_validator
 
 
 class WeaviateConfig(BaseModel):

api/core/indexing_runner.py

@@ -5,34 +5,34 @@ import re
 import threading
 import time
 import uuid
-from typing import Optional, List, cast, Type, Union, Literal, AbstractSet, Collection, Any
-
-from flask import current_app, Flask
-from flask_login import current_user
-from langchain.schema import Document
-from langchain.text_splitter import TextSplitter, TS, TokenTextSplitter
-from sqlalchemy.orm.exc import ObjectDeletedError
+from typing import AbstractSet, Any, Collection, List, Literal, Optional, Type, Union, cast
 
 from core.data_loader.file_extractor import FileExtractor
 from core.data_loader.loader.notion import NotionLoader
 from core.docstore.dataset_docstore import DatasetDocumentStore
+from core.errors.error import ProviderTokenNotInitError
 from core.generator.llm_generator import LLMGenerator
 from core.index.index import IndexBuilder
 from core.model_manager import ModelManager
-from core.errors.error import ProviderTokenNotInitError
 from core.model_runtime.entities.model_entities import ModelType, PriceType
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
 from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
-from core.spiltter.fixed_text_splitter import FixedRecursiveCharacterTextSplitter, EnhanceRecursiveCharacterTextSplitter
+from core.spiltter.fixed_text_splitter import EnhanceRecursiveCharacterTextSplitter, FixedRecursiveCharacterTextSplitter
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from extensions.ext_storage import storage
+from flask import Flask, current_app
+from flask_login import current_user
+from langchain.schema import Document
+from langchain.text_splitter import TS, TextSplitter, TokenTextSplitter
 from libs import helper
+from models.dataset import Dataset, DatasetProcessRule
 from models.dataset import Document as DatasetDocument
-from models.dataset import Dataset, DocumentSegment, DatasetProcessRule
+from models.dataset import DocumentSegment
 from models.model import UploadFile
 from models.source import DataSourceBinding
+from sqlalchemy.orm.exc import ObjectDeletedError
 
 
 class IndexingRunner:

api/core/memory/token_buffer_memory.py

@@ -1,7 +1,7 @@
 from core.file.message_file_parser import MessageFileParser
 from core.model_manager import ModelInstance
-from core.model_runtime.entities.message_entities import PromptMessage, TextPromptMessageContent, UserPromptMessage, \
-    AssistantPromptMessage, PromptMessageRole
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageRole,
+                                                          TextPromptMessageContent, UserPromptMessage)
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers import model_provider_factory
 from extensions.ext_database import db

api/core/model_manager.py

@@ -1,10 +1,10 @@
-from typing import Optional, Union, Generator, cast, List, IO
+from typing import IO, Generator, List, Optional, Union, cast
 
 from core.entities.provider_configuration import ProviderModelBundle
 from core.errors.error import ProviderTokenNotInitError
 from core.model_runtime.callbacks.base_callback import Callback
 from core.model_runtime.entities.llm_entities import LLMResult
-from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.entities.rerank_entities import RerankResult
 from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult

api/core/model_runtime/callbacks/base_callback.py

@@ -1,8 +1,8 @@
 from abc import ABC
-from typing import Optional, List
+from typing import List, Optional
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
-from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
 _TEXT_COLOR_MAPPING = {

api/core/model_runtime/callbacks/logging_callback.py

@@ -1,11 +1,11 @@
 import json
 import logging
 import sys
-from typing import Optional, List
+from typing import List, Optional
 
 from core.model_runtime.callbacks.base_callback import Callback
-from core.model_runtime.entities.llm_entities import LLMResultChunk, LLMResult
-from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
 logger = logging.getLogger(__name__)

api/core/model_runtime/entities/defaults.py

@@ -2,7 +2,6 @@ from typing import Dict
 
 from core.model_runtime.entities.model_entities import DefaultParameterName
 
-
 PARAMETER_RULE_TEMPLATE: Dict[DefaultParameterName, dict] = {
     DefaultParameterName.TEMPERATURE: {
         'label': {

api/core/model_runtime/entities/llm_entities.py

@@ -2,10 +2,9 @@ from decimal import Decimal
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
-
 from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage
 from core.model_runtime.entities.model_entities import ModelUsage, PriceInfo
+from pydantic import BaseModel
 
 
 class LLMMode(Enum):

api/core/model_runtime/entities/model_entities.py

@@ -2,9 +2,8 @@ from decimal import Decimal
 from enum import Enum
 from typing import Any, Optional
 
-from pydantic import BaseModel
-
 from core.model_runtime.entities.common_entities import I18nObject
+from pydantic import BaseModel
 
 
 class ModelType(Enum):

api/core/model_runtime/entities/provider_entities.py

@@ -1,10 +1,9 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
-
 from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.model_entities import ModelType, ProviderModel, AIModelEntity
+from core.model_runtime.entities.model_entities import AIModelEntity, ModelType, ProviderModel
+from pydantic import BaseModel
 
 
 class ConfigurateMethod(Enum):

api/core/model_runtime/entities/text_embedding_entities.py

@@ -1,8 +1,7 @@
 from decimal import Decimal
 
-from pydantic import BaseModel
-
 from core.model_runtime.entities.model_entities import ModelUsage
+from pydantic import BaseModel
 
 
 class EmbeddingUsage(ModelUsage):

api/core/model_runtime/model_providers/__base/ai_model.py

@@ -6,14 +6,13 @@ from abc import ABC, abstractmethod
 from typing import Optional
 
 import yaml
-from pydantic import ValidationError
-
-from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
-from core.model_runtime.entities.model_entities import PriceInfo, AIModelEntity, PriceType, PriceConfig, \
-    DefaultParameterName, FetchFrom, ModelType
 from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError
+from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
+from core.model_runtime.entities.model_entities import (AIModelEntity, DefaultParameterName, FetchFrom, ModelType,
+                                                        PriceConfig, PriceInfo, PriceType)
+from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
+from pydantic import ValidationError
 
 
 class AIModel(ABC):

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -2,15 +2,14 @@ import logging
 import os
 import time
 from abc import abstractmethod
-from typing import Optional, Generator, Union, List
+from typing import Generator, List, Optional, Union
 
 from core.model_runtime.callbacks.base_callback import Callback
 from core.model_runtime.callbacks.logging_callback import LoggingCallback
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage
-from core.model_runtime.entities.model_entities import ModelPropertyKey, PriceType, ParameterType, ParameterRule, \
-    ModelType
-from core.model_runtime.entities.llm_entities import LLMResult, LLMMode, LLMUsage, \
-    LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, PromptMessageTool
+from core.model_runtime.entities.model_entities import (ModelPropertyKey, ModelType, ParameterRule, ParameterType,
+                                                        PriceType)
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
 logger = logging.getLogger(__name__)

api/core/model_runtime/model_providers/__base/model_provider.py

@@ -1,11 +1,10 @@
 import importlib
 import os
 from abc import ABC, abstractmethod
-from typing import Optional, Dict
+from typing import Dict, Optional
 
 import yaml
-
-from core.model_runtime.entities.model_entities import ModelType, AIModelEntity
+from core.model_runtime.entities.model_entities import AIModelEntity, ModelType
 from core.model_runtime.entities.provider_entities import ProviderEntity
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 

api/core/model_runtime/model_providers/__base/speech2text_model.py

@@ -1,6 +1,6 @@
 import os
 from abc import abstractmethod
-from typing import Optional, IO
+from typing import IO, Optional
 
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.ai_model import AIModel

api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py

@@ -1,7 +1,8 @@
-from transformers import GPT2Tokenizer as TransformerGPT2Tokenizer
-from os.path import join, abspath, dirname
-from typing import Any
+from os.path import abspath, dirname, join
 from threading import Lock
+from typing import Any
+
+from transformers import GPT2Tokenizer as TransformerGPT2Tokenizer
 
 _tokenizer = None
 _lock = Lock()

api/core/model_runtime/model_providers/anthropic/llm/llm.py

@@ -1,18 +1,16 @@
-from typing import Optional, Generator, Union, List
+from typing import Generator, List, Optional, Union
 
 import anthropic
 from anthropic import Anthropic, Stream
-from anthropic.types import completion_create_params, Completion
-from httpx import Timeout
-
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, UserPromptMessage, AssistantPromptMessage, \
-    SystemPromptMessage
-from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, \
-    LLMResultChunkDelta
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from anthropic.types import Completion, completion_create_params
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool,
+                                                          SystemPromptMessage, UserPromptMessage)
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from httpx import Timeout
 
 
 class AnthropicLargeLanguageModel(LargeLanguageModel):

api/core/model_runtime/model_providers/azure_openai/_common.py

@@ -1,10 +1,8 @@
 import openai
-from httpx import Timeout
-
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.model_providers.azure_openai._constant import AZURE_OPENAI_API_VERSION
-
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from httpx import Timeout
 
 
 class _CommonAzureOpenAI:

api/core/model_runtime/model_providers/azure_openai/_constant.py

@@ -1,10 +1,9 @@
-from pydantic import BaseModel
-
-from core.model_runtime.entities.llm_entities import LLMMode
-from core.model_runtime.entities.model_entities import ModelFeature, ModelType, FetchFrom, ParameterRule, \
-    DefaultParameterName, PriceConfig, ModelPropertyKey
-from core.model_runtime.entities.model_entities import AIModelEntity, I18nObject
 from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
+from core.model_runtime.entities.llm_entities import LLMMode
+from core.model_runtime.entities.model_entities import (AIModelEntity, DefaultParameterName, FetchFrom, I18nObject,
+                                                        ModelFeature, ModelPropertyKey, ModelType, ParameterRule,
+                                                        PriceConfig)
+from pydantic import BaseModel
 
 AZURE_OPENAI_API_VERSION = '2023-12-01-preview'
 

api/core/model_runtime/model_providers/azure_openai/llm/llm.py

@@ -1,23 +1,22 @@
 import logging
-from typing import Optional, Generator, Union, List, cast
+from typing import Generator, List, Optional, Union, cast
 
 import tiktoken
-from openai import AzureOpenAI, Stream
-from openai.types import Completion
-from openai.types.chat import ChatCompletionChunk, ChatCompletion, ChatCompletionMessageToolCall
-from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall, ChoiceDeltaFunctionCall
-from openai.types.chat.chat_completion_message import FunctionCall
-
-from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, \
-    LLMResultChunk, LLMResultChunkDelta
-from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, AssistantPromptMessage, \
-    UserPromptMessage, PromptMessageContentType, ImagePromptMessageContent, \
-    TextPromptMessageContent, SystemPromptMessage, ToolPromptMessage
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent,
+                                                          PromptMessage, PromptMessageContentType, PromptMessageTool,
+                                                          SystemPromptMessage, TextPromptMessageContent,
+                                                          ToolPromptMessage, UserPromptMessage)
 from core.model_runtime.entities.model_entities import AIModelEntity, ModelPropertyKey
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.model_providers.azure_openai._common import _CommonAzureOpenAI
 from core.model_runtime.model_providers.azure_openai._constant import LLM_BASE_MODELS, AzureBaseModel
+from openai import AzureOpenAI, Stream
+from openai.types import Completion
+from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessageToolCall
+from openai.types.chat.chat_completion_chunk import ChoiceDeltaFunctionCall, ChoiceDeltaToolCall
+from openai.types.chat.chat_completion_message import FunctionCall
 
 logger = logging.getLogger(__name__)
 

api/core/model_runtime/model_providers/azure_openai/text_embedding/text_embedding.py

@@ -4,14 +4,13 @@ from typing import Optional, Tuple
 
 import numpy as np
 import tiktoken
-from openai import AzureOpenAI
-
-from core.model_runtime.entities.model_entities import PriceType, AIModelEntity
-from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.entities.model_entities import AIModelEntity, PriceType
+from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
 from core.model_runtime.model_providers.azure_openai._common import _CommonAzureOpenAI
 from core.model_runtime.model_providers.azure_openai._constant import EMBEDDING_BASE_MODELS, AzureBaseModel
+from openai import AzureOpenAI
 
 
 class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel):

api/core/model_runtime/model_providers/baichuan/baichuan.py

@@ -1,7 +1,8 @@
-from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+import logging
+
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
-import logging
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
 
 logger = logging.getLogger(__name__)
 

api/core/model_runtime/model_providers/baichuan/llm/baichuan_tokenizer.py

@@ -1,5 +1,6 @@
 import re
 
+
 class BaichuanTokenizer(object):
     @classmethod
     def count_chinese_characters(cls, text: str) -> int:

api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py

@@ -1,12 +1,18 @@
-from os.path import join
-from typing import List, Optional, Generator, Union, Dict, Any
-from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import \
-    InsufficientAccountBalance, InvalidAPIKeyError, InternalServerError, RateLimitReachedError, InvalidAuthenticationError, BadRequestError
 from enum import Enum
-from json import dumps, loads
-from requests import post
-from time import time
 from hashlib import md5
+from json import dumps, loads
+from os.path import join
+from time import time
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import (BadRequestError,
+                                                                                   InsufficientAccountBalance,
+                                                                                   InternalServerError,
+                                                                                   InvalidAPIKeyError,
+                                                                                   InvalidAuthenticationError,
+                                                                                   RateLimitReachedError)
+from requests import post
+
 
 class BaichuanMessage:
     class Role(Enum):

api/core/model_runtime/model_providers/baichuan/llm/llm.py

@@ -1,14 +1,21 @@
 from typing import Generator, List, Optional, Union, cast
-from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage, LLMResultChunk, LLMResultChunkDelta
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage, UserPromptMessage, SystemPromptMessage
-from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool,
+                                                          SystemPromptMessage, UserPromptMessage)
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
-from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import \
-    InsufficientAccountBalance, InvalidAPIKeyError, InternalServerError, RateLimitReachedError, InvalidAuthenticationError, BadRequestError
-from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo import BaichuanModel, BaichuanMessage
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.model_providers.baichuan.llm.baichuan_tokenizer import BaichuanTokenizer
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo import BaichuanMessage, BaichuanModel
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import (BadRequestError,
+                                                                                   InsufficientAccountBalance,
+                                                                                   InternalServerError,
+                                                                                   InvalidAPIKeyError,
+                                                                                   InvalidAuthenticationError,
+                                                                                   RateLimitReachedError)
+
 
 class BaichuanLarguageModel(LargeLanguageModel):
     def _invoke(self, model: str, credentials: dict, 

api/core/model_runtime/model_providers/baichuan/text_embedding/text_embedding.py

@@ -1,19 +1,22 @@
+import time
+from json import dumps, loads
 from typing import Optional, Tuple
 
 from core.model_runtime.entities.model_entities import PriceType
-from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
-from core.model_runtime.errors.invoke import InvokeError, InvokeConnectionError, InvokeServerUnavailableError, \
-    InvokeRateLimitError, InvokeAuthorizationError, InvokeBadRequestError
 from core.model_runtime.model_providers.baichuan.llm.baichuan_tokenizer import BaichuanTokenizer
-from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import InvalidAPIKeyError, InsufficientAccountBalance, \
-    InvalidAuthenticationError, RateLimitReachedError, InternalServerError, BadRequestError
-
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import (BadRequestError,
+                                                                                   InsufficientAccountBalance,
+                                                                                   InternalServerError,
+                                                                                   InvalidAPIKeyError,
+                                                                                   InvalidAuthenticationError,
+                                                                                   RateLimitReachedError)
 from requests import post
-from json import dumps, loads
 
-import time
 
 class BaichuanTextEmbeddingModel(TextEmbeddingModel):
     """

api/core/model_runtime/model_providers/chatglm/llm/llm.py

@@ -1,30 +1,22 @@
-from typing import Generator, List, Optional
-from requests import post
-
-
-from os.path import join
-from typing import cast
+import logging
 from json import dumps
+from os.path import join
+from typing import Generator, List, Optional, cast
 
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, UserPromptMessage, AssistantPromptMessage, \
-    SystemPromptMessage
-from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, \
-    LLMResultChunkDelta
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageFunction,
+                                                          PromptMessageTool, SystemPromptMessage, UserPromptMessage)
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, AssistantPromptMessage, \
-    PromptMessageFunction, UserPromptMessage, SystemPromptMessage
 from core.model_runtime.utils import helper
-from openai import OpenAI, Stream, \
-    APIConnectionError, APITimeoutError, AuthenticationError, InternalServerError, \
-    RateLimitError, ConflictError, NotFoundError, UnprocessableEntityError, PermissionDeniedError
-from openai.types.chat import ChatCompletionChunk, ChatCompletion
-from openai.types.chat.chat_completion_message import FunctionCall
 from httpx import Timeout
-
-import logging
+from openai import (APIConnectionError, APITimeoutError, AuthenticationError, ConflictError, InternalServerError,
+                    NotFoundError, OpenAI, PermissionDeniedError, RateLimitError, Stream, UnprocessableEntityError)
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
+from openai.types.chat.chat_completion_message import FunctionCall
+from requests import post
 
 logger = logging.getLogger(__name__)
 

api/core/model_runtime/model_providers/cohere/rerank/rerank.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
 import cohere
-
-from core.model_runtime.entities.rerank_entities import RerankResult, RerankDocument
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError
+from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.rerank_model import RerankModel
 

api/core/model_runtime/model_providers/google/llm/llm.py

@@ -1,24 +1,21 @@
-from typing import Optional, Generator, Union, List
+import logging
+from typing import Generator, List, Optional, Union
 
-import google.generativeai as genai
 import google.api_core.exceptions as exceptions
+import google.generativeai as genai
 import google.generativeai.client as client
-from google.generativeai.types import HarmCategory, HarmBlockThreshold
-
-from google.generativeai.types import GenerateContentResponse, ContentType
-from google.generativeai.types.content_types import to_part
-
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, UserPromptMessage, AssistantPromptMessage, \
-    SystemPromptMessage, PromptMessageRole, PromptMessageContentType
-from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, \
-    LLMResultChunkDelta
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage,
+                                                          PromptMessageContentType, PromptMessageRole,
+                                                          PromptMessageTool, SystemPromptMessage, UserPromptMessage)
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers import google
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from google.generativeai.types import ContentType, GenerateContentResponse, HarmBlockThreshold, HarmCategory
+from google.generativeai.types.content_types import to_part
 
-import logging
 logger = logging.getLogger(__name__)
 
 class GoogleLargeLanguageModel(LargeLanguageModel):

api/core/model_runtime/model_providers/huggingface_hub/_common.py

@@ -1,6 +1,5 @@
-from huggingface_hub.utils import HfHubHTTPError, BadRequestError
-
 from core.model_runtime.errors.invoke import InvokeBadRequestError, InvokeError
+from huggingface_hub.utils import BadRequestError, HfHubHTTPError
 
 
 class _CommonHuggingfaceHub:

api/core/model_runtime/model_providers/huggingface_hub/llm/llm.py

@@ -1,19 +1,18 @@
-from typing import Optional, List, Union, Generator
-
-from huggingface_hub import InferenceClient
-from huggingface_hub.hf_api import HfApi
-from huggingface_hub.utils import BadRequestError
+from typing import Generator, List, Optional, Union
 
 from core.model_runtime.entities.common_entities import I18nObject
 from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
-from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMMode
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage, \
-    UserPromptMessage, SystemPromptMessage
-from core.model_runtime.entities.model_entities import ParameterRule, DefaultParameterName, AIModelEntity, ModelType, \
-    FetchFrom, ModelPropertyKey
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool,
+                                                          SystemPromptMessage, UserPromptMessage)
+from core.model_runtime.entities.model_entities import (AIModelEntity, DefaultParameterName, FetchFrom,
+                                                        ModelPropertyKey, ModelType, ParameterRule)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.model_providers.huggingface_hub._common import _CommonHuggingfaceHub
+from huggingface_hub import InferenceClient
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import BadRequestError
 
 
 class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel):

api/core/model_runtime/model_providers/huggingface_hub/text_embedding/text_embedding.py

@@ -4,15 +4,13 @@ from typing import Optional
 
 import numpy as np
 import requests
-from huggingface_hub import InferenceClient, HfApi
-
 from core.model_runtime.entities.common_entities import I18nObject
 from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType, PriceType
-from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
 from core.model_runtime.model_providers.huggingface_hub._common import _CommonHuggingfaceHub
-
+from huggingface_hub import HfApi, InferenceClient
 
 HUGGINGFACE_ENDPOINT_API = 'https://api.endpoints.huggingface.cloud/v2/endpoint/'
 

api/core/model_runtime/model_providers/jina/text_embedding/jina_tokenizer.py

@@ -1,5 +1,7 @@
+from os.path import abspath, dirname, join
+
 from transformers import AutoTokenizer
-from os.path import join, abspath, dirname
+
 
 class JinaTokenizer:
     @staticmethod

api/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py

@@ -1,17 +1,16 @@
+import time
+from json import JSONDecodeError, dumps
 from typing import Optional
 
 from core.model_runtime.entities.model_entities import PriceType
-from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
-from core.model_runtime.errors.invoke import InvokeError, InvokeConnectionError, InvokeServerUnavailableError, \
-    InvokeRateLimitError, InvokeAuthorizationError, InvokeBadRequestError
 from core.model_runtime.model_providers.jina.text_embedding.jina_tokenizer import JinaTokenizer
-
 from requests import post
-from json import dumps, JSONDecodeError
 
-import time
 
 class JinaTextEmbeddingModel(TextEmbeddingModel):
     """

api/core/model_runtime/model_providers/localai/llm/llm.py

@@ -1,22 +1,24 @@
-from typing import Generator, List, Optional, Union, cast
-from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage, LLMResultChunk, LLMResultChunkDelta, LLMMode
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage, UserPromptMessage, SystemPromptMessage
-from core.model_runtime.entities.model_entities import AIModelEntity, ParameterRule, ParameterType, FetchFrom, ModelType, ModelPropertyKey
-from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
-from core.model_runtime.errors.validate import CredentialsValidateFailedError
-from openai import OpenAI, Stream, \
-    APIConnectionError, APITimeoutError, AuthenticationError, InternalServerError, \
-    RateLimitError, ConflictError, NotFoundError, UnprocessableEntityError, PermissionDeniedError
-from openai.types.chat import ChatCompletionChunk, ChatCompletion
-from openai.types.completion import Completion
-from openai.types.chat.chat_completion_message import FunctionCall
-from httpx import Timeout
 from os.path import join
+from typing import Generator, List, Optional, Union, cast
 
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool,
+                                                          SystemPromptMessage, UserPromptMessage)
+from core.model_runtime.entities.model_entities import (AIModelEntity, FetchFrom, ModelPropertyKey, ModelType,
+                                                        ParameterRule, ParameterType)
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils import helper
+from httpx import Timeout
+from openai import (APIConnectionError, APITimeoutError, AuthenticationError, ConflictError, InternalServerError,
+                    NotFoundError, OpenAI, PermissionDeniedError, RateLimitError, Stream, UnprocessableEntityError)
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
+from openai.types.chat.chat_completion_message import FunctionCall
+from openai.types.completion import Completion
+
 
 class LocalAILarguageModel(LargeLanguageModel):
     def _invoke(self, model: str, credentials: dict, 

api/core/model_runtime/model_providers/localai/text_embedding/text_embedding.py

@@ -1,17 +1,16 @@
+import time
+from json import JSONDecodeError, dumps
+from os.path import join
 from typing import Optional
 
 from core.model_runtime.entities.model_entities import PriceType
-from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
-from core.model_runtime.errors.invoke import InvokeError, InvokeConnectionError, InvokeServerUnavailableError, \
-    InvokeRateLimitError, InvokeAuthorizationError, InvokeBadRequestError
-
 from requests import post
-from json import dumps, JSONDecodeError
-from os.path import join
 
-import time
 
 class LocalAITextEmbeddingModel(TextEmbeddingModel):
     """

api/core/model_runtime/model_providers/minimax/llm/chat_completion.py

@@ -1,12 +1,14 @@
-from core.model_runtime.model_providers.minimax.llm.errors import BadRequestError, InvalidAPIKeyError, \
-    InternalServerError, RateLimitReachedError, InvalidAuthenticationError, InsufficientAccountBalanceError
-from core.model_runtime.model_providers.minimax.llm.types import MinimaxMessage
-from typing import List, Dict, Any, Generator, Union
-
-from json import dumps, loads
-from requests import post, Response
-from time import time
 from hashlib import md5
+from json import dumps, loads
+from time import time
+from typing import Any, Dict, Generator, List, Union
+
+from core.model_runtime.model_providers.minimax.llm.errors import (BadRequestError, InsufficientAccountBalanceError,
+                                                                   InternalServerError, InvalidAPIKeyError,
+                                                                   InvalidAuthenticationError, RateLimitReachedError)
+from core.model_runtime.model_providers.minimax.llm.types import MinimaxMessage
+from requests import Response, post
+
 
 class MinimaxChatCompletion(object):
     """

api/core/model_runtime/model_providers/minimax/llm/chat_completion_pro.py

@@ -1,12 +1,14 @@
-from core.model_runtime.model_providers.minimax.llm.errors import BadRequestError, InvalidAPIKeyError, \
-    InternalServerError, RateLimitReachedError, InvalidAuthenticationError, InsufficientAccountBalanceError
-from core.model_runtime.model_providers.minimax.llm.types import MinimaxMessage
-from typing import List, Dict, Any, Generator, Union
-
-from json import dumps, loads
-from requests import post, Response
-from time import time
 from hashlib import md5
+from json import dumps, loads
+from time import time
+from typing import Any, Dict, Generator, List, Union
+
+from core.model_runtime.model_providers.minimax.llm.errors import (BadRequestError, InsufficientAccountBalanceError,
+                                                                   InternalServerError, InvalidAPIKeyError,
+                                                                   InvalidAuthenticationError, RateLimitReachedError)
+from core.model_runtime.model_providers.minimax.llm.types import MinimaxMessage
+from requests import Response, post
+
 
 class MinimaxChatCompletionPro(object):
     """

api/core/model_runtime/model_providers/minimax/llm/llm.py

@@ -1,17 +1,20 @@
 from typing import Generator, List, Optional, Union
-from core.model_runtime.model_providers.minimax.llm.errors import BadRequestError, InvalidAPIKeyError, \
-    InternalServerError, RateLimitReachedError, InvalidAuthenticationError, InsufficientAccountBalanceError
-from core.model_runtime.model_providers.minimax.llm.types import MinimaxMessage
+
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool,
+                                                          SystemPromptMessage, UserPromptMessage)
+from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType, ParameterRule, ParameterType
+from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
+                                              InvokeError, InvokeRateLimitError, InvokeServerUnavailableError)
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.model_providers.minimax.llm.chat_completion import MinimaxChatCompletion
 from core.model_runtime.model_providers.minimax.llm.chat_completion_pro import MinimaxChatCompletionPro
+from core.model_runtime.model_providers.minimax.llm.errors import (BadRequestError, InsufficientAccountBalanceError,
+                                                                   InternalServerError, InvalidAPIKeyError,
+                                                                   InvalidAuthenticationError, RateLimitReachedError)
+from core.model_runtime.model_providers.minimax.llm.types import MinimaxMessage
 
-from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage, LLMResultChunk, LLMResultChunkDelta, LLMMode
-from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage, UserPromptMessage, SystemPromptMessage
-from core.model_runtime.entities.model_entities import AIModelEntity, ParameterRule, ParameterType, FetchFrom, ModelType
-from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
-    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
-from core.model_runtime.errors.validate import CredentialsValidateFailedError
 
 class MinimaxLargeLanguageModel(LargeLanguageModel):
     model_apis = {

api/core/model_runtime/model_providers/minimax/llm/types.py

@@ -1,5 +1,6 @@
-from typing import Dict, Any
 from enum import Enum
+from typing import Any, Dict
+
 
 class MinimaxMessage:
     class Role(Enum):

api/core/model_runtime/model_providers/minimax/minimax.py

@@ -1,7 +1,8 @@
-from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+import logging
+
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
-import logging
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
 
 logger = logging.getLogger(__name__)