Feat/environment variables in workflow (#6515)

Co-authored-by: JzoNg <jzongcode@gmail.com>

api/core/app/app_config/base_app_config_manager.py

@@ -1,6 +1,7 @@
-from typing import Optional, Union
+from collections.abc import Mapping
+from typing import Any
 
-from core.app.app_config.entities import AppAdditionalFeatures, EasyUIBasedAppModelConfigFrom
+from core.app.app_config.entities import AppAdditionalFeatures
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.app.app_config.features.more_like_this.manager import MoreLikeThisConfigManager
 from core.app.app_config.features.opening_statement.manager import OpeningStatementConfigManager
@@ -10,37 +11,19 @@ from core.app.app_config.features.suggested_questions_after_answer.manager impor
     SuggestedQuestionsAfterAnswerConfigManager,
 )
 from core.app.app_config.features.text_to_speech.manager import TextToSpeechConfigManager
-from models.model import AppMode, AppModelConfig
+from models.model import AppMode
 
 
 class BaseAppConfigManager:
-
     @classmethod
-    def convert_to_config_dict(cls, config_from: EasyUIBasedAppModelConfigFrom,
-                               app_model_config: Union[AppModelConfig, dict],
-                               config_dict: Optional[dict] = None) -> dict:
-        """
-        Convert app model config to config dict
-        :param config_from: app model config from
-        :param app_model_config: app model config
-        :param config_dict: app model config dict
-        :return:
-        """
-        if config_from != EasyUIBasedAppModelConfigFrom.ARGS:
-            app_model_config_dict = app_model_config.to_dict()
-            config_dict = app_model_config_dict.copy()
-
-        return config_dict
-
-    @classmethod
-    def convert_features(cls, config_dict: dict, app_mode: AppMode) -> AppAdditionalFeatures:
+    def convert_features(cls, config_dict: Mapping[str, Any], app_mode: AppMode) -> AppAdditionalFeatures:
         """
         Convert app config to app model config
 
         :param config_dict: app config
         :param app_mode: app mode
         """
-        config_dict = config_dict.copy()
+        config_dict = dict(config_dict.items())
 
         additional_features = AppAdditionalFeatures()
         additional_features.show_retrieve_source = RetrievalResourceConfigManager.convert(

api/core/app/app_config/features/file_upload/manager.py

@@ -1,11 +1,12 @@
-from typing import Optional
+from collections.abc import Mapping
+from typing import Any, Optional
 
 from core.app.app_config.entities import FileExtraConfig
 
 
 class FileUploadConfigManager:
     @classmethod
-    def convert(cls, config: dict, is_vision: bool = True) -> Optional[FileExtraConfig]:
+    def convert(cls, config: Mapping[str, Any], is_vision: bool = True) -> Optional[FileExtraConfig]:
         """
         Convert model config to model config
 

api/core/app/app_config/features/text_to_speech/manager.py

@@ -3,13 +3,13 @@ from core.app.app_config.entities import TextToSpeechEntity
 
 class TextToSpeechConfigManager:
     @classmethod
-    def convert(cls, config: dict) -> bool:
+    def convert(cls, config: dict):
         """
         Convert model config to model config
 
         :param config: model config args
         """
-        text_to_speech = False
+        text_to_speech = None
         text_to_speech_dict = config.get('text_to_speech')
         if text_to_speech_dict:
             if text_to_speech_dict.get('enabled'):

api/core/app/apps/advanced_chat/app_generator.py

@@ -1,3 +1,4 @@
+import contextvars
 import logging
 import os
 import threading
@@ -8,6 +9,7 @@ from typing import Union
 from flask import Flask, current_app
 from pydantic import ValidationError
 
+import contexts
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
 from core.app.apps.advanced_chat.app_runner import AdvancedChatAppRunner
@@ -107,6 +109,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             extras=extras,
             trace_manager=trace_manager
         )
+        contexts.tenant_id.set(application_generate_entity.app_config.tenant_id)
 
         return self._generate(
             app_model=app_model,
@@ -173,6 +176,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
                 inputs=args['inputs']
             )
         )
+        contexts.tenant_id.set(application_generate_entity.app_config.tenant_id)
 
         return self._generate(
             app_model=app_model,
@@ -225,6 +229,8 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             'queue_manager': queue_manager,
             'conversation_id': conversation.id,
             'message_id': message.id,
+            'user': user,
+            'context': contextvars.copy_context()
         })
 
         worker_thread.start()
@@ -249,7 +255,9 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
                          application_generate_entity: AdvancedChatAppGenerateEntity,
                          queue_manager: AppQueueManager,
                          conversation_id: str,
-                         message_id: str) -> None:
+                         message_id: str,
+                         user: Account,
+                         context: contextvars.Context) -> None:
         """
         Generate worker in a new thread.
         :param flask_app: Flask app
@@ -259,6 +267,8 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         :param message_id: message ID
         :return:
         """
+        for var, val in context.items():
+            var.set(val)
         with flask_app.app_context():
             try:
                 runner = AdvancedChatAppRunner()

api/core/app/apps/advanced_chat/app_runner.py

@@ -1,7 +1,8 @@
 import logging
 import os
 import time
-from typing import Optional, cast
+from collections.abc import Mapping
+from typing import Any, Optional, cast
 
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig
 from core.app.apps.advanced_chat.workflow_event_trigger_callback import WorkflowEventTriggerCallback
@@ -14,6 +15,7 @@ from core.app.entities.app_invoke_entities import (
 )
 from core.app.entities.queue_entities import QueueAnnotationReplyEvent, QueueStopEvent, QueueTextChunkEvent
 from core.moderation.base import ModerationException
+from core.workflow.callbacks.base_workflow_callback import WorkflowCallback
 from core.workflow.entities.node_entities import SystemVariable
 from core.workflow.nodes.base_node import UserFrom
 from core.workflow.workflow_engine_manager import WorkflowEngineManager
@@ -87,7 +89,7 @@ class AdvancedChatAppRunner(AppRunner):
 
         db.session.close()
 
-        workflow_callbacks = [WorkflowEventTriggerCallback(
+        workflow_callbacks: list[WorkflowCallback] = [WorkflowEventTriggerCallback(
             queue_manager=queue_manager,
             workflow=workflow
         )]
@@ -161,7 +163,7 @@ class AdvancedChatAppRunner(AppRunner):
             self, queue_manager: AppQueueManager,
             app_record: App,
             app_generate_entity: AdvancedChatAppGenerateEntity,
-            inputs: dict,
+            inputs: Mapping[str, Any],
             query: str,
             message_id: str
     ) -> bool:

api/core/app/apps/advanced_chat/generate_response_converter.py

@@ -1,9 +1,11 @@
 import json
 from collections.abc import Generator
-from typing import cast
+from typing import Any, cast
 
 from core.app.apps.base_app_generate_response_converter import AppGenerateResponseConverter
 from core.app.entities.task_entities import (
+    AppBlockingResponse,
+    AppStreamResponse,
     ChatbotAppBlockingResponse,
     ChatbotAppStreamResponse,
     ErrorStreamResponse,
@@ -18,12 +20,13 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter):
     _blocking_response_type = ChatbotAppBlockingResponse
 
     @classmethod
-    def convert_blocking_full_response(cls, blocking_response: ChatbotAppBlockingResponse) -> dict:
+    def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]:
         """
         Convert blocking full response.
         :param blocking_response: blocking response
         :return:
         """
+        blocking_response = cast(ChatbotAppBlockingResponse, blocking_response)
         response = {
             'event': 'message',
             'task_id': blocking_response.task_id,
@@ -39,7 +42,7 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter):
         return response
 
     @classmethod
-    def convert_blocking_simple_response(cls, blocking_response: ChatbotAppBlockingResponse) -> dict:
+    def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]:
         """
         Convert blocking simple response.
         :param blocking_response: blocking response
@@ -53,8 +56,7 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter):
         return response
 
     @classmethod
-    def convert_stream_full_response(cls, stream_response: Generator[ChatbotAppStreamResponse, None, None]) \
-            -> Generator[str, None, None]:
+    def convert_stream_full_response(cls, stream_response: Generator[AppStreamResponse, None, None]) -> Generator[str, Any, None]:
         """
         Convert stream full response.
         :param stream_response: stream response
@@ -83,8 +85,7 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter):
             yield json.dumps(response_chunk)
 
     @classmethod
-    def convert_stream_simple_response(cls, stream_response: Generator[ChatbotAppStreamResponse, None, None]) \
-            -> Generator[str, None, None]:
+    def convert_stream_simple_response(cls, stream_response: Generator[AppStreamResponse, None, None]) -> Generator[str, Any, None]:
         """
         Convert stream simple response.
         :param stream_response: stream response

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -118,7 +118,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         self._stream_generate_routes = self._get_stream_generate_routes()
         self._conversation_name_generate_thread = None
 
-    def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]:
+    def process(self):
         """
         Process generate task pipeline.
         :return:
@@ -141,8 +141,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         else:
             return self._to_blocking_response(generator)
 
-    def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) \
-            -> ChatbotAppBlockingResponse:
+    def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) -> ChatbotAppBlockingResponse:
         """
         Process blocking response.
         :return:
@@ -172,8 +171,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
         raise Exception('Queue listening stopped unexpectedly.')
 
-    def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) \
-            -> Generator[ChatbotAppStreamResponse, None, None]:
+    def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) -> Generator[ChatbotAppStreamResponse, Any, None]:
         """
         To stream response.
         :return:

api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py

@@ -14,13 +14,13 @@ from core.app.entities.queue_entities import (
     QueueWorkflowStartedEvent,
     QueueWorkflowSucceededEvent,
 )
-from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback
+from core.workflow.callbacks.base_workflow_callback import WorkflowCallback
 from core.workflow.entities.base_node_data_entities import BaseNodeData
 from core.workflow.entities.node_entities import NodeType
 from models.workflow import Workflow
 
 
-class WorkflowEventTriggerCallback(BaseWorkflowCallback):
+class WorkflowEventTriggerCallback(WorkflowCallback):
 
     def __init__(self, queue_manager: AppQueueManager, workflow: Workflow):
         self._queue_manager = queue_manager

api/core/app/apps/base_app_generate_response_converter.py

@@ -1,7 +1,7 @@
 import logging
 from abc import ABC, abstractmethod
 from collections.abc import Generator
-from typing import Union
+from typing import Any, Union
 
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.app.entities.task_entities import AppBlockingResponse, AppStreamResponse
@@ -15,44 +15,41 @@ class AppGenerateResponseConverter(ABC):
     @classmethod
     def convert(cls, response: Union[
         AppBlockingResponse,
-        Generator[AppStreamResponse, None, None]
-    ], invoke_from: InvokeFrom) -> Union[
-        dict,
-        Generator[str, None, None]
-    ]:
+        Generator[AppStreamResponse, Any, None]
+    ], invoke_from: InvokeFrom):
         if invoke_from in [InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API]:
-            if isinstance(response, cls._blocking_response_type):
+            if isinstance(response, AppBlockingResponse):
                 return cls.convert_blocking_full_response(response)
             else:
-                def _generate():
+                def _generate_full_response() -> Generator[str, Any, None]:
                     for chunk in cls.convert_stream_full_response(response):
                         if chunk == 'ping':
                             yield f'event: {chunk}\n\n'
                         else:
                             yield f'data: {chunk}\n\n'
 
-                return _generate()
+                return _generate_full_response()
         else:
-            if isinstance(response, cls._blocking_response_type):
+            if isinstance(response, AppBlockingResponse):
                 return cls.convert_blocking_simple_response(response)
             else:
-                def _generate():
+                def _generate_simple_response() -> Generator[str, Any, None]:
                     for chunk in cls.convert_stream_simple_response(response):
                         if chunk == 'ping':
                             yield f'event: {chunk}\n\n'
                         else:
                             yield f'data: {chunk}\n\n'
 
-                return _generate()
+                return _generate_simple_response()
 
     @classmethod
     @abstractmethod
-    def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict:
+    def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]:
         raise NotImplementedError
 
     @classmethod
     @abstractmethod
-    def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict:
+    def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]:
         raise NotImplementedError
 
     @classmethod
@@ -68,7 +65,7 @@ class AppGenerateResponseConverter(ABC):
         raise NotImplementedError
 
     @classmethod
-    def _get_simple_metadata(cls, metadata: dict) -> dict:
+    def _get_simple_metadata(cls, metadata: dict[str, Any]):
         """
         Get simple metadata.
         :param metadata: metadata

api/core/app/apps/workflow/app_generator.py

@@ -1,3 +1,4 @@
+import contextvars
 import logging
 import os
 import threading
@@ -8,6 +9,7 @@ from typing import Union
 from flask import Flask, current_app
 from pydantic import ValidationError
 
+import contexts
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.app.apps.base_app_generator import BaseAppGenerator
 from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedException, PublishFrom
@@ -38,7 +40,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
         invoke_from: InvokeFrom,
         stream: bool = True,
         call_depth: int = 0,
-    ) -> Union[dict, Generator[dict, None, None]]:
+    ):
         """
         Generate App response.
 
@@ -86,6 +88,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             call_depth=call_depth,
             trace_manager=trace_manager
         )
+        contexts.tenant_id.set(application_generate_entity.app_config.tenant_id)
 
         return self._generate(
             app_model=app_model,
@@ -126,7 +129,8 @@ class WorkflowAppGenerator(BaseAppGenerator):
         worker_thread = threading.Thread(target=self._generate_worker, kwargs={
             'flask_app': current_app._get_current_object(),
             'application_generate_entity': application_generate_entity,
-            'queue_manager': queue_manager
+            'queue_manager': queue_manager,
+            'context': contextvars.copy_context()
         })
 
         worker_thread.start()
@@ -150,8 +154,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
                                   node_id: str,
                                   user: Account,
                                   args: dict,
-                                  stream: bool = True) \
-            -> Union[dict, Generator[dict, None, None]]:
+                                  stream: bool = True):
         """
         Generate App response.
 
@@ -193,6 +196,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
                 inputs=args['inputs']
             )
         )
+        contexts.tenant_id.set(application_generate_entity.app_config.tenant_id)
 
         return self._generate(
             app_model=app_model,
@@ -205,7 +209,8 @@ class WorkflowAppGenerator(BaseAppGenerator):
 
     def _generate_worker(self, flask_app: Flask,
                          application_generate_entity: WorkflowAppGenerateEntity,
-                         queue_manager: AppQueueManager) -> None:
+                         queue_manager: AppQueueManager,
+                         context: contextvars.Context) -> None:
         """
         Generate worker in a new thread.
         :param flask_app: Flask app
@@ -213,6 +218,8 @@ class WorkflowAppGenerator(BaseAppGenerator):
         :param queue_manager: queue manager
         :return:
         """
+        for var, val in context.items():
+            var.set(val)
         with flask_app.app_context():
             try:
                 # workflow app

api/core/app/apps/workflow/app_runner.py

@@ -10,6 +10,7 @@ from core.app.entities.app_invoke_entities import (
     InvokeFrom,
     WorkflowAppGenerateEntity,
 )
+from core.workflow.callbacks.base_workflow_callback import WorkflowCallback
 from core.workflow.entities.node_entities import SystemVariable
 from core.workflow.nodes.base_node import UserFrom
 from core.workflow.workflow_engine_manager import WorkflowEngineManager
@@ -57,7 +58,7 @@ class WorkflowAppRunner:
 
         db.session.close()
 
-        workflow_callbacks = [WorkflowEventTriggerCallback(
+        workflow_callbacks: list[WorkflowCallback] = [WorkflowEventTriggerCallback(
             queue_manager=queue_manager,
             workflow=workflow
         )]

api/core/app/apps/workflow/workflow_event_trigger_callback.py

@@ -14,13 +14,13 @@ from core.app.entities.queue_entities import (
     QueueWorkflowStartedEvent,
     QueueWorkflowSucceededEvent,
 )
-from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback
+from core.workflow.callbacks.base_workflow_callback import WorkflowCallback
 from core.workflow.entities.base_node_data_entities import BaseNodeData
 from core.workflow.entities.node_entities import NodeType
 from models.workflow import Workflow
 
 
-class WorkflowEventTriggerCallback(BaseWorkflowCallback):
+class WorkflowEventTriggerCallback(WorkflowCallback):
 
     def __init__(self, queue_manager: AppQueueManager, workflow: Workflow):
         self._queue_manager = queue_manager

api/core/app/apps/workflow_logging_callback.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from core.app.entities.queue_entities import AppQueueEvent
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback
+from core.workflow.callbacks.base_workflow_callback import WorkflowCallback
 from core.workflow.entities.base_node_data_entities import BaseNodeData
 from core.workflow.entities.node_entities import NodeType
 
@@ -15,7 +15,7 @@ _TEXT_COLOR_MAPPING = {
 }
 
 
-class WorkflowLoggingCallback(BaseWorkflowCallback):
+class WorkflowLoggingCallback(WorkflowCallback):
 
     def __init__(self) -> None:
         self.current_node_id = None

api/core/app/entities/app_invoke_entities.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from enum import Enum
 from typing import Any, Optional
 
@@ -76,7 +77,7 @@ class AppGenerateEntity(BaseModel):
     # app config
     app_config: AppConfig
 
-    inputs: dict[str, Any]
+    inputs: Mapping[str, Any]
     files: list[FileVar] = []
     user_id: str
 
@@ -140,7 +141,7 @@ class AdvancedChatAppGenerateEntity(AppGenerateEntity):
     app_config: WorkflowUIBasedAppConfig
 
     conversation_id: Optional[str] = None
-    query: Optional[str] = None
+    query: str
 
     class SingleIterationRunEntity(BaseModel):
         """

api/core/app/segments/__init__.py

@@ -0,0 +1,27 @@
+from .segment_group import SegmentGroup
+from .segments import Segment
+from .types import SegmentType
+from .variables import (
+    ArrayVariable,
+    FileVariable,
+    FloatVariable,
+    IntegerVariable,
+    ObjectVariable,
+    SecretVariable,
+    StringVariable,
+    Variable,
+)
+
+__all__ = [
+    'IntegerVariable',
+    'FloatVariable',
+    'ObjectVariable',
+    'SecretVariable',
+    'FileVariable',
+    'StringVariable',
+    'ArrayVariable',
+    'Variable',
+    'SegmentType',
+    'SegmentGroup',
+    'Segment'
+]

api/core/app/segments/factory.py

@@ -0,0 +1,64 @@
+from collections.abc import Mapping
+from typing import Any
+
+from core.file.file_obj import FileVar
+
+from .segments import Segment, StringSegment
+from .types import SegmentType
+from .variables import (
+    ArrayVariable,
+    FileVariable,
+    FloatVariable,
+    IntegerVariable,
+    ObjectVariable,
+    SecretVariable,
+    StringVariable,
+    Variable,
+)
+
+
+def build_variable_from_mapping(m: Mapping[str, Any], /) -> Variable:
+    if (value_type := m.get('value_type')) is None:
+        raise ValueError('missing value type')
+    if not m.get('name'):
+        raise ValueError('missing name')
+    if (value := m.get('value')) is None:
+        raise ValueError('missing value')
+    match value_type:
+        case SegmentType.STRING:
+            return StringVariable.model_validate(m)
+        case SegmentType.NUMBER if isinstance(value, int):
+            return IntegerVariable.model_validate(m)
+        case SegmentType.NUMBER if isinstance(value, float):
+            return FloatVariable.model_validate(m)
+        case SegmentType.SECRET:
+            return SecretVariable.model_validate(m)
+        case SegmentType.NUMBER if not isinstance(value, float | int):
+            raise ValueError(f'invalid number value {value}')
+    raise ValueError(f'not supported value type {value_type}')
+
+
+def build_anonymous_variable(value: Any, /) -> Variable:
+    if isinstance(value, str):
+        return StringVariable(name='anonymous', value=value)
+    if isinstance(value, int):
+        return IntegerVariable(name='anonymous', value=value)
+    if isinstance(value, float):
+        return FloatVariable(name='anonymous', value=value)
+    if isinstance(value, dict):
+        # TODO: Limit the depth of the object
+        obj = {k: build_anonymous_variable(v) for k, v in value.items()}
+        return ObjectVariable(name='anonymous', value=obj)
+    if isinstance(value, list):
+        # TODO: Limit the depth of the array
+        elements = [build_anonymous_variable(v) for v in value]
+        return ArrayVariable(name='anonymous', value=elements)
+    if isinstance(value, FileVar):
+        return FileVariable(name='anonymous', value=value)
+    raise ValueError(f'not supported value {value}')
+
+
+def build_segment(value: Any, /) -> Segment:
+    if isinstance(value, str):
+        return StringSegment(value=value)
+    raise ValueError(f'not supported value {value}')

api/core/app/segments/parser.py

@@ -0,0 +1,17 @@
+import re
+
+from core.app.segments import SegmentGroup, factory
+from core.workflow.entities.variable_pool import VariablePool
+
+VARIABLE_PATTERN = re.compile(r'\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10})#\}\}')
+
+
+def convert_template(*, template: str, variable_pool: VariablePool):
+    parts = re.split(VARIABLE_PATTERN, template)
+    segments = []
+    for part in parts:
+        if '.' in part and (value := variable_pool.get(part.split('.'))):
+            segments.append(value)
+        else:
+            segments.append(factory.build_segment(part))
+    return SegmentGroup(segments=segments)

api/core/app/segments/segment_group.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+
+from .segments import Segment
+
+
+class SegmentGroup(BaseModel):
+    segments: list[Segment]
+
+    @property
+    def text(self):
+        return ''.join([segment.text for segment in self.segments])
+
+    @property
+    def log(self):
+        return ''.join([segment.log for segment in self.segments])
+
+    @property
+    def markdown(self):
+        return ''.join([segment.markdown for segment in self.segments])

api/core/app/segments/segments.py

@@ -0,0 +1,39 @@
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, field_validator
+
+from .types import SegmentType
+
+
+class Segment(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    value_type: SegmentType
+    value: Any
+
+    @field_validator('value_type')
+    def validate_value_type(cls, value):
+        """
+        This validator checks if the provided value is equal to the default value of the 'value_type' field.
+        If the value is different, a ValueError is raised.
+        """
+        if value != cls.model_fields['value_type'].default:
+            raise ValueError("Cannot modify 'value_type'")
+        return value
+
+    @property
+    def text(self) -> str:
+        return str(self.value)
+
+    @property
+    def log(self) -> str:
+        return str(self.value)
+
+    @property
+    def markdown(self) -> str:
+        return str(self.value)
+
+
+class StringSegment(Segment):
+    value_type: SegmentType = SegmentType.STRING
+    value: str

api/core/app/segments/types.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+
+class SegmentType(str, Enum):
+    STRING = 'string'
+    NUMBER = 'number'
+    FILE = 'file'
+
+    SECRET = 'secret'
+
+    OBJECT = 'object'
+
+    ARRAY = 'array'
+    ARRAY_STRING = 'array[string]'
+    ARRAY_NUMBER = 'array[number]'
+    ARRAY_OBJECT = 'array[object]'
+    ARRAY_FILE = 'array[file]'

api/core/app/segments/variables.py

@@ -0,0 +1,83 @@
+import json
+from collections.abc import Mapping, Sequence
+
+from pydantic import Field
+
+from core.file.file_obj import FileVar
+from core.helper import encrypter
+
+from .segments import Segment, StringSegment
+from .types import SegmentType
+
+
+class Variable(Segment):
+    """
+    A variable is a segment that has a name.
+    """
+
+    id: str = Field(
+        default='',
+        description="Unique identity for variable. It's only used by environment variables now.",
+    )
+    name: str
+
+
+class StringVariable(StringSegment, Variable):
+    pass
+
+
+class FloatVariable(Variable):
+    value_type: SegmentType = SegmentType.NUMBER
+    value: float
+
+
+class IntegerVariable(Variable):
+    value_type: SegmentType = SegmentType.NUMBER
+    value: int
+
+
+class ObjectVariable(Variable):
+    value_type: SegmentType = SegmentType.OBJECT
+    value: Mapping[str, Variable]
+
+    @property
+    def text(self) -> str:
+        # TODO: Process variables.
+        return json.dumps(self.model_dump()['value'], ensure_ascii=False)
+
+    @property
+    def log(self) -> str:
+        # TODO: Process variables.
+        return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2)
+
+    @property
+    def markdown(self) -> str:
+        # TODO: Use markdown code block
+        return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2)
+
+
+class ArrayVariable(Variable):
+    value_type: SegmentType = SegmentType.ARRAY
+    value: Sequence[Variable]
+
+    @property
+    def markdown(self) -> str:
+        return '\n'.join(['- ' + item.markdown for item in self.value])
+
+
+class FileVariable(Variable):
+    value_type: SegmentType = SegmentType.FILE
+    # TODO: embed FileVar in this model.
+    value: FileVar
+
+    @property
+    def markdown(self) -> str:
+        return self.value.to_markdown()
+
+
+class SecretVariable(StringVariable):
+    value_type: SegmentType = SegmentType.SECRET
+
+    @property
+    def log(self) -> str:
+        return encrypter.obfuscated_token(self.value)