FEAT: NEW WORKFLOW ENGINE (#3160)

Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Yeuoly <admin@srmxy.cn>
Co-authored-by: JzoNg <jzongcode@gmail.com>
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
Co-authored-by: jyong <jyong@dify.ai>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: jyong <718720800@qq.com>

api/core/workflow/entities/base_node_data_entities.py

@@ -0,0 +1,9 @@
+from abc import ABC
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class BaseNodeData(ABC, BaseModel):
+    title: str
+    desc: Optional[str] = None

api/core/workflow/entities/node_entities.py

@@ -0,0 +1,85 @@
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+from models.workflow import WorkflowNodeExecutionStatus
+
+
+class NodeType(Enum):
+    """
+    Node Types.
+    """
+    START = 'start'
+    END = 'end'
+    ANSWER = 'answer'
+    LLM = 'llm'
+    KNOWLEDGE_RETRIEVAL = 'knowledge-retrieval'
+    IF_ELSE = 'if-else'
+    CODE = 'code'
+    TEMPLATE_TRANSFORM = 'template-transform'
+    QUESTION_CLASSIFIER = 'question-classifier'
+    HTTP_REQUEST = 'http-request'
+    TOOL = 'tool'
+    VARIABLE_ASSIGNER = 'variable-assigner'
+
+    @classmethod
+    def value_of(cls, value: str) -> 'NodeType':
+        """
+        Get value of given node type.
+
+        :param value: node type value
+        :return: node type
+        """
+        for node_type in cls:
+            if node_type.value == value:
+                return node_type
+        raise ValueError(f'invalid node type value {value}')
+
+
+class SystemVariable(Enum):
+    """
+    System Variables.
+    """
+    QUERY = 'query'
+    FILES = 'files'
+    CONVERSATION = 'conversation'
+
+    @classmethod
+    def value_of(cls, value: str) -> 'SystemVariable':
+        """
+        Get value of given system variable.
+
+        :param value: system variable value
+        :return: system variable
+        """
+        for system_variable in cls:
+            if system_variable.value == value:
+                return system_variable
+        raise ValueError(f'invalid system variable value {value}')
+
+
+class NodeRunMetadataKey(Enum):
+    """
+    Node Run Metadata Key.
+    """
+    TOTAL_TOKENS = 'total_tokens'
+    TOTAL_PRICE = 'total_price'
+    CURRENCY = 'currency'
+    TOOL_INFO = 'tool_info'
+
+
+class NodeRunResult(BaseModel):
+    """
+    Node Run Result.
+    """
+    status: WorkflowNodeExecutionStatus = WorkflowNodeExecutionStatus.RUNNING
+
+    inputs: Optional[dict] = None  # node inputs
+    process_data: Optional[dict] = None  # process data
+    outputs: Optional[dict] = None  # node outputs
+    metadata: Optional[dict[NodeRunMetadataKey, Any]] = None  # node metadata
+
+    edge_source_handle: Optional[str] = None  # source handle id of node with multiple branches
+
+    error: Optional[str] = None  # error message if status is failed

api/core/workflow/entities/variable_entities.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+
+
+class VariableSelector(BaseModel):
+    """
+    Variable Selector.
+    """
+    variable: str
+    value_selector: list[str]

api/core/workflow/entities/variable_pool.py

@@ -0,0 +1,94 @@
+from enum import Enum
+from typing import Any, Optional, Union
+
+from core.file.file_obj import FileVar
+from core.workflow.entities.node_entities import SystemVariable
+
+VariableValue = Union[str, int, float, dict, list, FileVar]
+
+
+class ValueType(Enum):
+    """
+    Value Type Enum
+    """
+    STRING = "string"
+    NUMBER = "number"
+    OBJECT = "object"
+    ARRAY_STRING = "array[string]"
+    ARRAY_NUMBER = "array[number]"
+    ARRAY_OBJECT = "array[object]"
+    ARRAY_FILE = "array[file]"
+    FILE = "file"
+
+
+class VariablePool:
+    variables_mapping = {}
+    user_inputs: dict
+    system_variables: dict[SystemVariable, Any]
+
+    def __init__(self, system_variables: dict[SystemVariable, Any],
+                 user_inputs: dict) -> None:
+        # system variables
+        # for example:
+        # {
+        #     'query': 'abc',
+        #     'files': []
+        # }
+        self.user_inputs = user_inputs
+        self.system_variables = system_variables
+        for system_variable, value in system_variables.items():
+            self.append_variable('sys', [system_variable.value], value)
+
+    def append_variable(self, node_id: str, variable_key_list: list[str], value: VariableValue) -> None:
+        """
+        Append variable
+        :param node_id: node id
+        :param variable_key_list: variable key list, like: ['result', 'text']
+        :param value: value
+        :return:
+        """
+        if node_id not in self.variables_mapping:
+            self.variables_mapping[node_id] = {}
+
+        variable_key_list_hash = hash(tuple(variable_key_list))
+
+        self.variables_mapping[node_id][variable_key_list_hash] = value
+
+    def get_variable_value(self, variable_selector: list[str],
+                           target_value_type: Optional[ValueType] = None) -> Optional[VariableValue]:
+        """
+        Get variable
+        :param variable_selector: include node_id and variables
+        :param target_value_type: target value type
+        :return:
+        """
+        if len(variable_selector) < 2:
+            raise ValueError('Invalid value selector')
+
+        node_id = variable_selector[0]
+        if node_id not in self.variables_mapping:
+            return None
+
+        # fetch variable keys, pop node_id
+        variable_key_list = variable_selector[1:]
+
+        variable_key_list_hash = hash(tuple(variable_key_list))
+
+        value = self.variables_mapping[node_id].get(variable_key_list_hash)
+
+        if target_value_type:
+            if target_value_type == ValueType.STRING:
+                return str(value)
+            elif target_value_type == ValueType.NUMBER:
+                return int(value)
+            elif target_value_type == ValueType.OBJECT:
+                if not isinstance(value, dict):
+                    raise ValueError('Invalid value type: object')
+            elif target_value_type in [ValueType.ARRAY_STRING,
+                                       ValueType.ARRAY_NUMBER,
+                                       ValueType.ARRAY_OBJECT,
+                                       ValueType.ARRAY_FILE]:
+                if not isinstance(value, list):
+                    raise ValueError(f'Invalid value type: {target_value_type.value}')
+
+        return value

api/core/workflow/entities/workflow_entities.py

@@ -0,0 +1,49 @@
+from typing import Optional
+
+from core.workflow.entities.node_entities import NodeRunResult
+from core.workflow.entities.variable_pool import VariablePool
+from core.workflow.nodes.base_node import BaseNode, UserFrom
+from models.workflow import Workflow, WorkflowType
+
+
+class WorkflowNodeAndResult:
+    node: BaseNode
+    result: Optional[NodeRunResult] = None
+
+    def __init__(self, node: BaseNode, result: Optional[NodeRunResult] = None):
+        self.node = node
+        self.result = result
+
+
+class WorkflowRunState:
+    tenant_id: str
+    app_id: str
+    workflow_id: str
+    workflow_type: WorkflowType
+    user_id: str
+    user_from: UserFrom
+
+    start_at: float
+    variable_pool: VariablePool
+
+    total_tokens: int = 0
+
+    workflow_nodes_and_results: list[WorkflowNodeAndResult]
+
+    def __init__(self, workflow: Workflow,
+                 start_at: float,
+                 variable_pool: VariablePool,
+                 user_id: str,
+                 user_from: UserFrom):
+        self.workflow_id = workflow.id
+        self.tenant_id = workflow.tenant_id
+        self.app_id = workflow.app_id
+        self.workflow_type = WorkflowType.value_of(workflow.type)
+        self.user_id = user_id
+        self.user_from = user_from
+
+        self.start_at = start_at
+        self.variable_pool = variable_pool
+
+        self.total_tokens = 0
+        self.workflow_nodes_and_results = []