FEAT: NEW WORKFLOW ENGINE (#3160)

Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Yeuoly <admin@srmxy.cn>
Co-authored-by: JzoNg <jzongcode@gmail.com>
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
Co-authored-by: jyong <jyong@dify.ai>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: jyong <718720800@qq.com>

api/core/helper/code_executor/code_executor.py

@@ -0,0 +1,87 @@
+from os import environ
+from typing import Literal, Optional
+
+from httpx import post
+from pydantic import BaseModel
+from yarl import URL
+
+from core.helper.code_executor.javascript_transformer import NodeJsTemplateTransformer
+from core.helper.code_executor.jina2_transformer import Jinja2TemplateTransformer
+from core.helper.code_executor.python_transformer import PythonTemplateTransformer
+
+# Code Executor
+CODE_EXECUTION_ENDPOINT = environ.get('CODE_EXECUTION_ENDPOINT', '')
+CODE_EXECUTION_API_KEY = environ.get('CODE_EXECUTION_API_KEY', '')
+
+CODE_EXECUTION_TIMEOUT= (10, 60)
+
+class CodeExecutionException(Exception):
+    pass
+
+class CodeExecutionResponse(BaseModel):
+    class Data(BaseModel):
+        stdout: Optional[str]
+        error: Optional[str]
+
+    code: int
+    message: str
+    data: Data
+
+class CodeExecutor:
+    @classmethod
+    def execute_code(cls, language: Literal['python3', 'javascript', 'jinja2'], code: str, inputs: dict) -> dict:
+        """
+        Execute code
+        :param language: code language
+        :param code: code
+        :param inputs: inputs
+        :return:
+        """
+        template_transformer = None
+        if language == 'python3':
+            template_transformer = PythonTemplateTransformer
+        elif language == 'jinja2':
+            template_transformer = Jinja2TemplateTransformer
+        elif language == 'javascript':
+            template_transformer = NodeJsTemplateTransformer
+        else:
+            raise CodeExecutionException('Unsupported language')
+
+        runner, preload = template_transformer.transform_caller(code, inputs)
+        url = URL(CODE_EXECUTION_ENDPOINT) / 'v1' / 'sandbox' / 'run'
+        headers = {
+            'X-Api-Key': CODE_EXECUTION_API_KEY
+        }
+        data = {
+            'language': 'python3' if language == 'jinja2' else
+                        'nodejs' if language == 'javascript' else
+                        'python3' if language == 'python3' else None,
+            'code': runner,
+            'preload': preload
+        }
+
+        try:
+            response = post(str(url), json=data, headers=headers, timeout=CODE_EXECUTION_TIMEOUT)
+            if response.status_code == 503:
+                raise CodeExecutionException('Code execution service is unavailable')
+            elif response.status_code != 200:
+                raise Exception(f'Failed to execute code, got status code {response.status_code}, please check if the sandbox service is running')
+        except CodeExecutionException as e:
+            raise e
+        except Exception as e:
+            raise CodeExecutionException('Failed to execute code, this is likely a network issue, please check if the sandbox service is running')
+        
+        try:
+            response = response.json()
+        except:
+            raise CodeExecutionException('Failed to parse response')
+        
+        response = CodeExecutionResponse(**response)
+
+        if response.code != 0:
+            raise CodeExecutionException(response.message)
+        
+        if response.data.error:
+            raise CodeExecutionException(response.data.error)
+        
+        return template_transformer.transform_response(response.data.stdout)

api/core/helper/code_executor/javascript_transformer.py

@@ -0,0 +1,54 @@
+import json
+import re
+
+from core.helper.code_executor.template_transformer import TemplateTransformer
+
+NODEJS_RUNNER = """// declare main function here
+{{code}}
+
+// execute main function, and return the result
+// inputs is a dict, unstructured inputs
+output = main({{inputs}})
+
+// convert output to json and print
+output = JSON.stringify(output)
+
+result = `<<RESULT>>${output}<<RESULT>>`
+
+console.log(result)
+"""
+
+NODEJS_PRELOAD = """"""
+
+class NodeJsTemplateTransformer(TemplateTransformer):
+    @classmethod
+    def transform_caller(cls, code: str, inputs: dict) -> tuple[str, str]:
+        """
+        Transform code to python runner
+        :param code: code
+        :param inputs: inputs
+        :return:
+        """
+        
+        # transform inputs to json string
+        inputs_str = json.dumps(inputs, indent=4)
+
+        # replace code and inputs
+        runner = NODEJS_RUNNER.replace('{{code}}', code)
+        runner = runner.replace('{{inputs}}', inputs_str)
+
+        return runner, NODEJS_PRELOAD
+    
+    @classmethod
+    def transform_response(cls, response: str) -> dict:
+        """
+        Transform response to dict
+        :param response: response
+        :return:
+        """
+        # extract result
+        result = re.search(r'<<RESULT>>(.*)<<RESULT>>', response, re.DOTALL)
+        if not result:
+            raise ValueError('Failed to parse result')
+        result = result.group(1)
+        return json.loads(result)

api/core/helper/code_executor/jina2_transformer.py

@@ -0,0 +1,84 @@
+import json
+import re
+
+from core.helper.code_executor.template_transformer import TemplateTransformer
+
+PYTHON_RUNNER = """
+import jinja2
+
+template = jinja2.Template('''{{code}}''')
+
+def main(**inputs):
+    return template.render(**inputs)
+
+# execute main function, and return the result
+output = main(**{{inputs}})
+
+result = f'''<<RESULT>>{output}<<RESULT>>'''
+
+print(result)
+
+"""
+
+JINJA2_PRELOAD_TEMPLATE = """{% set fruits = ['Apple'] %}
+{{ 'a' }}
+{% for fruit in fruits %}
+    <li>{{ fruit }}</li>
+{% endfor %}
+{% if fruits|length > 1 %}
+1
+{% endif %}
+{% for i in range(5) %}
+    {% if i == 3 %}{{ i }}{% else %}{% endif %}
+{% endfor %}
+    {% for i in range(3) %}
+        {{ i + 1 }}
+    {% endfor %}
+{% macro say_hello() %}a{{ 'b' }}{% endmacro %}
+{{ s }}{{ say_hello() }}"""
+
+JINJA2_PRELOAD = f"""
+import jinja2
+
+def _jinja2_preload_():
+    # prepare jinja2 environment, load template and render before to avoid sandbox issue
+    template = jinja2.Template('''{JINJA2_PRELOAD_TEMPLATE}''')
+    template.render(s='a')
+
+if __name__ == '__main__':
+    _jinja2_preload_()
+
+"""
+
+class Jinja2TemplateTransformer(TemplateTransformer):
+    @classmethod
+    def transform_caller(cls, code: str, inputs: dict) -> tuple[str, str]:
+        """
+        Transform code to python runner
+        :param code: code
+        :param inputs: inputs
+        :return:
+        """
+
+        # transform jinja2 template to python code
+        runner = PYTHON_RUNNER.replace('{{code}}', code)
+        runner = runner.replace('{{inputs}}', json.dumps(inputs, indent=4))
+
+        return runner, JINJA2_PRELOAD
+    
+    @classmethod
+    def transform_response(cls, response: str) -> dict:
+        """
+        Transform response to dict
+        :param response: response
+        :return:
+        """
+        # extract result
+        result = re.search(r'<<RESULT>>(.*)<<RESULT>>', response, re.DOTALL)
+        if not result:
+            raise ValueError('Failed to parse result')
+        result = result.group(1)
+
+        return {
+            'result': result
+        }

api/core/helper/code_executor/python_transformer.py

@@ -0,0 +1,57 @@
+import json
+import re
+
+from core.helper.code_executor.template_transformer import TemplateTransformer
+
+PYTHON_RUNNER = """# declare main function here
+{{code}}
+
+# execute main function, and return the result
+# inputs is a dict, and it
+output = main(**{{inputs}})
+
+# convert output to json and print
+output = json.dumps(output, indent=4)
+
+result = f'''<<RESULT>>
+{output}
+<<RESULT>>'''
+
+print(result)
+"""
+
+PYTHON_PRELOAD = """"""
+
+
+class PythonTemplateTransformer(TemplateTransformer):
+    @classmethod
+    def transform_caller(cls, code: str, inputs: dict) -> tuple[str, str]:
+        """
+        Transform code to python runner
+        :param code: code
+        :param inputs: inputs
+        :return:
+        """
+        
+        # transform inputs to json string
+        inputs_str = json.dumps(inputs, indent=4)
+
+        # replace code and inputs
+        runner = PYTHON_RUNNER.replace('{{code}}', code)
+        runner = runner.replace('{{inputs}}', inputs_str)
+
+        return runner, PYTHON_PRELOAD
+    
+    @classmethod
+    def transform_response(cls, response: str) -> dict:
+        """
+        Transform response to dict
+        :param response: response
+        :return:
+        """
+        # extract result
+        result = re.search(r'<<RESULT>>(.*?)<<RESULT>>', response, re.DOTALL)
+        if not result:
+            raise ValueError('Failed to parse result')
+        result = result.group(1)
+        return json.loads(result)

api/core/helper/code_executor/template_transformer.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+
+
+class TemplateTransformer(ABC):
+    @classmethod
+    @abstractmethod
+    def transform_caller(cls, code: str, inputs: dict) -> tuple[str, str]:
+        """
+        Transform code to python runner
+        :param code: code
+        :param inputs: inputs
+        :return: runner, preload
+        """
+        pass
+    
+    @classmethod
+    @abstractmethod
+    def transform_response(cls, response: str) -> dict:
+        """
+        Transform response to dict
+        :param response: response
+        :return:
+        """
+        pass

api/core/helper/moderation.py

@@ -1,7 +1,7 @@
 import logging
 import random
 
-from core.entities.application_entities import ModelConfigEntity
+from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
 from core.model_runtime.errors.invoke import InvokeBadRequestError
 from core.model_runtime.model_providers.openai.moderation.moderation import OpenAIModerationModel
 from extensions.ext_hosting_provider import hosting_configuration
@@ -10,7 +10,7 @@ from models.provider import ProviderType
 logger = logging.getLogger(__name__)
 
 
-def check_moderation(model_config: ModelConfigEntity, text: str) -> bool:
+def check_moderation(model_config: ModelConfigWithCredentialsEntity, text: str) -> bool:
     moderation_config = hosting_configuration.moderation_config
     if (moderation_config and moderation_config.enabled is True
             and 'openai' in hosting_configuration.provider_map

api/core/helper/ssrf_proxy.py

@@ -38,6 +38,10 @@ def patch(url, *args, **kwargs):
     return _patch(url=url, *args, proxies=httpx_proxies, **kwargs)
 
 def delete(url, *args, **kwargs):
+    if 'follow_redirects' in kwargs:
+        if kwargs['follow_redirects']:
+            kwargs['allow_redirects'] = kwargs['follow_redirects']
+        kwargs.pop('follow_redirects')
     return _delete(url=url, *args, proxies=requests_proxies, **kwargs)
 
 def head(url, *args, **kwargs):