Feat/dataset service api (#1245)

Co-authored-by: jyong <jyong@dify.ai>
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>

api/fields/app_fields.py

@@ -0,0 +1,138 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+app_detail_kernel_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+}
+
+related_app_list = {
+    'data': fields.List(fields.Nested(app_detail_kernel_fields)),
+    'total': fields.Integer,
+}
+
+model_config_fields = {
+    'opening_statement': fields.String,
+    'suggested_questions': fields.Raw(attribute='suggested_questions_list'),
+    'suggested_questions_after_answer': fields.Raw(attribute='suggested_questions_after_answer_dict'),
+    'speech_to_text': fields.Raw(attribute='speech_to_text_dict'),
+    'retriever_resource': fields.Raw(attribute='retriever_resource_dict'),
+    'more_like_this': fields.Raw(attribute='more_like_this_dict'),
+    'sensitive_word_avoidance': fields.Raw(attribute='sensitive_word_avoidance_dict'),
+    'model': fields.Raw(attribute='model_dict'),
+    'user_input_form': fields.Raw(attribute='user_input_form_list'),
+    'dataset_query_variable': fields.String,
+    'pre_prompt': fields.String,
+    'agent_mode': fields.Raw(attribute='agent_mode_dict'),
+}
+
+app_detail_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'enable_site': fields.Boolean,
+    'enable_api': fields.Boolean,
+    'api_rpm': fields.Integer,
+    'api_rph': fields.Integer,
+    'is_demo': fields.Boolean,
+    'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
+    'created_at': TimestampField
+}
+
+prompt_config_fields = {
+    'prompt_template': fields.String,
+}
+
+model_config_partial_fields = {
+    'model': fields.Raw(attribute='model_dict'),
+    'pre_prompt': fields.String,
+}
+
+app_partial_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'enable_site': fields.Boolean,
+    'enable_api': fields.Boolean,
+    'is_demo': fields.Boolean,
+    'model_config': fields.Nested(model_config_partial_fields, attribute='app_model_config'),
+    'created_at': TimestampField
+}
+
+app_pagination_fields = {
+    'page': fields.Integer,
+    'limit': fields.Integer(attribute='per_page'),
+    'total': fields.Integer,
+    'has_more': fields.Boolean(attribute='has_next'),
+    'data': fields.List(fields.Nested(app_partial_fields), attribute='items')
+}
+
+template_fields = {
+    'name': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'mode': fields.String,
+    'model_config': fields.Nested(model_config_fields),
+}
+
+template_list_fields = {
+    'data': fields.List(fields.Nested(template_fields)),
+}
+
+site_fields = {
+    'access_token': fields.String(attribute='code'),
+    'code': fields.String,
+    'title': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'default_language': fields.String,
+    'customize_domain': fields.String,
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'customize_token_strategy': fields.String,
+    'prompt_public': fields.Boolean,
+    'app_base_url': fields.String,
+}
+
+app_detail_fields_with_site = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'enable_site': fields.Boolean,
+    'enable_api': fields.Boolean,
+    'api_rpm': fields.Integer,
+    'api_rph': fields.Integer,
+    'is_demo': fields.Boolean,
+    'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
+    'site': fields.Nested(site_fields),
+    'api_base_url': fields.String,
+    'created_at': TimestampField
+}
+
+app_site_fields = {
+    'app_id': fields.String,
+    'access_token': fields.String(attribute='code'),
+    'code': fields.String,
+    'title': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'default_language': fields.String,
+    'customize_domain': fields.String,
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'customize_token_strategy': fields.String,
+    'prompt_public': fields.Boolean
+}

api/fields/conversation_fields.py

@@ -0,0 +1,182 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+
+class MessageTextField(fields.Raw):
+    def format(self, value):
+        return value[0]['text'] if value else ''
+
+
+account_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'email': fields.String
+}
+
+feedback_fields = {
+    'rating': fields.String,
+    'content': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account': fields.Nested(account_fields, allow_null=True),
+}
+
+annotation_fields = {
+    'content': fields.String,
+    'account': fields.Nested(account_fields, allow_null=True),
+    'created_at': TimestampField
+}
+
+message_detail_fields = {
+    'id': fields.String,
+    'conversation_id': fields.String,
+    'inputs': fields.Raw,
+    'query': fields.String,
+    'message': fields.Raw,
+    'message_tokens': fields.Integer,
+    'answer': fields.String,
+    'answer_tokens': fields.Integer,
+    'provider_response_latency': fields.Float,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account_id': fields.String,
+    'feedbacks': fields.List(fields.Nested(feedback_fields)),
+    'annotation': fields.Nested(annotation_fields, allow_null=True),
+    'created_at': TimestampField
+}
+
+feedback_stat_fields = {
+    'like': fields.Integer,
+    'dislike': fields.Integer
+}
+
+model_config_fields = {
+    'opening_statement': fields.String,
+    'suggested_questions': fields.Raw,
+    'model': fields.Raw,
+    'user_input_form': fields.Raw,
+    'pre_prompt': fields.String,
+    'agent_mode': fields.Raw,
+}
+
+simple_configs_fields = {
+    'prompt_template': fields.String,
+}
+
+simple_model_config_fields = {
+    'model': fields.Raw(attribute='model_dict'),
+    'pre_prompt': fields.String,
+}
+
+simple_message_detail_fields = {
+    'inputs': fields.Raw,
+    'query': fields.String,
+    'message': MessageTextField,
+    'answer': fields.String,
+}
+
+conversation_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_end_user_session_id': fields.String(),
+    'from_account_id': fields.String,
+    'read_at': TimestampField,
+    'created_at': TimestampField,
+    'annotation': fields.Nested(annotation_fields, allow_null=True),
+    'model_config': fields.Nested(simple_model_config_fields),
+    'user_feedback_stats': fields.Nested(feedback_stat_fields),
+    'admin_feedback_stats': fields.Nested(feedback_stat_fields),
+    'message': fields.Nested(simple_message_detail_fields, attribute='first_message')
+}
+
+conversation_pagination_fields = {
+    'page': fields.Integer,
+    'limit': fields.Integer(attribute='per_page'),
+    'total': fields.Integer,
+    'has_more': fields.Boolean(attribute='has_next'),
+    'data': fields.List(fields.Nested(conversation_fields), attribute='items')
+}
+
+conversation_message_detail_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account_id': fields.String,
+    'created_at': TimestampField,
+    'model_config': fields.Nested(model_config_fields),
+    'message': fields.Nested(message_detail_fields, attribute='first_message'),
+}
+
+simple_model_config_fields = {
+    'model': fields.Raw(attribute='model_dict'),
+    'pre_prompt': fields.String,
+}
+
+conversation_with_summary_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_end_user_session_id': fields.String,
+    'from_account_id': fields.String,
+    'summary': fields.String(attribute='summary_or_query'),
+    'read_at': TimestampField,
+    'created_at': TimestampField,
+    'annotated': fields.Boolean,
+    'model_config': fields.Nested(simple_model_config_fields),
+    'message_count': fields.Integer,
+    'user_feedback_stats': fields.Nested(feedback_stat_fields),
+    'admin_feedback_stats': fields.Nested(feedback_stat_fields)
+}
+
+conversation_with_summary_pagination_fields = {
+    'page': fields.Integer,
+    'limit': fields.Integer(attribute='per_page'),
+    'total': fields.Integer,
+    'has_more': fields.Boolean(attribute='has_next'),
+    'data': fields.List(fields.Nested(conversation_with_summary_fields), attribute='items')
+}
+
+conversation_detail_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account_id': fields.String,
+    'created_at': TimestampField,
+    'annotated': fields.Boolean,
+    'model_config': fields.Nested(model_config_fields),
+    'message_count': fields.Integer,
+    'user_feedback_stats': fields.Nested(feedback_stat_fields),
+    'admin_feedback_stats': fields.Nested(feedback_stat_fields)
+}
+
+simple_conversation_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'inputs': fields.Raw,
+    'status': fields.String,
+    'introduction': fields.String,
+    'created_at': TimestampField
+}
+
+conversation_infinite_scroll_pagination_fields = {
+    'limit': fields.Integer,
+    'has_more': fields.Boolean,
+    'data': fields.List(fields.Nested(simple_conversation_fields))
+}
+
+conversation_with_model_config_fields = {
+    **simple_conversation_fields,
+    'model_config': fields.Raw,
+}
+
+conversation_with_model_config_infinite_scroll_pagination_fields = {
+    'limit': fields.Integer,
+    'has_more': fields.Boolean,
+    'data': fields.List(fields.Nested(conversation_with_model_config_fields))
+}

api/fields/data_source_fields.py

@@ -0,0 +1,65 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+integrate_icon_fields = {
+    'type': fields.String,
+    'url': fields.String,
+    'emoji': fields.String
+}
+
+integrate_page_fields = {
+    'page_name': fields.String,
+    'page_id': fields.String,
+    'page_icon': fields.Nested(integrate_icon_fields, allow_null=True),
+    'is_bound': fields.Boolean,
+    'parent_id': fields.String,
+    'type': fields.String
+}
+
+integrate_workspace_fields = {
+    'workspace_name': fields.String,
+    'workspace_id': fields.String,
+    'workspace_icon': fields.String,
+    'pages': fields.List(fields.Nested(integrate_page_fields))
+}
+
+integrate_notion_info_list_fields = {
+    'notion_info': fields.List(fields.Nested(integrate_workspace_fields)),
+}
+
+integrate_icon_fields = {
+    'type': fields.String,
+    'url': fields.String,
+    'emoji': fields.String
+}
+
+integrate_page_fields = {
+    'page_name': fields.String,
+    'page_id': fields.String,
+    'page_icon': fields.Nested(integrate_icon_fields, allow_null=True),
+    'parent_id': fields.String,
+    'type': fields.String
+}
+
+integrate_workspace_fields = {
+    'workspace_name': fields.String,
+    'workspace_id': fields.String,
+    'workspace_icon': fields.String,
+    'pages': fields.List(fields.Nested(integrate_page_fields)),
+    'total': fields.Integer
+}
+
+integrate_fields = {
+    'id': fields.String,
+    'provider': fields.String,
+    'created_at': TimestampField,
+    'is_bound': fields.Boolean,
+    'disabled': fields.Boolean,
+    'link': fields.String,
+    'source_info': fields.Nested(integrate_workspace_fields)
+}
+
+integrate_list_fields = {
+    'data': fields.List(fields.Nested(integrate_fields)),
+}

api/fields/dataset_fields.py

@@ -0,0 +1,43 @@
+from flask_restful import fields
+from libs.helper import TimestampField
+
+dataset_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'description': fields.String,
+    'permission': fields.String,
+    'data_source_type': fields.String,
+    'indexing_technique': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+}
+
+dataset_detail_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'description': fields.String,
+    'provider': fields.String,
+    'permission': fields.String,
+    'data_source_type': fields.String,
+    'indexing_technique': fields.String,
+    'app_count': fields.Integer,
+    'document_count': fields.Integer,
+    'word_count': fields.Integer,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'updated_by': fields.String,
+    'updated_at': TimestampField,
+    'embedding_model': fields.String,
+    'embedding_model_provider': fields.String,
+    'embedding_available': fields.Boolean
+}
+
+dataset_query_detail_fields = {
+    "id": fields.String,
+    "content": fields.String,
+    "source": fields.String,
+    "source_app_id": fields.String,
+    "created_by_role": fields.String,
+    "created_by": fields.String,
+    "created_at": TimestampField
+}

api/fields/document_fields.py

@@ -0,0 +1,76 @@
+from flask_restful import fields
+
+from fields.dataset_fields import dataset_fields
+from libs.helper import TimestampField
+
+document_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'data_source_type': fields.String,
+    'data_source_info': fields.Raw(attribute='data_source_info_dict'),
+    'dataset_process_rule_id': fields.String,
+    'name': fields.String,
+    'created_from': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'tokens': fields.Integer,
+    'indexing_status': fields.String,
+    'error': fields.String,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'archived': fields.Boolean,
+    'display_status': fields.String,
+    'word_count': fields.Integer,
+    'hit_count': fields.Integer,
+    'doc_form': fields.String,
+}
+
+document_with_segments_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'data_source_type': fields.String,
+    'data_source_info': fields.Raw(attribute='data_source_info_dict'),
+    'dataset_process_rule_id': fields.String,
+    'name': fields.String,
+    'created_from': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'tokens': fields.Integer,
+    'indexing_status': fields.String,
+    'error': fields.String,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'archived': fields.Boolean,
+    'display_status': fields.String,
+    'word_count': fields.Integer,
+    'hit_count': fields.Integer,
+    'completed_segments': fields.Integer,
+    'total_segments': fields.Integer
+}
+
+dataset_and_document_fields = {
+    'dataset': fields.Nested(dataset_fields),
+    'documents': fields.List(fields.Nested(document_fields)),
+    'batch': fields.String
+}
+
+document_status_fields = {
+    'id': fields.String,
+    'indexing_status': fields.String,
+    'processing_started_at': TimestampField,
+    'parsing_completed_at': TimestampField,
+    'cleaning_completed_at': TimestampField,
+    'splitting_completed_at': TimestampField,
+    'completed_at': TimestampField,
+    'paused_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField,
+    'completed_segments': fields.Integer,
+    'total_segments': fields.Integer,
+}
+
+document_status_fields_list = {
+    'data': fields.List(fields.Nested(document_status_fields))
+}

api/fields/file_fields.py

@@ -0,0 +1,18 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+upload_config_fields = {
+    'file_size_limit': fields.Integer,
+    'batch_count_limit': fields.Integer
+}
+
+file_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'size': fields.Integer,
+    'extension': fields.String,
+    'mime_type': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+}

api/fields/hit_testing_fields.py

@@ -0,0 +1,41 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+document_fields = {
+    'id': fields.String,
+    'data_source_type': fields.String,
+    'name': fields.String,
+    'doc_type': fields.String,
+}
+
+segment_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'document_id': fields.String,
+    'content': fields.String,
+    'answer': fields.String,
+    'word_count': fields.Integer,
+    'tokens': fields.Integer,
+    'keywords': fields.List(fields.String),
+    'index_node_id': fields.String,
+    'index_node_hash': fields.String,
+    'hit_count': fields.Integer,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'status': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'indexing_at': TimestampField,
+    'completed_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField,
+    'document': fields.Nested(document_fields),
+}
+
+hit_testing_record_fields = {
+    'segment': fields.Nested(segment_fields),
+    'score': fields.Float,
+    'tsne_position': fields.Raw
+}

api/fields/installed_app_fields.py

@@ -0,0 +1,25 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+app_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String
+}
+
+installed_app_fields = {
+    'id': fields.String,
+    'app': fields.Nested(app_fields),
+    'app_owner_tenant_id': fields.String,
+    'is_pinned': fields.Boolean,
+    'last_used_at': TimestampField,
+    'editable': fields.Boolean,
+    'uninstallable': fields.Boolean,
+}
+
+installed_app_list_fields = {
+    'installed_apps': fields.List(fields.Nested(installed_app_fields))
+}

api/fields/message_fields.py

@@ -0,0 +1,43 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+feedback_fields = {
+    'rating': fields.String
+}
+
+retriever_resource_fields = {
+    'id': fields.String,
+    'message_id': fields.String,
+    'position': fields.Integer,
+    'dataset_id': fields.String,
+    'dataset_name': fields.String,
+    'document_id': fields.String,
+    'document_name': fields.String,
+    'data_source_type': fields.String,
+    'segment_id': fields.String,
+    'score': fields.Float,
+    'hit_count': fields.Integer,
+    'word_count': fields.Integer,
+    'segment_position': fields.Integer,
+    'index_node_hash': fields.String,
+    'content': fields.String,
+    'created_at': TimestampField
+}
+
+message_fields = {
+    'id': fields.String,
+    'conversation_id': fields.String,
+    'inputs': fields.Raw,
+    'query': fields.String,
+    'answer': fields.String,
+    'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
+    'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
+    'created_at': TimestampField
+}
+
+message_infinite_scroll_pagination_fields = {
+    'limit': fields.Integer,
+    'has_more': fields.Boolean,
+    'data': fields.List(fields.Nested(message_fields))
+}

api/fields/segment_fields.py

@@ -0,0 +1,32 @@
+from flask_restful import fields
+from libs.helper import TimestampField
+
+segment_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'document_id': fields.String,
+    'content': fields.String,
+    'answer': fields.String,
+    'word_count': fields.Integer,
+    'tokens': fields.Integer,
+    'keywords': fields.List(fields.String),
+    'index_node_id': fields.String,
+    'index_node_hash': fields.String,
+    'hit_count': fields.Integer,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'status': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'indexing_at': TimestampField,
+    'completed_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField
+}
+
+segment_list_response = {
+    'data': fields.List(fields.Nested(segment_fields)),
+    'has_more': fields.Boolean,
+    'limit': fields.Integer
+}