chore(api/controllers): Apply Ruff Formatter. (#7645)

api/controllers/service_api/dataset/dataset.py

@@ -16,7 +16,7 @@ from services.dataset_service import DatasetService
 
 def _validate_name(name):
     if not name or len(name) < 1 or len(name) > 40:
-        raise ValueError('Name must be between 1 to 40 characters.')
+        raise ValueError("Name must be between 1 to 40 characters.")
     return name
 
 
@@ -26,24 +26,18 @@ class DatasetListApi(DatasetApiResource):
     def get(self, tenant_id):
         """Resource for getting datasets."""
 
-        page = request.args.get('page', default=1, type=int)
-        limit = request.args.get('limit', default=20, type=int)
-        provider = request.args.get('provider', default="vendor")
-        search = request.args.get('keyword', default=None, type=str)
-        tag_ids = request.args.getlist('tag_ids')
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        provider = request.args.get("provider", default="vendor")
+        search = request.args.get("keyword", default=None, type=str)
+        tag_ids = request.args.getlist("tag_ids")
 
-        datasets, total = DatasetService.get_datasets(page, limit, provider,
-                                                      tenant_id, current_user, search, tag_ids)
+        datasets, total = DatasetService.get_datasets(page, limit, provider, tenant_id, current_user, search, tag_ids)
         # check embedding setting
         provider_manager = ProviderManager()
-        configurations = provider_manager.get_configurations(
-            tenant_id=current_user.current_tenant_id
-        )
+        configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id)
 
-        embedding_models = configurations.get_models(
-            model_type=ModelType.TEXT_EMBEDDING,
-            only_active=True
-        )
+        embedding_models = configurations.get_models(model_type=ModelType.TEXT_EMBEDDING, only_active=True)
 
         model_names = []
         for embedding_model in embedding_models:
@@ -51,50 +45,59 @@ class DatasetListApi(DatasetApiResource):
 
         data = marshal(datasets, dataset_detail_fields)
         for item in data:
-            if item['indexing_technique'] == 'high_quality':
+            if item["indexing_technique"] == "high_quality":
                 item_model = f"{item['embedding_model']}:{item['embedding_model_provider']}"
                 if item_model in model_names:
-                    item['embedding_available'] = True
+                    item["embedding_available"] = True
                 else:
-                    item['embedding_available'] = False
+                    item["embedding_available"] = False
             else:
-                item['embedding_available'] = True
-        response = {
-            'data': data,
-            'has_more': len(datasets) == limit,
-            'limit': limit,
-            'total': total,
-            'page': page
-        }
+                item["embedding_available"] = True
+        response = {"data": data, "has_more": len(datasets) == limit, "limit": limit, "total": total, "page": page}
         return response, 200
 
-
     def post(self, tenant_id):
         """Resource for creating datasets."""
         parser = reqparse.RequestParser()
-        parser.add_argument('name', nullable=False, required=True,
-                            help='type is required. Name must be between 1 to 40 characters.',
-                            type=_validate_name)
-        parser.add_argument('indexing_technique', type=str, location='json',
-                            choices=Dataset.INDEXING_TECHNIQUE_LIST,
-                            help='Invalid indexing technique.')
-        parser.add_argument('permission', type=str, location='json', choices=(
-            DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM), help='Invalid permission.', required=False, nullable=False)
+        parser.add_argument(
+            "name",
+            nullable=False,
+            required=True,
+            help="type is required. Name must be between 1 to 40 characters.",
+            type=_validate_name,
+        )
+        parser.add_argument(
+            "indexing_technique",
+            type=str,
+            location="json",
+            choices=Dataset.INDEXING_TECHNIQUE_LIST,
+            help="Invalid indexing technique.",
+        )
+        parser.add_argument(
+            "permission",
+            type=str,
+            location="json",
+            choices=(DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM),
+            help="Invalid permission.",
+            required=False,
+            nullable=False,
+        )
         args = parser.parse_args()
 
         try:
             dataset = DatasetService.create_empty_dataset(
                 tenant_id=tenant_id,
-                name=args['name'],
-                indexing_technique=args['indexing_technique'],
+                name=args["name"],
+                indexing_technique=args["indexing_technique"],
                 account=current_user,
-                permission=args['permission']
+                permission=args["permission"],
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
 
         return marshal(dataset, dataset_detail_fields), 200
 
+
 class DatasetApi(DatasetApiResource):
     """Resource for dataset."""
 
@@ -106,7 +109,7 @@ class DatasetApi(DatasetApiResource):
             dataset_id (UUID): The ID of the dataset to be deleted.
 
         Returns:
-            dict: A dictionary with a key 'result' and a value 'success' 
+            dict: A dictionary with a key 'result' and a value 'success'
                   if the dataset was successfully deleted. Omitted in HTTP response.
             int: HTTP status code 204 indicating that the operation was successful.
 
@@ -118,11 +121,12 @@ class DatasetApi(DatasetApiResource):
 
         try:
             if DatasetService.delete_dataset(dataset_id_str, current_user):
-                return {'result': 'success'}, 204
+                return {"result": "success"}, 204
             else:
                 raise NotFound("Dataset not found.")
         except services.errors.dataset.DatasetInUseError:
             raise DatasetInUseError()
 
-api.add_resource(DatasetListApi, '/datasets')
-api.add_resource(DatasetApi, '/datasets/<uuid:dataset_id>')
+
+api.add_resource(DatasetListApi, "/datasets")
+api.add_resource(DatasetApi, "/datasets/<uuid:dataset_id>")

api/controllers/service_api/dataset/document.py

@@ -27,47 +27,40 @@ from services.file_service import FileService
 class DocumentAddByTextApi(DatasetApiResource):
     """Resource for documents."""
 
-    @cloud_edition_billing_resource_check('vector_space', 'dataset')
-    @cloud_edition_billing_resource_check('documents', 'dataset')
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_resource_check("documents", "dataset")
     def post(self, tenant_id, dataset_id):
         """Create document by text."""
         parser = reqparse.RequestParser()
-        parser.add_argument('name', type=str, required=True, nullable=False, location='json')
-        parser.add_argument('text', type=str, required=True, nullable=False, location='json')
-        parser.add_argument('process_rule', type=dict, required=False, nullable=True, location='json')
-        parser.add_argument('original_document_id', type=str, required=False, location='json')
-        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
-        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False,
-                            location='json')
-        parser.add_argument('indexing_technique', type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, nullable=False,
-                            location='json')
-        parser.add_argument('retrieval_model', type=dict, required=False, nullable=False,
-                            location='json')
+        parser.add_argument("name", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("text", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("process_rule", type=dict, required=False, nullable=True, location="json")
+        parser.add_argument("original_document_id", type=str, required=False, location="json")
+        parser.add_argument("doc_form", type=str, default="text_model", required=False, nullable=False, location="json")
+        parser.add_argument(
+            "doc_language", type=str, default="English", required=False, nullable=False, location="json"
+        )
+        parser.add_argument(
+            "indexing_technique", type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, nullable=False, location="json"
+        )
+        parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
         args = parser.parse_args()
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
-            raise ValueError('Dataset is not exist.')
+            raise ValueError("Dataset is not exist.")
 
-        if not dataset.indexing_technique and not args['indexing_technique']:
-            raise ValueError('indexing_technique is required.')
+        if not dataset.indexing_technique and not args["indexing_technique"]:
+            raise ValueError("indexing_technique is required.")
 
-        upload_file = FileService.upload_text(args.get('text'), args.get('name'))
+        upload_file = FileService.upload_text(args.get("text"), args.get("name"))
         data_source = {
-            'type': 'upload_file',
-            'info_list': {
-                'data_source_type': 'upload_file',
-                'file_info_list': {
-                    'file_ids': [upload_file.id]
-                }
-            }
+            "type": "upload_file",
+            "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
         }
-        args['data_source'] = data_source
+        args["data_source"] = data_source
         # validate args
         DocumentService.document_create_args_validate(args)
 
@@ -76,60 +69,49 @@ class DocumentAddByTextApi(DatasetApiResource):
                 dataset=dataset,
                 document_data=args,
                 account=current_user,
-                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
-                created_from='api'
+                dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
+                created_from="api",
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         document = documents[0]
 
-        documents_and_batch_fields = {
-            'document': marshal(document, document_fields),
-            'batch': batch
-        }
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
         return documents_and_batch_fields, 200
 
 
 class DocumentUpdateByTextApi(DatasetApiResource):
     """Resource for update documents."""
 
-    @cloud_edition_billing_resource_check('vector_space', 'dataset')
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
     def post(self, tenant_id, dataset_id, document_id):
         """Update document by text."""
         parser = reqparse.RequestParser()
-        parser.add_argument('name', type=str, required=False, nullable=True, location='json')
-        parser.add_argument('text', type=str, required=False, nullable=True, location='json')
-        parser.add_argument('process_rule', type=dict, required=False, nullable=True, location='json')
-        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
-        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False,
-                            location='json')
-        parser.add_argument('retrieval_model', type=dict, required=False, nullable=False,
-                            location='json')
+        parser.add_argument("name", type=str, required=False, nullable=True, location="json")
+        parser.add_argument("text", type=str, required=False, nullable=True, location="json")
+        parser.add_argument("process_rule", type=dict, required=False, nullable=True, location="json")
+        parser.add_argument("doc_form", type=str, default="text_model", required=False, nullable=False, location="json")
+        parser.add_argument(
+            "doc_language", type=str, default="English", required=False, nullable=False, location="json"
+        )
+        parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
         args = parser.parse_args()
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
-            raise ValueError('Dataset is not exist.')
+            raise ValueError("Dataset is not exist.")
 
-        if args['text']:
-            upload_file = FileService.upload_text(args.get('text'), args.get('name'))
+        if args["text"]:
+            upload_file = FileService.upload_text(args.get("text"), args.get("name"))
             data_source = {
-                'type': 'upload_file',
-                'info_list': {
-                    'data_source_type': 'upload_file',
-                    'file_info_list': {
-                        'file_ids': [upload_file.id]
-                    }
-                }
+                "type": "upload_file",
+                "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
             }
-            args['data_source'] = data_source
+            args["data_source"] = data_source
         # validate args
-        args['original_document_id'] = str(document_id)
+        args["original_document_id"] = str(document_id)
         DocumentService.document_create_args_validate(args)
 
         try:
@@ -137,65 +119,53 @@ class DocumentUpdateByTextApi(DatasetApiResource):
                 dataset=dataset,
                 document_data=args,
                 account=current_user,
-                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
-                created_from='api'
+                dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
+                created_from="api",
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         document = documents[0]
 
-        documents_and_batch_fields = {
-            'document': marshal(document, document_fields),
-            'batch': batch
-        }
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
         return documents_and_batch_fields, 200
 
 
 class DocumentAddByFileApi(DatasetApiResource):
     """Resource for documents."""
-    @cloud_edition_billing_resource_check('vector_space', 'dataset')
-    @cloud_edition_billing_resource_check('documents', 'dataset')
+
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_resource_check("documents", "dataset")
     def post(self, tenant_id, dataset_id):
         """Create document by upload file."""
         args = {}
-        if 'data' in request.form:
-            args = json.loads(request.form['data'])
-        if 'doc_form' not in args:
-            args['doc_form'] = 'text_model'
-        if 'doc_language' not in args:
-            args['doc_language'] = 'English'
+        if "data" in request.form:
+            args = json.loads(request.form["data"])
+        if "doc_form" not in args:
+            args["doc_form"] = "text_model"
+        if "doc_language" not in args:
+            args["doc_language"] = "English"
         # get dataset info
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
-            raise ValueError('Dataset is not exist.')
-        if not dataset.indexing_technique and not args.get('indexing_technique'):
-            raise ValueError('indexing_technique is required.')
+            raise ValueError("Dataset is not exist.")
+        if not dataset.indexing_technique and not args.get("indexing_technique"):
+            raise ValueError("indexing_technique is required.")
 
         # save file info
-        file = request.files['file']
+        file = request.files["file"]
         # check file
-        if 'file' not in request.files:
+        if "file" not in request.files:
             raise NoFileUploadedError()
 
         if len(request.files) > 1:
             raise TooManyFilesError()
 
         upload_file = FileService.upload_file(file, current_user)
-        data_source = {
-            'type': 'upload_file',
-            'info_list': {
-                'file_info_list': {
-                    'file_ids': [upload_file.id]
-                }
-            }
-        }
-        args['data_source'] = data_source
+        data_source = {"type": "upload_file", "info_list": {"file_info_list": {"file_ids": [upload_file.id]}}}
+        args["data_source"] = data_source
         # validate args
         DocumentService.document_create_args_validate(args)
 
@@ -204,63 +174,49 @@ class DocumentAddByFileApi(DatasetApiResource):
                 dataset=dataset,
                 document_data=args,
                 account=dataset.created_by_account,
-                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
-                created_from='api'
+                dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
+                created_from="api",
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         document = documents[0]
-        documents_and_batch_fields = {
-            'document': marshal(document, document_fields),
-            'batch': batch
-        }
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
         return documents_and_batch_fields, 200
 
 
 class DocumentUpdateByFileApi(DatasetApiResource):
     """Resource for update documents."""
 
-    @cloud_edition_billing_resource_check('vector_space', 'dataset')
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
     def post(self, tenant_id, dataset_id, document_id):
         """Update document by upload file."""
         args = {}
-        if 'data' in request.form:
-            args = json.loads(request.form['data'])
-        if 'doc_form' not in args:
-            args['doc_form'] = 'text_model'
-        if 'doc_language' not in args:
-            args['doc_language'] = 'English'
+        if "data" in request.form:
+            args = json.loads(request.form["data"])
+        if "doc_form" not in args:
+            args["doc_form"] = "text_model"
+        if "doc_language" not in args:
+            args["doc_language"] = "English"
 
         # get dataset info
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
-            raise ValueError('Dataset is not exist.')
-        if 'file' in request.files:
+            raise ValueError("Dataset is not exist.")
+        if "file" in request.files:
             # save file info
-            file = request.files['file']
-
+            file = request.files["file"]
 
             if len(request.files) > 1:
                 raise TooManyFilesError()
 
             upload_file = FileService.upload_file(file, current_user)
-            data_source = {
-                'type': 'upload_file',
-                'info_list': {
-                    'file_info_list': {
-                        'file_ids': [upload_file.id]
-                    }
-                }
-            }
-            args['data_source'] = data_source
+            data_source = {"type": "upload_file", "info_list": {"file_info_list": {"file_ids": [upload_file.id]}}}
+            args["data_source"] = data_source
         # validate args
-        args['original_document_id'] = str(document_id)
+        args["original_document_id"] = str(document_id)
         DocumentService.document_create_args_validate(args)
 
         try:
@@ -268,16 +224,13 @@ class DocumentUpdateByFileApi(DatasetApiResource):
                 dataset=dataset,
                 document_data=args,
                 account=dataset.created_by_account,
-                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
-                created_from='api'
+                dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
+                created_from="api",
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         document = documents[0]
-        documents_and_batch_fields = {
-            'document': marshal(document, document_fields),
-            'batch': batch
-        }
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
         return documents_and_batch_fields, 200
 
 
@@ -289,13 +242,10 @@ class DocumentDeleteApi(DatasetApiResource):
         tenant_id = str(tenant_id)
 
         # get dataset info
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
-            raise ValueError('Dataset is not exist.')
+            raise ValueError("Dataset is not exist.")
 
         document = DocumentService.get_document(dataset.id, document_id)
 
@@ -311,44 +261,39 @@ class DocumentDeleteApi(DatasetApiResource):
             # delete document
             DocumentService.delete_document(document)
         except services.errors.document.DocumentIndexingError:
-            raise DocumentIndexingError('Cannot delete document during indexing.')
+            raise DocumentIndexingError("Cannot delete document during indexing.")
 
-        return {'result': 'success'}, 200
+        return {"result": "success"}, 200
 
 
 class DocumentListApi(DatasetApiResource):
     def get(self, tenant_id, dataset_id):
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        page = request.args.get('page', default=1, type=int)
-        limit = request.args.get('limit', default=20, type=int)
-        search = request.args.get('keyword', default=None, type=str)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        search = request.args.get("keyword", default=None, type=str)
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
-            raise NotFound('Dataset not found.')
+            raise NotFound("Dataset not found.")
 
-        query = Document.query.filter_by(
-            dataset_id=str(dataset_id), tenant_id=tenant_id)
+        query = Document.query.filter_by(dataset_id=str(dataset_id), tenant_id=tenant_id)
 
         if search:
-            search = f'%{search}%'
+            search = f"%{search}%"
             query = query.filter(Document.name.like(search))
 
         query = query.order_by(desc(Document.created_at))
 
-        paginated_documents = query.paginate(
-            page=page, per_page=limit, max_per_page=100, error_out=False)
+        paginated_documents = query.paginate(page=page, per_page=limit, max_per_page=100, error_out=False)
         documents = paginated_documents.items
 
         response = {
-            'data': marshal(documents, document_fields),
-            'has_more': len(documents) == limit,
-            'limit': limit,
-            'total': paginated_documents.total,
-            'page': page
+            "data": marshal(documents, document_fields),
+            "has_more": len(documents) == limit,
+            "limit": limit,
+            "total": paginated_documents.total,
+            "page": page,
         }
 
         return response
@@ -360,38 +305,36 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         batch = str(batch)
         tenant_id = str(tenant_id)
         # get dataset
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
-            raise NotFound('Dataset not found.')
+            raise NotFound("Dataset not found.")
         # get documents
         documents = DocumentService.get_batch_documents(dataset_id, batch)
         if not documents:
-            raise NotFound('Documents not found.')
+            raise NotFound("Documents not found.")
         documents_status = []
         for document in documents:
-            completed_segments = DocumentSegment.query.filter(DocumentSegment.completed_at.isnot(None),
-                                                              DocumentSegment.document_id == str(document.id),
-                                                              DocumentSegment.status != 're_segment').count()
-            total_segments = DocumentSegment.query.filter(DocumentSegment.document_id == str(document.id),
-                                                          DocumentSegment.status != 're_segment').count()
+            completed_segments = DocumentSegment.query.filter(
+                DocumentSegment.completed_at.isnot(None),
+                DocumentSegment.document_id == str(document.id),
+                DocumentSegment.status != "re_segment",
+            ).count()
+            total_segments = DocumentSegment.query.filter(
+                DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment"
+            ).count()
             document.completed_segments = completed_segments
             document.total_segments = total_segments
             if document.is_paused:
-                document.indexing_status = 'paused'
+                document.indexing_status = "paused"
             documents_status.append(marshal(document, document_status_fields))
-        data = {
-            'data': documents_status
-        }
+        data = {"data": documents_status}
         return data
 
 
-api.add_resource(DocumentAddByTextApi, '/datasets/<uuid:dataset_id>/document/create_by_text')
-api.add_resource(DocumentAddByFileApi, '/datasets/<uuid:dataset_id>/document/create_by_file')
-api.add_resource(DocumentUpdateByTextApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text')
-api.add_resource(DocumentUpdateByFileApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file')
-api.add_resource(DocumentDeleteApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>')
-api.add_resource(DocumentListApi, '/datasets/<uuid:dataset_id>/documents')
-api.add_resource(DocumentIndexingStatusApi, '/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status')
+api.add_resource(DocumentAddByTextApi, "/datasets/<uuid:dataset_id>/document/create_by_text")
+api.add_resource(DocumentAddByFileApi, "/datasets/<uuid:dataset_id>/document/create_by_file")
+api.add_resource(DocumentUpdateByTextApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text")
+api.add_resource(DocumentUpdateByFileApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file")
+api.add_resource(DocumentDeleteApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
+api.add_resource(DocumentListApi, "/datasets/<uuid:dataset_id>/documents")
+api.add_resource(DocumentIndexingStatusApi, "/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status")

api/controllers/service_api/dataset/error.py

@@ -2,78 +2,78 @@ from libs.exception import BaseHTTPException
 
 
 class NoFileUploadedError(BaseHTTPException):
-    error_code = 'no_file_uploaded'
+    error_code = "no_file_uploaded"
     description = "Please upload your file."
     code = 400
 
 
 class TooManyFilesError(BaseHTTPException):
-    error_code = 'too_many_files'
+    error_code = "too_many_files"
     description = "Only one file is allowed."
     code = 400
 
 
 class FileTooLargeError(BaseHTTPException):
-    error_code = 'file_too_large'
+    error_code = "file_too_large"
     description = "File size exceeded. {message}"
     code = 413
 
 
 class UnsupportedFileTypeError(BaseHTTPException):
-    error_code = 'unsupported_file_type'
+    error_code = "unsupported_file_type"
     description = "File type not allowed."
     code = 415
 
 
 class HighQualityDatasetOnlyError(BaseHTTPException):
-    error_code = 'high_quality_dataset_only'
+    error_code = "high_quality_dataset_only"
     description = "Current operation only supports 'high-quality' datasets."
     code = 400
 
 
 class DatasetNotInitializedError(BaseHTTPException):
-    error_code = 'dataset_not_initialized'
+    error_code = "dataset_not_initialized"
     description = "The dataset is still being initialized or indexing. Please wait a moment."
     code = 400
 
 
 class ArchivedDocumentImmutableError(BaseHTTPException):
-    error_code = 'archived_document_immutable'
+    error_code = "archived_document_immutable"
     description = "The archived document is not editable."
     code = 403
 
 
 class DatasetNameDuplicateError(BaseHTTPException):
-    error_code = 'dataset_name_duplicate'
+    error_code = "dataset_name_duplicate"
     description = "The dataset name already exists. Please modify your dataset name."
     code = 409
 
 
 class InvalidActionError(BaseHTTPException):
-    error_code = 'invalid_action'
+    error_code = "invalid_action"
     description = "Invalid action."
     code = 400
 
 
 class DocumentAlreadyFinishedError(BaseHTTPException):
-    error_code = 'document_already_finished'
+    error_code = "document_already_finished"
     description = "The document has been processed. Please refresh the page or go to the document details."
     code = 400
 
 
 class DocumentIndexingError(BaseHTTPException):
-    error_code = 'document_indexing'
+    error_code = "document_indexing"
     description = "The document is being processed and cannot be edited."
     code = 400
 
 
 class InvalidMetadataError(BaseHTTPException):
-    error_code = 'invalid_metadata'
+    error_code = "invalid_metadata"
     description = "The metadata content is incorrect. Please check and verify."
     code = 400
 
 
 class DatasetInUseError(BaseHTTPException):
-    error_code = 'dataset_in_use'
+    error_code = "dataset_in_use"
     description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
     code = 409

api/controllers/service_api/dataset/segment.py

@@ -21,52 +21,47 @@ from services.dataset_service import DatasetService, DocumentService, SegmentSer
 class SegmentApi(DatasetApiResource):
     """Resource for segments."""
 
-    @cloud_edition_billing_resource_check('vector_space', 'dataset')
-    @cloud_edition_billing_knowledge_limit_check('add_segment', 'dataset')
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
     def post(self, tenant_id, dataset_id, document_id):
         """Create single segment."""
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
-            raise NotFound('Dataset not found.')
+            raise NotFound("Dataset not found.")
         # check document
         document_id = str(document_id)
         document = DocumentService.get_document(dataset.id, document_id)
         if not document:
-            raise NotFound('Document not found.')
+            raise NotFound("Document not found.")
         # check embedding model setting
-        if dataset.indexing_technique == 'high_quality':
+        if dataset.indexing_technique == "high_quality":
             try:
                 model_manager = ModelManager()
                 model_manager.get_model_instance(
                     tenant_id=current_user.current_tenant_id,
                     provider=dataset.embedding_model_provider,
                     model_type=ModelType.TEXT_EMBEDDING,
-                    model=dataset.embedding_model
+                    model=dataset.embedding_model,
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
                     "No Embedding Model available. Please configure a valid provider "
-                    "in the Settings -> Model Provider.")
-            except ProviderTokenNotInitError as ex:   
+                    "in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
         # validate args
         parser = reqparse.RequestParser()
-        parser.add_argument('segments', type=list, required=False, nullable=True, location='json')
+        parser.add_argument("segments", type=list, required=False, nullable=True, location="json")
         args = parser.parse_args()
-        if args['segments'] is not None:
-            for args_item in args['segments']:
+        if args["segments"] is not None:
+            for args_item in args["segments"]:
                 SegmentService.segment_create_args_validate(args_item, document)
-            segments = SegmentService.multi_create_segment(args['segments'], document, dataset)
-            return {
-                'data': marshal(segments, segment_fields),
-                'doc_form': document.doc_form
-            }, 200
+            segments = SegmentService.multi_create_segment(args["segments"], document, dataset)
+            return {"data": marshal(segments, segment_fields), "doc_form": document.doc_form}, 200
         else:
             return {"error": "Segemtns is required"}, 400
 
@@ -75,61 +70,53 @@ class SegmentApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
-            raise NotFound('Dataset not found.')
+            raise NotFound("Dataset not found.")
         # check document
         document_id = str(document_id)
         document = DocumentService.get_document(dataset.id, document_id)
         if not document:
-            raise NotFound('Document not found.')
+            raise NotFound("Document not found.")
         # check embedding model setting
-        if dataset.indexing_technique == 'high_quality':
+        if dataset.indexing_technique == "high_quality":
             try:
                 model_manager = ModelManager()
                 model_manager.get_model_instance(
                     tenant_id=current_user.current_tenant_id,
                     provider=dataset.embedding_model_provider,
                     model_type=ModelType.TEXT_EMBEDDING,
-                    model=dataset.embedding_model
+                    model=dataset.embedding_model,
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
                     "No Embedding Model available. Please configure a valid provider "
-                    "in the Settings -> Model Provider.")
+                    "in the Settings -> Model Provider."
+                )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
 
         parser = reqparse.RequestParser()
-        parser.add_argument('status', type=str,
-                            action='append', default=[], location='args')
-        parser.add_argument('keyword', type=str, default=None, location='args')
+        parser.add_argument("status", type=str, action="append", default=[], location="args")
+        parser.add_argument("keyword", type=str, default=None, location="args")
         args = parser.parse_args()
 
-        status_list = args['status']
-        keyword = args['keyword']
+        status_list = args["status"]
+        keyword = args["keyword"]
 
         query = DocumentSegment.query.filter(
-            DocumentSegment.document_id == str(document_id),
-            DocumentSegment.tenant_id == current_user.current_tenant_id
+            DocumentSegment.document_id == str(document_id), DocumentSegment.tenant_id == current_user.current_tenant_id
         )
 
         if status_list:
             query = query.filter(DocumentSegment.status.in_(status_list))
 
         if keyword:
-            query = query.where(DocumentSegment.content.ilike(f'%{keyword}%'))
+            query = query.where(DocumentSegment.content.ilike(f"%{keyword}%"))
 
         total = query.count()
         segments = query.order_by(DocumentSegment.position).all()
-        return {
-            'data': marshal(segments, segment_fields),
-            'doc_form': document.doc_form,
-            'total': total
-        }, 200
+        return {"data": marshal(segments, segment_fields), "doc_form": document.doc_form, "total": total}, 200
 
 
 class DatasetSegmentApi(DatasetApiResource):
@@ -137,48 +124,41 @@ class DatasetSegmentApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
-            raise NotFound('Dataset not found.')
+            raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id = str(document_id)
         document = DocumentService.get_document(dataset_id, document_id)
         if not document:
-            raise NotFound('Document not found.')
+            raise NotFound("Document not found.")
         # check segment
         segment = DocumentSegment.query.filter(
-            DocumentSegment.id == str(segment_id),
-            DocumentSegment.tenant_id == current_user.current_tenant_id
+            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
         ).first()
         if not segment:
-            raise NotFound('Segment not found.')
+            raise NotFound("Segment not found.")
         SegmentService.delete_segment(segment, document, dataset)
-        return {'result': 'success'}, 200
+        return {"result": "success"}, 200
 
-    @cloud_edition_billing_resource_check('vector_space', 'dataset')
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
     def post(self, tenant_id, dataset_id, document_id, segment_id):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(
-            Dataset.tenant_id == tenant_id,
-            Dataset.id == dataset_id
-        ).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
-            raise NotFound('Dataset not found.')
+            raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id = str(document_id)
         document = DocumentService.get_document(dataset_id, document_id)
         if not document:
-            raise NotFound('Document not found.')
-        if dataset.indexing_technique == 'high_quality':
+            raise NotFound("Document not found.")
+        if dataset.indexing_technique == "high_quality":
             # check embedding model setting
             try:
                 model_manager = ModelManager()
@@ -186,35 +166,34 @@ class DatasetSegmentApi(DatasetApiResource):
                     tenant_id=current_user.current_tenant_id,
                     provider=dataset.embedding_model_provider,
                     model_type=ModelType.TEXT_EMBEDDING,
-                    model=dataset.embedding_model
+                    model=dataset.embedding_model,
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
                     "No Embedding Model available. Please configure a valid provider "
-                    "in the Settings -> Model Provider.")
+                    "in the Settings -> Model Provider."
+                )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
             # check segment
         segment_id = str(segment_id)
         segment = DocumentSegment.query.filter(
-            DocumentSegment.id == str(segment_id),
-            DocumentSegment.tenant_id == current_user.current_tenant_id
+            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
         ).first()
         if not segment:
-            raise NotFound('Segment not found.')
+            raise NotFound("Segment not found.")
 
         # validate args
         parser = reqparse.RequestParser()
-        parser.add_argument('segment', type=dict, required=False, nullable=True, location='json')
+        parser.add_argument("segment", type=dict, required=False, nullable=True, location="json")
         args = parser.parse_args()
 
-        SegmentService.segment_create_args_validate(args['segment'], document)
-        segment = SegmentService.update_segment(args['segment'], segment, document, dataset)
-        return {
-            'data': marshal(segment, segment_fields),
-            'doc_form': document.doc_form
-        }, 200
+        SegmentService.segment_create_args_validate(args["segment"], document)
+        segment = SegmentService.update_segment(args["segment"], segment, document, dataset)
+        return {"data": marshal(segment, segment_fields), "doc_form": document.doc_form}, 200
 
 
-api.add_resource(SegmentApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments')
-api.add_resource(DatasetSegmentApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>')
+api.add_resource(SegmentApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
+api.add_resource(
+    DatasetSegmentApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>"
+)