enhancement: introduce Ruff for Python linter for reordering and removing unused imports with automated pre-commit and sytle check (#2366)

api/tasks/add_document_to_index_task.py

@@ -4,13 +4,14 @@ import time
 
 import click
 from celery import shared_task
+from langchain.schema import Document
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.schema import Document
 from models.dataset import Document as DatasetDocument
 from models.dataset import DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/annotation/add_annotation_to_index_task.py

@@ -3,8 +3,9 @@ import time
 
 import click
 from celery import shared_task
-from core.index.index import IndexBuilder
 from langchain.schema import Document
+
+from core.index.index import IndexBuilder
 from models.dataset import Dataset
 from services.dataset_service import DatasetCollectionBindingService
 

api/tasks/annotation/batch_import_annotations_task.py

@@ -1,17 +1,17 @@
-import json
 import logging
 import time
 
 import click
 from celery import shared_task
+from langchain.schema import Document
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.schema import Document
 from models.dataset import Dataset
 from models.model import App, AppAnnotationSetting, MessageAnnotation
 from services.dataset_service import DatasetCollectionBindingService
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/annotation/delete_annotation_index_task.py

@@ -1,9 +1,9 @@
-import datetime
 import logging
 import time
 
 import click
 from celery import shared_task
+
 from core.index.index import IndexBuilder
 from models.dataset import Dataset
 from services.dataset_service import DatasetCollectionBindingService

api/tasks/annotation/disable_annotation_reply_task.py

@@ -1,15 +1,15 @@
-import datetime
 import logging
 import time
 
 import click
 from celery import shared_task
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from models.dataset import Dataset
-from models.model import App, AppAnnotationSetting, MessageAnnotation
-from werkzeug.exceptions import NotFound
+from models.model import App, AppAnnotationSetting
 
 
 @shared_task(queue='dataset')

api/tasks/annotation/enable_annotation_reply_task.py

@@ -4,14 +4,15 @@ import time
 
 import click
 from celery import shared_task
+from langchain.schema import Document
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.schema import Document
 from models.dataset import Dataset
 from models.model import App, AppAnnotationSetting, MessageAnnotation
 from services.dataset_service import DatasetCollectionBindingService
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/annotation/update_annotation_to_index_task.py

@@ -3,8 +3,9 @@ import time
 
 import click
 from celery import shared_task
-from core.index.index import IndexBuilder
 from langchain.schema import Document
+
+from core.index.index import IndexBuilder
 from models.dataset import Dataset
 from services.dataset_service import DatasetCollectionBindingService
 

api/tasks/batch_create_segment_to_index_task.py

@@ -6,6 +6,8 @@ from typing import List, cast
 
 import click
 from celery import shared_task
+from sqlalchemy import func
+
 from core.indexing_runner import IndexingRunner
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
@@ -14,7 +16,6 @@ from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from libs import helper
 from models.dataset import Dataset, Document, DocumentSegment
-from sqlalchemy import func
 
 
 @shared_task(queue='dataset')

api/tasks/clean_dataset_task.py

@@ -3,12 +3,17 @@ import time
 
 import click
 from celery import shared_task
+
 from core.index.index import IndexBuilder
-from core.index.vector_index.vector_index import VectorIndex
 from extensions.ext_database import db
-from flask import current_app
-from models.dataset import (AppDatasetJoin, Dataset, DatasetKeywordTable, DatasetProcessRule, DatasetQuery, Document,
-                            DocumentSegment)
+from models.dataset import (
+    AppDatasetJoin,
+    Dataset,
+    DatasetProcessRule,
+    DatasetQuery,
+    Document,
+    DocumentSegment,
+)
 
 
 @shared_task(queue='dataset')

api/tasks/clean_document_task.py

@@ -3,6 +3,7 @@ import time
 
 import click
 from celery import shared_task
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from models.dataset import Dataset, DocumentSegment

api/tasks/clean_notion_document_task.py

@@ -4,6 +4,7 @@ from typing import List
 
 import click
 from celery import shared_task
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from models.dataset import Dataset, Document, DocumentSegment

api/tasks/create_segment_to_index_task.py

@@ -5,12 +5,13 @@ from typing import List, Optional
 
 import click
 from celery import shared_task
+from langchain.schema import Document
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.schema import Document
 from models.dataset import DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/deal_dataset_vector_index_task.py

@@ -3,12 +3,12 @@ import time
 
 import click
 from celery import shared_task
+from langchain.schema import Document
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
-from langchain.schema import Document
-from models.dataset import Dataset
+from models.dataset import Dataset, DocumentSegment
 from models.dataset import Document as DatasetDocument
-from models.dataset import DocumentSegment
 
 
 @shared_task(queue='dataset')

api/tasks/delete_segment_from_index_task.py

@@ -3,11 +3,11 @@ import time
 
 import click
 from celery import shared_task
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from models.dataset import Dataset, Document, DocumentSegment
-from werkzeug.exceptions import NotFound
+from models.dataset import Dataset, Document
 
 
 @shared_task(queue='dataset')

api/tasks/disable_segment_from_index_task.py

@@ -3,11 +3,12 @@ import time
 
 import click
 from celery import shared_task
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from models.dataset import DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/document_indexing_sync_task.py

@@ -4,13 +4,14 @@ import time
 
 import click
 from celery import shared_task
+from werkzeug.exceptions import NotFound
+
 from core.data_loader.loader.notion import NotionLoader
 from core.index.index import IndexBuilder
 from core.indexing_runner import DocumentIsPausedException, IndexingRunner
 from extensions.ext_database import db
 from models.dataset import Dataset, Document, DocumentSegment
 from models.source import DataSourceBinding
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/document_indexing_task.py

@@ -4,10 +4,10 @@ import time
 
 import click
 from celery import shared_task
+
 from core.indexing_runner import DocumentIsPausedException, IndexingRunner
 from extensions.ext_database import db
 from models.dataset import Document
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/document_indexing_update_task.py

@@ -4,11 +4,12 @@ import time
 
 import click
 from celery import shared_task
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from core.indexing_runner import DocumentIsPausedException, IndexingRunner
 from extensions.ext_database import db
 from models.dataset import Dataset, Document, DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/enable_segment_to_index_task.py

@@ -4,12 +4,13 @@ import time
 
 import click
 from celery import shared_task
+from langchain.schema import Document
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.schema import Document
 from models.dataset import DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/mail_invite_member_task.py

@@ -3,10 +3,10 @@ import time
 
 import click
 from celery import shared_task
-from constants.languages import languages
-from extensions.ext_mail import mail
 from flask import current_app, render_template
 
+from extensions.ext_mail import mail
+
 
 @shared_task(queue='mail')
 def send_invite_member_mail_task(language: str, to: str, token: str, inviter_name: str, workspace_name: str):

api/tasks/recover_document_indexing_task.py

@@ -3,10 +3,11 @@ import time
 
 import click
 from celery import shared_task
+from werkzeug.exceptions import NotFound
+
 from core.indexing_runner import DocumentIsPausedException, IndexingRunner
 from extensions.ext_database import db
 from models.dataset import Document
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/remove_document_from_index_task.py

@@ -3,11 +3,12 @@ import time
 
 import click
 from celery import shared_task
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from models.dataset import Document, DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/update_segment_index_task.py

@@ -5,12 +5,13 @@ from typing import List, Optional
 
 import click
 from celery import shared_task
+from langchain.schema import Document
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.schema import Document
 from models.dataset import DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')

api/tasks/update_segment_keyword_index_task.py

@@ -1,16 +1,15 @@
 import datetime
 import logging
 import time
-from typing import List, Optional
 
 import click
 from celery import shared_task
+from werkzeug.exceptions import NotFound
+
 from core.index.index import IndexBuilder
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.schema import Document
 from models.dataset import DocumentSegment
-from werkzeug.exceptions import NotFound
 
 
 @shared_task(queue='dataset')