hz/ebiz-dify-ai
1
0
Fork 0
mirror of http://112.124.100.131/huang.ze/ebiz-dify-ai.git synced 2025-12-25 02:33:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: apply ruff's pyupgrade linter rules to modernize Python code with targeted version (#2419)

Browse Source
This commit is contained in:
Bowen Liang
2024-02-09 15:21:33 +08:00
committed by GitHub
parent 589099a005
commit 063191889d
246 changed files with 912 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
api/core/data_loader/file_extractor.py
View File

@@ -1,6 +1,6 @@
import tempfile
from pathlib import Path
from typing import List, Optional, Union
from typing import Optional, Union
import requests
from flask import current_app
@@ -28,7 +28,7 @@ USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM
class FileExtractor:
@classmethod
def load(cls, upload_file: UploadFile, return_text: bool = False, is_automatic: bool = False) -> Union[List[Document], str]:
def load(cls, upload_file: UploadFile, return_text: bool = False, is_automatic: bool = False) -> Union[list[Document], str]:
with tempfile.TemporaryDirectory() as temp_dir:
suffix = Path(upload_file.key).suffix
file_path = f"{temp_dir}/{next(tempfile._get_candidate_names())}{suffix}"
@@ -37,7 +37,7 @@ class FileExtractor:
return cls.load_from_file(file_path, return_text, upload_file, is_automatic)
@classmethod
def load_from_url(cls, url: str, return_text: bool = False) -> Union[List[Document], str]:
def load_from_url(cls, url: str, return_text: bool = False) -> Union[list[Document], str]:
response = requests.get(url, headers={
"User-Agent": USER_AGENT
})
@@ -53,7 +53,7 @@ class FileExtractor:
@classmethod
def load_from_file(cls, file_path: str, return_text: bool = False,
upload_file: Optional[UploadFile] = None,
is_automatic: bool = False) -> Union[List[Document], str]:
is_automatic: bool = False) -> Union[list[Document], str]:
input_file = Path(file_path)
delimiter = '\n'
file_extension = input_file.suffix.lower()

6
api/core/data_loader/loader/csv_loader.py
View File

@@ -1,6 +1,6 @@
import csv
import logging
from typing import Dict, List, Optional
from typing import Optional
from langchain.document_loaders import CSVLoader as LCCSVLoader
from langchain.document_loaders.helpers import detect_file_encodings
@@ -14,7 +14,7 @@ class CSVLoader(LCCSVLoader):
self,
file_path: str,
source_column: Optional[str] = None,
csv_args: Optional[Dict] = None,
csv_args: Optional[dict] = None,
encoding: Optional[str] = None,
autodetect_encoding: bool = True,
):
@@ -24,7 +24,7 @@ class CSVLoader(LCCSVLoader):
self.csv_args = csv_args or {}
self.autodetect_encoding = autodetect_encoding
def load(self) -> List[Document]:
def load(self) -> list[Document]:
"""Load data into document objects."""
try:
with open(self.file_path, newline="", encoding=self.encoding) as csvfile:

3
api/core/data_loader/loader/excel.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
@@ -23,7 +22,7 @@ class ExcelLoader(BaseLoader):
"""Initialize with file path."""
self._file_path = file_path
def load(self) -> List[Document]:
def load(self) -> list[Document]:
data = []
keys = []
wb = load_workbook(filename=self._file_path, read_only=True)

3
api/core/data_loader/loader/html.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from bs4 import BeautifulSoup
from langchain.document_loaders.base import BaseLoader
@@ -23,7 +22,7 @@ class HTMLLoader(BaseLoader):
"""Initialize with file path."""
self._file_path = file_path
def load(self) -> List[Document]:
def load(self) -> list[Document]:
return [Document(page_content=self._load_as_text())]
def _load_as_text(self) -> str:

12
api/core/data_loader/loader/markdown.py
View File

@@ -1,6 +1,6 @@
import logging
import re
from typing import List, Optional, Tuple, cast
from typing import Optional, cast
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.helpers import detect_file_encodings
@@ -42,7 +42,7 @@ class MarkdownLoader(BaseLoader):
self._encoding = encoding
self._autodetect_encoding = autodetect_encoding
def load(self) -> List[Document]:
def load(self) -> list[Document]:
tups = self.parse_tups(self._file_path)
documents = []
for header, value in tups:
@@ -54,13 +54,13 @@ class MarkdownLoader(BaseLoader):
return documents
def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:
def markdown_to_tups(self, markdown_text: str) -> list[tuple[Optional[str], str]]:
"""Convert a markdown file to a dictionary.
The keys are the headers and the values are the text under each header.
"""
markdown_tups: List[Tuple[Optional[str], str]] = []
markdown_tups: list[tuple[Optional[str], str]] = []
lines = markdown_text.split("\n")
current_header = None
@@ -103,11 +103,11 @@ class MarkdownLoader(BaseLoader):
content = re.sub(pattern, r"\1", content)
return content
def parse_tups(self, filepath: str) -> List[Tuple[Optional[str], str]]:
def parse_tups(self, filepath: str) -> list[tuple[Optional[str], str]]:
"""Parse file into tuples."""
content = ""
try:
with open(filepath, "r", encoding=self._encoding) as f:
with open(filepath, encoding=self._encoding) as f:
content = f.read()
except UnicodeDecodeError as e:
if self._autodetect_encoding:

20
api/core/data_loader/loader/notion.py
View File

@@ -1,6 +1,6 @@
import json
import logging
from typing import Any, Dict, List, Optional
from typing import Any, Optional
import requests
from flask import current_app
@@ -67,7 +67,7 @@ class NotionLoader(BaseLoader):
document_model=document_model
)
def load(self) -> List[Document]:
def load(self) -> list[Document]:
self.update_last_edited_time(
self._document_model
)
@@ -78,7 +78,7 @@ class NotionLoader(BaseLoader):
def _load_data_as_documents(
self, notion_obj_id: str, notion_page_type: str
) -> List[Document]:
) -> list[Document]:
docs = []
if notion_page_type == 'database':
# get all the pages in the database
@@ -94,8 +94,8 @@ class NotionLoader(BaseLoader):
return docs
def _get_notion_database_data(
self, database_id: str, query_dict: Dict[str, Any] = {}
) -> List[Document]:
self, database_id: str, query_dict: dict[str, Any] = {}
) -> list[Document]:
"""Get all the pages from a Notion database."""
res = requests.post(
DATABASE_URL_TMPL.format(database_id=database_id),
@@ -149,12 +149,12 @@ class NotionLoader(BaseLoader):
return database_content_list
def _get_notion_block_data(self, page_id: str) -> List[str]:
def _get_notion_block_data(self, page_id: str) -> list[str]:
result_lines_arr = []
cur_block_id = page_id
while True:
block_url = BLOCK_CHILD_URL_TMPL.format(block_id=cur_block_id)
query_dict: Dict[str, Any] = {}
query_dict: dict[str, Any] = {}
res = requests.request(
"GET",
@@ -216,7 +216,7 @@ class NotionLoader(BaseLoader):
cur_block_id = block_id
while True:
block_url = BLOCK_CHILD_URL_TMPL.format(block_id=cur_block_id)
query_dict: Dict[str, Any] = {}
query_dict: dict[str, Any] = {}
res = requests.request(
"GET",
@@ -280,7 +280,7 @@ class NotionLoader(BaseLoader):
cur_block_id = block_id
while not done:
block_url = BLOCK_CHILD_URL_TMPL.format(block_id=cur_block_id)
query_dict: Dict[str, Any] = {}
query_dict: dict[str, Any] = {}
res = requests.request(
"GET",
@@ -346,7 +346,7 @@ class NotionLoader(BaseLoader):
else:
retrieve_page_url = RETRIEVE_PAGE_URL_TMPL.format(page_id=obj_id)
query_dict: Dict[str, Any] = {}
query_dict: dict[str, Any] = {}
res = requests.request(
"GET",

4
api/core/data_loader/loader/pdf.py
View File

@@ -1,5 +1,5 @@
import logging
from typing import List, Optional
from typing import Optional
from langchain.document_loaders import PyPDFium2Loader
from langchain.document_loaders.base import BaseLoader
@@ -28,7 +28,7 @@ class PdfLoader(BaseLoader):
self._file_path = file_path
self._upload_file = upload_file
def load(self) -> List[Document]:
def load(self) -> list[Document]:
plaintext_file_key = ''
plaintext_file_exists = False
if self._upload_file:

3
api/core/data_loader/loader/unstructured/unstructured_eml.py
View File

@@ -1,6 +1,5 @@
import base64
import logging
from typing import List
from bs4 import BeautifulSoup
from langchain.document_loaders.base import BaseLoader
@@ -24,7 +23,7 @@ class UnstructuredEmailLoader(BaseLoader):
self._file_path = file_path
self._api_url = api_url
def load(self) -> List[Document]:
def load(self) -> list[Document]:
from unstructured.partition.email import partition_email
elements = partition_email(filename=self._file_path, api_url=self._api_url)

3
api/core/data_loader/loader/unstructured/unstructured_markdown.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
@@ -34,7 +33,7 @@ class UnstructuredMarkdownLoader(BaseLoader):
self._file_path = file_path
self._api_url = api_url
def load(self) -> List[Document]:
def load(self) -> list[Document]:
from unstructured.partition.md import partition_md
elements = partition_md(filename=self._file_path, api_url=self._api_url)

3
api/core/data_loader/loader/unstructured/unstructured_msg.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
@@ -24,7 +23,7 @@ class UnstructuredMsgLoader(BaseLoader):
self._file_path = file_path
self._api_url = api_url
def load(self) -> List[Document]:
def load(self) -> list[Document]:
from unstructured.partition.msg import partition_msg
elements = partition_msg(filename=self._file_path, api_url=self._api_url)

3
api/core/data_loader/loader/unstructured/unstructured_ppt.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
@@ -23,7 +22,7 @@ class UnstructuredPPTLoader(BaseLoader):
self._file_path = file_path
self._api_url = api_url
def load(self) -> List[Document]:
def load(self) -> list[Document]:
from unstructured.partition.ppt import partition_ppt
elements = partition_ppt(filename=self._file_path, api_url=self._api_url)

3
api/core/data_loader/loader/unstructured/unstructured_pptx.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
@@ -22,7 +21,7 @@ class UnstructuredPPTXLoader(BaseLoader):
self._file_path = file_path
self._api_url = api_url
def load(self) -> List[Document]:
def load(self) -> list[Document]:
from unstructured.partition.pptx import partition_pptx
elements = partition_pptx(filename=self._file_path, api_url=self._api_url)

3
api/core/data_loader/loader/unstructured/unstructured_text.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
@@ -24,7 +23,7 @@ class UnstructuredTextLoader(BaseLoader):
self._file_path = file_path
self._api_url = api_url
def load(self) -> List[Document]:
def load(self) -> list[Document]:
from unstructured.partition.text import partition_text
elements = partition_text(filename=self._file_path, api_url=self._api_url)

3
api/core/data_loader/loader/unstructured/unstructured_xml.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import List
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
@@ -24,7 +23,7 @@ class UnstructuredXmlLoader(BaseLoader):
self._file_path = file_path
self._api_url = api_url
def load(self) -> List[Document]:
def load(self) -> list[Document]:
from unstructured.partition.xml import partition_xml
elements = partition_xml(filename=self._file_path, xml_keep_tags=True, api_url=self._api_url)