hz/ebiz-dify-ai
1
0
Fork 0
mirror of http://112.124.100.131/huang.ze/ebiz-dify-ai.git synced 2025-12-11 11:56:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix special code (#473)

Browse Source
This commit is contained in:
Jyong
2023-06-28 13:58:36 +08:00
committed by GitHub
parent 97e9ebd29a
commit 2eea114ac0
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/core/indexing_runner.py
View File

@@ -235,7 +235,8 @@ class IndexingRunner:
if len(preview_texts) < 5:
preview_texts.append(document.page_content)
tokens += TokenCalculator.get_num_tokens(self.embedding_model_name, document.page_content)
tokens += TokenCalculator.get_num_tokens(self.embedding_model_name,
self.filter_string(document.page_content))
return {
"total_segments": total_segments,
@@ -345,6 +346,8 @@ class IndexingRunner:
return text_docs
def filter_string(self, text):
text = text.replace('<|', '<')
text = text.replace('|>', '>')
pattern = re.compile('[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\xFF]')
return pattern.sub('', text)