feat: support LLM process document file (#10966)

Co-authored-by: -LAN- <laipz8200@outlook.com>
2025-12-16 06:16:53 +08:00 · 2024-11-22 19:32:44 +08:00
parent 556de444e8
commit 08ac36812b
37 changed files with 233 additions and 88 deletions
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-001.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-001.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 1048576
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-002.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-002.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 1048576
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0827.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0827.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 1048576
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0924.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0924.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 1048576
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-exp-0827.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-exp-0827.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 1048576
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-latest.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-latest.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 1048576
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 1048576
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-001.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-001.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 2097152
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-002.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-002.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 2097152
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0801.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0801.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 2097152
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0827.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0827.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 2097152
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-latest.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-latest.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 2097152
--- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 2097152
--- a/api/core/model_runtime/model_providers/google/llm/gemini-exp-1114.yaml
+++ b/api/core/model_runtime/model_providers/google/llm/gemini-exp-1114.yaml
@@ -7,6 +7,7 @@ features:
  - vision
  - tool-call
  - stream-tool-call
+  - document
 model_properties:
  mode: chat
  context_size: 32767
--- a/api/core/model_runtime/model_providers/google/llm/llm.py
+++ b/api/core/model_runtime/model_providers/google/llm/llm.py
@@ -16,6 +16,7 @@ from PIL import Image
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
 from core.model_runtime.entities.message_entities import (
    AssistantPromptMessage,
+    DocumentPromptMessageContent,
    ImagePromptMessageContent,
    PromptMessage,
    PromptMessageContentType,
@@ -35,6 +36,21 @@ from core.model_runtime.errors.invoke import (
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel

+GOOGLE_AVAILABLE_MIMETYPE = [
+    "application/pdf",
+    "application/x-javascript",
+    "text/javascript",
+    "application/x-python",
+    "text/x-python",
+    "text/plain",
+    "text/html",
+    "text/css",
+    "text/md",
+    "text/csv",
+    "text/xml",
+    "text/rtf",
+]
+

 class GoogleLargeLanguageModel(LargeLanguageModel):
    def _invoke(
@@ -370,6 +386,12 @@ class GoogleLargeLanguageModel(LargeLanguageModel):
                                raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}")
                        blob = {"inline_data": {"mime_type": mime_type, "data": base64_data}}
                        glm_content["parts"].append(blob)
+                    elif c.type == PromptMessageContentType.DOCUMENT:
+                        message_content = cast(DocumentPromptMessageContent, c)
+                        if message_content.mime_type not in GOOGLE_AVAILABLE_MIMETYPE:
+                            raise ValueError(f"Unsupported mime type {message_content.mime_type}")
+                        blob = {"inline_data": {"mime_type": message_content.mime_type, "data": message_content.data}}
+                        glm_content["parts"].append(blob)

            return glm_content
        elif isinstance(message, AssistantPromptMessage):
--- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml
+++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml
@@ -6,6 +6,7 @@ model_type: llm
 features:
  - vision
  - agent-thought
+  - video
 model_properties:
  mode: chat
  context_size: 32000
--- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max.yaml
+++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max.yaml
@@ -6,6 +6,7 @@ model_type: llm
 features:
  - vision
  - agent-thought
+  - video
 model_properties:
  mode: chat
  context_size: 32000
--- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus-0809.yaml
+++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus-0809.yaml
@@ -6,6 +6,7 @@ model_type: llm
 features:
  - vision
  - agent-thought
+  - video
 model_properties:
  mode: chat
  context_size: 32768
--- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus.yaml
+++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus.yaml
@@ -6,6 +6,7 @@ model_type: llm
 features:
  - vision
  - agent-thought
+  - video
 model_properties:
  mode: chat
  context_size: 8000
--- a/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml
+++ b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml
@@ -6,6 +6,7 @@ model_properties:
  mode: chat
 features:
  - vision
+  - video
 parameter_rules:
  - name: temperature
    use_template: temperature