feat: [frontend] support vision (#1518)

Co-authored-by: Joel <iamjoel007@gmail.com>

web/app/components/app/configuration/index.tsx

@@ -25,19 +25,19 @@ import type {
 } from '@/models/debug'
 import type { ExternalDataTool } from '@/models/common'
 import type { DataSet } from '@/models/datasets'
-import type { ModelConfig as BackendModelConfig } from '@/types/app'
+import type { ModelConfig as BackendModelConfig, VisionSettings } from '@/types/app'
 import ConfigContext from '@/context/debug-configuration'
 import ConfigModel from '@/app/components/app/configuration/config-model'
 import Config from '@/app/components/app/configuration/config'
 import Debug from '@/app/components/app/configuration/debug'
 import Confirm from '@/app/components/base/confirm'
-import { ProviderEnum } from '@/app/components/header/account-setting/model-page/declarations'
+import { ModelFeature, ProviderEnum } from '@/app/components/header/account-setting/model-page/declarations'
 import { ToastContext } from '@/app/components/base/toast'
 import { fetchAppDetail, updateAppModelConfig } from '@/service/apps'
 import { promptVariablesToUserInputsForm, userInputsFormToPromptVariables } from '@/utils/model-config'
 import { fetchDatasets } from '@/service/datasets'
 import { useProviderContext } from '@/context/provider-context'
-import { AppType, ModelModeType } from '@/types/app'
+import { AppType, ModelModeType, Resolution, TransferMethod } from '@/types/app'
 import { FlipBackward } from '@/app/components/base/icons/src/vender/line/arrows'
 import { PromptMode } from '@/models/debug'
 import { DEFAULT_CHAT_PROMPT_CONFIG, DEFAULT_COMPLETION_PROMPT_CONFIG } from '@/config'
@@ -198,6 +198,7 @@ const Configuration: FC = () => {
   }
 
   const { textGenerationModelList } = useProviderContext()
+  const currModel = textGenerationModelList.find(item => item.model_name === modelConfig.model_id)
   const hasSetCustomAPIKEY = !!textGenerationModelList?.find(({ model_provider: provider }) => {
     if (provider.provider_type === 'system' && provider.quota_type === 'paid')
       return true
@@ -271,7 +272,8 @@ const Configuration: FC = () => {
     id: modelId,
     provider,
     mode: modeMode,
-  }: { id: string; provider: ProviderEnum; mode: ModelModeType }) => {
+    features,
+  }: { id: string; provider: ProviderEnum; mode: ModelModeType; features: string[] }) => {
     if (isAdvancedMode) {
       const appMode = mode
 
@@ -297,10 +299,31 @@ const Configuration: FC = () => {
     })
 
     setModelConfig(newModelConfig)
+    const supportVision = features && features.includes(ModelFeature.vision)
+    // eslint-disable-next-line @typescript-eslint/no-use-before-define
+    setVisionConfig({
+      // eslint-disable-next-line @typescript-eslint/no-use-before-define
+      ...visionConfig,
+      enabled: supportVision,
+    }, true)
+  }
+
+  const isShowVisionConfig = !!currModel?.features.includes(ModelFeature.vision)
+  const [visionConfig, doSetVisionConfig] = useState({
+    enabled: false,
+    number_limits: 2,
+    detail: Resolution.low,
+    transfer_methods: [TransferMethod.local_file],
+  })
+
+  const setVisionConfig = (config: VisionSettings, notNoticeFormattingChanged?: boolean) => {
+    doSetVisionConfig(config)
+    if (!notNoticeFormattingChanged)
+      setFormattingChanged(true)
   }
 
   useEffect(() => {
-    fetchAppDetail({ url: '/apps', id: appId }).then(async (res) => {
+    fetchAppDetail({ url: '/apps', id: appId }).then(async (res: any) => {
       setMode(res.mode)
       const modelConfig = res.model_config
       const promptMode = modelConfig.prompt_type === PromptMode.advanced ? PromptMode.advanced : PromptMode.simple
@@ -362,6 +385,10 @@ const Configuration: FC = () => {
         },
         completionParams: model.completion_params,
       }
+
+      if (modelConfig.file_upload)
+        setVisionConfig(modelConfig.file_upload.image, true)
+
       syncToPublishedConfig(config)
       setPublishedConfig(config)
       setDatasetConfigs(modelConfig.dataset_configs)
@@ -459,6 +486,9 @@ const Configuration: FC = () => {
         completion_params: completionParams as any,
       },
       dataset_configs: datasetConfigs,
+      file_upload: {
+        image: visionConfig,
+      },
     }
 
     if (isAdvancedMode) {
@@ -557,6 +587,9 @@ const Configuration: FC = () => {
       datasetConfigs,
       setDatasetConfigs,
       hasSetContextVar,
+      isShowVisionConfig,
+      visionConfig,
+      setVisionConfig,
     }}
     >
       <>