feat: [frontend] support vision (#1518)

Co-authored-by: Joel <iamjoel007@gmail.com>

web/app/components/share/text-generation/index.tsx

@@ -26,6 +26,8 @@ import SavedItems from '@/app/components/app/text-generate/saved-items'
 import type { InstalledApp } from '@/models/explore'
 import { DEFAULT_VALUE_MAX_LEN, appDefaultIconBackground } from '@/config'
 import Toast from '@/app/components/base/toast'
+import type { VisionFile, VisionSettings } from '@/types/app'
+import { Resolution, TransferMethod } from '@/types/app'
 
 const GROUP_SIZE = 5 // to avoid RPM(Request per minute) limit. The group task finished then the next group.
 enum TaskStatus {
@@ -92,6 +94,14 @@ const TextGeneration: FC<IMainProps> = ({
   // send message task
   const [controlSend, setControlSend] = useState(0)
   const [controlStopResponding, setControlStopResponding] = useState(0)
+  const [visionConfig, setVisionConfig] = useState<VisionSettings>({
+    enabled: false,
+    number_limits: 2,
+    detail: Resolution.low,
+    transfer_methods: [TransferMethod.local_file],
+  })
+  const [completionFiles, setCompletionFiles] = useState<VisionFile[]>([])
+
   const handleSend = () => {
     setIsCallBatchAPI(false)
     setControlSend(Date.now())
@@ -338,7 +348,11 @@ const TextGeneration: FC<IMainProps> = ({
       setSiteInfo(siteInfo as SiteInfo)
       changeLanguage(siteInfo.default_language)
 
-      const { user_input_form, more_like_this, sensitive_word_avoidance }: any = appParams
+      const { user_input_form, more_like_this, file_upload, sensitive_word_avoidance }: any = appParams
+      setVisionConfig({
+        ...file_upload.image,
+        image_file_size_limit: appParams?.system_parameters?.image_file_size_limit,
+      })
       const prompt_variables = userInputsFormToPromptVariables(user_input_form)
       setPromptConfig({
         prompt_template: '', // placeholder for feture
@@ -378,6 +392,8 @@ const TextGeneration: FC<IMainProps> = ({
     handleSaveMessage={handleSaveMessage}
     taskId={task?.id}
     onCompleted={handleCompleted}
+    visionConfig={visionConfig}
+    completionFiles={completionFiles}
   />)
 
   const renderBatchRes = () => {
@@ -512,6 +528,8 @@ const TextGeneration: FC<IMainProps> = ({
                 onInputsChange={setInputs}
                 promptConfig={promptConfig}
                 onSend={handleSend}
+                visionConfig={visionConfig}
+                onVisionFilesChange={setCompletionFiles}
               />
             </div>
             <div className={cn(isInBatchTab ? 'block' : 'hidden')}>

web/app/components/share/text-generation/result/index.tsx

@@ -13,6 +13,7 @@ import Loading from '@/app/components/base/loading'
 import type { PromptConfig } from '@/models/debug'
 import type { InstalledApp } from '@/models/explore'
 import type { ModerationService } from '@/models/common'
+import { TransferMethod, type VisionFile, type VisionSettings } from '@/types/app'
 export type IResultProps = {
   isCallBatchAPI: boolean
   isPC: boolean
@@ -32,6 +33,8 @@ export type IResultProps = {
   onCompleted: (completionRes: string, taskId?: number, success?: boolean) => void
   enableModeration?: boolean
   moderationService?: (text: string) => ReturnType<ModerationService>
+  visionConfig: VisionSettings
+  completionFiles: VisionFile[]
 }
 
 const Result: FC<IResultProps> = ({
@@ -51,6 +54,8 @@ const Result: FC<IResultProps> = ({
   handleSaveMessage,
   taskId,
   onCompleted,
+  visionConfig,
+  completionFiles,
 }) => {
   const [isResponsing, { setTrue: setResponsingTrue, setFalse: setResponsingFalse }] = useBoolean(false)
   useEffect(() => {
@@ -108,6 +113,11 @@ const Result: FC<IResultProps> = ({
       logError(t('appDebug.errorMessage.valueOfVarRequired', { key: hasEmptyInput }))
       return false
     }
+
+    if (completionFiles.find(item => item.transfer_method === TransferMethod.local_file && !item.upload_file_id)) {
+      notify({ type: 'info', message: t('appDebug.errorMessage.waitForImgUpload') })
+      return false
+    }
     return !hasEmptyInput
   }
 
@@ -120,9 +130,20 @@ const Result: FC<IResultProps> = ({
     if (!checkCanSend())
       return
 
-    const data = {
+    const data: Record<string, any> = {
       inputs,
     }
+    if (visionConfig.enabled && completionFiles && completionFiles?.length > 0) {
+      data.files = completionFiles.map((item) => {
+        if (item.transfer_method === TransferMethod.local_file) {
+          return {
+            ...item,
+            url: '',
+          }
+        }
+        return item
+      })
+    }
 
     setMessageId(null)
     setFeedback({
@@ -145,7 +166,6 @@ const Result: FC<IResultProps> = ({
         setResponsingFalse()
         onCompleted(getCompletionRes(), taskId, false)
         isTimeout = true
-        console.log(`[#${taskId}]: timeout`)
       }
     }, 1000)
     sendCompletionMessage(data, {

web/app/components/share/text-generation/run-once/index.tsx

@@ -9,6 +9,8 @@ import type { SiteInfo } from '@/models/share'
 import type { PromptConfig } from '@/models/debug'
 import Button from '@/app/components/base/button'
 import { DEFAULT_VALUE_MAX_LEN } from '@/config'
+import TextGenerationImageUploader from '@/app/components/base/image-uploader/text-generation-image-uploader'
+import type { VisionFile, VisionSettings } from '@/types/app'
 
 export type IRunOnceProps = {
   siteInfo: SiteInfo
@@ -16,12 +18,16 @@ export type IRunOnceProps = {
   inputs: Record<string, any>
   onInputsChange: (inputs: Record<string, any>) => void
   onSend: () => void
+  visionConfig: VisionSettings
+  onVisionFilesChange: (files: VisionFile[]) => void
 }
 const RunOnce: FC<IRunOnceProps> = ({
   promptConfig,
   inputs,
   onInputsChange,
   onSend,
+  visionConfig,
+  onVisionFilesChange,
 }) => {
   const { t } = useTranslation()
 
@@ -73,6 +79,24 @@ const RunOnce: FC<IRunOnceProps> = ({
               </div>
             </div>
           ))}
+          {
+            visionConfig?.enabled && (
+              <div className="w-full mt-4">
+                <div className="text-gray-900 text-sm font-medium">Image Upload</div>
+                <div className='mt-2'>
+                  <TextGenerationImageUploader
+                    settings={visionConfig}
+                    onFilesChange={files => onVisionFilesChange(files.filter(file => file.progress !== -1).map(fileItem => ({
+                      type: 'image',
+                      transfer_method: fileItem.type,
+                      url: fileItem.url,
+                      upload_file_id: fileItem.fileId,
+                    })))}
+                  />
+                </div>
+              </div>
+            )
+          }
           {promptConfig.prompt_variables.length > 0 && (
             <div className='mt-4 h-[1px] bg-gray-100'></div>
           )}