external knowledge api (#8913)

Co-authored-by: Yi <yxiaoisme@gmail.com>

web/app/components/datasets/external-knowledge-base/create/ExternalApiSelect.tsx

@@ -0,0 +1,110 @@
+import React, { useEffect, useState } from 'react'
+import {
+  RiAddLine,
+  RiArrowDownSLine,
+} from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import { useRouter } from 'next/navigation'
+import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
+import { useModalContext } from '@/context/modal-context'
+import { useExternalKnowledgeApi } from '@/context/external-knowledge-api-context'
+
+type ApiItem = {
+  value: string
+  name: string
+  url: string
+}
+
+type ExternalApiSelectProps = {
+  items: ApiItem[]
+  value?: string
+  onSelect: (item: ApiItem) => void
+}
+
+const ExternalApiSelect: React.FC<ExternalApiSelectProps> = ({ items, value, onSelect }) => {
+  const { t } = useTranslation()
+  const [isOpen, setIsOpen] = useState(false)
+  const [selectedItem, setSelectedItem] = useState<ApiItem | null>(
+    items.find(item => item.value === value) || null,
+  )
+  const { setShowExternalKnowledgeAPIModal } = useModalContext()
+  const { mutateExternalKnowledgeApis } = useExternalKnowledgeApi()
+  const router = useRouter()
+
+  useEffect(() => {
+    const newSelectedItem = items.find(item => item.value === value) || null
+    setSelectedItem(newSelectedItem)
+  }, [value, items])
+
+  const handleAddNewAPI = () => {
+    setShowExternalKnowledgeAPIModal({
+      payload: { name: '', settings: { endpoint: '', api_key: '' } },
+      onSaveCallback: async () => {
+        mutateExternalKnowledgeApis()
+        router.refresh()
+      },
+      onCancelCallback: () => {
+        mutateExternalKnowledgeApis()
+      },
+      isEditMode: false,
+    })
+  }
+
+  const handleSelect = (item: ApiItem) => {
+    setSelectedItem(item)
+    onSelect(item)
+    setIsOpen(false)
+  }
+
+  return (
+    <div className="relative w-full">
+      <div
+        className={`flex items-center justify-between cursor-pointer px-2 py-1 gap-0.5 self-stretch rounded-lg 
+        bg-components-input-bg-normal hover:bg-state-base-hover-alt ${isOpen && 'bg-state-base-hover-alt'}`}
+        onClick={() => setIsOpen(!isOpen)}
+      >
+        {selectedItem
+          ? (
+            <div className="flex p-1 items-center gap-2 self-stretch rounded-lg">
+              <ApiConnectionMod className='text-text-secondary w-4 h-4' />
+              <div className='flex items-center flex-grow'>
+                <span className='text-components-input-text-filled text-ellipsis system-sm-regular overflow-hidden'>{selectedItem.name}</span>
+              </div>
+            </div>
+          )
+          : (
+            <span className='text-components-input-text-placeholder system-sm-regular'>{t('dataset.selectExternalKnowledgeAPI.placeholder')}</span>
+          )}
+        <RiArrowDownSLine className={`w-4 h-4 text-text-quaternary transition-transform ${isOpen ? 'text-text-secondary' : ''}`} />
+      </div>
+      {isOpen && (
+        <div className="absolute z-10 w-full mt-1 bg-components-panel-bg-blur border rounded-xl shadow-lg">
+          {items.map(item => (
+            <div
+              key={item.value}
+              className="flex p-1 items-center cursor-pointer"
+              onClick={() => handleSelect(item)}
+            >
+              <div className="flex p-2 items-center gap-2 self-stretch rounded-lg hover:bg-state-base-hover w-full">
+                <ApiConnectionMod className='text-text-secondary w-4 h-4' />
+                <span className='text-text-secondary text-ellipsis system-sm-medium overflow-hidden flex-grow'>{item.name}</span>
+                <span className='text-text-tertiary overflow-hidden text-right text-ellipsis system-xs-regular'>{item.url}</span>
+              </div>
+            </div>
+          ))}
+          <div className='flex p-1 flex-col items-start self-stretch'>
+            <div
+              className='flex p-2 items-center gap-2 self-stretch rounded-lg cursor-pointer hover:bg-state-base-hover'
+              onClick={handleAddNewAPI}
+            >
+              <RiAddLine className='text-text-secondary w-4 h-4' />
+              <span className='flex-grow overflow-hidden text-text-secondary text-ellipsis system-sm-medium'>{t('dataset.createNewExternalAPI')}</span>
+            </div>
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
+export default ExternalApiSelect

web/app/components/datasets/external-knowledge-base/create/ExternalApiSelection.tsx

@@ -0,0 +1,96 @@
+'use client'
+
+import React, { useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { RiAddLine } from '@remixicon/react'
+import { useRouter } from 'next/navigation'
+import ExternalApiSelect from './ExternalApiSelect'
+import Input from '@/app/components/base/input'
+import Button from '@/app/components/base/button'
+import { useModalContext } from '@/context/modal-context'
+import { useExternalKnowledgeApi } from '@/context/external-knowledge-api-context'
+
+type ExternalApiSelectionProps = {
+  external_knowledge_api_id: string
+  external_knowledge_id: string
+  onChange: (data: { external_knowledge_api_id?: string; external_knowledge_id?: string }) => void
+}
+
+const ExternalApiSelection: React.FC<ExternalApiSelectionProps> = ({ external_knowledge_api_id, external_knowledge_id, onChange }) => {
+  const { t } = useTranslation()
+  const router = useRouter()
+  const { externalKnowledgeApiList } = useExternalKnowledgeApi()
+  const [selectedApiId, setSelectedApiId] = useState(external_knowledge_api_id)
+  const { setShowExternalKnowledgeAPIModal } = useModalContext()
+  const { mutateExternalKnowledgeApis } = useExternalKnowledgeApi()
+
+  const apiItems = externalKnowledgeApiList.map(api => ({
+    value: api.id,
+    name: api.name,
+    url: api.settings.endpoint,
+  }))
+
+  useEffect(() => {
+    if (apiItems.length > 0) {
+      const newSelectedId = external_knowledge_api_id || apiItems[0].value
+      setSelectedApiId(newSelectedId)
+      if (newSelectedId !== external_knowledge_api_id)
+        onChange({ external_knowledge_api_id: newSelectedId, external_knowledge_id })
+    }
+  }, [apiItems, external_knowledge_api_id, external_knowledge_id, onChange])
+
+  const handleAddNewAPI = () => {
+    setShowExternalKnowledgeAPIModal({
+      payload: { name: '', settings: { endpoint: '', api_key: '' } },
+      onSaveCallback: async () => {
+        mutateExternalKnowledgeApis()
+        router.refresh()
+      },
+      onCancelCallback: () => {
+        mutateExternalKnowledgeApis()
+      },
+      isEditMode: false,
+    })
+  }
+
+  useEffect(() => {
+    if (!external_knowledge_api_id && apiItems.length > 0)
+      onChange({ external_knowledge_api_id: apiItems[0].value, external_knowledge_id })
+  }, [])
+
+  return (
+    <form className='flex flex-col gap-4 self-stretch'>
+      <div className='flex flex-col gap-1 self-stretch'>
+        <div className='flex flex-col self-stretch'>
+          <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalAPIPanelTitle')}</label>
+        </div>
+        {apiItems.length > 0
+          ? <ExternalApiSelect
+            items={apiItems}
+            value={selectedApiId}
+            onSelect={(e) => {
+              setSelectedApiId(e.value)
+              onChange({ external_knowledge_api_id: e.value, external_knowledge_id })
+            }}
+          />
+          : <Button variant={'tertiary'} onClick={handleAddNewAPI} className='justify-start gap-0.5'>
+            <RiAddLine className='w-4 h-4 text-text-tertiary' />
+            <span className='text-text-tertiary system-sm-regular'>{t('dataset.noExternalKnowledge')}</span>
+          </Button>
+        }
+      </div>
+      <div className='flex flex-col gap-1 self-stretch'>
+        <div className='flex flex-col self-stretch'>
+          <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalKnowledgeId')}</label>
+        </div>
+        <Input
+          value={external_knowledge_id}
+          onChange={e => onChange({ external_knowledge_id: e.target.value, external_knowledge_api_id })}
+          placeholder={t('dataset.externalKnowledgeIdPlaceholder') ?? ''}
+        />
+      </div>
+    </form>
+  )
+}
+
+export default ExternalApiSelection

web/app/components/datasets/external-knowledge-base/create/InfoPanel.tsx

@@ -0,0 +1,33 @@
+import { RiBookOpenLine } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+
+const InfoPanel = () => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex w-[360px] pt-[108px] pb-2 pr-8 flex-col items-start'>
+      <div className='flex min-w-[240px] w-full p-6 flex-col items-start gap-3 self-stretch rounded-xl bg-background-section'>
+        <div className='flex p-1 w-10 h-10 justify-center items-center gap-2 flex-grow self-stretch rounded-lg border-[0.5px] border-components-card-border bg-components-card-bg'>
+          <RiBookOpenLine className='w-5 h-5 text-text-accent' />
+        </div>
+        <p className='flex flex-col items-start gap-2 self-stretch'>
+          <span className='self-stretch text-text-secondary system-xl-semibold'>
+            {t('dataset.connectDatasetIntro.title')}
+          </span>
+          <span className='text-text-tertiary system-sm-regular'>
+            {t('dataset.connectDatasetIntro.content.front')}
+            <a className='text-text-accent system-sm-regular ml-1' href='https://docs.dify.ai/guides/knowledge-base/external-knowledge-api-documentation' target='_blank' rel="noopener noreferrer">
+              {t('dataset.connectDatasetIntro.content.link')}
+            </a>
+            {t('dataset.connectDatasetIntro.content.end')}
+          </span>
+          <a className='self-stretch text-text-accent system-sm-regular' href='https://docs.dify.ai/guides/knowledge-base/connect-external-knowledge' target='_blank' rel="noopener noreferrer">
+            {t('dataset.connectDatasetIntro.learnMore')}
+          </a>
+        </p>
+      </div>
+    </div>
+  )
+}
+
+export default InfoPanel

web/app/components/datasets/external-knowledge-base/create/KnowledgeBaseInfo.tsx

@@ -0,0 +1,65 @@
+import React from 'react'
+import { RiBookOpenLine } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import Input from '@/app/components/base/input'
+
+type KnowledgeBaseInfoProps = {
+  name: string
+  description?: string
+  onChange: (data: { name?: string; description?: string }) => void
+}
+
+const KnowledgeBaseInfo: React.FC<KnowledgeBaseInfoProps> = ({ name, description, onChange }) => {
+  const { t } = useTranslation()
+
+  const handleNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    onChange({ name: e.target.value })
+  }
+
+  const handleDescriptionChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
+    onChange({ description: e.target.value })
+  }
+
+  return (
+    <form className='flex flex-col gap-4 self-stretch'>
+      <div className='flex flex-col gap-4 self-stretch'>
+        <div className='flex flex-col gap-1 self-stretch'>
+          <div className='flex flex-col justify-center self-stretch'>
+            <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalKnowledgeName')}</label>
+          </div>
+          <Input
+            value={name}
+            onChange={handleNameChange}
+            placeholder={t('dataset.externalKnowledgeNamePlaceholder') ?? ''}
+          />
+        </div>
+        <div className='flex flex-col gap-1 self-stretch'>
+          <div className='flex flex-col justify-center self-stretch'>
+            <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalKnowledgeDescription')}</label>
+          </div>
+          <div className='flex flex-col gap-1 self-stretch'>
+            <textarea
+              value={description}
+              onChange={ e => handleDescriptionChange(e)}
+              placeholder={t('dataset.externalKnowledgeDescriptionPlaceholder') ?? ''}
+              className={`flex h-20 py-2 p-3 self-stretch items-start rounded-lg bg-components-input-bg-normal ${description ? 'text-components-input-text-filled' : 'text-components-input-text-placeholder'} system-sm-regular`}
+            />
+            <a
+              className='flex py-0.5 gap-1 self-stretch'
+              href='https://docs.dify.ai/features/datasets#how-to-write-a-good-dataset-description'
+              target="_blank"
+              rel="noopener noreferrer"
+            >
+              <div className='flex p-0.5 items-center gap-2'>
+                <RiBookOpenLine className='w-3 h-3 text-text-tertiary' />
+              </div>
+              <div className='flex-grow text-text-tertiary body-xs-regular'>{t('dataset.learnHowToWriteGoodKnowledgeDescription')}</div>
+            </a>
+          </div>
+        </div>
+      </div>
+    </form>
+  )
+}
+
+export default KnowledgeBaseInfo

web/app/components/datasets/external-knowledge-base/create/RetrievalSettings.tsx

@@ -0,0 +1,67 @@
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import TopKItem from '@/app/components/base/param-item/top-k-item'
+import ScoreThresholdItem from '@/app/components/base/param-item/score-threshold-item'
+import cn from '@/utils/classnames'
+
+type RetrievalSettingsProps = {
+  topK: number
+  scoreThreshold: number
+  scoreThresholdEnabled: boolean
+  isInHitTesting?: boolean
+  isInRetrievalSetting?: boolean
+  onChange: (data: { top_k?: number; score_threshold?: number; score_threshold_enabled?: boolean }) => void
+}
+
+const RetrievalSettings: FC<RetrievalSettingsProps> = ({
+  topK,
+  scoreThreshold,
+  scoreThresholdEnabled,
+  onChange,
+  isInHitTesting = false,
+  isInRetrievalSetting = false,
+}) => {
+  const { t } = useTranslation()
+
+  const handleScoreThresholdChange = (enabled: boolean) => {
+    onChange({ score_threshold_enabled: enabled })
+  }
+
+  return (
+    <div className={cn('flex flex-col gap-2 self-stretch', isInRetrievalSetting && 'w-full max-w-[480px]')}>
+      {!isInHitTesting && !isInRetrievalSetting && <div className='flex h-7 pt-1 flex-col gap-2 self-stretch'>
+        <label className='text-text-secondary system-sm-semibold'>{t('dataset.retrievalSettings')}</label>
+      </div>}
+      <div className={cn(
+        'flex gap-4 self-stretch',
+        {
+          'flex-col': isInHitTesting,
+          'flex-row': isInRetrievalSetting,
+          'flex-col sm:flex-row': !isInHitTesting && !isInRetrievalSetting,
+        },
+      )}>
+        <div className='flex flex-col gap-1 flex-grow'>
+          <TopKItem
+            className='grow'
+            value={topK}
+            onChange={(_key, v) => onChange({ top_k: v })}
+            enable={true}
+          />
+        </div>
+        <div className='flex flex-col gap-1 flex-grow'>
+          <ScoreThresholdItem
+            className='grow'
+            value={scoreThreshold}
+            onChange={(_key, v) => onChange({ score_threshold: v })}
+            enable={scoreThresholdEnabled}
+            hasSwitch={true}
+            onSwitchChange={(_key, v) => handleScoreThresholdChange(v)}
+          />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default RetrievalSettings

web/app/components/datasets/external-knowledge-base/create/declarations.ts

@@ -0,0 +1,12 @@
+export type CreateKnowledgeBaseReq = {
+  name: string
+  description?: string
+  external_knowledge_api_id: string
+  provider: 'external'
+  external_knowledge_id: string
+  external_retrieval_model: {
+    top_k: number
+    score_threshold: number
+    score_threshold_enabled: boolean
+  }
+}

web/app/components/datasets/external-knowledge-base/create/index.tsx

@@ -0,0 +1,128 @@
+'use client'
+
+import { useCallback, useState } from 'react'
+import { useRouter } from 'next/navigation'
+import { RiArrowLeftLine, RiArrowRightLine } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import KnowledgeBaseInfo from './KnowledgeBaseInfo'
+import ExternalApiSelection from './ExternalApiSelection'
+import RetrievalSettings from './RetrievalSettings'
+import InfoPanel from './InfoPanel'
+import type { CreateKnowledgeBaseReq } from './declarations'
+import Divider from '@/app/components/base/divider'
+import Button from '@/app/components/base/button'
+
+type ExternalKnowledgeBaseCreateProps = {
+  onConnect: (formValue: CreateKnowledgeBaseReq) => void
+  loading: boolean
+}
+
+const ExternalKnowledgeBaseCreate: React.FC<ExternalKnowledgeBaseCreateProps> = ({ onConnect, loading }) => {
+  const { t } = useTranslation()
+  const router = useRouter()
+  const [formData, setFormData] = useState<CreateKnowledgeBaseReq>({
+    name: '',
+    description: '',
+    external_knowledge_api_id: '',
+    external_knowledge_id: '',
+    external_retrieval_model: {
+      top_k: 2,
+      score_threshold: 0.5,
+      score_threshold_enabled: false,
+    },
+    provider: 'external',
+
+  })
+
+  const navBackHandle = useCallback(() => {
+    router.replace('/datasets')
+  }, [router])
+
+  const handleFormChange = (newData: CreateKnowledgeBaseReq) => {
+    setFormData(newData)
+  }
+
+  const isFormValid = formData.name.trim() !== ''
+    && formData.external_knowledge_api_id !== ''
+    && formData.external_knowledge_id !== ''
+    && formData.external_retrieval_model.top_k !== undefined
+    && formData.external_retrieval_model.score_threshold !== undefined
+
+  return (
+    <div className='flex flex-col flex-grow self-stretch rounded-t-2xl border-t border-effects-highlight bg-components-panel-bg'>
+      <div className='flex justify-center flex-grow self-stretch'>
+        <div className='flex w-full max-w-[960px] px-14 py-0 flex-col items-center'>
+          <div className='flex w-full max-w-[640px] pt-6 pb-8 flex-col grow items-center gap-4'>
+            <div className='relative flex flex-col py-2 items-center gap-[2px] self-stretch'>
+              <div className='flex-grow text-text-primary system-xl-semibold self-stretch'>{t('dataset.connectDataset')}</div>
+              <p className='text-text-tertiary system-sm-regular'>
+                <span>{t('dataset.connectHelper.helper1')}</span>
+                <span className='text-text-secondary system-sm-medium'>{t('dataset.connectHelper.helper2')}</span>
+                <span>{t('dataset.connectHelper.helper3')}</span>
+                <a className='self-stretch text-text-accent system-sm-regular' href='https://docs.dify.ai/guides/knowledge-base/connect-external-knowledge' target='_blank' rel="noopener noreferrer">
+                  {t('dataset.connectHelper.helper4')}
+                </a>
+                <span>{t('dataset.connectHelper.helper5')} </span>
+              </p>
+              <Button
+                className='flex w-8 h-8 p-2 items-center justify-center absolute left-[-44px] top-1 rounded-full'
+                variant='tertiary'
+                onClick={navBackHandle}
+              >
+                <RiArrowLeftLine className='w-4 h-4 text-text-tertiary' />
+              </Button>
+            </div>
+            <KnowledgeBaseInfo
+              name={formData.name}
+              description={formData.description ?? ''}
+              onChange={data => handleFormChange({
+                ...formData,
+                ...data,
+              })}
+            />
+            <Divider />
+            <ExternalApiSelection
+              external_knowledge_api_id={formData.external_knowledge_api_id}
+              external_knowledge_id={formData.external_knowledge_id}
+              onChange={data => handleFormChange({
+                ...formData,
+                ...data,
+              })}
+            />
+            <RetrievalSettings
+              topK={formData.external_retrieval_model.top_k}
+              scoreThreshold={formData.external_retrieval_model.score_threshold}
+              scoreThresholdEnabled={formData.external_retrieval_model.score_threshold_enabled}
+              onChange={data => handleFormChange({
+                ...formData,
+                external_retrieval_model: {
+                  ...formData.external_retrieval_model,
+                  ...data,
+                },
+              })}
+            />
+            <div className='flex py-2 justify-end items-center gap-2 self-stretch'>
+              <Button variant='secondary' onClick={navBackHandle}>
+                <div className='text-components-button-secondary-text system-sm-medium'>{t('dataset.externalKnowledgeForm.cancel')}</div>
+              </Button>
+              <Button
+                variant='primary'
+                onClick={() => {
+                  onConnect(formData)
+                }}
+                disabled={!isFormValid}
+                loading={loading}
+              >
+                <div className='text-components-button-primary-text system-sm-medium'>{t('dataset.externalKnowledgeForm.connect')}</div>
+                <RiArrowRightLine className='w-4 h-4 text-components-button-primary-text' />
+              </Button>
+            </div>
+          </div>
+        </div>
+        <InfoPanel />
+      </div>
+    </div>
+  )
+}
+
+export default ExternalKnowledgeBaseCreate