external knowledge api (#8913)

Co-authored-by: Yi <yxiaoisme@gmail.com>
2025-12-22 17:26:54 +08:00 · 2024-09-30 15:38:43 +08:00
parent 77aef9ff1d
commit 9d221a5e19
90 changed files with 4623 additions and 1171 deletions
--- a/web/app/components/datasets/hit-testing/hit-detail.tsx
+++ b/web/app/components/datasets/hit-testing/hit-detail.tsx
@@ -26,12 +26,15 @@ const HitDetail: FC<IHitDetailProps> = ({ segInfo }) => {
      )
    }

-    return segInfo?.content
+    return <div className='mb-4 text-md text-gray-800 h-full'>{segInfo?.content}</div>
  }

  return (
-    <div className='overflow-x-auto'>
-      <div className="bg-gray-25 p-6">
+    segInfo?.id === 'external'
+      ? <div className='w-full overflow-x-auto px-2'>
+        <div className={s.segModalContent}>{renderContent()}</div>
+      </div>
+      : <div className='overflow-x-auto'>
        <div className="flex items-center">
          <SegmentIndexTag
            positionId={segInfo?.position || ''}
@@ -59,7 +62,6 @@ const HitDetail: FC<IHitDetailProps> = ({ segInfo }) => {
            })}
        </div>
      </div>
-    </div>
  )
 }

--- a/web/app/components/datasets/hit-testing/index.tsx
+++ b/web/app/components/datasets/hit-testing/index.tsx
@@ -13,7 +13,7 @@ import s from './style.module.css'
 import HitDetail from './hit-detail'
 import ModifyRetrievalModal from './modify-retrieval-modal'
 import cn from '@/utils/classnames'
-import type { HitTestingResponse, HitTesting as HitTestingType } from '@/models/datasets'
+import type { ExternalKnowledgeBaseHitTestingResponse, ExternalKnowledgeBaseHitTesting as ExternalKnowledgeBaseHitTestingType, HitTestingResponse, HitTesting as HitTestingType } from '@/models/datasets'
 import Loading from '@/app/components/base/loading'
 import Modal from '@/app/components/base/modal'
 import Drawer from '@/app/components/base/drawer'
@@ -49,8 +49,10 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
  const isMobile = media === MediaType.mobile

  const [hitResult, setHitResult] = useState<HitTestingResponse | undefined>() // 初始化记录为空数组
+  const [externalHitResult, setExternalHitResult] = useState<ExternalKnowledgeBaseHitTestingResponse | undefined>()
  const [submitLoading, setSubmitLoading] = useState(false)
  const [currParagraph, setCurrParagraph] = useState<{ paraInfo?: HitTestingType; showModal: boolean }>({ showModal: false })
+  const [externalCurrParagraph, setExternalCurrParagraph] = useState<{ paraInfo?: ExternalKnowledgeBaseHitTestingType; showModal: boolean }>({ showModal: false })
  const [text, setText] = useState('')

  const [currPage, setCurrPage] = React.useState<number>(0)
@@ -66,12 +68,52 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
    setCurrParagraph({ paraInfo: detail, showModal: true })
  }

+  const onClickExternalCard = (detail: ExternalKnowledgeBaseHitTestingType) => {
+    setExternalCurrParagraph({ paraInfo: detail, showModal: true })
+  }
  const { dataset: currentDataset } = useContext(DatasetDetailContext)
+  const isExternal = currentDataset?.provider === 'external'

  const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict as RetrievalConfig)
  const [isShowModifyRetrievalModal, setIsShowModifyRetrievalModal] = useState(false)
  const [isShowRightPanel, { setTrue: showRightPanel, setFalse: hideRightPanel, set: setShowRightPanel }] = useBoolean(!isMobile)

+  const renderHitResults = (results: any[], onClickCard: (record: any) => void) => (
+    <>
+      <div className='text-gray-600 font-semibold mb-4'>{t('datasetHitTesting.hit.title')}</div>
+      <div className='overflow-auto flex-1'>
+        <div className={s.cardWrapper}>
+          {results.map((record, idx) => (
+            <SegmentCard
+              key={idx}
+              loading={false}
+              refSource= {{
+                title: record.title,
+                uri: record.metadata ? record.metadata['x-amz-bedrock-kb-source-uri'] : '',
+              }}
+              isExternal={isExternal}
+              detail={record.segment}
+              contentExternal={record.content}
+              score={record.score}
+              scene='hitTesting'
+              className='h-[216px] mb-4'
+              onClick={() => onClickCard(record)}
+            />
+          ))}
+        </div>
+      </div>
+    </>
+  )
+
+  const renderEmptyState = () => (
+    <div className='h-full flex flex-col justify-center items-center'>
+      <div className={cn(docStyle.commonIcon, docStyle.targetIcon, '!bg-gray-200 !h-14 !w-14')} />
+      <div className='text-gray-300 text-[13px] mt-3'>
+        {t('datasetHitTesting.hit.emptyTip')}
+      </div>
+    </div>
+  )
+
  useEffect(() => {
    setShowRightPanel(!isMobile)
  }, [isMobile, setShowRightPanel])
@@ -86,12 +128,14 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
        <Textarea
          datasetId={datasetId}
          setHitResult={setHitResult}
+          setExternalHitResult={setExternalHitResult}
          onSubmit={showRightPanel}
          onUpdateList={recordsMutate}
          loading={submitLoading}
          setLoading={setSubmitLoading}
          setText={setText}
          text={text}
+          isExternal={isExternal}
          onClickRetrievalMethod={() => setIsShowModifyRetrievalModal(true)}
          retrievalConfig={retrievalConfig}
          isEconomy={currentDataset?.indexing_technique === 'economy'}
@@ -159,47 +203,42 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
                className='h-[216px]'
              />
            </div>
-            : !hitResult?.records.length
-              ? (
-                <div className='h-full flex flex-col justify-center items-center'>
-                  <div className={cn(docStyle.commonIcon, docStyle.targetIcon, '!bg-gray-200 !h-14 !w-14')} />
-                  <div className='text-gray-300 text-[13px] mt-3'>
-                    {t('datasetHitTesting.hit.emptyTip')}
-                  </div>
-                </div>
-              )
-              : (
-                <>
-                  <div className='text-gray-600 font-semibold mb-4'>{t('datasetHitTesting.hit.title')}</div>
-                  <div className='overflow-auto flex-1'>
-                    <div className={s.cardWrapper}>
-                      {hitResult?.records.map((record, idx) => {
-                        return <SegmentCard
-                          key={idx}
-                          loading={false}
-                          detail={record.segment as any}
-                          score={record.score}
-                          scene='hitTesting'
-                          className='h-[216px] mb-4'
-                          onClick={() => onClickCard(record as any)}
-                        />
-                      })}
-                    </div>
-                  </div>
-                </>
-              )
+            : (
+              (() => {
+                if (!hitResult?.records.length && !externalHitResult?.records.length)
+                  return renderEmptyState()
+
+                if (hitResult?.records.length)
+                  return renderHitResults(hitResult.records, onClickCard)
+
+                return renderHitResults(externalHitResult?.records || [], onClickExternalCard)
+              })()
+            )
          }
        </div>
      </FloatRightContainer>
      <Modal
-        className='w-[520px] p-0'
+        className={isExternal ? 'py-10 px-8' : 'w-full'}
        closable
-        onClose={() => setCurrParagraph({ showModal: false })}
-        isShow={currParagraph.showModal}
+        onClose={() => {
+          setCurrParagraph({ showModal: false })
+          setExternalCurrParagraph({ showModal: false })
+        }}
+        isShow={currParagraph.showModal || externalCurrParagraph.showModal}
      >
-        {currParagraph.showModal && <HitDetail
-          segInfo={currParagraph.paraInfo?.segment}
-        />}
+        {currParagraph.showModal && (
+          <HitDetail
+            segInfo={currParagraph.paraInfo?.segment}
+          />
+        )}
+        {externalCurrParagraph.showModal && (
+          <HitDetail
+            segInfo={{
+              id: 'external',
+              content: externalCurrParagraph.paraInfo?.content,
+            }}
+          />
+        )}
      </Modal>
      <Drawer isOpen={isShowModifyRetrievalModal} onClose={() => setIsShowModifyRetrievalModal(false)} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'>
        <ModifyRetrievalModal
--- a/web/app/components/datasets/hit-testing/modify-external-retrieval-modal.tsx
+++ b/web/app/components/datasets/hit-testing/modify-external-retrieval-modal.tsx
@@ -0,0 +1,71 @@
+import { useState } from 'react'
+import {
+  RiCloseLine,
+} from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import RetrievalSettings from '../external-knowledge-base/create/RetrievalSettings'
+import Button from '@/app/components/base/button'
+import ActionButton from '@/app/components/base/action-button'
+
+type ModifyExternalRetrievalModalProps = {
+  onClose: () => void
+  onSave: (data: { top_k: number; score_threshold: number; score_threshold_enabled: boolean }) => void
+  initialTopK: number
+  initialScoreThreshold: number
+  initialScoreThresholdEnabled: boolean
+}
+
+const ModifyExternalRetrievalModal: React.FC<ModifyExternalRetrievalModalProps> = ({
+  onClose,
+  onSave,
+  initialTopK,
+  initialScoreThreshold,
+  initialScoreThresholdEnabled,
+}) => {
+  const { t } = useTranslation()
+  const [topK, setTopK] = useState(initialTopK)
+  const [scoreThreshold, setScoreThreshold] = useState(initialScoreThreshold)
+  const [scoreThresholdEnabled, setScoreThresholdEnabled] = useState(initialScoreThresholdEnabled)
+
+  const handleSettingsChange = (data: { top_k?: number; score_threshold?: number; score_threshold_enabled?: boolean }) => {
+    if (data.top_k !== undefined)
+      setTopK(data.top_k)
+    if (data.score_threshold !== undefined)
+      setScoreThreshold(data.score_threshold)
+    if (data.score_threshold_enabled !== undefined)
+      setScoreThresholdEnabled(data.score_threshold_enabled)
+  }
+
+  const handleSave = () => {
+    onSave({ top_k: topK, score_threshold: scoreThreshold, score_threshold_enabled: scoreThresholdEnabled })
+    onClose()
+  }
+
+  return (
+    <div className='absolute z-10 top-[36px] right-[14px] flex w-[320px] flex-col items-start rounded-2xl border-[0.5px]
+      border-components-panel-border bg-components-panel-bg shadows-shadow-2xl'
+    >
+      <div className='flex p-4 pb-2 items-center justify-between self-stretch'>
+        <div className='text-text-primary system-xl-semibold flex-grow'>{t('datasetHitTesting.settingTitle')}</div>
+        <ActionButton className='ml-auto' onClick={onClose}>
+          <RiCloseLine className='w-4 h-4 flex-shrink-0' />
+        </ActionButton>
+      </div>
+      <div className='flex p-4 pt-2 flex-col justify-center items-start gap-4 self-stretch'>
+        <RetrievalSettings
+          topK={topK}
+          scoreThreshold={scoreThreshold}
+          scoreThresholdEnabled={scoreThresholdEnabled}
+          onChange={handleSettingsChange}
+          isInHitTesting={true}
+        />
+      </div>
+      <div className='flex p-4 pt-2 justify-end items-end gap-1 w-full'>
+        <Button className='flex-shrink-0 min-w-[72px]' onClick={onClose}>{t('common.operation.cancel')}</Button>
+        <Button variant='primary' className='flex-shrink-0 min-w-[72px]' onClick={handleSave}>{t('common.operation.save')}</Button>
+      </div>
+    </div>
+  )
+}
+
+export default ModifyExternalRetrievalModal
--- a/web/app/components/datasets/hit-testing/textarea.tsx
+++ b/web/app/components/datasets/hit-testing/textarea.tsx
@@ -1,12 +1,17 @@
+import React, { useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import {
+  RiEqualizer2Line,
+} from '@remixicon/react'
 import Button from '../../base/button'
 import Tag from '../../base/tag'
 import { getIcon } from '../common/retrieval-method-info'
 import s from './style.module.css'
+import ModifyExternalRetrievalModal from './modify-external-retrieval-modal'
 import Tooltip from '@/app/components/base/tooltip'
 import cn from '@/utils/classnames'
-import type { HitTestingResponse } from '@/models/datasets'
-import { hitTesting } from '@/service/datasets'
+import type { ExternalKnowledgeBaseHitTestingResponse, HitTestingResponse } from '@/models/datasets'
+import { externalKnowledgeBaseHitTesting, hitTesting } from '@/service/datasets'
 import { asyncRunSafe } from '@/utils'
 import { RETRIEVE_METHOD, type RetrievalConfig } from '@/types/app'

@@ -14,10 +19,12 @@ type TextAreaWithButtonIProps = {
  datasetId: string
  onUpdateList: () => void
  setHitResult: (res: HitTestingResponse) => void
+  setExternalHitResult: (res: ExternalKnowledgeBaseHitTestingResponse) => void
  loading: boolean
  setLoading: (v: boolean) => void
  text: string
  setText: (v: string) => void
+  isExternal?: boolean
  onClickRetrievalMethod: () => void
  retrievalConfig: RetrievalConfig
  isEconomy: boolean
@@ -28,16 +35,29 @@ const TextAreaWithButton = ({
  datasetId,
  onUpdateList,
  setHitResult,
+  setExternalHitResult,
  setLoading,
  loading,
  text,
  setText,
+  isExternal = false,
  onClickRetrievalMethod,
  retrievalConfig,
  isEconomy,
  onSubmit: _onSubmit,
 }: TextAreaWithButtonIProps) => {
  const { t } = useTranslation()
+  const [isSettingsOpen, setIsSettingsOpen] = useState(false)
+  const [externalRetrievalSettings, setExternalRetrievalSettings] = useState({
+    top_k: 2,
+    score_threshold: 0.5,
+    score_threshold_enabled: false,
+  })
+
+  const handleSaveExternalRetrievalSettings = (data: { top_k: number; score_threshold: number; score_threshold_enabled: boolean }) => {
+    setExternalRetrievalSettings(data)
+    setIsSettingsOpen(false)
+  }

  function handleTextChange(event: any) {
    setText(event.target.value)
@@ -63,28 +83,70 @@ const TextAreaWithButton = ({
    _onSubmit && _onSubmit()
  }

+  const externalRetrievalTestingOnSubmit = async () => {
+    const [e, res] = await asyncRunSafe<ExternalKnowledgeBaseHitTestingResponse>(
+      externalKnowledgeBaseHitTesting({
+        datasetId,
+        query: text,
+        external_retrieval_model: {
+          top_k: externalRetrievalSettings.top_k,
+          score_threshold: externalRetrievalSettings.score_threshold,
+          score_threshold_enabled: externalRetrievalSettings.score_threshold_enabled,
+        },
+      }) as Promise<ExternalKnowledgeBaseHitTestingResponse>,
+    )
+    if (!e) {
+      setExternalHitResult(res)
+      onUpdateList?.()
+    }
+    setLoading(false)
+  }
+
  const retrievalMethod = isEconomy ? RETRIEVE_METHOD.invertedIndex : retrievalConfig.search_method
  const Icon = getIcon(retrievalMethod)
  return (
    <>
      <div className={s.wrapper}>
-        <div className='pt-2 rounded-tl-xl rounded-tr-xl bg-[#EEF4FF]'>
+        <div className='relative pt-2 rounded-tl-xl rounded-tr-xl bg-[#EEF4FF]'>
          <div className="px-4 pb-2 flex justify-between h-8 items-center">
            <span className="text-gray-800 font-semibold text-sm">
              {t('datasetHitTesting.input.title')}
            </span>
-            <Tooltip
-              popupContent={t('dataset.retrieval.changeRetrievalMethod')}
-            >
-              <div
-                onClick={onClickRetrievalMethod}
-                className='flex px-2 h-7 items-center space-x-1 bg-white hover:bg-[#ECE9FE] rounded-md shadow-sm cursor-pointer text-[#6927DA]'
+            {isExternal
+              ? <Button
+                variant='secondary'
+                size='small'
+                onClick={() => setIsSettingsOpen(!isSettingsOpen)}
              >
-                <Icon className='w-3.5 h-3.5'></Icon>
-                <div className='text-xs font-medium'>{t(`dataset.retrieval.${retrievalMethod}.title`)}</div>
-              </div>
-            </Tooltip>
+                <RiEqualizer2Line className='text-components-button-secondary-text w-3.5 h-3.5' />
+                <div className='flex px-[3px] justify-center items-center gap-1'>
+                  <span className='text-components-button-secondary-text system-xs-medium'>{t('datasetHitTesting.settingTitle')}</span>
+                </div>
+              </Button>
+              : <Tooltip
+                popupContent={t('dataset.retrieval.changeRetrievalMethod')}
+              >
+                <div
+                  onClick={onClickRetrievalMethod}
+                  className='flex px-2 h-7 items-center space-x-1 bg-white hover:bg-[#ECE9FE] rounded-md shadow-sm cursor-pointer text-[#6927DA]'
+                >
+                  <Icon className='w-3.5 h-3.5'></Icon>
+                  <div className='text-xs font-medium'>{t(`dataset.retrieval.${retrievalMethod}.title`)}</div>
+                </div>
+              </Tooltip>
+            }
          </div>
+          {
+            isSettingsOpen && (
+              <ModifyExternalRetrievalModal
+                onClose={() => setIsSettingsOpen(false)}
+                onSave={handleSaveExternalRetrievalSettings}
+                initialTopK={externalRetrievalSettings.top_k}
+                initialScoreThreshold={externalRetrievalSettings.score_threshold}
+                initialScoreThresholdEnabled={externalRetrievalSettings.score_threshold_enabled}
+              />
+            )
+          }
          <div className='h-2 rounded-tl-xl rounded-tr-xl bg-white'></div>
        </div>
        <div className='px-4 pb-11'>
@@ -122,7 +184,7 @@ const TextAreaWithButton = ({

            <div>
              <Button
-                onClick={onSubmit}
+                onClick={isExternal ? externalRetrievalTestingOnSubmit : onSubmit}
                variant="primary"
                loading={loading}
                disabled={(!text?.length || text?.length > 200)}
@@ -132,7 +194,6 @@ const TextAreaWithButton = ({
            </div>
          </div>
        </div>
-
      </div>
    </>
  )