feat: upgrade knowledge metadata (#16063)

Support filter knowledge by metadata.

Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: NFish <douxc512@gmail.com>

web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts

@@ -6,13 +6,28 @@ import {
 } from 'react'
 import produce from 'immer'
 import { isEqual } from 'lodash-es'
+import { v4 as uuid4 } from 'uuid'
 import type { ValueSelector, Var } from '../../types'
 import { BlockEnum, VarType } from '../../types'
 import {
-  useIsChatMode, useNodesReadOnly,
+  useIsChatMode,
+  useNodesReadOnly,
   useWorkflow,
 } from '../../hooks'
-import type { KnowledgeRetrievalNodeType, MultipleRetrievalConfig } from './types'
+import type {
+  HandleAddCondition,
+  HandleRemoveCondition,
+  HandleToggleConditionLogicalOperator,
+  HandleUpdateCondition,
+  KnowledgeRetrievalNodeType,
+  MetadataFilteringModeEnum,
+  MultipleRetrievalConfig,
+} from './types'
+import {
+  ComparisonOperator,
+  LogicalOperator,
+  MetadataFilteringVariableType,
+} from './types'
 import {
   getMultipleRetrievalConfig,
   getSelectedDatasetsMode,
@@ -25,6 +40,7 @@ import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-cr
 import useOneStepRun from '@/app/components/workflow/nodes/_base/hooks/use-one-step-run'
 import { useCurrentProviderAndModel, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import useAvailableVarList from '@/app/components/workflow/nodes/_base/hooks/use-available-var-list'
 
 const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
   const { nodesReadOnly: readOnly } = useNodesReadOnly()
@@ -196,13 +212,14 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     setInputs(newInputs)
   }, [inputs, setInputs, selectedDatasets, currentRerankModel, currentRerankProvider])
 
+  const [selectedDatasetsLoaded, setSelectedDatasetsLoaded] = useState(false)
   // datasets
   useEffect(() => {
     (async () => {
       const inputs = inputRef.current
       const datasetIds = inputs.dataset_ids
       if (datasetIds?.length > 0) {
-        const { data: dataSetsWithDetail } = await fetchDatasets({ url: '/datasets', params: { page: 1, ids: datasetIds } })
+        const { data: dataSetsWithDetail } = await fetchDatasets({ url: '/datasets', params: { page: 1, ids: datasetIds } as any })
         setSelectedDatasets(dataSetsWithDetail)
       }
       const newInputs = produce(inputs, (draft) => {
@@ -210,6 +227,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
         draft._datasets = selectedDatasets
       })
       setInputs(newInputs)
+      setSelectedDatasetsLoaded(true)
     })()
   // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
@@ -287,6 +305,113 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     })
   }, [runInputData, setRunInputData])
 
+  const handleMetadataFilterModeChange = useCallback((newMode: MetadataFilteringModeEnum) => {
+    setInputs(produce(inputRef.current, (draft) => {
+      draft.metadata_filtering_mode = newMode
+    }))
+  }, [setInputs])
+
+  const handleAddCondition = useCallback<HandleAddCondition>(({ name, type }) => {
+    let operator: ComparisonOperator = ComparisonOperator.is
+
+    if (type === MetadataFilteringVariableType.number)
+      operator = ComparisonOperator.equal
+
+    const newCondition = {
+      id: uuid4(),
+      name,
+      comparison_operator: operator,
+    }
+
+    const newInputs = produce(inputRef.current, (draft) => {
+      if (draft.metadata_filtering_conditions) {
+        draft.metadata_filtering_conditions.conditions.push(newCondition)
+      }
+      else {
+        draft.metadata_filtering_conditions = {
+          logical_operator: LogicalOperator.and,
+          conditions: [newCondition],
+        }
+      }
+    })
+    setInputs(newInputs)
+  }, [setInputs])
+
+  const handleRemoveCondition = useCallback<HandleRemoveCondition>((id) => {
+    const conditions = inputRef.current.metadata_filtering_conditions?.conditions || []
+    const index = conditions.findIndex(c => c.id === id)
+    const newInputs = produce(inputRef.current, (draft) => {
+      if (index > -1)
+        draft.metadata_filtering_conditions?.conditions.splice(index, 1)
+    })
+    setInputs(newInputs)
+  }, [setInputs])
+
+  const handleUpdateCondition = useCallback<HandleUpdateCondition>((id, newCondition) => {
+    const conditions = inputRef.current.metadata_filtering_conditions?.conditions || []
+    const index = conditions.findIndex(c => c.id === id)
+    const newInputs = produce(inputRef.current, (draft) => {
+      if (index > -1)
+        draft.metadata_filtering_conditions!.conditions[index] = newCondition
+    })
+    setInputs(newInputs)
+  }, [setInputs])
+
+  const handleToggleConditionLogicalOperator = useCallback<HandleToggleConditionLogicalOperator>(() => {
+    const oldLogicalOperator = inputRef.current.metadata_filtering_conditions?.logical_operator
+    const newLogicalOperator = oldLogicalOperator === LogicalOperator.and ? LogicalOperator.or : LogicalOperator.and
+    const newInputs = produce(inputRef.current, (draft) => {
+      draft.metadata_filtering_conditions!.logical_operator = newLogicalOperator
+    })
+    setInputs(newInputs)
+  }, [setInputs])
+
+  const handleMetadataModelChange = useCallback((model: { provider: string; modelId: string; mode?: string }) => {
+    const newInputs = produce(inputRef.current, (draft) => {
+      draft.metadata_model_config = {
+        provider: model.provider,
+        name: model.modelId,
+        mode: model.mode || 'chat',
+        completion_params: draft.metadata_model_config?.completion_params || { temperature: 0.7 },
+      }
+    })
+    setInputs(newInputs)
+  }, [setInputs])
+
+  const handleMetadataCompletionParamsChange = useCallback((newParams: Record<string, any>) => {
+    const newInputs = produce(inputRef.current, (draft) => {
+      draft.metadata_model_config = {
+        ...draft.metadata_model_config!,
+        completion_params: newParams,
+      }
+    })
+    setInputs(newInputs)
+  }, [setInputs])
+
+  const filterStringVar = useCallback((varPayload: Var) => {
+    return [VarType.string].includes(varPayload.type)
+  }, [])
+
+  const {
+    availableVars: availableStringVars,
+    availableNodesWithParent: availableStringNodesWithParent,
+  } = useAvailableVarList(id, {
+    onlyLeafNodeVar: false,
+    filterVar: filterStringVar,
+  })
+
+  const filterNumberVar = useCallback((varPayload: Var) => {
+    return [VarType.number].includes(varPayload.type)
+  }, [])
+
+  const {
+    availableVars: availableNumberVars,
+    availableNodesWithParent: availableNumberNodesWithParent,
+  } = useAvailableVarList(id, {
+    onlyLeafNodeVar: false,
+    filterVar: filterNumberVar,
+  })
+
   return {
     readOnly,
     inputs,
@@ -297,6 +422,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     handleModelChanged,
     handleCompletionParamsChange,
     selectedDatasets: selectedDatasets.filter(d => d.name),
+    selectedDatasetsLoaded,
     handleOnDatasetsChange,
     isShowSingleRun,
     hideSingleRun,
@@ -308,6 +434,17 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     runResult,
     rerankModelOpen,
     setRerankModelOpen,
+    handleMetadataFilterModeChange,
+    handleUpdateCondition,
+    handleAddCondition,
+    handleRemoveCondition,
+    handleToggleConditionLogicalOperator,
+    handleMetadataModelChange,
+    handleMetadataCompletionParamsChange,
+    availableStringVars,
+    availableStringNodesWithParent,
+    availableNumberVars,
+    availableNumberNodesWithParent,
   }
 }