feat(website-crawl): add jina reader as additional alternative for website crawling (#8761)

web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx

@@ -2,11 +2,12 @@
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import { useBoolean } from 'ahooks'
 import Panel from '../panel'
 import { DataSourceType } from '../panel/types'
 import ConfigFirecrawlModal from './config-firecrawl-modal'
+import ConfigJinaReaderModal from './config-jina-reader-modal'
 import cn from '@/utils/classnames'
+import s from '@/app/components/datasets/create/website/index.module.css'
 import { fetchDataSources, removeDataSourceApiKeyBinding } from '@/service/datasets'
 
 import type {
@@ -19,9 +20,11 @@ import {
 } from '@/models/common'
 import Toast from '@/app/components/base/toast'
 
-type Props = {}
+type Props = {
+  provider: DataSourceProvider
+}
 
-const DataSourceWebsite: FC<Props> = () => {
+const DataSourceWebsite: FC<Props> = ({ provider }) => {
   const { t } = useTranslation()
   const { isCurrentWorkspaceManager } = useAppContext()
   const [sources, setSources] = useState<DataSourceItem[]>([])
@@ -36,22 +39,26 @@ const DataSourceWebsite: FC<Props> = () => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
-  const [isShowConfig, {
-    setTrue: showConfig,
-    setFalse: hideConfig,
-  }] = useBoolean(false)
+  const [configTarget, setConfigTarget] = useState<DataSourceProvider | null>(null)
+  const showConfig = useCallback((provider: DataSourceProvider) => {
+    setConfigTarget(provider)
+  }, [setConfigTarget])
+
+  const hideConfig = useCallback(() => {
+    setConfigTarget(null)
+  }, [setConfigTarget])
 
   const handleAdded = useCallback(() => {
     checkSetApiKey()
     hideConfig()
   }, [checkSetApiKey, hideConfig])
 
-  const getIdByProvider = (provider: string): string | undefined => {
+  const getIdByProvider = (provider: DataSourceProvider): string | undefined => {
     const source = sources.find(item => item.provider === provider)
     return source?.id
   }
 
-  const handleRemove = useCallback((provider: string) => {
+  const handleRemove = useCallback((provider: DataSourceProvider) => {
     return async () => {
       const dataSourceId = getIdByProvider(provider)
       if (dataSourceId) {
@@ -69,22 +76,34 @@ const DataSourceWebsite: FC<Props> = () => {
     <>
       <Panel
         type={DataSourceType.website}
-        isConfigured={sources.length > 0}
-        onConfigure={showConfig}
+        provider={provider}
+        isConfigured={sources.find(item => item.provider === provider) !== undefined}
+        onConfigure={() => showConfig(provider)}
         readOnly={!isCurrentWorkspaceManager}
-        configuredList={sources.map(item => ({
+        configuredList={sources.filter(item => item.provider === provider).map(item => ({
           id: item.id,
           logo: ({ className }: { className: string }) => (
-            <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
+            item.provider === DataSourceProvider.fireCrawl
+              ? (
+                <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
+              )
+              : (
+                <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>
+                  <span className={s.jinaLogo} />
+                </div>
+              )
           ),
-          name: 'Firecrawl',
+          name: item.provider === DataSourceProvider.fireCrawl ? 'Firecrawl' : 'Jina Reader',
           isActive: true,
         }))}
-        onRemove={handleRemove(DataSourceProvider.fireCrawl)}
+        onRemove={handleRemove(provider)}
       />
-      {isShowConfig && (
+      {configTarget === DataSourceProvider.fireCrawl && (
         <ConfigFirecrawlModal onSaved={handleAdded} onCancel={hideConfig} />
       )}
+      {configTarget === DataSourceProvider.jinaReader && (
+        <ConfigJinaReaderModal onSaved={handleAdded} onCancel={hideConfig} />
+      )}
     </>
 
   )