hz/ebiz-dify-ai
1
0
Fork 0
mirror of http://112.124.100.131/huang.ze/ebiz-dify-ai.git synced 2025-12-24 18:23:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support firecrawl frontend code (#5226)

Browse Source
This commit is contained in:
Joel
2024-06-14 22:02:41 +08:00
committed by GitHub
parent 8d1386df0f
commit 28554350de
51 changed files with 1979 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
web/app/components/datasets/create/index.tsx
View File

@@ -8,7 +8,7 @@ import StepOne from './step-one'
import StepTwo from './step-two'
import StepThree from './step-three'
import { DataSourceType } from '@/models/datasets'
import type { DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
import { fetchDataSource } from '@/service/common'
import { fetchDatasetDetail } from '@/service/datasets'
import type { NotionPage } from '@/models/common'
@@ -19,6 +19,15 @@ type DatasetUpdateFormProps = {
datasetId?: string
}
const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
crawl_sub_pages: true,
only_main_content: true,
includes: '',
excludes: '',
limit: 10,
max_depth: '',
}
const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
const { t } = useTranslation()
const { setShowAccountSettingModal } = useModalContext()
@@ -36,9 +45,13 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
setNotionPages(value)
}
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
const updateFileList = (preparedFiles: FileItem[]) => {
setFiles(preparedFiles)
}
const [fireCrawlJobId, setFireCrawlJobId] = useState('')
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
const targetIndex = list.findIndex(file => file.fileID === fileItem.fileID)
@@ -108,20 +121,27 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
<StepsNavBar step={step} datasetId={datasetId} />
</div>
<div className="grow bg-white">
{step === 1 && <StepOne
hasConnection={hasConnection}
onSetting={() => setShowAccountSettingModal({ payload: 'data-source' })}
datasetId={datasetId}
dataSourceType={dataSourceType}
dataSourceTypeDisable={!!detail?.data_source_type}
changeType={setDataSourceType}
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
onStepChange={nextStep}
/>}
<div className={step === 1 ? 'block h-full' : 'hidden'}>
<StepOne
hasConnection={hasConnection}
onSetting={() => setShowAccountSettingModal({ payload: 'data-source' })}
datasetId={datasetId}
dataSourceType={dataSourceType}
dataSourceTypeDisable={!!detail?.data_source_type}
changeType={setDataSourceType}
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
onStepChange={nextStep}
websitePages={websitePages}
updateWebsitePages={setWebsitePages}
onFireCrawlJobIdChange={setFireCrawlJobId}
crawlOptions={crawlOptions}
onCrawlOptionsChange={setCrawlOptions}
/>
</div>
{(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
isAPIKeySet={!!embeddingsDefaultModel}
onSetting={() => setShowAccountSettingModal({ payload: 'provider' })}
@@ -130,9 +150,12 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
dataSourceType={dataSourceType}
files={fileList.map(file => file.file)}
notionPages={notionPages}
websitePages={websitePages}
fireCrawlJobId={fireCrawlJobId}
onStepChange={changeStep}
updateIndexingTypeCache={updateIndexingTypeCache}
updateResultCache={updateResultCache}
crawlOptions={crawlOptions}
/>}
{step === 3 && <StepThree
datasetId={datasetId}