mirror of
http://112.124.100.131/huang.ze/ebiz-dify-ai.git
synced 2025-12-09 10:56:52 +08:00
Feat/dataset notion import (#392)
Co-authored-by: StyleZhang <jasonapring2015@outlook.com> Co-authored-by: JzoNg <jzongcode@gmail.com>
This commit is contained in:
Jyong
@@ -0,0 +1,3 @@
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#374151" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 249 B |
4
web/app/components/datasets/create/assets/normal.svg
Normal file
4
web/app/components/datasets/create/assets/normal.svg
Normal file
@@ -0,0 +1,4 @@
|
||||
<svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M5.75 4.5C6.7165 4.5 7.5 3.7165 7.5 2.75C7.5 1.7835 6.7165 1 5.75 1C4.7835 1 4 1.7835 4 2.75C4 3.7165 4.7835 4.5 5.75 4.5Z" fill="#444CE7"/>
|
||||
<path d="M3.48775 4.314C3.36842 4.14172 3.30875 4.05558 3.24448 4.02712C3.18679 4.00157 3.12605 3.99844 3.06603 4.01794C2.99918 4.03965 2.94661 4.10099 2.84146 4.22367C2.41951 4.71598 2.13172 5.32705 2.03543 6.00009H2C1.72386 6.00009 1.5 5.77623 1.5 5.50009C1.5 5.31565 1.59961 5.15388 1.75036 5.06668C1.98939 4.9284 2.07107 4.62254 1.9328 4.38351C1.79453 4.14448 1.48867 4.0628 1.24964 4.20107C0.802591 4.45967 0.5 4.94425 0.5 5.50009C0.5 6.32852 1.17157 7.00009 2 7.00009H2.03545C2.14342 7.75422 2.49192 8.43113 2.99997 8.94961L2.99997 10.1117C2.99994 10.1712 2.99992 10.2424 3.00504 10.305C3.01097 10.3776 3.02619 10.4816 3.08171 10.5906C3.15362 10.7317 3.26835 10.8465 3.40948 10.9184C3.51845 10.9739 3.62245 10.9891 3.69505 10.9951C3.7577 11.0002 3.82881 11.0001 3.88836 11.0001H4.86154C4.92109 11.0001 4.99224 11.0002 5.05488 10.9951C5.12749 10.9891 5.23149 10.9739 5.34046 10.9184C5.48158 10.8465 5.59632 10.7317 5.66822 10.5906C5.72375 10.4816 5.73897 10.3776 5.7449 10.305C5.75002 10.2424 5.75 10.1712 5.74997 10.1117L5.74997 10.0001H6.24998L6.24997 10.1115C6.24995 10.1711 6.24992 10.2422 6.25504 10.3048C6.26097 10.3775 6.2762 10.4815 6.33172 10.5904C6.40363 10.7315 6.51836 10.8463 6.65948 10.9182C6.76846 10.9737 6.87245 10.9889 6.94506 10.9949C7.0077 11 7.0788 11 7.13835 10.9999H8.11159C8.17113 11 8.24229 11 8.30493 10.9949C8.37753 10.9889 8.48153 10.9737 8.5905 10.9182C8.73162 10.8463 8.84636 10.7315 8.91827 10.5904C8.97379 10.4815 8.98901 10.3775 8.99494 10.3048C9.00006 10.2422 9.00004 10.1711 9.00001 10.1115L9.00001 9.66299C9.55312 9.40029 10.0258 8.99721 10.3726 8.49993L10.6116 8.49994C10.6711 8.49996 10.7423 8.49999 10.8049 8.49487C10.8775 8.48893 10.9815 8.47371 11.0905 8.41819C11.2316 8.34628 11.3464 8.23155 11.4183 8.09043C11.4738 7.98145 11.489 7.87746 11.4949 7.80485C11.5001 7.74221 11.5 7.67109 11.5 7.61154V5.88181C11.5 5.82509 11.5001 5.75735 11.4954 5.69761C11.49 5.62851 11.4763 5.5294 11.4257 5.42448C11.352 5.27143 11.2285 5.14794 11.0755 5.07422C10.9705 5.02369 10.8714 5.00992 10.8023 5.00454C10.7577 5.00106 10.7087 5.0002 10.6631 4.99999C10.4953 4.64662 10.2702 4.32616 10 4.05044L10 3.51615C10 3.43874 10.0001 3.35111 9.99335 3.27574C9.98593 3.19252 9.96656 3.06385 9.88754 2.93633C9.78902 2.77733 9.63465 2.66089 9.4547 2.60984C9.31038 2.56889 9.18134 2.58561 9.09929 2.60134C9.02497 2.61559 8.94073 2.63969 8.8663 2.66098L8.78839 2.68324C8.6859 2.71252 8.63465 2.72716 8.59861 2.75356C8.5638 2.77904 8.54252 2.80415 8.52309 2.84266C8.50297 2.88255 8.49603 2.94339 8.48215 3.06506C8.32585 4.43546 7.16224 5.5 5.75 5.5C4.81225 5.5 3.98413 5.03063 3.48775 4.314Z" fill="#444CE7"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
11
web/app/components/datasets/create/assets/star.svg
Normal file
11
web/app/components/datasets/create/assets/star.svg
Normal file
@@ -0,0 +1,11 @@
|
||||
<svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M6 0.5C6.27614 0.5 6.5 0.723858 6.5 1V2C6.5 2.27614 6.27614 2.5 6 2.5C5.72386 2.5 5.5 2.27614 5.5 2V1C5.5 0.723858 5.72386 0.5 6 0.5Z" fill="#FB6514"/>
|
||||
<path d="M2.81791 2.11092C2.62265 1.91566 2.30606 1.91566 2.1108 2.11092C1.91554 2.30619 1.91554 2.62277 2.1108 2.81803L2.81791 3.52514C3.01317 3.7204 3.32975 3.7204 3.52502 3.52514C3.72028 3.32988 3.72028 3.01329 3.52502 2.81803L2.81791 2.11092Z" fill="#FB6514"/>
|
||||
<path d="M0.5 6C0.5 5.72386 0.723858 5.5 1 5.5H2C2.27614 5.5 2.5 5.72386 2.5 6C2.5 6.27614 2.27614 6.5 2 6.5H1C0.723858 6.5 0.5 6.27614 0.5 6Z" fill="#FB6514"/>
|
||||
<path d="M10 5.5C9.72386 5.5 9.5 5.72386 9.5 6C9.5 6.27614 9.72386 6.5 10 6.5H11C11.2761 6.5 11.5 6.27614 11.5 6C11.5 5.72386 11.2761 5.5 11 5.5H10Z" fill="#FB6514"/>
|
||||
<path d="M9.18192 8.47482C8.98666 8.27955 8.67008 8.27955 8.47482 8.47482C8.27955 8.67008 8.27955 8.98666 8.47482 9.18192L9.18192 9.88903C9.37718 10.0843 9.69377 10.0843 9.88903 9.88903C10.0843 9.69377 10.0843 9.37718 9.88903 9.18192L9.18192 8.47482Z" fill="#FB6514"/>
|
||||
<path d="M9.88903 2.81803C10.0843 2.62277 10.0843 2.30619 9.88903 2.11092C9.69377 1.91566 9.37718 1.91566 9.18192 2.11092L8.47482 2.81803C8.27955 3.01329 8.27955 3.32988 8.47482 3.52514C8.67008 3.7204 8.98666 3.7204 9.18192 3.52514L9.88903 2.81803Z" fill="#FB6514"/>
|
||||
<path d="M6 9.5C6.27614 9.5 6.5 9.72386 6.5 10V11C6.5 11.2761 6.27614 11.5 6 11.5C5.72386 11.5 5.5 11.2761 5.5 11V10C5.5 9.72386 5.72386 9.5 6 9.5Z" fill="#FB6514"/>
|
||||
<path d="M3.52502 9.18192C3.72028 8.98666 3.72028 8.67008 3.52502 8.47482C3.32975 8.27955 3.01317 8.27955 2.81791 8.47482L2.1108 9.18192C1.91554 9.37718 1.91554 9.69377 2.1108 9.88903C2.30606 10.0843 2.62265 10.0843 2.81791 9.88903L3.52502 9.18192Z" fill="#FB6514"/>
|
||||
<path d="M6.44837 3.27869C6.36413 3.10804 6.19032 3 6.00001 3C5.8097 3 5.6359 3.10804 5.55166 3.27869L4.89538 4.60823L3.4277 4.82276C3.23942 4.85028 3.08308 4.98228 3.02439 5.16328C2.9657 5.34429 3.01484 5.54291 3.15115 5.67568L4.21275 6.70968L3.96221 8.17048C3.93004 8.35807 4.00716 8.54766 4.16115 8.65953C4.31514 8.77139 4.51928 8.78613 4.68774 8.69754L6.00001 8.00742L7.31229 8.69754C7.48075 8.78613 7.68489 8.77139 7.83888 8.65953C7.99287 8.54766 8.06999 8.35807 8.03782 8.17048L7.78728 6.70968L8.84888 5.67568C8.98519 5.54291 9.03433 5.34429 8.97564 5.16328C8.91695 4.98228 8.76061 4.85028 8.57233 4.82276L7.10465 4.60823L6.44837 3.27869Z" fill="#FB6514"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.4 KiB |
@@ -0,0 +1,111 @@
|
||||
.progressContainer {
|
||||
@apply relative pb-4 w-full;
|
||||
border-bottom: 0.5px solid #EAECF0;
|
||||
}
|
||||
.sourceItem {
|
||||
position: relative;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 4px;
|
||||
padding: 0 4px;
|
||||
height: 24px;
|
||||
background: #EFF4FF;
|
||||
border-radius: 6px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.sourceItem.error {
|
||||
background: #FEE4E2;
|
||||
}
|
||||
.sourceItem.success {
|
||||
background: #D1FADF;
|
||||
}
|
||||
.progressbar {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
height: 100%;
|
||||
background-color: #B2CCFF;
|
||||
}
|
||||
.sourceItem .info {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
.sourceItem .info .name {
|
||||
font-weight: 500;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
color: #101828;
|
||||
}
|
||||
.sourceItem.success .info .name {
|
||||
color: #05603A;
|
||||
}
|
||||
.sourceItem .percent {
|
||||
font-weight: 500;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
color: #344054;
|
||||
}
|
||||
.sourceItem .error {
|
||||
color: #D92D20;
|
||||
}
|
||||
.sourceItem .success {
|
||||
color: #05603A;
|
||||
}
|
||||
|
||||
|
||||
.cost {
|
||||
@apply flex justify-between items-center text-xs text-gray-700;
|
||||
}
|
||||
.embeddingStatus {
|
||||
@apply flex items-center justify-between text-gray-900 font-medium text-sm mr-2;
|
||||
}
|
||||
.commonIcon {
|
||||
@apply w-3 h-3 mr-1 inline-block align-middle;
|
||||
}
|
||||
.highIcon {
|
||||
mask-image: url(../assets/star.svg);
|
||||
@apply bg-orange-500;
|
||||
}
|
||||
.economyIcon {
|
||||
background-color: #444ce7;
|
||||
mask-image: url(../assets/normal.svg);
|
||||
}
|
||||
.tokens {
|
||||
@apply text-xs font-medium px-1;
|
||||
}
|
||||
.price {
|
||||
color: #f79009;
|
||||
@apply text-xs font-medium;
|
||||
}
|
||||
|
||||
.fileIcon {
|
||||
@apply w-4 h-4 mr-1 bg-center bg-no-repeat;
|
||||
background-image: url(../assets/unknow.svg);
|
||||
background-size: 16px;
|
||||
}
|
||||
.fileIcon.csv {
|
||||
background-image: url(../assets/csv.svg);
|
||||
}
|
||||
|
||||
.fileIcon.xlsx,
|
||||
.fileIcon.xls {
|
||||
background-image: url(../assets/xlsx.svg);
|
||||
}
|
||||
.fileIcon.pdf {
|
||||
background-image: url(../assets/pdf.svg);
|
||||
}
|
||||
.fileIcon.html,
|
||||
.fileIcon.htm {
|
||||
background-image: url(../assets/html.svg);
|
||||
}
|
||||
.fileIcon.md,
|
||||
.fileIcon.markdown {
|
||||
background-image: url(../assets/md.svg);
|
||||
}
|
||||
.fileIcon.txt {
|
||||
background-image: url(../assets/txt.svg);
|
||||
}
|
||||
.fileIcon.json {
|
||||
background-image: url(../assets/json.svg);
|
||||
}
|
||||
242
web/app/components/datasets/create/embedding-process/index.tsx
Normal file
242
web/app/components/datasets/create/embedding-process/index.tsx
Normal file
@@ -0,0 +1,242 @@
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useEffect, useMemo } from 'react'
|
||||
import useSWR from 'swr'
|
||||
import { useRouter } from 'next/navigation'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { omit } from 'lodash-es'
|
||||
import { ArrowRightIcon } from '@heroicons/react/24/solid'
|
||||
import { useGetState } from 'ahooks'
|
||||
import cn from 'classnames'
|
||||
import s from './index.module.css'
|
||||
import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
|
||||
import Button from '@/app/components/base/button'
|
||||
import type { FullDocumentDetail, IndexingStatusResponse, ProcessRuleResponse } from '@/models/datasets'
|
||||
import { formatNumber } from '@/utils/format'
|
||||
import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchIndexingEstimateBatch, fetchProcessRule } from '@/service/datasets'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import NotionIcon from '@/app/components/base/notion-icon'
|
||||
|
||||
type Props = {
|
||||
datasetId: string
|
||||
batchId: string
|
||||
documents?: FullDocumentDetail[]
|
||||
indexingType?: string
|
||||
}
|
||||
|
||||
const RuleDetail: FC<{ sourceData?: ProcessRuleResponse }> = ({ sourceData }) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const segmentationRuleMap = {
|
||||
mode: t('datasetDocuments.embedding.mode'),
|
||||
segmentLength: t('datasetDocuments.embedding.segmentLength'),
|
||||
textCleaning: t('datasetDocuments.embedding.textCleaning'),
|
||||
}
|
||||
|
||||
const getRuleName = (key: string) => {
|
||||
if (key === 'remove_extra_spaces')
|
||||
return t('datasetCreation.stepTwo.removeExtraSpaces')
|
||||
|
||||
if (key === 'remove_urls_emails')
|
||||
return t('datasetCreation.stepTwo.removeUrlEmails')
|
||||
|
||||
if (key === 'remove_stopwords')
|
||||
return t('datasetCreation.stepTwo.removeStopwords')
|
||||
}
|
||||
|
||||
const getValue = useCallback((field: string) => {
|
||||
let value: string | number | undefined = '-'
|
||||
switch (field) {
|
||||
case 'mode':
|
||||
value = sourceData?.mode === 'automatic' ? (t('datasetDocuments.embedding.automatic') as string) : (t('datasetDocuments.embedding.custom') as string)
|
||||
break
|
||||
case 'segmentLength':
|
||||
value = sourceData?.rules?.segmentation?.max_tokens
|
||||
break
|
||||
default:
|
||||
value = sourceData?.mode === 'automatic'
|
||||
? (t('datasetDocuments.embedding.automatic') as string)
|
||||
// eslint-disable-next-line array-callback-return
|
||||
: sourceData?.rules?.pre_processing_rules?.map((rule) => {
|
||||
if (rule.enabled)
|
||||
return getRuleName(rule.id)
|
||||
}).filter(Boolean).join(';')
|
||||
break
|
||||
}
|
||||
return value
|
||||
}, [sourceData])
|
||||
|
||||
return <div className='flex flex-col pt-8 pb-10 first:mt-0'>
|
||||
{Object.keys(segmentationRuleMap).map((field) => {
|
||||
return <FieldInfo
|
||||
key={field}
|
||||
label={segmentationRuleMap[field as keyof typeof segmentationRuleMap]}
|
||||
displayedValue={String(getValue(field))}
|
||||
/>
|
||||
})}
|
||||
</div>
|
||||
}
|
||||
|
||||
const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], indexingType }) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const getFirstDocument = documents[0]
|
||||
|
||||
const [indexingStatusBatchDetail, setIndexingStatusDetail, getIndexingStatusDetail] = useGetState<IndexingStatusResponse[]>([])
|
||||
const fetchIndexingStatus = async () => {
|
||||
const status = await doFetchIndexingStatus({ datasetId, batchId })
|
||||
setIndexingStatusDetail(status.data)
|
||||
}
|
||||
|
||||
const [runId, setRunId, getRunId] = useGetState<any>(null)
|
||||
|
||||
const stopQueryStatus = () => {
|
||||
clearInterval(getRunId())
|
||||
}
|
||||
|
||||
const startQueryStatus = () => {
|
||||
const runId = setInterval(() => {
|
||||
const indexingStatusBatchDetail = getIndexingStatusDetail()
|
||||
const isCompleted = indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error'].includes(indexingStatusDetail.indexing_status))
|
||||
if (isCompleted) {
|
||||
stopQueryStatus()
|
||||
return
|
||||
}
|
||||
fetchIndexingStatus()
|
||||
}, 2500)
|
||||
setRunId(runId)
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
fetchIndexingStatus()
|
||||
startQueryStatus()
|
||||
return () => {
|
||||
stopQueryStatus()
|
||||
}
|
||||
}, [])
|
||||
|
||||
// get rule
|
||||
const { data: ruleDetail, error: ruleError } = useSWR({
|
||||
action: 'fetchProcessRule',
|
||||
params: { documentId: getFirstDocument.id },
|
||||
}, apiParams => fetchProcessRule(omit(apiParams, 'action')), {
|
||||
revalidateOnFocus: false,
|
||||
})
|
||||
// get cost
|
||||
const { data: indexingEstimateDetail, error: indexingEstimateErr } = useSWR({
|
||||
action: 'fetchIndexingEstimateBatch',
|
||||
datasetId,
|
||||
batchId,
|
||||
}, apiParams => fetchIndexingEstimateBatch(omit(apiParams, 'action')), {
|
||||
revalidateOnFocus: false,
|
||||
})
|
||||
|
||||
const router = useRouter()
|
||||
const navToDocumentList = () => {
|
||||
router.push(`/datasets/${datasetId}/documents`)
|
||||
}
|
||||
|
||||
const isEmbedding = useMemo(() => {
|
||||
return indexingStatusBatchDetail.some((indexingStatusDetail: { indexing_status: any }) => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''))
|
||||
}, [indexingStatusBatchDetail])
|
||||
const isEmbeddingCompleted = useMemo(() => {
|
||||
return indexingStatusBatchDetail.every((indexingStatusDetail: { indexing_status: any }) => ['completed', 'error'].includes(indexingStatusDetail?.indexing_status || ''))
|
||||
}, [indexingStatusBatchDetail])
|
||||
|
||||
const getSourceName = (id: string) => {
|
||||
const doc = documents.find(document => document.id === id)
|
||||
return doc?.name
|
||||
}
|
||||
const getFileType = (name?: string) => name?.split('.').pop() || 'txt'
|
||||
const getSourcePercent = (detail: IndexingStatusResponse) => {
|
||||
const completedCount = detail.completed_segments || 0
|
||||
const totalCount = detail.total_segments || 0
|
||||
if (totalCount === 0)
|
||||
return 0
|
||||
const percent = Math.round(completedCount * 100 / totalCount)
|
||||
return percent > 100 ? 100 : percent
|
||||
}
|
||||
const getSourceType = (id: string) => {
|
||||
const doc = documents.find(document => document.id === id)
|
||||
return doc?.data_source_type as DataSourceType
|
||||
}
|
||||
const getIcon = (id: string) => {
|
||||
const doc = documents.find(document => document.id === id) as any // TODO type fix
|
||||
|
||||
return doc.data_source_info.notion_page_icon
|
||||
}
|
||||
const isSourceEmbedding = (detail: IndexingStatusResponse) => ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'].includes(detail.indexing_status || '')
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className='h-5 flex justify-between items-center mb-5'>
|
||||
<div className={s.embeddingStatus}>
|
||||
{isEmbedding && t('datasetDocuments.embedding.processing')}
|
||||
{isEmbeddingCompleted && t('datasetDocuments.embedding.completed')}
|
||||
</div>
|
||||
<div className={s.cost}>
|
||||
{indexingType === 'high_quaility' && (
|
||||
<div className='flex items-center'>
|
||||
<div className={cn(s.commonIcon, s.highIcon)} />
|
||||
{t('datasetDocuments.embedding.highQuality')} · {t('datasetDocuments.embedding.estimate')}
|
||||
<span className={s.tokens}>{formatNumber(indexingEstimateDetail?.tokens || 0)}</span>tokens
|
||||
(<span className={s.price}>${formatNumber(indexingEstimateDetail?.total_price || 0)}</span>)
|
||||
</div>
|
||||
)}
|
||||
{indexingType === 'economy' && (
|
||||
<div className='flex items-center'>
|
||||
<div className={cn(s.commonIcon, s.economyIcon)} />
|
||||
{t('datasetDocuments.embedding.economy')} · {t('datasetDocuments.embedding.estimate')}
|
||||
<span className={s.tokens}>0</span>tokens
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className={s.progressContainer}>
|
||||
{indexingStatusBatchDetail.map(indexingStatusDetail => (
|
||||
<div className={cn(
|
||||
s.sourceItem,
|
||||
indexingStatusDetail.indexing_status === 'error' && s.error,
|
||||
indexingStatusDetail.indexing_status === 'completed' && s.success,
|
||||
)}>
|
||||
{isSourceEmbedding(indexingStatusDetail) && (
|
||||
<div className={s.progressbar} style={{ width: `${getSourcePercent(indexingStatusDetail)}%` }}/>
|
||||
)}
|
||||
<div className={s.info}>
|
||||
{getSourceType(indexingStatusDetail.id) === DataSourceType.FILE && (
|
||||
<div className={cn(s.fileIcon, s[getFileType(getSourceName(indexingStatusDetail.id))])}/>
|
||||
)}
|
||||
{getSourceType(indexingStatusDetail.id) === DataSourceType.NOTION && (
|
||||
<NotionIcon
|
||||
className='shrink-0 mr-1'
|
||||
type='page'
|
||||
src={getIcon(indexingStatusDetail.id)}
|
||||
/>
|
||||
)}
|
||||
<div className={s.name}>{getSourceName(indexingStatusDetail.id)}</div>
|
||||
</div>
|
||||
<div className='shrink-0'>
|
||||
{isSourceEmbedding(indexingStatusDetail) && (
|
||||
<div className={s.percent}>{`${getSourcePercent(indexingStatusDetail)}%`}</div>
|
||||
)}
|
||||
{indexingStatusDetail.indexing_status === 'error' && (
|
||||
<div className={cn(s.percent, s.error)}>Error</div>
|
||||
)}
|
||||
{indexingStatusDetail.indexing_status === 'completed' && (
|
||||
<div className={cn(s.percent, s.success)}>100%</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<RuleDetail sourceData={ruleDetail} />
|
||||
<div className='flex items-center gap-2 mt-10'>
|
||||
<Button className='w-fit' type='primary' onClick={navToDocumentList}>
|
||||
<span>{t('datasetCreation.stepThree.navTo')}</span>
|
||||
<ArrowRightIcon className='h-4 w-4 ml-2 stroke-current stroke-1' />
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default EmbeddingProcess
|
||||
@@ -11,6 +11,9 @@
|
||||
}
|
||||
|
||||
.previewHeader .title {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
color: #101828;
|
||||
font-weight: 600;
|
||||
font-size: 18px;
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
'use client'
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import React, { useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import { XMarkIcon } from '@heroicons/react/20/solid'
|
||||
import s from './index.module.css'
|
||||
import type { File } from '@/models/datasets'
|
||||
import { fetchFilePreview } from '@/service/common'
|
||||
|
||||
import cn from 'classnames'
|
||||
import s from './index.module.css'
|
||||
|
||||
type IProps = {
|
||||
file?: File,
|
||||
file?: File
|
||||
notionPage?: any
|
||||
hidePreview: () => void
|
||||
}
|
||||
|
||||
const FilePreview = ({
|
||||
file,
|
||||
hidePreview,
|
||||
}: IProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [previewContent, setPreviewContent] = useState('')
|
||||
@@ -28,23 +31,27 @@ const FilePreview = ({
|
||||
}
|
||||
|
||||
const getFileName = (currentFile?: File) => {
|
||||
if (!currentFile) {
|
||||
if (!currentFile)
|
||||
return ''
|
||||
}
|
||||
|
||||
const arr = currentFile.name.split('.')
|
||||
return arr.slice(0, -1).join()
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (file) {
|
||||
if (file)
|
||||
getPreviewContent(file.id)
|
||||
}
|
||||
}, [file])
|
||||
|
||||
return (
|
||||
<div className={cn(s.filePreview)}>
|
||||
<div className={cn(s.previewHeader)}>
|
||||
<div className={cn(s.title)}>{t('datasetCreation.stepOne.filePreview')}</div>
|
||||
<div className={cn(s.title)}>
|
||||
<span>{t('datasetCreation.stepOne.filePreview')}</span>
|
||||
<div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
|
||||
<XMarkIcon className='h-4 w-4'></XMarkIcon>
|
||||
</div>
|
||||
</div>
|
||||
<div className={cn(s.fileName)}>
|
||||
<span>{getFileName(file)}</span><span className={cn(s.filetype)}>.{file?.extension}</span>
|
||||
</div>
|
||||
|
||||
@@ -1,32 +1,44 @@
|
||||
'use client'
|
||||
import React, { useState, useCallback, useEffect } from 'react'
|
||||
import React, { useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import type { DataSet, File, createDocumentResponse } from '@/models/datasets'
|
||||
import { fetchTenantInfo } from '@/service/common'
|
||||
import { fetchDataDetail } from '@/service/datasets'
|
||||
|
||||
import AppUnavailable from '../../base/app-unavailable'
|
||||
import StepsNavBar from './steps-nav-bar'
|
||||
import StepOne from './step-one'
|
||||
import StepTwo from './step-two'
|
||||
import StepThree from './step-three'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import type { DataSet, File, createDocumentResponse } from '@/models/datasets'
|
||||
import { fetchDataSource, fetchTenantInfo } from '@/service/common'
|
||||
import { fetchDataDetail } from '@/service/datasets'
|
||||
import type { DataSourceNotionPage } from '@/models/common'
|
||||
|
||||
import AccountSetting from '@/app/components/header/account-setting'
|
||||
import AppUnavailable from '../../base/app-unavailable'
|
||||
|
||||
type Page = DataSourceNotionPage & { workspace_id: string }
|
||||
|
||||
type DatasetUpdateFormProps = {
|
||||
datasetId?: string;
|
||||
datasetId?: string
|
||||
}
|
||||
|
||||
const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [hasSetAPIKEY, setHasSetAPIKEY] = useState(true)
|
||||
const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
|
||||
const [hasConnection, setHasConnection] = useState(true)
|
||||
const [isShowDataSourceSetting, { setTrue: showDataSourceSetting, setFalse: hideDataSourceSetting }] = useBoolean()
|
||||
const [dataSourceType, setDataSourceType] = useState<DataSourceType>(DataSourceType.FILE)
|
||||
const [step, setStep] = useState(1)
|
||||
const [indexingTypeCache, setIndexTypeCache] = useState('')
|
||||
const [file, setFile] = useState<File | undefined>()
|
||||
const [result, setResult] = useState<createDocumentResponse | undefined>()
|
||||
const [hasError, setHasError] = useState(false)
|
||||
|
||||
const [notionPages, setNotionPages] = useState<Page[]>([])
|
||||
const updateNotionPages = (value: Page[]) => {
|
||||
setNotionPages(value)
|
||||
}
|
||||
|
||||
const updateFile = (file?: File) => {
|
||||
setFile(file)
|
||||
}
|
||||
@@ -50,9 +62,15 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
|
||||
const hasSetKey = data.providers.some(({ is_valid }) => is_valid)
|
||||
setHasSetAPIKEY(hasSetKey)
|
||||
}
|
||||
const checkNotionConnection = async () => {
|
||||
const { data } = await fetchDataSource({ url: '/data-source/integrates' })
|
||||
const hasConnection = data.filter(item => item.provider === 'notion') || []
|
||||
setHasConnection(hasConnection.length > 0)
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
checkAPIKey()
|
||||
checkNotionConnection()
|
||||
}, [])
|
||||
|
||||
const [detail, setDetail] = useState<DataSet | null>(null)
|
||||
@@ -62,16 +80,16 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
|
||||
try {
|
||||
const detail = await fetchDataDetail(datasetId)
|
||||
setDetail(detail)
|
||||
} catch (e) {
|
||||
}
|
||||
catch (e) {
|
||||
setHasError(true)
|
||||
}
|
||||
}
|
||||
})()
|
||||
}, [datasetId])
|
||||
|
||||
if (hasError) {
|
||||
if (hasError)
|
||||
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
|
||||
}
|
||||
|
||||
return (
|
||||
<div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
|
||||
@@ -80,9 +98,16 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
|
||||
</div>
|
||||
<div className="grow bg-white">
|
||||
{step === 1 && <StepOne
|
||||
hasConnection={hasConnection}
|
||||
onSetting={showDataSourceSetting}
|
||||
datasetId={datasetId}
|
||||
dataSourceType={dataSourceType}
|
||||
dataSourceTypeDisable={!!detail?.data_source_type}
|
||||
changeType={setDataSourceType}
|
||||
file={file}
|
||||
updateFile={updateFile}
|
||||
notionPages={notionPages}
|
||||
updateNotionPages={updateNotionPages}
|
||||
onStepChange={nextStep}
|
||||
/>}
|
||||
{(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
|
||||
@@ -90,7 +115,9 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
|
||||
onSetting={showSetAPIKey}
|
||||
indexingType={detail?.indexing_technique || ''}
|
||||
datasetId={datasetId}
|
||||
dataSourceType={dataSourceType}
|
||||
file={file}
|
||||
notionPages={notionPages}
|
||||
onStepChange={changeStep}
|
||||
updateIndexingTypeCache={updateIndexingTypeCache}
|
||||
updateResultCache={updateResultCache}
|
||||
@@ -106,6 +133,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
|
||||
await checkAPIKey()
|
||||
hideSetAPIkey()
|
||||
}} />}
|
||||
{isShowDataSourceSetting && <AccountSetting activeTab="data-source" onCancel={hideDataSourceSetting}/>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
.filePreview {
|
||||
@apply flex flex-col border-l border-gray-200 shrink-0;
|
||||
width: 528px;
|
||||
background-color: #fcfcfd;
|
||||
}
|
||||
|
||||
.previewHeader {
|
||||
@apply border-b border-gray-200 shrink-0;
|
||||
margin: 42px 32px 0;
|
||||
padding-bottom: 16px;
|
||||
}
|
||||
|
||||
.previewHeader .title {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
color: #101828;
|
||||
font-weight: 600;
|
||||
font-size: 18px;
|
||||
line-height: 28px;
|
||||
}
|
||||
|
||||
.previewHeader .fileName {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
font-weight: 400;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
color: #1D2939;
|
||||
}
|
||||
|
||||
.previewHeader .filetype {
|
||||
color: #667085;
|
||||
}
|
||||
|
||||
.previewContent {
|
||||
@apply overflow-y-auto grow;
|
||||
padding: 20px 32px;
|
||||
font-weight: 400;
|
||||
font-size: 16px;
|
||||
line-height: 24px;
|
||||
color: #344054;
|
||||
}
|
||||
|
||||
.previewContent .loading {
|
||||
width: 100%;
|
||||
height: 180px;
|
||||
background: #f9fafb center no-repeat url(../assets/Loading.svg);
|
||||
background-size: contain;
|
||||
}
|
||||
.fileContent {
|
||||
white-space: pre-line;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
'use client'
|
||||
import React, { useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import { XMarkIcon } from '@heroicons/react/20/solid'
|
||||
import s from './index.module.css'
|
||||
import type { DataSourceNotionPage } from '@/models/common'
|
||||
import NotionIcon from '@/app/components/base/notion-icon'
|
||||
import { fetchNotionPagePreview } from '@/service/datasets'
|
||||
|
||||
type Page = DataSourceNotionPage & { workspace_id: string }
|
||||
type IProps = {
|
||||
currentPage?: Page
|
||||
hidePreview: () => void
|
||||
}
|
||||
|
||||
const NotionPagePreview = ({
|
||||
currentPage,
|
||||
hidePreview,
|
||||
}: IProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [previewContent, setPreviewContent] = useState('')
|
||||
const [loading, setLoading] = useState(true)
|
||||
|
||||
const getPreviewContent = async () => {
|
||||
if (!currentPage)
|
||||
return
|
||||
try {
|
||||
const res = await fetchNotionPagePreview({
|
||||
workspaceID: currentPage.workspace_id,
|
||||
pageID: currentPage.page_id,
|
||||
pageType: currentPage.type,
|
||||
})
|
||||
setPreviewContent(res.content)
|
||||
setLoading(false)
|
||||
}
|
||||
catch {}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (currentPage) {
|
||||
setLoading(true)
|
||||
getPreviewContent()
|
||||
}
|
||||
}, [currentPage])
|
||||
|
||||
return (
|
||||
<div className={cn(s.filePreview)}>
|
||||
<div className={cn(s.previewHeader)}>
|
||||
<div className={cn(s.title)}>
|
||||
<span>{t('datasetCreation.stepOne.pagePreview')}</span>
|
||||
<div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
|
||||
<XMarkIcon className='h-4 w-4'></XMarkIcon>
|
||||
</div>
|
||||
</div>
|
||||
<div className={cn(s.fileName)}>
|
||||
<NotionIcon
|
||||
className='shrink-0 mr-1'
|
||||
type='page'
|
||||
src={currentPage?.page_icon}
|
||||
/>
|
||||
{currentPage?.page_name}
|
||||
</div>
|
||||
</div>
|
||||
<div className={cn(s.previewContent)}>
|
||||
{loading && <div className={cn(s.loading)}/>}
|
||||
{!loading && (
|
||||
<div className={cn(s.fileContent)}>{previewContent}</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default NotionPagePreview
|
||||
@@ -107,3 +107,53 @@
|
||||
background: center no-repeat url(../assets/folder-plus.svg);
|
||||
background-size: contain;
|
||||
}
|
||||
|
||||
.notionConnectionTip {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
padding: 24px;
|
||||
max-width: 640px;
|
||||
background: #F9FAFB;
|
||||
border-radius: 16px;
|
||||
}
|
||||
|
||||
.notionIcon {
|
||||
display: flex;
|
||||
padding: 12px;
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
background: #fff center no-repeat url(../assets/notion.svg);
|
||||
background-size: 24px;
|
||||
border: 0.5px solid #EAECF5;
|
||||
box-shadow: 0px 12px 16px -4px rgba(16, 24, 40, 0.08), 0px 4px 6px -2px rgba(16, 24, 40, 0.03);
|
||||
border-radius: 12px;
|
||||
}
|
||||
|
||||
.notionConnectionTip .title {
|
||||
position: relative;
|
||||
margin: 24px 0 4px;
|
||||
font-style: normal;
|
||||
font-weight: 600;
|
||||
font-size: 16px;
|
||||
line-height: 24px;
|
||||
color: #374151;
|
||||
}
|
||||
.notionConnectionTip .title::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: -6px;
|
||||
right: -12px;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
background: center no-repeat url(../assets/Icon-3-dots.svg);
|
||||
background-size: contain;
|
||||
}
|
||||
.notionConnectionTip .tip {
|
||||
margin-bottom: 20px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-size: 13px;
|
||||
line-height: 18px;
|
||||
color: #6B7280;
|
||||
}
|
||||
|
||||
@@ -1,36 +1,82 @@
|
||||
'use client'
|
||||
import React, { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { File } from '@/models/datasets'
|
||||
import cn from 'classnames'
|
||||
import FilePreview from '../file-preview'
|
||||
import FileUploader from '../file-uploader'
|
||||
import NotionPagePreview from '../notion-page-preview'
|
||||
import EmptyDatasetCreationModal from '../empty-dataset-creation-modal'
|
||||
import Button from '@/app/components/base/button'
|
||||
|
||||
import cn from 'classnames'
|
||||
import s from './index.module.css'
|
||||
import type { File } from '@/models/datasets'
|
||||
import type { DataSourceNotionPage } from '@/models/common'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
|
||||
|
||||
type IStepOneProps = {
|
||||
datasetId?: string,
|
||||
file?: File,
|
||||
updateFile: (file?: File) => void,
|
||||
onStepChange: () => void,
|
||||
datasetId?: string
|
||||
dataSourceType: DataSourceType
|
||||
dataSourceTypeDisable: Boolean
|
||||
hasConnection: boolean
|
||||
onSetting: () => void
|
||||
file?: File
|
||||
updateFile: (file?: File) => void
|
||||
notionPages?: any[]
|
||||
updateNotionPages: (value: any[]) => void
|
||||
onStepChange: () => void
|
||||
changeType: (type: DataSourceType) => void
|
||||
}
|
||||
|
||||
type Page = DataSourceNotionPage & { workspace_id: string }
|
||||
|
||||
type NotionConnectorProps = {
|
||||
onSetting: () => void
|
||||
}
|
||||
export const NotionConnector = ({ onSetting }: NotionConnectorProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className={s.notionConnectionTip}>
|
||||
<span className={s.notionIcon}/>
|
||||
<div className={s.title}>{t('datasetCreation.stepOne.notionSyncTitle')}</div>
|
||||
<div className={s.tip}>{t('datasetCreation.stepOne.notionSyncTip')}</div>
|
||||
<Button className='h-8' type='primary' onClick={onSetting}>{t('datasetCreation.stepOne.connect')}</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const StepOne = ({
|
||||
datasetId,
|
||||
dataSourceType,
|
||||
dataSourceTypeDisable,
|
||||
changeType,
|
||||
hasConnection,
|
||||
onSetting,
|
||||
onStepChange,
|
||||
file,
|
||||
updateFile,
|
||||
notionPages = [],
|
||||
updateNotionPages,
|
||||
}: IStepOneProps) => {
|
||||
const [dataSourceType, setDataSourceType] = useState('FILE')
|
||||
const [showModal, setShowModal] = useState(false)
|
||||
const [showFilePreview, setShowFilePreview] = useState(true)
|
||||
const [currentNotionPage, setCurrentNotionPage] = useState<Page | undefined>()
|
||||
const { t } = useTranslation()
|
||||
|
||||
const hidePreview = () => setShowFilePreview(false)
|
||||
|
||||
const modalShowHandle = () => setShowModal(true)
|
||||
|
||||
const modalCloseHandle = () => setShowModal(false)
|
||||
|
||||
const updateCurrentPage = (page: Page) => {
|
||||
setCurrentNotionPage(page)
|
||||
}
|
||||
|
||||
const hideNotionPagePreview = () => {
|
||||
setCurrentNotionPage(undefined)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className='flex w-full h-full'>
|
||||
<div className='grow overflow-y-auto relative'>
|
||||
@@ -38,41 +84,76 @@ const StepOne = ({
|
||||
<div className={s.form}>
|
||||
<div className={s.dataSourceTypeList}>
|
||||
<div
|
||||
className={cn(s.dataSourceItem, dataSourceType === 'FILE' && s.active)}
|
||||
onClick={() => setDataSourceType('FILE')}
|
||||
className={cn(
|
||||
s.dataSourceItem,
|
||||
dataSourceType === DataSourceType.FILE && s.active,
|
||||
dataSourceTypeDisable && dataSourceType !== DataSourceType.FILE && s.disabled,
|
||||
)}
|
||||
onClick={() => {
|
||||
if (dataSourceTypeDisable)
|
||||
return
|
||||
changeType(DataSourceType.FILE)
|
||||
hidePreview()
|
||||
}}
|
||||
>
|
||||
<span className={cn(s.datasetIcon)}/>
|
||||
<span className={cn(s.datasetIcon)} />
|
||||
{t('datasetCreation.stepOne.dataSourceType.file')}
|
||||
</div>
|
||||
<div
|
||||
className={cn(s.dataSourceItem, s.disabled, dataSourceType === 'notion' && s.active)}
|
||||
// onClick={() => setDataSourceType('notion')}
|
||||
className={cn(
|
||||
s.dataSourceItem,
|
||||
dataSourceType === DataSourceType.NOTION && s.active,
|
||||
dataSourceTypeDisable && dataSourceType !== DataSourceType.NOTION && s.disabled,
|
||||
)}
|
||||
onClick={() => {
|
||||
if (dataSourceTypeDisable)
|
||||
return
|
||||
changeType(DataSourceType.NOTION)
|
||||
hidePreview()
|
||||
}}
|
||||
>
|
||||
<span className={s.comingTag}>Coming soon</span>
|
||||
<span className={cn(s.datasetIcon, s.notion)}/>
|
||||
<span className={cn(s.datasetIcon, s.notion)} />
|
||||
{t('datasetCreation.stepOne.dataSourceType.notion')}
|
||||
</div>
|
||||
<div
|
||||
className={cn(s.dataSourceItem, s.disabled, dataSourceType === 'web' && s.active)}
|
||||
// onClick={() => setDataSourceType('web')}
|
||||
className={cn(s.dataSourceItem, s.disabled, dataSourceType === DataSourceType.WEB && s.active)}
|
||||
// onClick={() => changeType(DataSourceType.WEB)}
|
||||
>
|
||||
<span className={s.comingTag}>Coming soon</span>
|
||||
<span className={cn(s.datasetIcon, s.web)}/>
|
||||
<span className={cn(s.datasetIcon, s.web)} />
|
||||
{t('datasetCreation.stepOne.dataSourceType.web')}
|
||||
</div>
|
||||
</div>
|
||||
<FileUploader onFileUpdate={updateFile} file={file} />
|
||||
<Button disabled={!file} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
|
||||
{dataSourceType === DataSourceType.FILE && (
|
||||
<>
|
||||
<FileUploader onFileUpdate={updateFile} file={file} />
|
||||
<Button disabled={!file} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
|
||||
</>
|
||||
)}
|
||||
{dataSourceType === DataSourceType.NOTION && (
|
||||
<>
|
||||
{!hasConnection && <NotionConnector onSetting={onSetting} />}
|
||||
{hasConnection && (
|
||||
<>
|
||||
<div className='mb-8 w-[640px]'>
|
||||
<NotionPageSelector value={notionPages.map(page => page.page_id)} onSelect={updateNotionPages} onPreview={updateCurrentPage} />
|
||||
</div>
|
||||
<Button disabled={!notionPages.length} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{!datasetId && (
|
||||
<>
|
||||
<div className={s.dividerLine}/>
|
||||
<div className={s.dividerLine} />
|
||||
<div onClick={modalShowHandle} className={s.OtherCreationOption}>{t('datasetCreation.stepOne.emptyDatasetCreation')}</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<EmptyDatasetCreationModal show={showModal} onHide={modalCloseHandle}/>
|
||||
<EmptyDatasetCreationModal show={showModal} onHide={modalCloseHandle} />
|
||||
</div>
|
||||
{file && <FilePreview file={file} />}
|
||||
{file && showFilePreview && <FilePreview file={file} hidePreview={hidePreview} />}
|
||||
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
'use client'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { createDocumentResponse } from '@/models/datasets'
|
||||
import EmbeddingDetail from '../../documents/detail/embedding'
|
||||
|
||||
import cn from 'classnames'
|
||||
import EmbeddingProcess from '../embedding-process'
|
||||
|
||||
import s from './index.module.css'
|
||||
import type { FullDocumentDetail, createDocumentResponse } from '@/models/datasets'
|
||||
|
||||
type StepThreeProps = {
|
||||
datasetId?: string,
|
||||
datasetName?: string,
|
||||
indexingType?: string,
|
||||
datasetId?: string
|
||||
datasetName?: string
|
||||
indexingType?: string
|
||||
creationCache?: createDocumentResponse
|
||||
}
|
||||
|
||||
@@ -38,12 +38,11 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
|
||||
<div className={s.content}>{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}</div>
|
||||
</div>
|
||||
)}
|
||||
<EmbeddingDetail
|
||||
datasetId={datasetId || creationCache?.dataset?.id}
|
||||
documentId={creationCache?.document.id}
|
||||
<EmbeddingProcess
|
||||
datasetId={datasetId || creationCache?.dataset?.id || ''}
|
||||
batchId={creationCache?.batch || ''}
|
||||
documents={creationCache?.documents as FullDocumentDetail[]}
|
||||
indexingType={indexingType || creationCache?.dataset?.indexing_technique}
|
||||
stopPosition='bottom'
|
||||
detail={creationCache?.document}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -58,4 +57,4 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
|
||||
)
|
||||
}
|
||||
|
||||
export default StepThree;
|
||||
export default StepThree
|
||||
|
||||
@@ -14,9 +14,26 @@
|
||||
}
|
||||
|
||||
.fixed {
|
||||
padding-top: 12px;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
background: rgba(255, 255, 255, 0.9);
|
||||
border-bottom: 0.5px solid #EAECF0;
|
||||
backdrop-filter: blur(4px);
|
||||
animation: fix 0.5s;
|
||||
}
|
||||
|
||||
@keyframes fix {
|
||||
from {
|
||||
padding-top: 42px;
|
||||
font-size: 18px;
|
||||
line-height: 28px;
|
||||
}
|
||||
to {
|
||||
padding-top: 12px;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
}
|
||||
}
|
||||
|
||||
.form {
|
||||
@@ -273,11 +290,11 @@
|
||||
@apply bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-white placeholder:text-gray-400;
|
||||
}
|
||||
|
||||
.file {
|
||||
.source {
|
||||
@apply flex justify-between items-center mt-8 px-6 py-4 rounded-xl bg-gray-50;
|
||||
}
|
||||
|
||||
.file .divider {
|
||||
.source .divider {
|
||||
@apply shrink-0 mx-4 w-px bg-gray-200;
|
||||
height: 42px;
|
||||
}
|
||||
@@ -318,9 +335,19 @@
|
||||
.fileIcon.json {
|
||||
background-image: url(../assets/json.svg);
|
||||
}
|
||||
|
||||
.fileContent {
|
||||
flex: 1 1 50%;
|
||||
.sourceContent {
|
||||
flex: 1 1 auto;
|
||||
}
|
||||
.sourceCount {
|
||||
@apply shrink-0 ml-1;
|
||||
font-weight: 500;
|
||||
font-size: 13px;
|
||||
line-height: 18px;
|
||||
color: #667085;
|
||||
}
|
||||
.segmentCount {
|
||||
flex: 1 1 30%;
|
||||
max-width: 120px;
|
||||
}
|
||||
|
||||
.divider {
|
||||
|
||||
@@ -6,9 +6,10 @@ import { useBoolean } from 'ahooks'
|
||||
import { XMarkIcon } from '@heroicons/react/20/solid'
|
||||
import cn from 'classnames'
|
||||
import Link from 'next/link'
|
||||
import { groupBy } from 'lodash-es'
|
||||
import PreviewItem from './preview-item'
|
||||
import s from './index.module.css'
|
||||
import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
|
||||
import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
|
||||
import {
|
||||
createDocument,
|
||||
createFirstDocument,
|
||||
@@ -20,6 +21,11 @@ import Loading from '@/app/components/base/loading'
|
||||
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { formatNumber } from '@/utils/format'
|
||||
import type { DataSourceNotionPage } from '@/models/common'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import NotionIcon from '@/app/components/base/notion-icon'
|
||||
|
||||
type Page = DataSourceNotionPage & { workspace_id: string }
|
||||
|
||||
type StepTwoProps = {
|
||||
isSetting?: boolean
|
||||
@@ -28,7 +34,9 @@ type StepTwoProps = {
|
||||
onSetting: () => void
|
||||
datasetId?: string
|
||||
indexingType?: string
|
||||
dataSourceType: DataSourceType
|
||||
file?: File
|
||||
notionPages?: Page[]
|
||||
onStepChange?: (delta: number) => void
|
||||
updateIndexingTypeCache?: (type: string) => void
|
||||
updateResultCache?: (res: createDocumentResponse) => void
|
||||
@@ -52,7 +60,9 @@ const StepTwo = ({
|
||||
onSetting,
|
||||
datasetId,
|
||||
indexingType,
|
||||
dataSourceType,
|
||||
file,
|
||||
notionPages = [],
|
||||
onStepChange,
|
||||
updateIndexingTypeCache,
|
||||
updateResultCache,
|
||||
@@ -169,12 +179,54 @@ const StepTwo = ({
|
||||
return processRule
|
||||
}
|
||||
|
||||
const getNotionInfo = () => {
|
||||
const workspacesMap = groupBy(notionPages, 'workspace_id')
|
||||
const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
|
||||
return {
|
||||
workspaceId,
|
||||
pages: workspacesMap[workspaceId],
|
||||
}
|
||||
})
|
||||
return workspaces.map((workspace) => {
|
||||
return {
|
||||
workspace_id: workspace.workspaceId,
|
||||
pages: workspace.pages.map((page) => {
|
||||
const { page_id, page_name, page_icon, type } = page
|
||||
return {
|
||||
page_id,
|
||||
page_name,
|
||||
page_icon,
|
||||
type,
|
||||
}
|
||||
}),
|
||||
}
|
||||
}) as NotionInfo[]
|
||||
}
|
||||
|
||||
const getFileIndexingEstimateParams = () => {
|
||||
const params = {
|
||||
file_id: file?.id,
|
||||
dataset_id: datasetId,
|
||||
indexing_technique: getIndexing_technique(),
|
||||
process_rule: getProcessRule(),
|
||||
let params
|
||||
if (dataSourceType === DataSourceType.FILE) {
|
||||
params = {
|
||||
info_list: {
|
||||
data_source_type: dataSourceType,
|
||||
file_info_list: {
|
||||
// TODO multi files
|
||||
file_ids: [file?.id || ''],
|
||||
},
|
||||
},
|
||||
indexing_technique: getIndexing_technique(),
|
||||
process_rule: getProcessRule(),
|
||||
}
|
||||
}
|
||||
if (dataSourceType === DataSourceType.NOTION) {
|
||||
params = {
|
||||
info_list: {
|
||||
data_source_type: dataSourceType,
|
||||
notion_info_list: getNotionInfo(),
|
||||
},
|
||||
indexing_technique: getIndexing_technique(),
|
||||
process_rule: getProcessRule(),
|
||||
}
|
||||
}
|
||||
return params
|
||||
}
|
||||
@@ -190,13 +242,22 @@ const StepTwo = ({
|
||||
else {
|
||||
params = {
|
||||
data_source: {
|
||||
type: 'upload_file',
|
||||
info: file?.id,
|
||||
name: file?.name,
|
||||
type: dataSourceType,
|
||||
info_list: {
|
||||
data_source_type: dataSourceType,
|
||||
},
|
||||
},
|
||||
indexing_technique: getIndexing_technique(),
|
||||
process_rule: getProcessRule(),
|
||||
} as CreateDocumentReq
|
||||
if (dataSourceType === DataSourceType.FILE) {
|
||||
params.data_source.info_list.file_info_list = {
|
||||
// TODO multi files
|
||||
file_ids: [file?.id || ''],
|
||||
}
|
||||
}
|
||||
if (dataSourceType === DataSourceType.NOTION)
|
||||
params.data_source.info_list.notion_info_list = getNotionInfo()
|
||||
}
|
||||
return params
|
||||
}
|
||||
@@ -249,9 +310,7 @@ const StepTwo = ({
|
||||
body: params,
|
||||
})
|
||||
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
|
||||
updateResultCache && updateResultCache({
|
||||
document: res,
|
||||
})
|
||||
updateResultCache && updateResultCache(res)
|
||||
}
|
||||
onStepChange && onStepChange(+1)
|
||||
isSetting && onSave && onSave()
|
||||
@@ -319,7 +378,6 @@ const StepTwo = ({
|
||||
<div className={cn(s.form)}>
|
||||
<div className={s.label}>{t('datasetCreation.stepTwo.segmentation')}</div>
|
||||
<div className='max-w-[640px]'>
|
||||
|
||||
<div
|
||||
className={cn(
|
||||
s.radioItem,
|
||||
@@ -467,16 +525,41 @@ const StepTwo = ({
|
||||
<Link className='text-[#155EEF]' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
|
||||
</div>
|
||||
)}
|
||||
<div className={s.file}>
|
||||
<div className={s.fileContent}>
|
||||
<div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.fileName')}</div>
|
||||
<div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
|
||||
<span className={cn(s.fileIcon, file && s[file.extension])} />
|
||||
{getFileName(file?.name || '')}
|
||||
</div>
|
||||
{/* TODO multi files */}
|
||||
<div className={s.source}>
|
||||
<div className={s.sourceContent}>
|
||||
{dataSourceType === DataSourceType.FILE && (
|
||||
<>
|
||||
<div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.fileSource')}</div>
|
||||
<div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
|
||||
<span className={cn(s.fileIcon, file && s[file.extension])} />
|
||||
{getFileName(file?.name || '')}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{dataSourceType === DataSourceType.NOTION && (
|
||||
<>
|
||||
<div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.notionSource')}</div>
|
||||
<div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
|
||||
<NotionIcon
|
||||
className='shrink-0 mr-1'
|
||||
type='page'
|
||||
src={notionPages[0]?.page_icon}
|
||||
/>
|
||||
{notionPages[0]?.page_name}
|
||||
{notionPages.length > 1 && (
|
||||
<span className={s.sourceCount}>
|
||||
<span>{t('datasetCreation.stepTwo.other')}</span>
|
||||
<span>{notionPages.length - 1}</span>
|
||||
<span>{t('datasetCreation.stepTwo.notionUnit')}</span>
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className={s.divider} />
|
||||
<div className={s.fileContent}>
|
||||
<div className={s.segmentCount}>
|
||||
<div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateSegment')}</div>
|
||||
<div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user