feat: support firecrawl frontend code (#5226)

web/models/common.ts

@@ -172,6 +172,39 @@ export type DataSourceNotion = {
   source_info: DataSourceNotionWorkspace
 }
 
+export enum DataSourceCategory {
+  website = 'website',
+}
+export enum WebsiteProvider {
+  fireCrawl = 'firecrawl',
+}
+
+export type WebsiteCredentials = {
+  auth_type: 'bearer'
+  config: {
+    base_url: string
+    api_key: string
+  }
+}
+
+export type FirecrawlConfig = {
+  api_key: string
+  base_url: string
+}
+
+export type DataSourceWebsiteItem = {
+  id: string
+  category: DataSourceCategory.website
+  provider: WebsiteProvider
+  credentials: WebsiteCredentials
+  disabled: boolean
+  created_at: number
+  updated_at: number
+}
+export type DataSourceWebsite = {
+  settings: DataSourceWebsiteItem[]
+}
+
 export type GithubRepo = {
   stargazers_count: number
 }

web/models/datasets.ts

@@ -5,7 +5,7 @@ import type { Tag } from '@/app/components/base/tag-management/constant'
 export enum DataSourceType {
   FILE = 'upload_file',
   NOTION = 'notion_import',
-  WEB = 'web_import',
+  WEB = 'website_crawl',
 }
 
 export type DataSet = {
@@ -39,6 +39,22 @@ export type CustomFile = File & {
   created_at?: number
 }
 
+export type CrawlOptions = {
+  crawl_sub_pages: boolean
+  only_main_content: boolean
+  includes: string
+  excludes: string
+  limit: number | string
+  max_depth: number | string
+}
+
+export type CrawlResultItem = {
+  title: string
+  markdown: string
+  description: string
+  source_url: string
+}
+
 export type FileItem = {
   fileID: string
   file: CustomFile
@@ -149,6 +165,8 @@ export type DataSourceInfo = {
     extension: string
   }
   notion_page_icon?: string
+  job_id: string
+  url: string
 }
 
 export type InitialDocumentDetail = {
@@ -219,6 +237,11 @@ export type DataSource = {
     file_info_list?: {
       file_ids: string[]
     }
+    website_info_list?: {
+      provider: string
+      job_id: string
+      urls: string[]
+    }
   }
 }