feat(website-crawl): add jina reader as additional alternative for website crawling (#8761)

api/controllers/console/datasets/website.py

@@ -14,7 +14,9 @@ class WebsiteCrawlApi(Resource):
     @account_initialization_required
     def post(self):
         parser = reqparse.RequestParser()
-        parser.add_argument("provider", type=str, choices=["firecrawl"], required=True, nullable=True, location="json")
+        parser.add_argument(
+            "provider", type=str, choices=["firecrawl", "jinareader"], required=True, nullable=True, location="json"
+        )
         parser.add_argument("url", type=str, required=True, nullable=True, location="json")
         parser.add_argument("options", type=dict, required=True, nullable=True, location="json")
         args = parser.parse_args()
@@ -33,7 +35,7 @@ class WebsiteCrawlStatusApi(Resource):
     @account_initialization_required
     def get(self, job_id: str):
         parser = reqparse.RequestParser()
-        parser.add_argument("provider", type=str, choices=["firecrawl"], required=True, location="args")
+        parser.add_argument("provider", type=str, choices=["firecrawl", "jinareader"], required=True, location="args")
         args = parser.parse_args()
         # get crawl status
         try:

api/core/rag/extractor/extract_processor.py

@@ -12,6 +12,7 @@ from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.extractor.excel_extractor import ExcelExtractor
 from core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor
 from core.rag.extractor.html_extractor import HtmlExtractor
+from core.rag.extractor.jina_reader_extractor import JinaReaderWebExtractor
 from core.rag.extractor.markdown_extractor import MarkdownExtractor
 from core.rag.extractor.notion_extractor import NotionExtractor
 from core.rag.extractor.pdf_extractor import PdfExtractor
@@ -171,6 +172,15 @@ class ExtractProcessor:
                     only_main_content=extract_setting.website_info.only_main_content,
                 )
                 return extractor.extract()
+            elif extract_setting.website_info.provider == "jinareader":
+                extractor = JinaReaderWebExtractor(
+                    url=extract_setting.website_info.url,
+                    job_id=extract_setting.website_info.job_id,
+                    tenant_id=extract_setting.website_info.tenant_id,
+                    mode=extract_setting.website_info.mode,
+                    only_main_content=extract_setting.website_info.only_main_content,
+                )
+                return extractor.extract()
             else:
                 raise ValueError(f"Unsupported website provider: {extract_setting.website_info.provider}")
         else:

api/core/rag/extractor/jina_reader_extractor.py

@@ -0,0 +1,35 @@
+from core.rag.extractor.extractor_base import BaseExtractor
+from core.rag.models.document import Document
+from services.website_service import WebsiteService
+
+
+class JinaReaderWebExtractor(BaseExtractor):
+    """
+    Crawl and scrape websites and return content in clean llm-ready markdown.
+    """
+
+    def __init__(self, url: str, job_id: str, tenant_id: str, mode: str = "crawl", only_main_content: bool = False):
+        """Initialize with url, api_key, base_url and mode."""
+        self._url = url
+        self.job_id = job_id
+        self.tenant_id = tenant_id
+        self.mode = mode
+        self.only_main_content = only_main_content
+
+    def extract(self) -> list[Document]:
+        """Extract content from the URL."""
+        documents = []
+        if self.mode == "crawl":
+            crawl_data = WebsiteService.get_crawl_url_data(self.job_id, "jinareader", self._url, self.tenant_id)
+            if crawl_data is None:
+                return []
+            document = Document(
+                page_content=crawl_data.get("content", ""),
+                metadata={
+                    "source_url": crawl_data.get("url"),
+                    "description": crawl_data.get("description"),
+                    "title": crawl_data.get("title"),
+                },
+            )
+            documents.append(document)
+        return documents

api/services/auth/api_key_auth_factory.py

@@ -1,10 +1,13 @@
 from services.auth.firecrawl import FirecrawlAuth
+from services.auth.jina import JinaAuth
 
 
 class ApiKeyAuthFactory:
     def __init__(self, provider: str, credentials: dict):
         if provider == "firecrawl":
             self.auth = FirecrawlAuth(credentials)
+        elif provider == "jinareader":
+            self.auth = JinaAuth(credentials)
         else:
             raise ValueError("Invalid provider")
 

api/services/auth/jina.py

@@ -0,0 +1,44 @@
+import json
+
+import requests
+
+from services.auth.api_key_auth_base import ApiKeyAuthBase
+
+
+class JinaAuth(ApiKeyAuthBase):
+    def __init__(self, credentials: dict):
+        super().__init__(credentials)
+        auth_type = credentials.get("auth_type")
+        if auth_type != "bearer":
+            raise ValueError("Invalid auth type, Jina Reader auth type must be Bearer")
+        self.api_key = credentials.get("config").get("api_key", None)
+
+        if not self.api_key:
+            raise ValueError("No API key provided")
+
+    def validate_credentials(self):
+        headers = self._prepare_headers()
+        options = {
+            "url": "https://example.com",
+        }
+        response = self._post_request("https://r.jina.ai", options, headers)
+        if response.status_code == 200:
+            return True
+        else:
+            self._handle_error(response)
+
+    def _prepare_headers(self):
+        return {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}
+
+    def _post_request(self, url, data, headers):
+        return requests.post(url, headers=headers, json=data)
+
+    def _handle_error(self, response):
+        if response.status_code in {402, 409, 500}:
+            error_message = response.json().get("error", "Unknown error occurred")
+            raise Exception(f"Failed to authorize. Status code: {response.status_code}. Error: {error_message}")
+        else:
+            if response.text:
+                error_message = json.loads(response.text).get("error", "Unknown error occurred")
+                raise Exception(f"Failed to authorize. Status code: {response.status_code}. Error: {error_message}")
+            raise Exception(f"Unexpected error occurred while trying to authorize. Status code: {response.status_code}")

api/services/website_service.py

@@ -1,6 +1,7 @@
 import datetime
 import json
 
+import requests
 from flask_login import current_user
 
 from core.helper import encrypter
@@ -65,6 +66,35 @@ class WebsiteService:
             time = str(datetime.datetime.now().timestamp())
             redis_client.setex(website_crawl_time_cache_key, 3600, time)
             return {"status": "active", "job_id": job_id}
+        elif provider == "jinareader":
+            api_key = encrypter.decrypt_token(
+                tenant_id=current_user.current_tenant_id, token=credentials.get("config").get("api_key")
+            )
+            crawl_sub_pages = options.get("crawl_sub_pages", False)
+            if not crawl_sub_pages:
+                response = requests.get(
+                    f"https://r.jina.ai/{url}",
+                    headers={"Accept": "application/json", "Authorization": f"Bearer {api_key}"},
+                )
+                if response.json().get("code") != 200:
+                    raise ValueError("Failed to crawl")
+                return {"status": "active", "data": response.json().get("data")}
+            else:
+                response = requests.post(
+                    "https://adaptivecrawl-kir3wx7b3a-uc.a.run.app",
+                    json={
+                        "url": url,
+                        "maxPages": options.get("limit", 1),
+                        "useSitemap": options.get("use_sitemap", True),
+                    },
+                    headers={
+                        "Content-Type": "application/json",
+                        "Authorization": f"Bearer {api_key}",
+                    },
+                )
+                if response.json().get("code") != 200:
+                    raise ValueError("Failed to crawl")
+                return {"status": "active", "job_id": response.json().get("data", {}).get("taskId")}
         else:
             raise ValueError("Invalid provider")
 
@@ -93,6 +123,42 @@ class WebsiteService:
                     time_consuming = abs(end_time - float(start_time))
                     crawl_status_data["time_consuming"] = f"{time_consuming:.2f}"
                     redis_client.delete(website_crawl_time_cache_key)
+        elif provider == "jinareader":
+            api_key = encrypter.decrypt_token(
+                tenant_id=current_user.current_tenant_id, token=credentials.get("config").get("api_key")
+            )
+            response = requests.post(
+                "https://adaptivecrawlstatus-kir3wx7b3a-uc.a.run.app",
+                headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
+                json={"taskId": job_id},
+            )
+            data = response.json().get("data", {})
+            crawl_status_data = {
+                "status": data.get("status", "active"),
+                "job_id": job_id,
+                "total": len(data.get("urls", [])),
+                "current": len(data.get("processed", [])) + len(data.get("failed", [])),
+                "data": [],
+                "time_consuming": data.get("duration", 0) / 1000,
+            }
+
+            if crawl_status_data["status"] == "completed":
+                response = requests.post(
+                    "https://adaptivecrawlstatus-kir3wx7b3a-uc.a.run.app",
+                    headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
+                    json={"taskId": job_id, "urls": list(data.get("processed", {}).keys())},
+                )
+                data = response.json().get("data", {})
+                formatted_data = [
+                    {
+                        "title": item.get("data", {}).get("title"),
+                        "source_url": item.get("data", {}).get("url"),
+                        "description": item.get("data", {}).get("description"),
+                        "markdown": item.get("data", {}).get("content"),
+                    }
+                    for item in data.get("processed", {}).values()
+                ]
+                crawl_status_data["data"] = formatted_data
         else:
             raise ValueError("Invalid provider")
         return crawl_status_data
@@ -119,6 +185,40 @@ class WebsiteService:
                     if item.get("source_url") == url:
                         return item
             return None
+        elif provider == "jinareader":
+            file_key = "website_files/" + job_id + ".txt"
+            if storage.exists(file_key):
+                data = storage.load_once(file_key)
+                if data:
+                    data = json.loads(data.decode("utf-8"))
+            elif not job_id:
+                response = requests.get(
+                    f"https://r.jina.ai/{url}",
+                    headers={"Accept": "application/json", "Authorization": f"Bearer {api_key}"},
+                )
+                if response.json().get("code") != 200:
+                    raise ValueError("Failed to crawl")
+                return response.json().get("data")
+            else:
+                api_key = encrypter.decrypt_token(tenant_id=tenant_id, token=credentials.get("config").get("api_key"))
+                response = requests.post(
+                    "https://adaptivecrawlstatus-kir3wx7b3a-uc.a.run.app",
+                    headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
+                    json={"taskId": job_id},
+                )
+                data = response.json().get("data", {})
+                if data.get("status") != "completed":
+                    raise ValueError("Crawl job is not completed")
+
+                response = requests.post(
+                    "https://adaptivecrawlstatus-kir3wx7b3a-uc.a.run.app",
+                    headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
+                    json={"taskId": job_id, "urls": list(data.get("processed", {}).keys())},
+                )
+                data = response.json().get("data", {})
+                for item in data.get("processed", {}).values():
+                    if item.get("data", {}).get("url") == url:
+                        return item.get("data", {})
         else:
             raise ValueError("Invalid provider")