deepdata: core product code (backend, frontend, runtime, deepflow module)

deepflow/src/crawler/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+from .article import Article
+from .crawler import Crawler
+from .jina_client import JinaClient
+from .readability_extractor import ReadabilityExtractor
+
+__all__ = ["Article", "Crawler", "JinaClient", "ReadabilityExtractor"]

deepflow/src/crawler/article.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+import re
+from urllib.parse import urljoin
+
+from markdownify import markdownify as md
+
+
+class Article:
+    url: str
+
+    def __init__(self, title: str, html_content: str):
+        self.title = title
+        self.html_content = html_content
+
+    def to_markdown(self, including_title: bool = True) -> str:
+        markdown = ""
+        if including_title:
+            markdown += f"# {self.title}\n\n"
+        
+        if self.html_content is None or not str(self.html_content).strip():
+            markdown += "*No content available*\n"
+        else:
+            markdown += md(self.html_content)
+        
+        return markdown
+
+    def to_message(self) -> list[dict]:
+        image_pattern = r"!\[.*?\]\((.*?)\)"
+
+        content: list[dict[str, str]] = []
+        markdown = self.to_markdown()
+        
+        if not markdown or not markdown.strip():
+            return [{"type": "text", "text": "No content available"}]
+        
+        parts = re.split(image_pattern, markdown)
+
+        for i, part in enumerate(parts):
+            if i % 2 == 1:
+                image_url = urljoin(self.url, part.strip())
+                content.append({"type": "image_url", "image_url": {"url": image_url}})
+            else:
+                text_part = part.strip()
+                if text_part:
+                    content.append({"type": "text", "text": text_part})
+
+        # If after processing all parts, content is still empty, provide a fallback message.
+        if not content:
+            content = [{"type": "text", "text": "No content available"}]
+        
+        return content

deepflow/src/crawler/crawler.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+import logging
+
+from .article import Article
+from .jina_client import JinaClient
+from .readability_extractor import ReadabilityExtractor
+
+logger = logging.getLogger(__name__)
+
+
+class Crawler:
+    def crawl(self, url: str) -> Article:
+        # To help LLMs better understand content, we extract clean
+        # articles from HTML, convert them to markdown, and split
+        # them into text and image blocks for one single and unified
+        # LLM message.
+        #
+        # Jina is not the best crawler on readability, however it's
+        # much easier and free to use.
+        #
+        # Instead of using Jina's own markdown converter, we'll use
+        # our own solution to get better readability results.
+        try:
+            jina_client = JinaClient()
+            html = jina_client.crawl(url, return_format="html")
+        except Exception as e:
+            logger.error(f"Failed to fetch URL {url} from Jina: {repr(e)}")
+            raise
+        
+        try:
+            extractor = ReadabilityExtractor()
+            article = extractor.extract_article(html)
+        except Exception as e:
+            logger.error(f"Failed to extract article from {url}: {repr(e)}")
+            raise
+        
+        article.url = url
+        return article

deepflow/src/crawler/jina_client.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+import logging
+import os
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class JinaClient:
+    def crawl(self, url: str, return_format: str = "html") -> str:
+        headers = {
+            "Content-Type": "application/json",
+            "X-Return-Format": return_format,
+        }
+        if os.getenv("JINA_API_KEY"):
+            headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
+        else:
+            logger.warning(
+                "Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information."
+            )
+        data = {"url": url}
+        response = requests.post("https://r.jina.ai/", headers=headers, json=data)
+        
+        if response.status_code != 200:
+            raise ValueError(f"Jina API returned status {response.status_code}: {response.text}")
+        
+        if not response.text or not response.text.strip():
+            raise ValueError("Jina API returned empty response")
+        
+        return response.text

deepflow/src/crawler/readability_extractor.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+import logging
+
+from readabilipy import simple_json_from_html_string
+
+from .article import Article
+
+logger = logging.getLogger(__name__)
+
+
+class ReadabilityExtractor:
+    def extract_article(self, html: str) -> Article:
+        article = simple_json_from_html_string(html, use_readability=True)
+        
+        content = article.get("content")
+        if not content or not str(content).strip():
+            logger.warning("Readability extraction returned empty content")
+            content = "<p>No content could be extracted from this page</p>"
+        
+        title = article.get("title")
+        if not title or not str(title).strip():
+            title = "Untitled"
+        
+        return Article(
+            title=title,
+            html_content=content,
+        )