deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,113 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+from src.crawler.article import Article
+
+
+class DummyMarkdownify:
+    """A dummy markdownify replacement for patching if needed."""
+
+    @staticmethod
+    def markdownify(html):
+        return html
+
+
+def test_to_markdown_includes_title(monkeypatch):
+    article = Article("Test Title", "<p>Hello <b>world</b>!</p>")
+    result = article.to_markdown(including_title=True)
+    assert result.startswith("# Test Title")
+    assert "Hello" in result
+
+
+def test_to_markdown_excludes_title():
+    article = Article("Test Title", "<p>Hello <b>world</b>!</p>")
+    result = article.to_markdown(including_title=False)
+    assert not result.startswith("# Test Title")
+    assert "Hello" in result
+
+
+def test_to_message_with_text_only():
+    article = Article("Test Title", "<p>Hello world!</p>")
+    article.url = "https://example.com/"
+    result = article.to_message()
+    assert isinstance(result, list)
+    assert any(item["type"] == "text" for item in result)
+    assert all("type" in item for item in result)
+
+
+def test_to_message_with_image(monkeypatch):
+    html = '<p>Intro</p><img src="img/pic.png"/>'
+    article = Article("Title", html)
+    article.url = "https://host.com/path/"
+    # The markdownify library will convert <img> to markdown image syntax
+    result = article.to_message()
+    # Should have both text and image_url types
+    types = [item["type"] for item in result]
+    assert "image_url" in types
+    assert "text" in types
+    # Check that the image_url is correctly joined
+    image_items = [item for item in result if item["type"] == "image_url"]
+    assert image_items
+    assert image_items[0]["image_url"]["url"] == "https://host.com/path/img/pic.png"
+
+
+def test_to_message_multiple_images():
+    html = '<p>Start</p><img src="a.png"/><p>Mid</p><img src="b.jpg"/>End'
+    article = Article("Title", html)
+    article.url = "http://x/"
+    result = article.to_message()
+    image_urls = [
+        item["image_url"]["url"] for item in result if item["type"] == "image_url"
+    ]
+    assert "http://x/a.png" in image_urls
+    assert "http://x/b.jpg" in image_urls
+    text_items = [item for item in result if item["type"] == "text"]
+    assert any("Start" in item["text"] for item in text_items)
+    assert any("Mid" in item["text"] for item in text_items)
+
+
+def test_to_message_handles_empty_html():
+    article = Article("Empty", "")
+    article.url = "http://test/"
+    result = article.to_message()
+    assert isinstance(result, list)
+    assert result[0]["type"] == "text"
+
+
+def test_to_markdown_handles_none_content():
+    article = Article("Test Title", None)
+    result = article.to_markdown(including_title=True)
+    assert "# Test Title" in result
+    assert "No content available" in result
+
+
+def test_to_markdown_handles_empty_string():
+    article = Article("Test Title", "")
+    result = article.to_markdown(including_title=True)
+    assert "# Test Title" in result
+    assert "No content available" in result
+
+
+def test_to_markdown_handles_whitespace_only():
+    article = Article("Test Title", "   \n  \t  ")
+    result = article.to_markdown(including_title=True)
+    assert "# Test Title" in result
+    assert "No content available" in result
+
+
+def test_to_message_handles_none_content():
+    article = Article("Title", None)
+    article.url = "http://test/"
+    result = article.to_message()
+    assert isinstance(result, list)
+    assert len(result) > 0
+    assert result[0]["type"] == "text"
+    assert "No content available" in result[0]["text"]
+
+
+def test_to_message_handles_whitespace_only_content():
+    article = Article("Title", "   \n  ")
+    article.url = "http://test/"
+    result = article.to_message()
+    assert isinstance(result, list)
+    assert result[0]["type"] == "text"
+    assert "No content available" in result[0]["text"]
@@ -0,0 +1,70 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+import src.crawler as crawler_module
+
+
+def test_crawler_sets_article_url(monkeypatch):
+    """Test that the crawler sets the article.url field correctly."""
+
+    class DummyArticle:
+        def __init__(self):
+            self.url = None
+
+        def to_markdown(self):
+            return "# Dummy"
+
+    class DummyJinaClient:
+        def crawl(self, url, return_format=None):
+            return "<html>dummy</html>"
+
+    class DummyReadabilityExtractor:
+        def extract_article(self, html):
+            return DummyArticle()
+
+    monkeypatch.setattr("src.crawler.crawler.JinaClient", DummyJinaClient)
+    monkeypatch.setattr(
+        "src.crawler.crawler.ReadabilityExtractor", DummyReadabilityExtractor
+    )
+
+    crawler = crawler_module.Crawler()
+    url = "http://example.com"
+    article = crawler.crawl(url)
+    assert article.url == url
+    assert article.to_markdown() == "# Dummy"
+
+
+def test_crawler_calls_dependencies(monkeypatch):
+    """Test that Crawler calls JinaClient.crawl and ReadabilityExtractor.extract_article."""
+    calls = {}
+
+    class DummyJinaClient:
+        def crawl(self, url, return_format=None):
+            calls["jina"] = (url, return_format)
+            return "<html>dummy</html>"
+
+    class DummyReadabilityExtractor:
+        def extract_article(self, html):
+            calls["extractor"] = html
+
+            class DummyArticle:
+                url = None
+
+                def to_markdown(self):
+                    return "# Dummy"
+
+            return DummyArticle()
+
+    monkeypatch.setattr("src.crawler.crawler.JinaClient", DummyJinaClient)
+    monkeypatch.setattr(
+        "src.crawler.crawler.ReadabilityExtractor", DummyReadabilityExtractor
+    )
+
+    crawler = crawler_module.Crawler()
+    url = "http://example.com"
+    crawler.crawl(url)
+    assert "jina" in calls
+    assert calls["jina"][0] == url
+    assert calls["jina"][1] == "html"
+    assert "extractor" in calls
+    assert calls["extractor"] == "<html>dummy</html>"
@@ -0,0 +1,108 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+from unittest.mock import Mock, patch
+
+import pytest
+
+from src.crawler.jina_client import JinaClient
+
+
+class TestJinaClient:
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_success(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = "<html><body>Test</body></html>"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act
+        result = client.crawl("https://example.com")
+
+        # Assert
+        assert result == "<html><body>Test</body></html>"
+        mock_post.assert_called_once()
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_http_error(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 500
+        mock_response.text = "Internal Server Error"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "status 500" in str(exc_info.value)
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_empty_response(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = ""
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "empty response" in str(exc_info.value)
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_whitespace_only_response(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = "   \n  \t  "
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "empty response" in str(exc_info.value)
+
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_not_found(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 404
+        mock_response.text = "Not Found"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act & Assert
+        with pytest.raises(ValueError) as exc_info:
+            client.crawl("https://example.com")
+
+        assert "status 404" in str(exc_info.value)
+
+    @patch.dict("os.environ", {}, clear=True)
+    @patch("src.crawler.jina_client.requests.post")
+    def test_crawl_without_api_key_logs_warning(self, mock_post):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = "<html>Test</html>"
+        mock_post.return_value = mock_response
+
+        client = JinaClient()
+
+        # Act
+        result = client.crawl("https://example.com")
+
+        # Assert
+        assert result == "<html>Test</html>"
@@ -0,0 +1,104 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+from unittest.mock import patch
+
+from src.crawler.readability_extractor import ReadabilityExtractor
+
+
+class TestReadabilityExtractor:
+    @patch("src.crawler.readability_extractor.simple_json_from_html_string")
+    def test_extract_article_with_valid_content(self, mock_simple_json):
+        # Arrange
+        mock_simple_json.return_value = {
+            "title": "Test Article",
+            "content": "<p>Article content</p>",
+        }
+        extractor = ReadabilityExtractor()
+
+        # Act
+        article = extractor.extract_article("<html>test</html>")
+
+        # Assert
+        assert article.title == "Test Article"
+        assert article.html_content == "<p>Article content</p>"
+
+    @patch("src.crawler.readability_extractor.simple_json_from_html_string")
+    def test_extract_article_with_none_content(self, mock_simple_json):
+        # Arrange
+        mock_simple_json.return_value = {
+            "title": "Test Article",
+            "content": None,
+        }
+        extractor = ReadabilityExtractor()
+
+        # Act
+        article = extractor.extract_article("<html>test</html>")
+
+        # Assert
+        assert article.title == "Test Article"
+        assert article.html_content == "<p>No content could be extracted from this page</p>"
+
+    @patch("src.crawler.readability_extractor.simple_json_from_html_string")
+    def test_extract_article_with_empty_content(self, mock_simple_json):
+        # Arrange
+        mock_simple_json.return_value = {
+            "title": "Test Article",
+            "content": "",
+        }
+        extractor = ReadabilityExtractor()
+
+        # Act
+        article = extractor.extract_article("<html>test</html>")
+
+        # Assert
+        assert article.title == "Test Article"
+        assert article.html_content == "<p>No content could be extracted from this page</p>"
+
+    @patch("src.crawler.readability_extractor.simple_json_from_html_string")
+    def test_extract_article_with_whitespace_only_content(self, mock_simple_json):
+        # Arrange
+        mock_simple_json.return_value = {
+            "title": "Test Article",
+            "content": "   \n  \t  ",
+        }
+        extractor = ReadabilityExtractor()
+
+        # Act
+        article = extractor.extract_article("<html>test</html>")
+
+        # Assert
+        assert article.title == "Test Article"
+        assert article.html_content == "<p>No content could be extracted from this page</p>"
+
+    @patch("src.crawler.readability_extractor.simple_json_from_html_string")
+    def test_extract_article_with_none_title(self, mock_simple_json):
+        # Arrange
+        mock_simple_json.return_value = {
+            "title": None,
+            "content": "<p>Article content</p>",
+        }
+        extractor = ReadabilityExtractor()
+
+        # Act
+        article = extractor.extract_article("<html>test</html>")
+
+        # Assert
+        assert article.title == "Untitled"
+        assert article.html_content == "<p>Article content</p>"
+
+    @patch("src.crawler.readability_extractor.simple_json_from_html_string")
+    def test_extract_article_with_empty_title(self, mock_simple_json):
+        # Arrange
+        mock_simple_json.return_value = {
+            "title": "",
+            "content": "<p>Article content</p>",
+        }
+        extractor = ReadabilityExtractor()
+
+        # Act
+        article = extractor.extract_article("<html>test</html>")
+
+        # Assert
+        assert article.title == "Untitled"
+        assert article.html_content == "<p>Article content</p>"