deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,133 @@
+import json
+from unittest.mock import Mock, patch
+
+from src.tools.crawl import crawl_tool
+
+
+class TestCrawlTool:
+    @patch("src.tools.crawl.Crawler")
+    def test_crawl_tool_success(self, mock_crawler_class):
+        # Arrange
+        mock_crawler = Mock()
+        mock_article = Mock()
+        mock_article.to_markdown.return_value = (
+            "# Test Article\nThis is test content." * 100
+        )
+        mock_crawler.crawl.return_value = mock_article
+        mock_crawler_class.return_value = mock_crawler
+
+        url = "https://example.com"
+
+        # Act
+        result = crawl_tool(url)
+
+        # Assert
+        assert isinstance(result, str)
+        result_dict = json.loads(result)
+        assert result_dict["url"] == url
+        assert "crawled_content" in result_dict
+        assert len(result_dict["crawled_content"]) <= 1000
+        mock_crawler_class.assert_called_once()
+        mock_crawler.crawl.assert_called_once_with(url)
+        mock_article.to_markdown.assert_called_once()
+
+    @patch("src.tools.crawl.Crawler")
+    def test_crawl_tool_short_content(self, mock_crawler_class):
+        # Arrange
+        mock_crawler = Mock()
+        mock_article = Mock()
+        short_content = "Short content"
+        mock_article.to_markdown.return_value = short_content
+        mock_crawler.crawl.return_value = mock_article
+        mock_crawler_class.return_value = mock_crawler
+
+        url = "https://example.com"
+
+        # Act
+        result = crawl_tool(url)
+
+        # Assert
+        result_dict = json.loads(result)
+        assert result_dict["crawled_content"] == short_content
+
+    @patch("src.tools.crawl.Crawler")
+    @patch("src.tools.crawl.logger")
+    def test_crawl_tool_crawler_exception(self, mock_logger, mock_crawler_class):
+        # Arrange
+        mock_crawler = Mock()
+        mock_crawler.crawl.side_effect = Exception("Network error")
+        mock_crawler_class.return_value = mock_crawler
+
+        url = "https://example.com"
+
+        # Act
+        result = crawl_tool(url)
+
+        # Assert
+        assert isinstance(result, str)
+        assert "Failed to crawl" in result
+        assert "Network error" in result
+        mock_logger.error.assert_called_once()
+
+    @patch("src.tools.crawl.Crawler")
+    @patch("src.tools.crawl.logger")
+    def test_crawl_tool_crawler_instantiation_exception(
+        self, mock_logger, mock_crawler_class
+    ):
+        # Arrange
+        mock_crawler_class.side_effect = Exception("Crawler init error")
+
+        url = "https://example.com"
+
+        # Act
+        result = crawl_tool(url)
+
+        # Assert
+        assert isinstance(result, str)
+        assert "Failed to crawl" in result
+        assert "Crawler init error" in result
+        mock_logger.error.assert_called_once()
+
+    @patch("src.tools.crawl.Crawler")
+    @patch("src.tools.crawl.logger")
+    def test_crawl_tool_markdown_conversion_exception(
+        self, mock_logger, mock_crawler_class
+    ):
+        # Arrange
+        mock_crawler = Mock()
+        mock_article = Mock()
+        mock_article.to_markdown.side_effect = Exception("Markdown conversion error")
+        mock_crawler.crawl.return_value = mock_article
+        mock_crawler_class.return_value = mock_crawler
+
+        url = "https://example.com"
+
+        # Act
+        result = crawl_tool(url)
+
+        # Assert
+        assert isinstance(result, str)
+        assert "Failed to crawl" in result
+        assert "Markdown conversion error" in result
+        mock_logger.error.assert_called_once()
+
+    @patch("src.tools.crawl.Crawler")
+    def test_crawl_tool_with_none_content(self, mock_crawler_class):
+        # Arrange
+        mock_crawler = Mock()
+        mock_article = Mock()
+        mock_article.to_markdown.return_value = "# Article\n\n*No content available*\n"
+        mock_crawler.crawl.return_value = mock_article
+        mock_crawler_class.return_value = mock_crawler
+
+        url = "https://example.com"
+
+        # Act
+        result = crawl_tool(url)
+
+        # Assert
+        assert isinstance(result, str)
+        result_dict = json.loads(result)
+        assert result_dict["url"] == url
+        assert "crawled_content" in result_dict
+        assert "No content available" in result_dict["crawled_content"]