feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更，主要内容：新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
@@ -0,0 +1,276 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: MIT
+
+import os
+from unittest.mock import patch
+
+import pytest
+
+from src.config import SearchEngine
+from src.tools.search import get_web_search_tool
+
+
+class TestGetWebSearchTool:
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    def test_get_web_search_tool_tavily(self):
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.name == "web_search"
+        assert tool.max_results == 5
+        assert tool.include_raw_content is True
+        assert tool.include_images is True
+        assert tool.include_image_descriptions is True
+        assert tool.include_answer is False
+        assert tool.search_depth == "advanced"
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.DUCKDUCKGO.value)
+    def test_get_web_search_tool_duckduckgo(self):
+        tool = get_web_search_tool(max_search_results=3)
+        assert tool.name == "web_search"
+        assert tool.max_results == 3
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.BRAVE_SEARCH.value)
+    @patch.dict(os.environ, {"BRAVE_SEARCH_API_KEY": "test_api_key"})
+    def test_get_web_search_tool_brave(self):
+        tool = get_web_search_tool(max_search_results=4)
+        assert tool.name == "web_search"
+        assert tool.search_wrapper.api_key == "test_api_key"
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.ARXIV.value)
+    def test_get_web_search_tool_arxiv(self):
+        tool = get_web_search_tool(max_search_results=2)
+        assert tool.name == "web_search"
+        assert tool.api_wrapper.top_k_results == 2
+        assert tool.api_wrapper.load_max_docs == 2
+        assert tool.api_wrapper.load_all_available_meta is True
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", "unsupported_engine")
+    def test_get_web_search_tool_unsupported_engine(self):
+        with pytest.raises(
+            ValueError, match="Unsupported search engine: unsupported_engine"
+        ):
+            get_web_search_tool(max_search_results=1)
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.BRAVE_SEARCH.value)
+    @patch.dict(os.environ, {}, clear=True)
+    def test_get_web_search_tool_brave_no_api_key(self):
+        tool = get_web_search_tool(max_search_results=1)
+        assert tool.search_wrapper.api_key == ""
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_get_web_search_tool_tavily_with_custom_config(self, mock_config):
+        """Test Tavily tool with custom configuration values."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {
+                "include_answer": True,
+                "search_depth": "basic",
+                "include_raw_content": False,
+                "include_images": False,
+                "include_image_descriptions": True,
+                "include_domains": ["example.com"],
+                "exclude_domains": ["spam.com"],
+            }
+        }
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.name == "web_search"
+        assert tool.max_results == 5
+        assert tool.include_answer is True
+        assert tool.search_depth == "basic"
+        assert tool.include_raw_content is False
+        assert tool.include_images is False
+        # include_image_descriptions should be False because include_images is False
+        assert tool.include_image_descriptions is False
+        assert tool.include_domains == ["example.com"]
+        assert tool.exclude_domains == ["spam.com"]
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_get_web_search_tool_tavily_with_empty_config(self, mock_config):
+        """Test Tavily tool uses defaults when config is empty."""
+        mock_config.return_value = {"SEARCH_ENGINE": {}}
+        tool = get_web_search_tool(max_search_results=10)
+        assert tool.name == "web_search"
+        assert tool.max_results == 10
+        assert tool.include_answer is False
+        assert tool.search_depth == "advanced"
+        assert tool.include_raw_content is True
+        assert tool.include_images is True
+        assert tool.include_image_descriptions is True
+        assert tool.include_domains == []
+        assert tool.exclude_domains == []
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_get_web_search_tool_tavily_image_descriptions_disabled_when_images_disabled(
+        self, mock_config
+    ):
+        """Test that include_image_descriptions is False when include_images is False."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {
+                "include_images": False,
+                "include_image_descriptions": True,  # This should be ignored
+            }
+        }
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.include_images is False
+        assert tool.include_image_descriptions is False
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_get_web_search_tool_tavily_partial_config(self, mock_config):
+        """Test Tavily tool with partial configuration."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {
+                "include_answer": True,
+                "include_domains": ["trusted.com"],
+            }
+        }
+        tool = get_web_search_tool(max_search_results=3)
+        assert tool.include_answer is True
+        assert tool.search_depth == "advanced"  # default
+        assert tool.include_raw_content is True  # default
+        assert tool.include_domains == ["trusted.com"]
+        assert tool.exclude_domains == []  # default
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_get_web_search_tool_tavily_with_no_config_file(self, mock_config):
+        """Test Tavily tool when config file doesn't exist."""
+        mock_config.return_value = {}
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.name == "web_search"
+        assert tool.max_results == 5
+        assert tool.include_answer is False
+        assert tool.search_depth == "advanced"
+        assert tool.include_raw_content is True
+        assert tool.include_images is True
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_get_web_search_tool_tavily_multiple_domains(self, mock_config):
+        """Test Tavily tool with multiple domains in include/exclude lists."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {
+                "include_domains": ["example.com", "trusted.com", "gov.cn"],
+                "exclude_domains": ["spam.com", "scam.org"],
+            }
+        }
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.include_domains == ["example.com", "trusted.com", "gov.cn"]
+        assert tool.exclude_domains == ["spam.com", "scam.org"]
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_with_no_search_engine_section(self, mock_config):
+        """Test Tavily tool when SEARCH_ENGINE section doesn't exist in config."""
+        mock_config.return_value = {"OTHER_CONFIG": {}}
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.name == "web_search"
+        assert tool.max_results == 5
+        assert tool.include_answer is False
+        assert tool.search_depth == "advanced"
+        assert tool.include_raw_content is True
+        assert tool.include_images is True
+        assert tool.include_domains == []
+        assert tool.exclude_domains == []
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_with_completely_empty_config(self, mock_config):
+        """Test Tavily tool with completely empty config."""
+        mock_config.return_value = {}
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.name == "web_search"
+        assert tool.max_results == 5
+        assert tool.include_answer is False
+        assert tool.search_depth == "advanced"
+        assert tool.include_raw_content is True
+        assert tool.include_images is True
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_with_only_include_answer_param(self, mock_config):
+        """Test Tavily tool with only include_answer parameter specified."""
+        mock_config.return_value = {"SEARCH_ENGINE": {"include_answer": True}}
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.include_answer is True
+        assert tool.search_depth == "advanced"
+        assert tool.include_raw_content is True
+        assert tool.include_images is True
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_with_only_search_depth_param(self, mock_config):
+        """Test Tavily tool with only search_depth parameter specified."""
+        mock_config.return_value = {"SEARCH_ENGINE": {"search_depth": "basic"}}
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.search_depth == "basic"
+        assert tool.include_answer is False
+        assert tool.include_raw_content is True
+        assert tool.include_images is True
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_with_only_include_domains_param(self, mock_config):
+        """Test Tavily tool with only include_domains parameter specified."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {"include_domains": ["example.com"]}
+        }
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.include_domains == ["example.com"]
+        assert tool.exclude_domains == []
+        assert tool.include_answer is False
+        assert tool.search_depth == "advanced"
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_with_explicit_false_boolean_values(self, mock_config):
+        """Test that explicitly False boolean values are respected (not treated as missing)."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {
+                "include_answer": False,
+                "include_raw_content": False,
+                "include_images": False,
+            }
+        }
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.include_answer is False
+        assert tool.include_raw_content is False
+        assert tool.include_images is False
+        assert tool.include_image_descriptions is False
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_with_empty_domain_lists(self, mock_config):
+        """Test that empty domain lists are treated as optional."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {
+                "include_domains": [],
+                "exclude_domains": [],
+            }
+        }
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.include_domains == []
+        assert tool.exclude_domains == []
+
+    @patch("src.tools.search.SELECTED_SEARCH_ENGINE", SearchEngine.TAVILY.value)
+    @patch("src.tools.search.load_yaml_config")
+    def test_tavily_all_parameters_optional_mix(self, mock_config):
+        """Test that any combination of optional parameters works."""
+        mock_config.return_value = {
+            "SEARCH_ENGINE": {
+                "include_answer": True,
+                "include_images": False,
+                # Deliberately omit search_depth, include_raw_content, domains
+            }
+        }
+        tool = get_web_search_tool(max_search_results=5)
+        assert tool.include_answer is True
+        assert tool.include_images is False
+        assert (
+            tool.include_image_descriptions is False
+        )  # should be False since include_images is False
+        assert tool.search_depth == "advanced"  # default
+        assert tool.include_raw_content is True  # default
+        assert tool.include_domains == []  # default
+        assert tool.exclude_domains == []  # default