feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更，主要内容：新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
@@ -0,0 +1,128 @@
+import inspect
+import json
+from typing import Dict, List, Any
+from data_juicer.ops.base_op import OPERATORS
+
+# ========= 配置区 =========
+# 1. 基类公共参数：所有Mapper/Filter通用，独立公共面板
+BASE_COMMON_PARAMS = {
+    "video_key": {"type": "str", "default": "video_path", "desc": "样本中视频列表字段名"},
+    "audio_key": {"type": "str", "default": "audios", "desc": "样本中音频列表字段名"},
+    "text_key": {"type": "str", "default": "text", "desc": "样本文本字段名"},
+}
+
+# 2. 高级运行时参数：全局自动注入，默认折叠，仅高级用户修改
+ADVANCED_RUNTIME_PARAMS = {
+    "accelerator": {"type": "str", "default": None, "desc": "加速设备 cpu/gpu"},
+    "num_proc": {"type": "int", "default": 1, "desc": "单算子并行进程数"},
+    "cpu_required": {"type": "float", "default": None, "desc": "Ray每个worker需要CPU资源"},
+    "gpu_required": {"type": "float", "default": None, "desc": "Ray每个worker需要GPU资源"},
+    "mem_required": {"type": "str", "default": None, "desc": "内存资源限制"},
+    "work_dir": {"type": "str", "default": None, "desc": "算子工作目录"},
+    "skip_op_error": {"type": "bool", "default": False, "desc": "是否跳过算子执行异常"},
+    "turbo": {"type": "bool", "default": False, "desc": "turbo加速开关"},
+    "history_key": {"type": "str", "default": "history", "desc": "对话历史字段"},
+    "prompt_key": {"type": "str", "default": "prompt", "desc": "prompt字段"},
+    "query_key": {"type": "str", "default": "query", "desc": "query字段"},
+    "response_key": {"type": "str", "default": "response", "desc": "response字段"},
+    "instruction_key": {"type": "str", "default": "instruction", "desc": "指令字段"},
+    "index_key": {"type": "str", "default": None, "desc": "索引字段"},
+}
+
+# 3. 已知枚举字典（DJ1.4.4 视频算子，可持续扩充）
+OP_ENUM_MAP: Dict[str, Dict[str, List[Any]]] = {
+    "video_extract_frames_mapper": {
+        "output_format": ["path", "bytes"],
+        "frame_sampling_method": ["all_keyframes", "uniform"],
+    },
+    "video_duration_filter": {
+        "any_or_all": ["any", "all"],
+    },
+    "video_aesthetics_filter": {
+        "reduce_mode": ["avg", "max", "min"],
+        "any_or_all": ["any", "all"],
+    },
+    "video_frames_text_similarity_filter": {
+        "reduce_mode": ["avg", "max", "min"],
+        "any_or_all": ["any", "all"],
+    }
+}
+
+# ======================
+def get_op_param_schema(op_name: str, op_cls) -> Dict[str, Any]:
+    """获取单个算子的完整schema"""
+    sig = inspect.signature(op_cls.__init__)
+    native_params = []
+    enum_map = OP_ENUM_MAP.get(op_name, {})
+
+    for param_name, param in sig.parameters.items():
+        # 过滤 self, *args, **kwargs
+        if param_name in ("self", "args", "kwargs"):
+            continue
+
+        default_val = param.default
+        if default_val is inspect.Parameter.empty:
+            default_str = "required"
+        else:
+            default_str = repr(default_val)
+
+        anno = param.annotation
+        enum_list = enum_map.get(param_name, None)
+
+        native_params.append({
+            "name": param_name,
+            "annotation": str(anno),
+            "default": default_str,
+            "enum": enum_list,
+            "ui_type": "select" if enum_list else "input",
+            "description": ""
+        })
+
+    op_info = {
+        "op_name": op_name,
+        "class_name": op_cls.__name__,
+        "op_category": op_cls.__bases__[0].__name__,
+        "accelerator": getattr(op_cls, "_accelerator", "cpu"),
+        "is_batched_op": getattr(op_cls, "_batched_op", False),
+        "docstring": op_cls.__doc__.strip().splitlines()[0] if op_cls.__doc__ else "",
+        "native_params": native_params,  # 主表单：算子原生参数
+        "base_common_params": BASE_COMMON_PARAMS,  # 公共面板
+        "advanced_runtime_params": ADVANCED_RUNTIME_PARAMS  # 高级折叠面板
+    }
+    return op_info
+
+
+def dump_all_ops_schema(save_json_path: str = "dj_ops_schema.json"):
+    """遍历全部算子，生成schema并保存json"""
+    # 加载全部算子，保证懒加载算子注册
+    from data_juicer.ops import load
+    all_ops = []
+    for op_name, op_cls in sorted(OPERATORS.modules.items()):
+        schema = get_op_param_schema(op_name, op_cls)
+        all_ops.append(schema)
+
+    final_payload = {
+        "version": "data-juicer 1.4.4",
+        "total_ops_count": len(all_ops),
+        "operators": all_ops
+    }
+    # 写入json
+    with open(save_json_path, "w", encoding="utf-8") as f:
+        json.dump(final_payload, f, ensure_ascii=False, indent=2)
+    print(f"✅ Schema saved to {save_json_path}, total operators: {len(all_ops)}")
+    return final_payload
+
+
+def filter_op_by_keyword(schema_data, keyword: str):
+    """快速筛选，例如 keyword=video 只返回视频算子"""
+    res = [op for op in schema_data["operators"] if keyword in op["op_name"]]
+    print(f"\n===== filter: contains [{keyword}], count={len(res)} =====")
+    for item in res:
+        print(f"OpName: {item['op_name']}, native params: {[p['name'] for p in item['native_params']]}")
+    return res
+
+
+if __name__ == "__main__":
+    schema_data = dump_all_ops_schema()
+    # 筛选所有video算子，打印预览
+    video_ops = filter_op_by_keyword(schema_data, "video")