feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更，主要内容：新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
@@ -0,0 +1,30 @@
+from markitdown import MarkItDown
+from io import BytesIO
+from PIL import Image
+import base64
+
+def image_callback(image: Image.Image) -> str:
+    print("======进入image_callback======")
+    max_width = 1200
+    if image.width > max_width:
+        ratio = max_width / image.width
+        new_size = (max_width, int(image.height * ratio))
+        image = image.resize(new_size, Image.Resampling.LANCZOS)
+    buf = BytesIO()
+    image.save(buf, format="PNG", optimize=True)
+    buf.seek(0)
+    b64_data = base64.b64encode(buf.read()).decode("utf-8")
+    return f"data:image/png;base64,{b64_data}"
+
+
+if __name__ == "__main__":
+    pptx_file = "test_ppt.pptx"
+    out_md = "pptx_out_ok.md"
+    md_converter = MarkItDown(image_callback=image_callback)
+    res = md_converter.convert(source=pptx_file)
+    md_text = res.text_content
+    print("===生成的原始md===")
+    print(md_text)
+    with open(out_md, "w", encoding="utf-8") as f:
+        f.write(md_text)
+    print(f"✅转换完成：{out_md}")
@@ -0,0 +1,85 @@
+from markitdown import MarkItDown
+from openpyxl import load_workbook
+from io import BytesIO
+from PIL import Image
+import base64
+from zipfile import ZipFile
+
+def extract_xlsx_all_images_base64(xlsx_path: str, max_width=1200):
+    img_list = []
+    wb = load_workbook(xlsx_path)
+    # 1.先读取单元格内嵌图片
+    for sheet_name in wb.sheetnames:
+        ws = wb[sheet_name]
+        for img in ws._images:
+            img_bytes = img.ref
+            pil_img = Image.open(BytesIO(img_bytes))
+            if pil_img.width > max_width:
+                ratio = max_width / pil_img.width
+                new_size = (max_width, int(pil_img.height * ratio))
+                pil_img = pil_img.resize(new_size, Image.Resampling.LANCZOS)
+            buf = BytesIO()
+            pil_img.save(buf, format="PNG", optimize=True)
+            b64_str = base64.b64encode(buf.getvalue()).decode("utf-8")
+            uri = f"data:image/png;base64,{b64_str}"
+            img_list.append({"sheet": sheet_name, "data_uri": uri})
+    wb.close()
+
+    # 2.解析xlsx压缩包内drawing.xml，提取浮动形状图片
+    with ZipFile(xlsx_path, "r") as zf:
+        file_list = zf.namelist()
+        # 遍历所有drawing，匹配media图片
+        drawing_files = [f for f in file_list if f.startswith("xl/drawings/drawing") and f.endswith(".xml")]
+        for drawing_path in drawing_files:
+            # 解析drawing.xml获取图片rId
+            import xml.etree.ElementTree as ET
+            ns_a = {"a": "http://schemas.openxmlformats.org/drawingml/2006/main"}
+            ns_xdr = {"xdr": "http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing"}
+            xml_data = zf.read(drawing_path)
+            root = ET.fromstring(xml_data)
+            for blip in root.findall(".//a:blip", ns_a):
+                r_id = blip.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed")
+                if not r_id:
+                    continue
+                # 读取drawing的关系文件
+                rel_path = drawing_path.replace(".xml", ".rels")
+                if rel_path not in file_list:
+                    continue
+                rel_xml = zf.read(rel_path)
+                rel_root = ET.fromstring(rel_xml)
+                ns_rel = {"r": "http://schemas.openxmlformats.org/package/2006/relationships"}
+                for rel in rel_root.findall(".//r:Relationship", ns_rel):
+                    if rel.get("Id") == r_id:
+                        target = rel.get("Target")
+                        media_path = f"xl/drawings/{target}"
+                        if media_path in file_list:
+                            media_bytes = zf.read(media_path)
+                            pil_img = Image.open(BytesIO(media_bytes))
+                            if pil_img.width > max_width:
+                                ratio = max_width / pil_img.width
+                                new_size = (max_width, int(pil_img.height * ratio))
+                                pil_img = pil_img.resize(new_size, Image.Resampling.LANCZOS)
+                            buf = BytesIO()
+                            pil_img.save(buf, format="PNG", optimize=True)
+                            b64_str = base64.b64encode(buf.getvalue()).decode("utf-8")
+                            uri = f"data:image/png;base64,{b64_str}"
+                            # 缺陷：无法精确归属到sheet名称，统一标记为浮动图片
+                            img_list.append({"sheet": "FloatingShape", "data_uri": uri})
+    return img_list
+
+
+if __name__ == "__main__":
+    xlsx_file = "test_xlsx.xlsx"
+    out_md = "xlsx_out.md"
+    md = MarkItDown()
+    res = md.convert(source=xlsx_file)
+    md_text = res.text_content
+    images = extract_xlsx_all_images_base64(xlsx_file)
+    print(f"一共提取图片：{len(images)}")
+    if len(images) > 0:
+        md_text += "\n\n## 提取的内嵌图片\n"
+        for item in images:
+            md_text += f"\n【Sheet：{item['sheet']}】\n![]({item['data_uri']})\n"
+    with open(out_md, "w", encoding="utf-8") as f:
+        f.write(md_text)
+    print(f"✅转换完成，输出文件：{out_md}")
@@ -0,0 +1,5 @@
+<!-- Slide number: 1 -->
+aaa
+#
+
+![](图片3.jpg)
@@ -0,0 +1,5 @@
+<!-- Slide number: 1 -->
+aaa
+#
+
+![](图片3.jpg)
@@ -0,0 +1,4 @@
+[{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/1.mp4"], "text": "<__dj__video>"},
+{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/2.mp4"], "text": "<__dj__video>"},
+{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/3.mp4"], "text": "<__dj__video>"},
+{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/3-副本.mp4"], "text": "<__dj__video>"}]
@@ -0,0 +1,4 @@
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/videos/1.mp4"], "text": "<__dj__video>"}
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/2.mp4"], "text": "<__dj__video>"}
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/videos/3.mp4"], "text": "<__dj__video>"}
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/videos/3-副本.mp4"], "text": "<__dj__video>"}
@@ -0,0 +1,128 @@
+import inspect
+import json
+from typing import Dict, List, Any
+from data_juicer.ops.base_op import OPERATORS
+
+# ========= 配置区 =========
+# 1. 基类公共参数：所有Mapper/Filter通用，独立公共面板
+BASE_COMMON_PARAMS = {
+    "video_key": {"type": "str", "default": "video_path", "desc": "样本中视频列表字段名"},
+    "audio_key": {"type": "str", "default": "audios", "desc": "样本中音频列表字段名"},
+    "text_key": {"type": "str", "default": "text", "desc": "样本文本字段名"},
+}
+
+# 2. 高级运行时参数：全局自动注入，默认折叠，仅高级用户修改
+ADVANCED_RUNTIME_PARAMS = {
+    "accelerator": {"type": "str", "default": None, "desc": "加速设备 cpu/gpu"},
+    "num_proc": {"type": "int", "default": 1, "desc": "单算子并行进程数"},
+    "cpu_required": {"type": "float", "default": None, "desc": "Ray每个worker需要CPU资源"},
+    "gpu_required": {"type": "float", "default": None, "desc": "Ray每个worker需要GPU资源"},
+    "mem_required": {"type": "str", "default": None, "desc": "内存资源限制"},
+    "work_dir": {"type": "str", "default": None, "desc": "算子工作目录"},
+    "skip_op_error": {"type": "bool", "default": False, "desc": "是否跳过算子执行异常"},
+    "turbo": {"type": "bool", "default": False, "desc": "turbo加速开关"},
+    "history_key": {"type": "str", "default": "history", "desc": "对话历史字段"},
+    "prompt_key": {"type": "str", "default": "prompt", "desc": "prompt字段"},
+    "query_key": {"type": "str", "default": "query", "desc": "query字段"},
+    "response_key": {"type": "str", "default": "response", "desc": "response字段"},
+    "instruction_key": {"type": "str", "default": "instruction", "desc": "指令字段"},
+    "index_key": {"type": "str", "default": None, "desc": "索引字段"},
+}
+
+# 3. 已知枚举字典（DJ1.4.4 视频算子，可持续扩充）
+OP_ENUM_MAP: Dict[str, Dict[str, List[Any]]] = {
+    "video_extract_frames_mapper": {
+        "output_format": ["path", "bytes"],
+        "frame_sampling_method": ["all_keyframes", "uniform"],
+    },
+    "video_duration_filter": {
+        "any_or_all": ["any", "all"],
+    },
+    "video_aesthetics_filter": {
+        "reduce_mode": ["avg", "max", "min"],
+        "any_or_all": ["any", "all"],
+    },
+    "video_frames_text_similarity_filter": {
+        "reduce_mode": ["avg", "max", "min"],
+        "any_or_all": ["any", "all"],
+    }
+}
+
+# ======================
+def get_op_param_schema(op_name: str, op_cls) -> Dict[str, Any]:
+    """获取单个算子的完整schema"""
+    sig = inspect.signature(op_cls.__init__)
+    native_params = []
+    enum_map = OP_ENUM_MAP.get(op_name, {})
+
+    for param_name, param in sig.parameters.items():
+        # 过滤 self, *args, **kwargs
+        if param_name in ("self", "args", "kwargs"):
+            continue
+
+        default_val = param.default
+        if default_val is inspect.Parameter.empty:
+            default_str = "required"
+        else:
+            default_str = repr(default_val)
+
+        anno = param.annotation
+        enum_list = enum_map.get(param_name, None)
+
+        native_params.append({
+            "name": param_name,
+            "annotation": str(anno),
+            "default": default_str,
+            "enum": enum_list,
+            "ui_type": "select" if enum_list else "input",
+            "description": ""
+        })
+
+    op_info = {
+        "op_name": op_name,
+        "class_name": op_cls.__name__,
+        "op_category": op_cls.__bases__[0].__name__,
+        "accelerator": getattr(op_cls, "_accelerator", "cpu"),
+        "is_batched_op": getattr(op_cls, "_batched_op", False),
+        "docstring": op_cls.__doc__.strip().splitlines()[0] if op_cls.__doc__ else "",
+        "native_params": native_params,  # 主表单：算子原生参数
+        "base_common_params": BASE_COMMON_PARAMS,  # 公共面板
+        "advanced_runtime_params": ADVANCED_RUNTIME_PARAMS  # 高级折叠面板
+    }
+    return op_info
+
+
+def dump_all_ops_schema(save_json_path: str = "dj_ops_schema.json"):
+    """遍历全部算子，生成schema并保存json"""
+    # 加载全部算子，保证懒加载算子注册
+    from data_juicer.ops import load
+    all_ops = []
+    for op_name, op_cls in sorted(OPERATORS.modules.items()):
+        schema = get_op_param_schema(op_name, op_cls)
+        all_ops.append(schema)
+
+    final_payload = {
+        "version": "data-juicer 1.4.4",
+        "total_ops_count": len(all_ops),
+        "operators": all_ops
+    }
+    # 写入json
+    with open(save_json_path, "w", encoding="utf-8") as f:
+        json.dump(final_payload, f, ensure_ascii=False, indent=2)
+    print(f"✅ Schema saved to {save_json_path}, total operators: {len(all_ops)}")
+    return final_payload
+
+
+def filter_op_by_keyword(schema_data, keyword: str):
+    """快速筛选，例如 keyword=video 只返回视频算子"""
+    res = [op for op in schema_data["operators"] if keyword in op["op_name"]]
+    print(f"\n===== filter: contains [{keyword}], count={len(res)} =====")
+    for item in res:
+        print(f"OpName: {item['op_name']}, native params: {[p['name'] for p in item['native_params']]}")
+    return res
+
+
+if __name__ == "__main__":
+    schema_data = dump_all_ops_schema()
+    # 筛选所有video算子，打印预览
+    video_ops = filter_op_by_keyword(schema_data, "video")
@@ -0,0 +1,12 @@
+2026-09-20 15:54:25.045 | INFO     | data_juicer.config.config:init_setup_from_cfg:682 - dataset_path config is set and a valid local path
+2026-09-20 15:54:25.045 | DEBUG    | data_juicer.utils.lazy_loader:_load:406 - Loading ray...
+2026-09-20 15:54:25.047 | DEBUG    | data_juicer.config.config:timing_context:43 - Initializing setup from config took 0.02 seconds
+2026-09-20 15:54:25.286 | DEBUG    | data_juicer.config.config:timing_context:43 - Updating operator process took 0.24 seconds
+2026-09-20 15:54:25.286 | INFO     | data_juicer.config.config:config_backup:998 - Back up the input config file [D:\xu_projects\for_mage\llm\repos\deepdata\test\ray\video_pipeline_ray.yaml] into the work_dir [D:\xu_projects\for_mage\llm\repos\deepdata\test\ray]
+2026-09-20 15:54:25.300 | INFO     | data_juicer.config.config:display_config:1019 - Configuration table: 
+2026-09-20 15:54:25.421 | DEBUG    | data_juicer.config.config:timing_context:43 - Total config initialization time took 1.39 seconds
+2026-09-20 15:55:14.068 | INFO     | data_juicer.core.executor.default_executor:__init__:54 - Using cache compression method: [None]
+2026-09-20 15:55:14.068 | INFO     | data_juicer.core.executor.default_executor:__init__:58 - Setting up dataset builder...
+2026-09-20 15:55:14.069 | INFO     | data_juicer.core.data.dataset_builder:__init__:48 - found dataset_path setting: D:\xu_projects\for_mage\llm\repos\deepdata\test\ray\input_videos_meta.jsonl
+2026-09-20 15:55:14.070 | INFO     | data_juicer.core.data.load_strategy:get_strategy_class:84 - Getting strategy class for exec: default, data_type: huggingface, data_source: None
+2026-09-20 15:55:14.070 | WARNING  | data_juicer.core.data.load_strategy:get_strategy_class:129 - No matching strategy found for combination exec: default, data_type: huggingface, data_source: *
@@ -0,0 +1,4 @@
+{"text": "2026-09-20 15:54:25.045 | DEBUG    | data_juicer.utils.lazy_loader:406 - Loading ray...\n", "record": {"elapsed": {"repr": "0:00:42.969853", "seconds": 42.969853}, "exception": null, "extra": {}, "file": {"name": "lazy_loader.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\utils\\lazy_loader.py"}, "function": "_load", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 406, "message": "Loading ray...", "module": "lazy_loader", "name": "data_juicer.utils.lazy_loader", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.045540+08:00", "timestamp": 1789890865.04554}}}
+{"text": "2026-09-20 15:54:25.047 | DEBUG    | data_juicer.config.config:43 - Initializing setup from config took 0.02 seconds\n", "record": {"elapsed": {"repr": "0:00:42.971854", "seconds": 42.971854}, "exception": null, "extra": {}, "file": {"name": "config.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\config\\config.py"}, "function": "timing_context", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 43, "message": "Initializing setup from config took 0.02 seconds", "module": "config", "name": "data_juicer.config.config", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.047541+08:00", "timestamp": 1789890865.047541}}}
+{"text": "2026-09-20 15:54:25.286 | DEBUG    | data_juicer.config.config:43 - Updating operator process took 0.24 seconds\n", "record": {"elapsed": {"repr": "0:00:43.210449", "seconds": 43.210449}, "exception": null, "extra": {}, "file": {"name": "config.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\config\\config.py"}, "function": "timing_context", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 43, "message": "Updating operator process took 0.24 seconds", "module": "config", "name": "data_juicer.config.config", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.286136+08:00", "timestamp": 1789890865.286136}}}
+{"text": "2026-09-20 15:54:25.421 | DEBUG    | data_juicer.config.config:43 - Total config initialization time took 1.39 seconds\n", "record": {"elapsed": {"repr": "0:00:43.345850", "seconds": 43.34585}, "exception": null, "extra": {}, "file": {"name": "config.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\config\\config.py"}, "function": "timing_context", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 43, "message": "Total config initialization time took 1.39 seconds", "module": "config", "name": "data_juicer.config.config", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.421537+08:00", "timestamp": 1789890865.421537}}}
@@ -0,0 +1 @@
+{"text": "2026-09-20 15:55:14.070 | WARNING  | data_juicer.core.data.load_strategy:129 - No matching strategy found for combination exec: default, data_type: huggingface, data_source: *\n", "record": {"elapsed": {"repr": "0:01:31.994509", "seconds": 91.994509}, "exception": null, "extra": {}, "file": {"name": "load_strategy.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\core\\data\\load_strategy.py"}, "function": "get_strategy_class", "level": {"icon": "⚠️", "name": "WARNING", "no": 30}, "line": 129, "message": "No matching strategy found for combination exec: default, data_type: huggingface, data_source: *", "module": "load_strategy", "name": "data_juicer.core.data.load_strategy", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:55:14.070196+08:00", "timestamp": 1789890914.070196}}}
@@ -0,0 +1,5 @@
+from data_juicer.config import init_configs
+from data_juicer.core import DefaultExecutor
+cfg = init_configs(args=["--config", "./video_pipeline_ray.yaml"])
+executor = DefaultExecutor(cfg)
+executor.run()
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+fast_ray_executor.py —— data-juicer Ray 模式「惰性 DAG」执行器
+
+已验证：253s → 7 个算子 0.02s 挂完（原 8 次全链重跑）
+
+【两个必须同时修的点】
+1) ray_dataset.py:159/196 调 self.data.columns() → Ray Data 内部以 limit(1)
+   执行整条 DAG（O(N^2)）。用静态列名覆盖 columns() 解决。
+2) 覆盖 columns() 会让 DJ "以为" meta/stats 已存在而跳过补列，
+   算子却要读 sample['__dj__meta__'] → KeyError。
+   所以必须【自己正确地把这两列补上】。
+
+【补列陷阱】pyarrow 无法从 [{}...] 推断 struct 类型（长度为 0）：
+   ✗ table.append_column(col, [{} for _ in range(len(table))])
+   ✓ table.append_column(col, pa.array([{}]*n, type=pa.struct([])))
+
+用法: python fast_ray_executor.py ./video_pipeline_ray.yaml
+"""
+import os, sys, time, glob, json
+import ray
+import pyarrow as pa
+
+
+def peek_columns(path):
+    """支持目录：扫描所有 jsonl/json 的第一行，取列名并集。"""
+    cols = set()
+    if os.path.isdir(path):
+        files = []
+        for ext in ("*.jsonl", "*.json"):
+            files.extend(sorted(glob.glob(os.path.join(path, "**", ext), recursive=True)))
+    else:
+        files = [path]
+
+    if not files:
+        print(f"[warn] 目录下没找到 jsonl/json: {path}")
+        return cols
+
+    for fp in files[:50]:                      # 最多扫 50 个，足够推断 schema
+        try:
+            with open(fp, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    if line.startswith("["):   # 兼容 json 数组
+                        line = line.lstrip("[").rstrip(",").rstrip("]")
+                    try:
+                        cols.update(json.loads(line).keys())
+                    except Exception:
+                        pass
+                    break                      # 每个文件只读第一行
+        except Exception as e:
+            print(f"[warn] 跳过 {fp}: {e}")
+    print(f"[列名] 扫描 {min(len(files),50)}/{len(files)} 个文件 -> {sorted(cols)}")
+    return cols
+
+
+def build_lazy_process(state, batch_size):
+    """state: dict(known=set(), ...)  用可变容器让 known 跨调用共享"""
+    known = state["known"]
+
+    def make_add_col(col):
+        def _fn(table):
+            if col in table.column_names:
+                return table
+            n = table.num_rows
+            # 关键：必须显式 pa.struct([])，否则 pyarrow 推断出长度 0
+            arr = pa.array([{} for _ in range(n)], type=pa.struct([]))
+            return table.append_column(col, arr)
+        return _fn
+
+    def lazy_process(self, operators, *, exporter=None, checkpointer=None, tracer=None):
+        if operators is None:
+            return self
+        if not isinstance(operators, list):
+            operators = [operators]
+
+        from data_juicer.utils.process_utils import calculate_ray_np
+        from data_juicer.utils.constant import Fields
+
+        calculate_ray_np(operators)
+
+        # ---- 先补 meta / stats 两列（惰性，不触发执行）----
+        for col in (Fields.meta, Fields.stats):
+            self.data = self.data.map_batches(
+                make_add_col(col), batch_format="pyarrow", batch_size=batch_size
+            )
+            known.add(col)
+        print(f"[预补列] {sorted(known)}")
+
+        # ---- 覆盖 columns()：返回已知列名，杜绝 schema 推理触发执行 ----
+        orig_columns = ray.data.Dataset.columns
+        ray.data.Dataset.columns = lambda ds: list(known)
+
+        try:
+            total = len(operators)
+            t_all = time.time()
+            print(f"\n[惰性 DAG] 开始挂接 {total} 个算子（每项应为毫秒级）")
+
+            for i, op in enumerate(operators, 1):
+                name = type(op).__name__
+                try:
+                    op.num_proc = 1      # concurrency=1 → actor 池=1 → 模型只加载一次
+                except Exception:
+                    pass
+
+                t = time.time()
+                self._run_single_op(op)  # Ray 分支：map_batches，不是 op.run
+                cost = time.time() - t
+
+                flag = "" if cost < 1.0 else "   <-- 异常! 此处触发了 action"
+                print(f"  [{i}/{total}] {name:<42s} {cost * 1000:8.1f} ms{flag}")
+
+                if not isinstance(self.data, ray.data.Dataset):
+                    raise RuntimeError(
+                        f"算子 [{name}] 把 ray.data.Dataset 转成了 {type(self.data).__name__}"
+                    )
+
+            print(f"[惰性 DAG] 构建总耗时: {time.time() - t_all:.2f}s（应接近 0）")
+        finally:
+            ray.data.Dataset.columns = orig_columns   # 还原，导出时用真实 schema
+
+        return self
+
+    return lazy_process
+
+
+def main():
+    cfg_path = sys.argv[1] if len(sys.argv) > 1 else "./video_pipeline_ray.yaml"
+
+    from data_juicer.config import init_configs
+    cfg = init_configs(args=["--config", cfg_path])
+
+    print("=" * 72)
+    print(f"  配置文件      : {cfg_path}")
+    print(f"  executor_type : {cfg.executor_type}")
+    print(f"  export_path   : {cfg.export_path}")
+    print("=" * 72)
+
+    ray.init(address=getattr(cfg, "ray_address", "auto"), ignore_reinit_error=True)
+    print(f"[Ray] 已连接: {ray.get_runtime_context().gcs_address}")
+
+    from data_juicer.core.data import ray_dataset as rd_mod
+
+    RayDataset = rd_mod.RayDataset
+    batch_size = getattr(rd_mod, "DEFAULT_BATCH_SIZE", 1024)
+
+    state = {"known": peek_columns(cfg.dataset_path)}
+    RayDataset.process = build_lazy_process(state, batch_size)
+
+    from data_juicer.core import ExecutorFactory
+    executor_cls = ExecutorFactory.create_executor(cfg.executor_type)
+    print(f"[执行器] {executor_cls.__name__}")
+    assert executor_cls.__name__ == "RayExecutor", "必须拿到 RayExecutor"
+
+    export_path = cfg.export_path
+    # 清掉上一轮遗留的分片（否则会被误合并进最终结果）
+    if os.path.isdir(export_path):
+        for f in glob.glob(os.path.join(export_path, "*.json")):
+            os.remove(f)
+    elif os.path.exists(export_path):
+        os.remove(export_path)          # Ray 的 write_json 是目录语义
+    os.makedirs(export_path, exist_ok=True)
+
+    executor = executor_cls(cfg)
+    t0 = time.time()
+    executor.run()
+    print(f"\n总耗时: {time.time() - t0:.1f}s")
+
+    shards = sorted(glob.glob(os.path.join(export_path, "*.json")))
+    if shards:
+        merged = export_path.rstrip("/") + ".jsonl"
+        n = 0
+        with open(merged, "w", encoding="utf-8") as out:
+            for s in shards:
+                for line in open(s, encoding="utf-8"):
+                    line = line.strip()
+                    if line:
+                        out.write(line + "\n")
+                        n += 1
+        print(f"\n已合并 {len(shards)} 个分片 -> {merged}（{n} 条样本）")
+        for line in open(merged, encoding="utf-8"):
+            d = json.loads(line)
+            vp = d.get("video_path", ["?"])[0].split("/")[-1]
+            print(f"  - {vp}")
+            print(f"    text : {d.get('text', '')[:70]}")
+            print(f"    meta : {d.get('__dj__meta__', {})}")
+            print(f"    stats: {d.get('__dj__stats__', {})}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,10 @@
+from data_juicer.config import init_configs
+from data_juicer.core import ExecutorFactory
+import sys
+
+cfg_path = sys.argv[1] if len(sys.argv) > 1 else "./video_pipeline_ray.yaml"
+cfg = init_configs(args=["--config", cfg_path])
+
+executor_cls = ExecutorFactory.create_executor(cfg.executor_type)
+print(">>> 执行器:", executor_cls.__name__)     # 必须打印 RayExecutor
+executor_cls(cfg).run()
@@ -0,0 +1,58 @@
+project_name: t-final
+dataset_path: "/home/apesea/xuhq/codes/DeepData/test/ray/input"
+executor_type: ray
+ray_address: 127.0.0.1:6388
+export_type: jsonl
+use_cache: false
+export_path: "/home/apesea/xuhq/codes/DeepData/test/ray/out_final"
+skip_op_error: false
+video_key: video_path
+open_monitor: false
+text_keys: ['text']
+np: 1
+keep_stats_in_res_ds: True          # ← 让 video_audio_tags 等 meta 字段导出
+
+process:
+  - video_extract_frames_mapper:
+      frame_num: 5
+      frame_sampling_method: uniform
+      video_key: video_path
+      output_format: path
+      frame_dir: "./frames"
+      num_proc: 1
+
+  - video_tagging_from_audio_mapper:
+      video_key: video_path
+      num_proc: 1
+      hf_ast: 'MIT/ast-finetuned-audioset-10-10-0.4593'
+
+  - video_duration_filter:
+      video_key: video_path
+      min_duration: 3
+      max_duration: 60
+      num_proc: 1
+
+  - video_aesthetics_filter:
+      video_key: video_path
+      min_score: 0.4
+      num_proc: 1
+
+  - video_captioning_from_frames_mapper:
+      video_key: video_path
+      hf_img2seq: Salesforce/blip-image-captioning-base   # ~990MB，稳
+      frame_sampling_method: uniform
+      frame_num: 3
+      keep_original_sample: false
+      caption_num: 1
+      keep_candidate_mode: random_any
+      num_proc: 1
+
+  - video_frames_text_similarity_filter:
+      video_key: video_path
+      min_score: 0.25
+      num_proc: 1
+
+  - ray_video_deduplicator:            # ← 不是 ray_video_deduplicator
+      video_key: video_path
+      backend: ray_actor
+
@@ -0,0 +1,36 @@
+import ray
+import time
+import os
+import sys
+
+if __name__ == "__main__":
+    # ✅ 关键修复：设置环境变量以缓解 Windows 端口冲突
+    os.environ["RAY_DEDUP_LOGS"] = "0"
+    
+    try:
+        # 连接现有集群
+        ray.init(address="auto")
+        print("✅ 成功连接到现有 Ray 集群")
+    except Exception as e:
+        print(f"❌ 连接失败: {e}")
+        sys.exit(1)
+
+    def process_data(row):
+        val = row.get('item')
+        time.sleep(1)
+        return {"result": f"Worker(PID:{os.getpid()}) 处理了 {val}"}
+
+    print("🚀 开始处理...")
+    ds = ray.data.from_items(list(range(10)))
+    
+    # ✅ 增加重试机制，避免端口抖动导致任务直接崩溃
+    ctx = ray.data.DataContext.get_current()
+    ctx.max_errored_blocks = 3  # 允许最多 3 个 Block 失败重试
+
+    try:
+        results = ds.map(process_data).take_all()
+        print("\n✅ 结果:")
+        for r in results:
+            print(r)
+    except Exception as e:
+        print(f"❌ 执行失败: {e}")
@@ -0,0 +1,6 @@
+## Sheet1
+| aaa | Unnamed: 1 | Unnamed: 2 | Unnamed: 3 |
+| --- | --- | --- | --- |
+| NaN | NaN | NaN | NaN |
+| NaN | NaN | NaN | NaN |
+| NaN | NaN | NaN | =DISPIMG("ID\_12B1BF73B293441E8EEB30BFE589089B",1) |
				`@@ -0,0 +1 @@`
				{"text": "2026-09-20 15:55:14.070 \| WARNING \| data_juicer.core.data.load_strategy:129 - No matching strategy found for combination exec: default, data_type: huggingface, data_source: \n", "record": {"elapsed": {"repr": "0:01:31.994509", "seconds": 91.994509}, "exception": null, "extra": {}, "file": {"name": "load_strategy.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\core\\data\\load_strategy.py"}, "function": "get_strategy_class", "level": {"icon": "⚠️", "name": "WARNING", "no": 30}, "line": 129, "message": "No matching strategy found for combination exec: default, data_type: huggingface, data_source: ", "module": "load_strategy", "name": "data_juicer.core.data.load_strategy", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:55:14.070196+08:00", "timestamp": 1789890914.070196}}}