feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更，主要内容：新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
@@ -0,0 +1,4 @@
+[{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/1.mp4"], "text": "<__dj__video>"},
+{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/2.mp4"], "text": "<__dj__video>"},
+{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/3.mp4"], "text": "<__dj__video>"},
+{"video_path": ["/home/apesea/xuhq/codes/DeepData/test/ray/videos/3-副本.mp4"], "text": "<__dj__video>"}]
@@ -0,0 +1,4 @@
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/videos/1.mp4"], "text": "<__dj__video>"}
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/2.mp4"], "text": "<__dj__video>"}
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/videos/3.mp4"], "text": "<__dj__video>"}
+{"video_path": ["/flow/61ad6137-e532-48f3-8324-7e874445e93a/videos/3-副本.mp4"], "text": "<__dj__video>"}
@@ -0,0 +1,128 @@
+import inspect
+import json
+from typing import Dict, List, Any
+from data_juicer.ops.base_op import OPERATORS
+
+# ========= 配置区 =========
+# 1. 基类公共参数：所有Mapper/Filter通用，独立公共面板
+BASE_COMMON_PARAMS = {
+    "video_key": {"type": "str", "default": "video_path", "desc": "样本中视频列表字段名"},
+    "audio_key": {"type": "str", "default": "audios", "desc": "样本中音频列表字段名"},
+    "text_key": {"type": "str", "default": "text", "desc": "样本文本字段名"},
+}
+
+# 2. 高级运行时参数：全局自动注入，默认折叠，仅高级用户修改
+ADVANCED_RUNTIME_PARAMS = {
+    "accelerator": {"type": "str", "default": None, "desc": "加速设备 cpu/gpu"},
+    "num_proc": {"type": "int", "default": 1, "desc": "单算子并行进程数"},
+    "cpu_required": {"type": "float", "default": None, "desc": "Ray每个worker需要CPU资源"},
+    "gpu_required": {"type": "float", "default": None, "desc": "Ray每个worker需要GPU资源"},
+    "mem_required": {"type": "str", "default": None, "desc": "内存资源限制"},
+    "work_dir": {"type": "str", "default": None, "desc": "算子工作目录"},
+    "skip_op_error": {"type": "bool", "default": False, "desc": "是否跳过算子执行异常"},
+    "turbo": {"type": "bool", "default": False, "desc": "turbo加速开关"},
+    "history_key": {"type": "str", "default": "history", "desc": "对话历史字段"},
+    "prompt_key": {"type": "str", "default": "prompt", "desc": "prompt字段"},
+    "query_key": {"type": "str", "default": "query", "desc": "query字段"},
+    "response_key": {"type": "str", "default": "response", "desc": "response字段"},
+    "instruction_key": {"type": "str", "default": "instruction", "desc": "指令字段"},
+    "index_key": {"type": "str", "default": None, "desc": "索引字段"},
+}
+
+# 3. 已知枚举字典（DJ1.4.4 视频算子，可持续扩充）
+OP_ENUM_MAP: Dict[str, Dict[str, List[Any]]] = {
+    "video_extract_frames_mapper": {
+        "output_format": ["path", "bytes"],
+        "frame_sampling_method": ["all_keyframes", "uniform"],
+    },
+    "video_duration_filter": {
+        "any_or_all": ["any", "all"],
+    },
+    "video_aesthetics_filter": {
+        "reduce_mode": ["avg", "max", "min"],
+        "any_or_all": ["any", "all"],
+    },
+    "video_frames_text_similarity_filter": {
+        "reduce_mode": ["avg", "max", "min"],
+        "any_or_all": ["any", "all"],
+    }
+}
+
+# ======================
+def get_op_param_schema(op_name: str, op_cls) -> Dict[str, Any]:
+    """获取单个算子的完整schema"""
+    sig = inspect.signature(op_cls.__init__)
+    native_params = []
+    enum_map = OP_ENUM_MAP.get(op_name, {})
+
+    for param_name, param in sig.parameters.items():
+        # 过滤 self, *args, **kwargs
+        if param_name in ("self", "args", "kwargs"):
+            continue
+
+        default_val = param.default
+        if default_val is inspect.Parameter.empty:
+            default_str = "required"
+        else:
+            default_str = repr(default_val)
+
+        anno = param.annotation
+        enum_list = enum_map.get(param_name, None)
+
+        native_params.append({
+            "name": param_name,
+            "annotation": str(anno),
+            "default": default_str,
+            "enum": enum_list,
+            "ui_type": "select" if enum_list else "input",
+            "description": ""
+        })
+
+    op_info = {
+        "op_name": op_name,
+        "class_name": op_cls.__name__,
+        "op_category": op_cls.__bases__[0].__name__,
+        "accelerator": getattr(op_cls, "_accelerator", "cpu"),
+        "is_batched_op": getattr(op_cls, "_batched_op", False),
+        "docstring": op_cls.__doc__.strip().splitlines()[0] if op_cls.__doc__ else "",
+        "native_params": native_params,  # 主表单：算子原生参数
+        "base_common_params": BASE_COMMON_PARAMS,  # 公共面板
+        "advanced_runtime_params": ADVANCED_RUNTIME_PARAMS  # 高级折叠面板
+    }
+    return op_info
+
+
+def dump_all_ops_schema(save_json_path: str = "dj_ops_schema.json"):
+    """遍历全部算子，生成schema并保存json"""
+    # 加载全部算子，保证懒加载算子注册
+    from data_juicer.ops import load
+    all_ops = []
+    for op_name, op_cls in sorted(OPERATORS.modules.items()):
+        schema = get_op_param_schema(op_name, op_cls)
+        all_ops.append(schema)
+
+    final_payload = {
+        "version": "data-juicer 1.4.4",
+        "total_ops_count": len(all_ops),
+        "operators": all_ops
+    }
+    # 写入json
+    with open(save_json_path, "w", encoding="utf-8") as f:
+        json.dump(final_payload, f, ensure_ascii=False, indent=2)
+    print(f"✅ Schema saved to {save_json_path}, total operators: {len(all_ops)}")
+    return final_payload
+
+
+def filter_op_by_keyword(schema_data, keyword: str):
+    """快速筛选，例如 keyword=video 只返回视频算子"""
+    res = [op for op in schema_data["operators"] if keyword in op["op_name"]]
+    print(f"\n===== filter: contains [{keyword}], count={len(res)} =====")
+    for item in res:
+        print(f"OpName: {item['op_name']}, native params: {[p['name'] for p in item['native_params']]}")
+    return res
+
+
+if __name__ == "__main__":
+    schema_data = dump_all_ops_schema()
+    # 筛选所有video算子，打印预览
+    video_ops = filter_op_by_keyword(schema_data, "video")
@@ -0,0 +1,12 @@
+2026-09-20 15:54:25.045 | INFO     | data_juicer.config.config:init_setup_from_cfg:682 - dataset_path config is set and a valid local path
+2026-09-20 15:54:25.045 | DEBUG    | data_juicer.utils.lazy_loader:_load:406 - Loading ray...
+2026-09-20 15:54:25.047 | DEBUG    | data_juicer.config.config:timing_context:43 - Initializing setup from config took 0.02 seconds
+2026-09-20 15:54:25.286 | DEBUG    | data_juicer.config.config:timing_context:43 - Updating operator process took 0.24 seconds
+2026-09-20 15:54:25.286 | INFO     | data_juicer.config.config:config_backup:998 - Back up the input config file [D:\xu_projects\for_mage\llm\repos\deepdata\test\ray\video_pipeline_ray.yaml] into the work_dir [D:\xu_projects\for_mage\llm\repos\deepdata\test\ray]
+2026-09-20 15:54:25.300 | INFO     | data_juicer.config.config:display_config:1019 - Configuration table: 
+2026-09-20 15:54:25.421 | DEBUG    | data_juicer.config.config:timing_context:43 - Total config initialization time took 1.39 seconds
+2026-09-20 15:55:14.068 | INFO     | data_juicer.core.executor.default_executor:__init__:54 - Using cache compression method: [None]
+2026-09-20 15:55:14.068 | INFO     | data_juicer.core.executor.default_executor:__init__:58 - Setting up dataset builder...
+2026-09-20 15:55:14.069 | INFO     | data_juicer.core.data.dataset_builder:__init__:48 - found dataset_path setting: D:\xu_projects\for_mage\llm\repos\deepdata\test\ray\input_videos_meta.jsonl
+2026-09-20 15:55:14.070 | INFO     | data_juicer.core.data.load_strategy:get_strategy_class:84 - Getting strategy class for exec: default, data_type: huggingface, data_source: None
+2026-09-20 15:55:14.070 | WARNING  | data_juicer.core.data.load_strategy:get_strategy_class:129 - No matching strategy found for combination exec: default, data_type: huggingface, data_source: *
@@ -0,0 +1,4 @@
+{"text": "2026-09-20 15:54:25.045 | DEBUG    | data_juicer.utils.lazy_loader:406 - Loading ray...\n", "record": {"elapsed": {"repr": "0:00:42.969853", "seconds": 42.969853}, "exception": null, "extra": {}, "file": {"name": "lazy_loader.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\utils\\lazy_loader.py"}, "function": "_load", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 406, "message": "Loading ray...", "module": "lazy_loader", "name": "data_juicer.utils.lazy_loader", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.045540+08:00", "timestamp": 1789890865.04554}}}
+{"text": "2026-09-20 15:54:25.047 | DEBUG    | data_juicer.config.config:43 - Initializing setup from config took 0.02 seconds\n", "record": {"elapsed": {"repr": "0:00:42.971854", "seconds": 42.971854}, "exception": null, "extra": {}, "file": {"name": "config.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\config\\config.py"}, "function": "timing_context", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 43, "message": "Initializing setup from config took 0.02 seconds", "module": "config", "name": "data_juicer.config.config", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.047541+08:00", "timestamp": 1789890865.047541}}}
+{"text": "2026-09-20 15:54:25.286 | DEBUG    | data_juicer.config.config:43 - Updating operator process took 0.24 seconds\n", "record": {"elapsed": {"repr": "0:00:43.210449", "seconds": 43.210449}, "exception": null, "extra": {}, "file": {"name": "config.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\config\\config.py"}, "function": "timing_context", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 43, "message": "Updating operator process took 0.24 seconds", "module": "config", "name": "data_juicer.config.config", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.286136+08:00", "timestamp": 1789890865.286136}}}
+{"text": "2026-09-20 15:54:25.421 | DEBUG    | data_juicer.config.config:43 - Total config initialization time took 1.39 seconds\n", "record": {"elapsed": {"repr": "0:00:43.345850", "seconds": 43.34585}, "exception": null, "extra": {}, "file": {"name": "config.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\config\\config.py"}, "function": "timing_context", "level": {"icon": "🐞", "name": "DEBUG", "no": 10}, "line": 43, "message": "Total config initialization time took 1.39 seconds", "module": "config", "name": "data_juicer.config.config", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:54:25.421537+08:00", "timestamp": 1789890865.421537}}}
@@ -0,0 +1 @@
+{"text": "2026-09-20 15:55:14.070 | WARNING  | data_juicer.core.data.load_strategy:129 - No matching strategy found for combination exec: default, data_type: huggingface, data_source: *\n", "record": {"elapsed": {"repr": "0:01:31.994509", "seconds": 91.994509}, "exception": null, "extra": {}, "file": {"name": "load_strategy.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\core\\data\\load_strategy.py"}, "function": "get_strategy_class", "level": {"icon": "⚠️", "name": "WARNING", "no": 30}, "line": 129, "message": "No matching strategy found for combination exec: default, data_type: huggingface, data_source: *", "module": "load_strategy", "name": "data_juicer.core.data.load_strategy", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:55:14.070196+08:00", "timestamp": 1789890914.070196}}}
@@ -0,0 +1,5 @@
+from data_juicer.config import init_configs
+from data_juicer.core import DefaultExecutor
+cfg = init_configs(args=["--config", "./video_pipeline_ray.yaml"])
+executor = DefaultExecutor(cfg)
+executor.run()
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+fast_ray_executor.py —— data-juicer Ray 模式「惰性 DAG」执行器
+
+已验证：253s → 7 个算子 0.02s 挂完（原 8 次全链重跑）
+
+【两个必须同时修的点】
+1) ray_dataset.py:159/196 调 self.data.columns() → Ray Data 内部以 limit(1)
+   执行整条 DAG（O(N^2)）。用静态列名覆盖 columns() 解决。
+2) 覆盖 columns() 会让 DJ "以为" meta/stats 已存在而跳过补列，
+   算子却要读 sample['__dj__meta__'] → KeyError。
+   所以必须【自己正确地把这两列补上】。
+
+【补列陷阱】pyarrow 无法从 [{}...] 推断 struct 类型（长度为 0）：
+   ✗ table.append_column(col, [{} for _ in range(len(table))])
+   ✓ table.append_column(col, pa.array([{}]*n, type=pa.struct([])))
+
+用法: python fast_ray_executor.py ./video_pipeline_ray.yaml
+"""
+import os, sys, time, glob, json
+import ray
+import pyarrow as pa
+
+
+def peek_columns(path):
+    """支持目录：扫描所有 jsonl/json 的第一行，取列名并集。"""
+    cols = set()
+    if os.path.isdir(path):
+        files = []
+        for ext in ("*.jsonl", "*.json"):
+            files.extend(sorted(glob.glob(os.path.join(path, "**", ext), recursive=True)))
+    else:
+        files = [path]
+
+    if not files:
+        print(f"[warn] 目录下没找到 jsonl/json: {path}")
+        return cols
+
+    for fp in files[:50]:                      # 最多扫 50 个，足够推断 schema
+        try:
+            with open(fp, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    if line.startswith("["):   # 兼容 json 数组
+                        line = line.lstrip("[").rstrip(",").rstrip("]")
+                    try:
+                        cols.update(json.loads(line).keys())
+                    except Exception:
+                        pass
+                    break                      # 每个文件只读第一行
+        except Exception as e:
+            print(f"[warn] 跳过 {fp}: {e}")
+    print(f"[列名] 扫描 {min(len(files),50)}/{len(files)} 个文件 -> {sorted(cols)}")
+    return cols
+
+
+def build_lazy_process(state, batch_size):
+    """state: dict(known=set(), ...)  用可变容器让 known 跨调用共享"""
+    known = state["known"]
+
+    def make_add_col(col):
+        def _fn(table):
+            if col in table.column_names:
+                return table
+            n = table.num_rows
+            # 关键：必须显式 pa.struct([])，否则 pyarrow 推断出长度 0
+            arr = pa.array([{} for _ in range(n)], type=pa.struct([]))
+            return table.append_column(col, arr)
+        return _fn
+
+    def lazy_process(self, operators, *, exporter=None, checkpointer=None, tracer=None):
+        if operators is None:
+            return self
+        if not isinstance(operators, list):
+            operators = [operators]
+
+        from data_juicer.utils.process_utils import calculate_ray_np
+        from data_juicer.utils.constant import Fields
+
+        calculate_ray_np(operators)
+
+        # ---- 先补 meta / stats 两列（惰性，不触发执行）----
+        for col in (Fields.meta, Fields.stats):
+            self.data = self.data.map_batches(
+                make_add_col(col), batch_format="pyarrow", batch_size=batch_size
+            )
+            known.add(col)
+        print(f"[预补列] {sorted(known)}")
+
+        # ---- 覆盖 columns()：返回已知列名，杜绝 schema 推理触发执行 ----
+        orig_columns = ray.data.Dataset.columns
+        ray.data.Dataset.columns = lambda ds: list(known)
+
+        try:
+            total = len(operators)
+            t_all = time.time()
+            print(f"\n[惰性 DAG] 开始挂接 {total} 个算子（每项应为毫秒级）")
+
+            for i, op in enumerate(operators, 1):
+                name = type(op).__name__
+                try:
+                    op.num_proc = 1      # concurrency=1 → actor 池=1 → 模型只加载一次
+                except Exception:
+                    pass
+
+                t = time.time()
+                self._run_single_op(op)  # Ray 分支：map_batches，不是 op.run
+                cost = time.time() - t
+
+                flag = "" if cost < 1.0 else "   <-- 异常! 此处触发了 action"
+                print(f"  [{i}/{total}] {name:<42s} {cost * 1000:8.1f} ms{flag}")
+
+                if not isinstance(self.data, ray.data.Dataset):
+                    raise RuntimeError(
+                        f"算子 [{name}] 把 ray.data.Dataset 转成了 {type(self.data).__name__}"
+                    )
+
+            print(f"[惰性 DAG] 构建总耗时: {time.time() - t_all:.2f}s（应接近 0）")
+        finally:
+            ray.data.Dataset.columns = orig_columns   # 还原，导出时用真实 schema
+
+        return self
+
+    return lazy_process
+
+
+def main():
+    cfg_path = sys.argv[1] if len(sys.argv) > 1 else "./video_pipeline_ray.yaml"
+
+    from data_juicer.config import init_configs
+    cfg = init_configs(args=["--config", cfg_path])
+
+    print("=" * 72)
+    print(f"  配置文件      : {cfg_path}")
+    print(f"  executor_type : {cfg.executor_type}")
+    print(f"  export_path   : {cfg.export_path}")
+    print("=" * 72)
+
+    ray.init(address=getattr(cfg, "ray_address", "auto"), ignore_reinit_error=True)
+    print(f"[Ray] 已连接: {ray.get_runtime_context().gcs_address}")
+
+    from data_juicer.core.data import ray_dataset as rd_mod
+
+    RayDataset = rd_mod.RayDataset
+    batch_size = getattr(rd_mod, "DEFAULT_BATCH_SIZE", 1024)
+
+    state = {"known": peek_columns(cfg.dataset_path)}
+    RayDataset.process = build_lazy_process(state, batch_size)
+
+    from data_juicer.core import ExecutorFactory
+    executor_cls = ExecutorFactory.create_executor(cfg.executor_type)
+    print(f"[执行器] {executor_cls.__name__}")
+    assert executor_cls.__name__ == "RayExecutor", "必须拿到 RayExecutor"
+
+    export_path = cfg.export_path
+    # 清掉上一轮遗留的分片（否则会被误合并进最终结果）
+    if os.path.isdir(export_path):
+        for f in glob.glob(os.path.join(export_path, "*.json")):
+            os.remove(f)
+    elif os.path.exists(export_path):
+        os.remove(export_path)          # Ray 的 write_json 是目录语义
+    os.makedirs(export_path, exist_ok=True)
+
+    executor = executor_cls(cfg)
+    t0 = time.time()
+    executor.run()
+    print(f"\n总耗时: {time.time() - t0:.1f}s")
+
+    shards = sorted(glob.glob(os.path.join(export_path, "*.json")))
+    if shards:
+        merged = export_path.rstrip("/") + ".jsonl"
+        n = 0
+        with open(merged, "w", encoding="utf-8") as out:
+            for s in shards:
+                for line in open(s, encoding="utf-8"):
+                    line = line.strip()
+                    if line:
+                        out.write(line + "\n")
+                        n += 1
+        print(f"\n已合并 {len(shards)} 个分片 -> {merged}（{n} 条样本）")
+        for line in open(merged, encoding="utf-8"):
+            d = json.loads(line)
+            vp = d.get("video_path", ["?"])[0].split("/")[-1]
+            print(f"  - {vp}")
+            print(f"    text : {d.get('text', '')[:70]}")
+            print(f"    meta : {d.get('__dj__meta__', {})}")
+            print(f"    stats: {d.get('__dj__stats__', {})}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,10 @@
+from data_juicer.config import init_configs
+from data_juicer.core import ExecutorFactory
+import sys
+
+cfg_path = sys.argv[1] if len(sys.argv) > 1 else "./video_pipeline_ray.yaml"
+cfg = init_configs(args=["--config", cfg_path])
+
+executor_cls = ExecutorFactory.create_executor(cfg.executor_type)
+print(">>> 执行器:", executor_cls.__name__)     # 必须打印 RayExecutor
+executor_cls(cfg).run()
@@ -0,0 +1,58 @@
+project_name: t-final
+dataset_path: "/home/apesea/xuhq/codes/DeepData/test/ray/input"
+executor_type: ray
+ray_address: 127.0.0.1:6388
+export_type: jsonl
+use_cache: false
+export_path: "/home/apesea/xuhq/codes/DeepData/test/ray/out_final"
+skip_op_error: false
+video_key: video_path
+open_monitor: false
+text_keys: ['text']
+np: 1
+keep_stats_in_res_ds: True          # ← 让 video_audio_tags 等 meta 字段导出
+
+process:
+  - video_extract_frames_mapper:
+      frame_num: 5
+      frame_sampling_method: uniform
+      video_key: video_path
+      output_format: path
+      frame_dir: "./frames"
+      num_proc: 1
+
+  - video_tagging_from_audio_mapper:
+      video_key: video_path
+      num_proc: 1
+      hf_ast: 'MIT/ast-finetuned-audioset-10-10-0.4593'
+
+  - video_duration_filter:
+      video_key: video_path
+      min_duration: 3
+      max_duration: 60
+      num_proc: 1
+
+  - video_aesthetics_filter:
+      video_key: video_path
+      min_score: 0.4
+      num_proc: 1
+
+  - video_captioning_from_frames_mapper:
+      video_key: video_path
+      hf_img2seq: Salesforce/blip-image-captioning-base   # ~990MB，稳
+      frame_sampling_method: uniform
+      frame_num: 3
+      keep_original_sample: false
+      caption_num: 1
+      keep_candidate_mode: random_any
+      num_proc: 1
+
+  - video_frames_text_similarity_filter:
+      video_key: video_path
+      min_score: 0.25
+      num_proc: 1
+
+  - ray_video_deduplicator:            # ← 不是 ray_video_deduplicator
+      video_key: video_path
+      backend: ray_actor
+
				`@@ -0,0 +1 @@`
				{"text": "2026-09-20 15:55:14.070 \| WARNING \| data_juicer.core.data.load_strategy:129 - No matching strategy found for combination exec: default, data_type: huggingface, data_source: \n", "record": {"elapsed": {"repr": "0:01:31.994509", "seconds": 91.994509}, "exception": null, "extra": {}, "file": {"name": "load_strategy.py", "path": "d:\\miniconda3\\envs\\python3.11\\Lib\\site-packages\\data_juicer\\core\\data\\load_strategy.py"}, "function": "get_strategy_class", "level": {"icon": "⚠️", "name": "WARNING", "no": 30}, "line": 129, "message": "No matching strategy found for combination exec: default, data_type: huggingface, data_source: ", "module": "load_strategy", "name": "data_juicer.core.data.load_strategy", "process": {"id": 3412, "name": "MainProcess"}, "thread": {"id": 24364, "name": "MainThread"}, "time": {"repr": "2026-09-20 15:55:14.070196+08:00", "timestamp": 1789890914.070196}}}