feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强,deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更,主要内容:

新增功能
- 修改密码:后端新增 ChangePasswordRequest DTO 与UserController/UserService
  对应接口,前端新增 ChangePasswordDialog 并在 Header 接入入口
- 文件预览组件重构:file-preview 目录拆分出 VideoPreview(支持 blobUrl 直传)、
  ImagePreview、AudioPreview,index.tsx 统一调度各类型预览器
- 新增 my_ops 算子模块:markitdown_parser、mineru_pdf_parser、text_quality_evaluator
- 新增 tools/minio-file-server MinIO 文件服务

功能增强
- 数据清洗:cleaning_task_service/scheduler/routes 重构,
  data_juicer_executor 与 wrapper 增强,OperatorTable/LogsTable/TaskDetail 同步调整
- 数据标注:annotation sync.py、project.py 接口与schema 更新
- 数据采集、算子市场、首页与菜单路由相关页面调整
- DataJuicer 算子接入(useOperatorOperations、dj_ops_schema)

结构调整
- deepflow/ 更名为 deep-flow/(419 个文件重命名)
- 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本

未纳入本次提交
- scripts-backup/、runtime/python-executor-backup/、components_backup/(备份目录)
- *.backup*、__pycache__ 产物
- zh_core_web_sm-3.8.0-py3-none-any.whl(46MB 构建产物)
- DataX/(内嵌独立仓库,远程亦未收录其源码)
This commit is contained in:
沃巴蒂
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
+146
View File
@@ -0,0 +1,146 @@
"""
从 py-data-juicer 包提取算子参数,生成PostgreSQL算子初始化SQL(填充settings)
依赖:pip install data-juicer
"""
import importlib
import inspect
import json
from typing import Dict, Any, List
from data_juicer.ops.base_op import BaseOP
from data_juicer.ops import ALL_OPS
# --------------------------
# 前端表单schema映射规则(参考原sql脚本settings结构)
# 原settings样例:
# {"repeatPhraseRatio": {
# "name": "文档词重复率",
# "description": "xxx",
# "type": "slider",
# "defaultVal":0.5,
# "min":0,"max":1,"step":0.1
# }}
# 支持类型:input / inputNumber / slider / switch / select / range
# --------------------------
def pydantic_field_to_form_schema(field) -> Dict[str, Any]:
"""
将data‑juicer算子pydantic参数Field转为前端表单配置
"""
finfo = field.field_info
field_name = field.name
desc = finfo.description or ""
default = finfo.default if finfo.default is not None else None
schema = {
"name": field_name,
"description": desc,
"defaultVal": default,
"required": finfo.is_required() is False
}
# 推断控件类型
field_type = finfo.annotation
if hasattr(finfo, "ge") or hasattr(finfo, "le"):
schema["min"] = getattr(finfo, "ge", None)
schema["max"] = getattr(finfo, "le", None)
if hasattr(finfo, "multiple_of"):
schema["step"] = finfo.multiple_of
schema["type"] = "slider"
elif field_type is bool:
schema["type"] = "switch"
schema["checkedLabel"] = "开启"
schema["unCheckedLabel"] = "关闭"
elif getattr(finfo, "enum", None):
# 枚举 -> select下拉框
schema["type"] = "select"
schema["options"] = [{"label": e.value, "value": e.value} for e in finfo.enum]
elif str(field_type).startswith("int") or str(field_type).startswith("float"):
schema["type"] = "inputNumber"
elif "List" in str(field_type) or "Tuple" in str(field_type):
schema["type"] = "range"
else:
schema["type"] = "input"
return schema
def extract_op_settings(op_cls: BaseOP) -> str | None:
"""
从算子类提取所有配置参数,生成settings JSON字符串
"""
if not hasattr(op_cls, "__pydantic_fields__"):
return None
pydantic_fields = op_cls.__pydantic_fields__
settings_dict = {}
for fname, field in pydantic_fields.items():
# skip 内部私有参数
if fname.startswith("_"):
continue
form_schema = pydantic_field_to_form_schema(field)
settings_dict[fname] = form_schema
if len(settings_dict) == 0:
return None
return json.dumps(settings_dict, ensure_ascii=False)
def gen_operator_sql(op_cls) -> str:
"""生成单条算子insert sql"""
op_id = op_cls._name # data‑juicer算子唯一id,例如 document_deduplicator
op_full_name = op_cls.__name__
# 获取注释作为描述
doc = inspect.getdoc(op_cls) or ""
# inputs/outputs
inputs = ",".join(op_cls.ACCEPTED_MODALITIES)
outputs = ",".join(op_cls.OUTPUT_MODALITIES)
version = "1.4.4"
settings_json = extract_op_settings(op_cls)
file_name = ""
file_size = None
is_star = False
created_by = "system"
updated_by = "system"
def escape_sql_str(s):
if s is None:
return "NULL"
return "'" + str(s).replace("'", "''") + "'"
cols = [
"id", "name", "description", "version", "inputs", "outputs",
"runtime", "settings", "file_name", "file_size", "is_star", "created_by", "updated_by"
]
vals = [
escape_sql_str(op_id),
escape_sql_str(op_full_name),
escape_sql_str(doc[:2000]),
escape_sql_str(version),
escape_sql_str(inputs),
escape_sql_str(outputs),
"NULL",
escape_sql_str(settings_json),
escape_sql_str(file_name),
"NULL" if file_size is None else str(file_size),
"false",
escape_sql_str(created_by),
escape_sql_str(updated_by),
]
sql_line = f"INSERT INTO t_operator ({','.join(cols)}) VALUES ({','.join(vals)}) ON CONFLICT DO NOTHING;"
return sql_line
def main():
output_sql_lines: List[str] = []
print(f"total op count: {len(ALL_OPS)}")
for op_cls in ALL_OPS.values():
sql = gen_operator_sql(op_cls)
output_sql_lines.append(sql)
# 输出完整脚本
final_sql = "\n".join(output_sql_lines)
with open("./data_juicer_ops_gen.sql", "w", encoding="utf‑8") as f:
f.write("-- auto generated data‑juicer operator init sql with settings\n")
f.write(final_sql)
print("Generated file: data_juicer_ops_gen.sql")
print(final_sql[:2000])
if __name__ == "__main__":
main()