figmar/deepdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

Browse Source 
本次同步 09-12 至 09-28 的代码变更，主要内容：

新增功能
- 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService
  对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口
- 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、
  ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器
- 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator
- 新增 tools/minio-file-server MinIO 文件服务

功能增强
- 数据清洗：cleaning_task_service/scheduler/routes 重构，
  data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整
- 数据标注：annotation sync.py、project.py 接口与schema 更新
- 数据采集、算子市场、首页与菜单路由相关页面调整
- DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）

结构调整
- deepflow/ 更名为 deep-flow/（419 个文件重命名）
- 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本

未纳入本次提交
- scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录）
- *.backup*、__pycache__ 产物
- zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物）
- DataX/（内嵌独立仓库，远程亦未收录其源码）
This commit is contained in:
沃巴蒂
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/deepdata-python/app/module/cleaning/service/cleaning_task_service.py
+278 -7
View File
@@ -131,6 +131,7 @@ class CleaningTaskService:
"""Count cleaning tasks using SQL COUNT"""
return await self.task_repo.count_tasks(db, status, keyword)
'''
async def get_task(self, db: AsyncSession, task_id: str) -> CleaningTaskDto:
"""Get task by ID"""
task = await self.task_repo.find_task_by_id(db, task_id)
@@ -168,6 +169,64 @@ class CleaningTaskService:
task.instance.append(OperatorInstanceDto(id=inst.operator_id))
return task
'''
async def get_task(self, db: AsyncSession, task_id: str) -> CleaningTaskDto:
"""Get task by ID"""
task = await self.task_repo.find_task_by_id(db, task_id)
if not task:
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
await self._set_process(db, task)
instances = await self.operator_instance_repo.find_operator_by_instance_id(
db, task_id
)
# Batch query operators
all_operators = await self.operator_service.get_operators(
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
)
operator_map = {op.id: op for op in all_operators}
task.instance = []
for inst in instances:
operator = operator_map.get(inst.operator_id)
if operator:
# 创建 DTO 并填充基本信息
operator_dto = OperatorInstanceDto(
id=operator.id,
name=operator.name,
description=operator.description,
inputs=operator.inputs,
outputs=operator.outputs,
settings=json.loads(operator.settings) if operator.settings else {},
categories=operator.categories,
)
# 🔴 核心修复：解析并应用 overrides
if inst.settings_override:
try:
operator_dto.overrides = json.loads(inst.settings_override)
except (json.JSONDecodeError, TypeError) as e:
logger.error(
f"Failed to parse settings_override for {inst.operator_id}: {e}"
)
task.instance.append(operator_dto)
else:
# 容错：如果找不到算子元数据
operator_dto = OperatorInstanceDto(id=inst.operator_id)
if inst.settings_override:
try:
operator_dto.overrides = json.loads(inst.settings_override)
except (json.JSONDecodeError, TypeError):
pass
task.instance.append(operator_dto)
return task
async def create_task(
self, db: AsyncSession, request: CreateCleaningTaskRequest
@@ -355,6 +414,7 @@ class CleaningTaskService:
logger.error(f"Failed to parse runtime config: {e}")
return {}
'''
async def scan_dataset(
self,
db: AsyncSession,
@@ -399,6 +459,72 @@ class CleaningTaskService:
"metadata": metadata_dict,
}
f.write(json.dumps(file_info, ensure_ascii=False) + "\n")
'''
async def scan_dataset(
self,
db: AsyncSession,
task_id: str,
src_dataset_id: str,
succeed_files: Set[str] | None = None,
) -> None:
"""Scan source dataset and create dataset.jsonl"""
target_file_path = Path(f"{FLOW_PATH}/{task_id}/dataset.jsonl")
target_file_path.parent.mkdir(parents=True, exist_ok=True)
logger.info(f"🔍 [Scan] 开始扫描数据集 {src_dataset_id} ...")
query = text("""
SELECT id, file_name, file_path, file_type, file_size, metadata
FROM t_dm_dataset_files
WHERE dataset_id = :dataset_id
ORDER BY created_at
""")
result = await db.execute(query, {"dataset_id": src_dataset_id})
files = result.fetchall()
logger.info(f"🔥 [Scan] 任务所属数据集: {src_dataset_id}")
logger.info(f"🔥 [Scan] 数据库中文件数: {len(files)}") # 🔴 加这行
total_found = len(files)
skipped_count = 0
written_count = 0
if total_found == 0:
logger.warning("⚠️ [Scan] 数据库中没有找到文件！请检查数据集是否正确上传了文件。")
with open(target_file_path, "w", encoding="utf-8") as f:
for file in files:
# 核心逻辑：如果文件已处理过，则跳过
if succeed_files and file.id in succeed_files:
skipped_count += 1
continue
written_count += 1
metadata_dict = {}
if file.metadata:
try:
parsed = json.loads(file.metadata)
if isinstance(parsed, dict):
metadata_dict = parsed
except (json.JSONDecodeError, TypeError):
pass
file_info = {
"fileId": file.id,
"fileName": file.file_name,
"filePath": file.file_path,
"fileType": file.file_type,
"fileSize": file.file_size,
"metadata": metadata_dict,
}
f.write(json.dumps(file_info, ensure_ascii=False) + "\n")
logger.info(f"✅ [Scan] 扫描完成: 共 {total_found} 个 | 跳过 {skipped_count} 个 | 写入 {written_count} 个。")
if written_count == 0 and total_found > 0:
logger.warning("⚠️ [Scan] 所有文件都被跳过（因为之前已处理成功）。如果修改了参数，请清空历史结果或添加新文件。")
async def get_task_results(
self, db: AsyncSession, task_id: str
@@ -406,17 +532,44 @@ class CleaningTaskService:
"""Get task results"""
return await self.result_repo.find_by_instance_id(db, task_id)
def _resolve_log_path(self, task_id: str, retry_count: int) -> Path | None:
"""
解析日志路径（严格模式）：
只查找特定的 output.log 或 output.log.N。
如果文件不存在，返回 None，让调用者（前端或路由）处理等待逻辑。
"""
safe_task_id = _sanitize_task_id(task_id)
flow_root = Path(FLOW_PATH).resolve()
task_dir = flow_root / safe_task_id
# 构建目标文件名
target_name = "output.log" if retry_count == 0 else f"output.log.{retry_count}"
target_path = task_dir / target_name
# 严格检查：存在则返回，不存在则返回 None
if target_path.exists():
return target_path
return None
async def get_task_log(
self, db: AsyncSession, task_id: str, retry_count: int
) -> List[CleaningTaskLog]:
"""Get task log"""
"""Get task log (严格匹配模式，禁止兜底旧日志)"""
safe_task_id = _sanitize_task_id(task_id)
safe_retry_count = _sanitize_retry_count(retry_count)
flow_root = Path(FLOW_PATH).resolve()
log_path = flow_root / safe_task_id / "output.log"
if safe_retry_count > 0:
log_path = flow_root / safe_task_id / f"output.log.{safe_retry_count}"
task_dir = flow_root / safe_task_id
# 1. 严格定位指定的日志文件路径
log_path = task_dir / ("output.log" if safe_retry_count == 0 else f"output.log.{safe_retry_count}")
# 🔴 核心修改：移除 glob 兜底逻辑
# 如果文件不存在，说明 Runtime 尚未生成或正在初始化写入。
# 直接返回空列表，由前端根据 TaskStatus 显示 Loading/等待状态。
if not log_path.exists():
return []
# 防止路径穿越：规范化后校验仍在 FLOW_PATH 下
resolved_log_path = log_path.resolve()
@@ -424,6 +577,7 @@ class CleaningTaskService:
logger.warning(f"Path traversal attempt detected: task_id={task_id}")
return []
# 二次校验（安全冗余）
if not resolved_log_path.exists():
return []
@@ -444,6 +598,7 @@ class CleaningTaskService:
return logs
def _get_log_level(
self, line: str, default_level: str, std_pattern, ex_pattern
) -> str:
@@ -503,9 +658,16 @@ class CleaningTaskService:
async def execute_task(self, db: AsyncSession, task_id: str) -> bool:
"""Execute task"""
logger.info(f"🚀 [Service] 任务 {task_id} 开始执行...")
# 目前是全量扫描源数据集，并不考虑已经处理过的文件
succeed_set = None
'''
succeeded = await self.result_repo.find_by_instance_id(db, task_id, "COMPLETED")
succeed_set = {res.src_file_id for res in succeeded}
logger.info(f"📊 [Execute] 发现 {len(succeed_set)} 个历史成功文件，将采用增量模式（跳过已处理文件）。")
'''
task = await self.task_repo.find_task_by_id(db, task_id)
if not task:
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
@@ -516,13 +678,16 @@ class CleaningTaskService:
task.file_count = src_dataset.fileCount
await self.task_repo.update_task(db, task)
await self.scan_dataset(db, task_id, task.src_dataset_id, succeed_set)
await self.scan_dataset(db, task_id, task.src_dataset_id, succeed_files=succeed_set)
await self.result_repo.delete_by_instance_id(db, task_id, "FAILED")
try:
return await self.scheduler.execute_task(
logger.info(f"[Service] Calling scheduler.execute_task for {task_id}, retry: {(task.retry_count or 0) + 1}")
result = await self.scheduler.execute_task(
db, task_id, (task.retry_count or 0) + 1
)
logger.info(f"[Service] Scheduler returned result for {task_id}: {result}")
return result
except Exception as e:
logger.error(
f"execute_task failed, task_id={task_id}, retry_count={(task.retry_count or 0) + 1}: {e}"
@@ -576,3 +741,109 @@ class CleaningTaskService:
result.append(operator_dto)
return result
async def update_task_config(self, db: AsyncSession, task_id: str, data: dict) -> None:
"""
更新任务配置（全量更新算子列表和参数）
"""
# 1. 检查任务是否存在
task = await self.task_repo.find_task_by_id(db, task_id)
if not task:
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
# 2. 运行中的任务不可修改
if task.status == CleaningTaskStatus.RUNNING:
raise BusinessError(
ErrorCodes.CLEANING_TASK_STATUS_INVALID,
"任务正在运行中，无法修改配置。请先停止任务。"
)
# 3. 更新任务基本信息 (名称、描述)
if "name" in data:
task.name = data["name"]
if "description" in data:
task.description = data["description"]
# 🔴 修复：删除 task.updated_at = datetime.now()
# 因为 CleaningTaskDto 中没有 updated_at 字段
await self.task_repo.update_task(db, task)
# 4. 更新算子配置 (先删后插)
instances_data = data.get("instance", [])
# 4a. 删除旧的算子关联
await self.operator_instance_repo.delete_by_instance_id(db, task_id)
if instances_data:
# 4b. 准备写入数据
wrapped_instances = []
for d in instances_data:
op_id = d.get("id")
if not op_id:
logger.warning(f"算子 ID 缺失，跳过")
continue
# 确保 overrides 不为 None
overrides = d.get("overrides")
if overrides is None:
overrides = {}
# 创建一个简单的对象适配 Repository 的 insert_instance 方法
class InstanceWrapper:
def __init__(self, oid, over):
self.id = oid
self.overrides = over
wrapped_instances.append(InstanceWrapper(op_id, overrides))
# 写入数据库
if wrapped_instances:
await self.operator_instance_repo.insert_instance(db, task_id, wrapped_instances)
# 5. 重新生成 Process YAML 文件
try:
all_operators = await self.operator_service.get_operators(
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
)
op_map = {op.id: op for op in all_operators}
enriched_instances = []
for d in instances_data:
op_id = d.get("id")
if not op_id: continue
op = op_map.get(op_id)
if op:
enriched_instances.append(
OperatorInstanceDto(
id=op_id,
name=d.get("name", op.name),
inputs=op.inputs,
outputs=op.outputs,
settings=op.settings,
overrides=d.get("overrides", {}),
categories=op.categories,
)
)
else:
# 容错：如果找不到元数据，使用基础信息
enriched_instances.append(
OperatorInstanceDto(
id=op_id,
name=d.get("name", "Unknown Operator"),
inputs="",
outputs="",
overrides=d.get("overrides", {}),
)
)
executor_type = self.validator.check_and_get_executor_type(enriched_instances)
# 复用已有的 prepare_task 生成 YAML
await self.prepare_task(task.dest_dataset_id, task_id, enriched_instances, op_map, executor_type)
except Exception as e:
logger.error(f"生成 YAML 配置失败: {e}", exc_info=True)
raise BusinessError(ErrorCodes.CLEANING_FILE_SYSTEM_ERROR, f"生成配置文件失败: {str(e)}")