沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
487 lines
16 KiB
Python
487 lines
16 KiB
Python
from typing import Optional
|
||
import re
|
||
import asyncio
|
||
import json
|
||
from pathlib import Path
|
||
from fastapi.responses import StreamingResponse
|
||
|
||
from fastapi import APIRouter, Depends
|
||
from fastapi.responses import StreamingResponse
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.logging import get_logger
|
||
from app.db.session import get_db
|
||
from app.module.cleaning.schema import (
|
||
CleaningTaskDto,
|
||
CreateCleaningTaskRequest,
|
||
CleaningResultDto,
|
||
CleaningTaskLog,
|
||
)
|
||
from app.module.cleaning.service import CleaningTaskService
|
||
from app.module.shared.schema import StandardResponse, PaginatedData
|
||
from fastapi import Query
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
# Module-level pattern for task_id sanitization (CodeQL sanitizer)
|
||
_TASK_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
|
||
|
||
|
||
def _sanitize_task_id(task_id: str) -> str:
|
||
"""Validate and return sanitized task_id for safe path construction.
|
||
|
||
CodeQL recognizes the return value of this function as sanitized,
|
||
allowing safe use in path construction.
|
||
"""
|
||
if not _TASK_ID_PATTERN.fullmatch(task_id):
|
||
raise ValueError(f"Invalid task_id: {task_id}")
|
||
return task_id
|
||
|
||
|
||
def _sanitize_retry_count(retry_count: int) -> int:
|
||
"""Validate and return sanitized retry_count for safe path construction.
|
||
|
||
CodeQL recognizes the return value of this function as sanitized,
|
||
allowing safe use in path construction.
|
||
"""
|
||
if retry_count < 0 or retry_count > 1000:
|
||
raise ValueError(f"Invalid retry_count: {retry_count}")
|
||
return retry_count
|
||
|
||
router = APIRouter(prefix="/cleaning/tasks", tags=["Cleaning Tasks"])
|
||
|
||
|
||
def _get_operator_service():
|
||
"""Get operator service"""
|
||
from app.module.operator.service import OperatorService
|
||
from app.module.operator.repository import (
|
||
OperatorRepository,
|
||
CategoryRelationRepository,
|
||
OperatorReleaseRepository,
|
||
)
|
||
from app.module.operator.parsers import ParserHolder
|
||
from app.module.shared.file_service import FileService
|
||
from app.module.shared.chunk_upload_repository import ChunkUploadRepository
|
||
|
||
return OperatorService(
|
||
operator_repo=OperatorRepository(None),
|
||
category_relation_repo=CategoryRelationRepository(None),
|
||
operator_release_repo=OperatorReleaseRepository(None),
|
||
parser_holder=ParserHolder(),
|
||
file_service=FileService(ChunkUploadRepository()),
|
||
)
|
||
|
||
|
||
def _get_task_service(db: AsyncSession) -> CleaningTaskService:
|
||
"""Get cleaning task service instance"""
|
||
from app.module.cleaning.service import (
|
||
CleaningTaskScheduler,
|
||
CleaningTaskValidator,
|
||
)
|
||
from app.module.cleaning.service.cleaning_task_scheduler import get_scheduler
|
||
from app.module.cleaning.repository import (
|
||
CleaningTaskRepository,
|
||
CleaningResultRepository,
|
||
OperatorInstanceRepository,
|
||
)
|
||
from app.module.dataset.service import DatasetManagementService
|
||
from app.module.shared.common.lineage import LineageService
|
||
|
||
scheduler = get_scheduler() # 使用全局单例，保持轮询状态跨请求共享
|
||
operator_service = _get_operator_service()
|
||
dataset_service = DatasetManagementService(db)
|
||
lineage_service = LineageService(db)
|
||
|
||
task_repo = CleaningTaskRepository(None)
|
||
|
||
return CleaningTaskService(
|
||
task_repo=task_repo,
|
||
result_repo=CleaningResultRepository(None),
|
||
operator_instance_repo=OperatorInstanceRepository(None),
|
||
operator_service=operator_service,
|
||
scheduler=scheduler,
|
||
validator=CleaningTaskValidator(task_repo=task_repo, template_repo=None),
|
||
dataset_service=dataset_service,
|
||
lineage_service=lineage_service,
|
||
)
|
||
|
||
|
||
@router.get(
|
||
"",
|
||
response_model=StandardResponse[PaginatedData[CleaningTaskDto]],
|
||
summary="查询清洗任务列表",
|
||
description="根据参数查询清洗任务列表（支持分页、状态过滤、关键词搜索）",
|
||
tags=["mcp"],
|
||
)
|
||
async def get_cleaning_tasks(
|
||
page: int = 0,
|
||
size: int = 10,
|
||
status: Optional[str] = None,
|
||
keyword: Optional[str] = None,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Query cleaning tasks"""
|
||
task_service = _get_task_service(db)
|
||
|
||
tasks = await task_service.get_tasks(db, status, keyword, page, size)
|
||
count = await task_service.count_tasks(db, status, keyword)
|
||
total_pages = (count + size - 1) // size if size > 0 else 0
|
||
|
||
return StandardResponse(
|
||
code="0",
|
||
message="success",
|
||
data=PaginatedData(
|
||
page=page,
|
||
size=size,
|
||
total_elements=count,
|
||
total_pages=total_pages,
|
||
content=tasks,
|
||
),
|
||
)
|
||
|
||
|
||
@router.post(
|
||
"",
|
||
response_model=StandardResponse[CleaningTaskDto],
|
||
summary="创建清洗任务",
|
||
description="根据模板ID或算子列表创建清洗任务",
|
||
tags=["mcp"],
|
||
)
|
||
async def create_cleaning_task(
|
||
request: CreateCleaningTaskRequest,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Create cleaning task"""
|
||
task_service = _get_task_service(db)
|
||
|
||
task = await task_service.create_task(db, request)
|
||
await db.commit()
|
||
|
||
await task_service.execute_task(db, task.id)
|
||
await db.commit()
|
||
|
||
return StandardResponse(code="0", message="success", data=task)
|
||
|
||
|
||
@router.get(
|
||
"/{task_id}",
|
||
response_model=StandardResponse[CleaningTaskDto],
|
||
summary="获取清洗任务详情",
|
||
description="根据ID获取清洗任务详细信息",
|
||
)
|
||
async def get_cleaning_task(
|
||
task_id: str,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Get cleaning task by ID"""
|
||
task_service = _get_task_service(db)
|
||
task = await task_service.get_task(db, task_id)
|
||
return StandardResponse(code="0", message="success", data=task)
|
||
|
||
|
||
@router.delete(
|
||
"/{task_id}",
|
||
response_model=StandardResponse[str],
|
||
summary="删除清洗任务",
|
||
description="删除指定的清洗任务",
|
||
)
|
||
async def delete_cleaning_task(
|
||
task_id: str,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Delete cleaning task"""
|
||
task_service = _get_task_service(db)
|
||
await task_service.delete_task(db, task_id)
|
||
await db.commit()
|
||
return StandardResponse(code="0", message="success", data=task_id)
|
||
|
||
|
||
@router.post(
|
||
"/{task_id}/stop",
|
||
response_model=StandardResponse[str],
|
||
summary="停止清洗任务",
|
||
description="停止正在运行的清洗任务",
|
||
)
|
||
async def stop_cleaning_task(
|
||
task_id: str,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Stop cleaning task"""
|
||
task_service = _get_task_service(db)
|
||
await task_service.stop_task(db, task_id)
|
||
await db.commit()
|
||
return StandardResponse(code="0", message="success", data=task_id)
|
||
|
||
|
||
@router.post(
|
||
"/{task_id}/execute",
|
||
response_model=StandardResponse[str],
|
||
summary="执行清洗任务",
|
||
description="重新执行清洗任务",
|
||
)
|
||
async def execute_cleaning_task(
|
||
task_id: str,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Execute cleaning task"""
|
||
logger.info(f"🔥 [Route] 收到执行请求: task_id={task_id}") # 🔴 加这行
|
||
task_service = _get_task_service(db)
|
||
await task_service.execute_task(db, task_id)
|
||
await db.commit()
|
||
return StandardResponse(code="0", message="success", data=task_id)
|
||
|
||
|
||
@router.get(
|
||
"/{task_id}/result",
|
||
response_model=StandardResponse[list[CleaningResultDto]],
|
||
summary="获取清洗任务结果",
|
||
description="获取指定清洗任务的执行结果",
|
||
)
|
||
async def get_cleaning_task_results(
|
||
task_id: str,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Get cleaning task results"""
|
||
task_service = _get_task_service(db)
|
||
results = await task_service.get_task_results(db, task_id)
|
||
return StandardResponse(code="0", message="success", data=results)
|
||
|
||
|
||
@router.get(
|
||
"/{task_id}/result/download",
|
||
summary="下载清洗任务结果文件压缩包",
|
||
description="下载指定清洗任务的源文件和处理后文件压缩包",
|
||
)
|
||
async def download_cleaning_result_files(
|
||
task_id: str,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Download cleaning task result files (both src and dest)"""
|
||
import zipfile
|
||
import io
|
||
from pathlib import Path
|
||
from fastapi.responses import StreamingResponse
|
||
from app.core.exception import BusinessError, ErrorCodes
|
||
|
||
task_service = _get_task_service(db)
|
||
|
||
task = await task_service.get_task(db, task_id)
|
||
|
||
results = await task_service.get_task_results(db, task_id)
|
||
if not results:
|
||
raise BusinessError(ErrorCodes.NOT_FOUND, f"Cleaning task {task_id} not found")
|
||
|
||
# 基础路径保持不变
|
||
src_base_path = Path("/dataset") / task.src_dataset_id
|
||
dest_base_path = Path("/dataset") / task.dest_dataset_id
|
||
|
||
zip_buffer = io.BytesIO()
|
||
with zipfile.ZipFile(zip_buffer, "w") as zipf:
|
||
for file_record in results:
|
||
# 1. 修复路径拼接逻辑：基于 base_path 拼接，防止路径累积
|
||
if file_record.src_name:
|
||
current_src_path = src_base_path / file_record.src_name
|
||
if current_src_path.exists():
|
||
zipf.write(current_src_path, arcname=f"src/{file_record.src_name}")
|
||
|
||
if file_record.dest_name:
|
||
current_dest_path = dest_base_path / file_record.dest_name
|
||
if current_dest_path.exists():
|
||
zipf.write(current_dest_path, arcname=f"dest/{file_record.dest_name}")
|
||
|
||
zip_buffer.seek(0)
|
||
|
||
# 2. 修复文件名生成：使用 task.name 更合理，避免使用循环变量
|
||
import re
|
||
safe_name = re.sub(r'[^\w\u4e00-\u9fff\-]', '_', task.name or task_id)
|
||
filename = f"cleaning_result_{safe_name}.zip"
|
||
|
||
return StreamingResponse(
|
||
zip_buffer,
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||
)
|
||
|
||
|
||
@router.get(
|
||
"/{task_id}/log/stream",
|
||
summary="流式获取清洗任务日志",
|
||
description="通过SSE流式获取清洗任务日志",
|
||
)
|
||
async def stream_cleaning_task_log(
|
||
task_id: str,
|
||
retry_count: int = Query(0, alias="retry_count"),
|
||
db: AsyncSession = Depends(get_db)
|
||
):
|
||
"""流式输出任务日志 (支持启动等待与自动结束)"""
|
||
task_service = _get_task_service(db)
|
||
|
||
# 1. 尝试解析路径（此时可能还没生成）
|
||
log_path = task_service._resolve_log_path(task_id, retry_count)
|
||
|
||
if not log_path:
|
||
# 2. 【新增】进入等待循环，给 Runtime 启动的时间
|
||
# 最多等待 60 秒。如果 60 秒还没生成，说明任务可能根本没启动
|
||
max_wait = 60
|
||
waited = 0
|
||
while waited < max_wait:
|
||
await asyncio.sleep(1)
|
||
log_path = task_service._resolve_log_path(task_id, retry_count)
|
||
if log_path:
|
||
break
|
||
waited += 1
|
||
|
||
if not log_path:
|
||
# 超时仍未生成，发送结束信号
|
||
async def empty_gen():
|
||
yield "data: " + json.dumps({"level": "WARN", "message": "[WAITING_TIMEOUT]"}) + "\n\n"
|
||
return StreamingResponse(empty_gen(), media_type="text/event-stream")
|
||
|
||
# 防止路径穿越
|
||
flow_root = Path("/flow").resolve()
|
||
if flow_root not in log_path.resolve().parents:
|
||
return StreamingResponse(
|
||
iter(["data: " + json.dumps({"level": "ERROR", "message": "PATH_ERROR"}) + "\n\n"]),
|
||
media_type="text/event-stream"
|
||
)
|
||
|
||
# 3. 文件已就绪，开始 Tail 读取
|
||
async def log_generator():
|
||
with open(log_path, "r", encoding="utf-8") as f:
|
||
# 跳到文件末尾
|
||
f.seek(0, 2)
|
||
|
||
while True:
|
||
line = f.readline()
|
||
if line:
|
||
# 解析日志级别并发送
|
||
level = "INFO"
|
||
if "ERROR" in line: level = "ERROR"
|
||
elif "WARNING" in line: level = "WARNING"
|
||
|
||
yield "data: " + json.dumps({"level": level, "message": line.rstrip()}) + "\n\n"
|
||
else:
|
||
# 没有新内容，检查文件是否已很久未更新（任务结束迹象）
|
||
# 简单策略：休眠后继续
|
||
await asyncio.sleep(0.5)
|
||
# 注：复杂的结束逻辑通常由前端在 TaskStatus 变更为 COMPLETED 时主动断开 SSE 来处理
|
||
|
||
return StreamingResponse(log_generator(), media_type="text/event-stream")
|
||
|
||
|
||
@router.get(
|
||
"/{task_id}/log/{retry_count}",
|
||
response_model=StandardResponse[list[CleaningTaskLog]],
|
||
summary="获取清洗任务日志",
|
||
description="获取指定清洗任务的执行日志",
|
||
)
|
||
async def get_cleaning_task_log(
|
||
task_id: str,
|
||
retry_count: int,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Get cleaning task log"""
|
||
task_service = _get_task_service(db)
|
||
logs = await task_service.get_task_log(db, task_id, retry_count)
|
||
return StandardResponse(code="0", message="success", data=logs)
|
||
|
||
|
||
@router.get(
|
||
"/{task_id}/log/{retry_count}/download",
|
||
summary="下载清洗任务日志文件",
|
||
description="下载指定清洗任务的日志文件",
|
||
)
|
||
async def download_cleaning_task_log(
|
||
task_id: str,
|
||
retry_count: int,
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Download cleaning task log file"""
|
||
from pathlib import Path
|
||
from fastapi.responses import FileResponse
|
||
|
||
flow_base = Path("/flow").resolve()
|
||
|
||
# 校验 task_id 并获取净化后的值（CodeQL 认可此返回值为安全）
|
||
try:
|
||
safe_task_id = _sanitize_task_id(task_id)
|
||
except ValueError:
|
||
logger.warning(f"Invalid task_id in download_log: task_id={task_id}")
|
||
from app.core.exception import BusinessError, ErrorCodes
|
||
raise BusinessError(
|
||
ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND,
|
||
f"Invalid task_id: {task_id}",
|
||
)
|
||
|
||
task_service = _get_task_service(db)
|
||
task = await task_service.task_repo.find_task_by_id(db, safe_task_id)
|
||
|
||
if not task:
|
||
from app.core.exception import BusinessError, ErrorCodes
|
||
|
||
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
|
||
|
||
# 校验并净化 retry_count（CodeQL 认可此返回值为安全）
|
||
safe_retry_count = _sanitize_retry_count(retry_count)
|
||
log_filename = "output.log" if safe_retry_count == 0 else f"output.log.{safe_retry_count}"
|
||
log_path = (flow_base / safe_task_id / log_filename).resolve()
|
||
|
||
# 防止路径穿越：parents 校验仍在 flow_base 下
|
||
if flow_base not in log_path.parents:
|
||
logger.warning(f"Path traversal attempt in download_log: task_id={task_id}")
|
||
from app.core.exception import BusinessError, ErrorCodes
|
||
raise BusinessError(
|
||
ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND,
|
||
f"Invalid task_id: {task_id}",
|
||
)
|
||
|
||
if not log_path.exists():
|
||
from app.core.exception import BusinessError, ErrorCodes
|
||
|
||
raise BusinessError(
|
||
ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND,
|
||
f"Log file not found for task {task_id}, retry {retry_count}",
|
||
)
|
||
|
||
# Generate filename with task name and retry count
|
||
import re
|
||
|
||
task_name = task.name or "未命名任务"
|
||
safe_task_name = re.sub(
|
||
r"[^\w\u4e00-\u9fff\-]", "_", task_name
|
||
) # Keep alphanumeric, Chinese, and hyphens
|
||
run_number = retry_count + 1 # retry_count is 0-indexed, so add 1 for display
|
||
filename = f"{safe_task_name}_第{run_number}次运行.log"
|
||
|
||
return FileResponse(
|
||
path=log_path,
|
||
media_type="text/plain",
|
||
filename=filename,
|
||
)
|
||
|
||
# ==========================================
|
||
# 新增：更新任务配置的路由
|
||
# ==========================================
|
||
@router.put(
|
||
"/{task_id}",
|
||
response_model=StandardResponse[str],
|
||
summary="更新任务配置",
|
||
description="全量更新任务的名称、描述以及关联的算子列表和参数配置",
|
||
)
|
||
async def update_task_config_route(
|
||
task_id: str,
|
||
data: dict, # 接收前端传来的完整 JSON 配置
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""Update task configuration and operator settings"""
|
||
# 获取 Service 实例
|
||
# 注意：这里需要复用路由文件中获取 task_service 的逻辑，通常如下：
|
||
from app.module.cleaning.interface.cleaning_task_routes import _get_task_service
|
||
task_service = _get_task_service(db)
|
||
|
||
# 调用 Service 层逻辑
|
||
await task_service.update_task_config(db, task_id, data)
|
||
await db.commit()
|
||
|
||
return StandardResponse(code="0", message="任务配置已更新", data=task_id)
|