from typing import Optional import re import asyncio import json from pathlib import Path from fastapi.responses import StreamingResponse from fastapi import APIRouter, Depends from fastapi.responses import StreamingResponse from sqlalchemy.ext.asyncio import AsyncSession from app.core.logging import get_logger from app.db.session import get_db from app.module.cleaning.schema import ( CleaningTaskDto, CreateCleaningTaskRequest, CleaningResultDto, CleaningTaskLog, ) from app.module.cleaning.service import CleaningTaskService from app.module.shared.schema import StandardResponse, PaginatedData from fastapi import Query logger = get_logger(__name__) # Module-level pattern for task_id sanitization (CodeQL sanitizer) _TASK_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$") def _sanitize_task_id(task_id: str) -> str: """Validate and return sanitized task_id for safe path construction. CodeQL recognizes the return value of this function as sanitized, allowing safe use in path construction. """ if not _TASK_ID_PATTERN.fullmatch(task_id): raise ValueError(f"Invalid task_id: {task_id}") return task_id def _sanitize_retry_count(retry_count: int) -> int: """Validate and return sanitized retry_count for safe path construction. CodeQL recognizes the return value of this function as sanitized, allowing safe use in path construction. """ if retry_count < 0 or retry_count > 1000: raise ValueError(f"Invalid retry_count: {retry_count}") return retry_count router = APIRouter(prefix="/cleaning/tasks", tags=["Cleaning Tasks"]) def _get_operator_service(): """Get operator service""" from app.module.operator.service import OperatorService from app.module.operator.repository import ( OperatorRepository, CategoryRelationRepository, OperatorReleaseRepository, ) from app.module.operator.parsers import ParserHolder from app.module.shared.file_service import FileService from app.module.shared.chunk_upload_repository import ChunkUploadRepository return OperatorService( operator_repo=OperatorRepository(None), category_relation_repo=CategoryRelationRepository(None), operator_release_repo=OperatorReleaseRepository(None), parser_holder=ParserHolder(), file_service=FileService(ChunkUploadRepository()), ) def _get_task_service(db: AsyncSession) -> CleaningTaskService: """Get cleaning task service instance""" from app.module.cleaning.service import ( CleaningTaskScheduler, CleaningTaskValidator, ) from app.module.cleaning.service.cleaning_task_scheduler import get_scheduler from app.module.cleaning.repository import ( CleaningTaskRepository, CleaningResultRepository, OperatorInstanceRepository, ) from app.module.dataset.service import DatasetManagementService from app.module.shared.common.lineage import LineageService scheduler = get_scheduler() # 使用全局单例,保持轮询状态跨请求共享 operator_service = _get_operator_service() dataset_service = DatasetManagementService(db) lineage_service = LineageService(db) task_repo = CleaningTaskRepository(None) return CleaningTaskService( task_repo=task_repo, result_repo=CleaningResultRepository(None), operator_instance_repo=OperatorInstanceRepository(None), operator_service=operator_service, scheduler=scheduler, validator=CleaningTaskValidator(task_repo=task_repo, template_repo=None), dataset_service=dataset_service, lineage_service=lineage_service, ) @router.get( "", response_model=StandardResponse[PaginatedData[CleaningTaskDto]], summary="查询清洗任务列表", description="根据参数查询清洗任务列表(支持分页、状态过滤、关键词搜索)", tags=["mcp"], ) async def get_cleaning_tasks( page: int = 0, size: int = 10, status: Optional[str] = None, keyword: Optional[str] = None, db: AsyncSession = Depends(get_db), ): """Query cleaning tasks""" task_service = _get_task_service(db) tasks = await task_service.get_tasks(db, status, keyword, page, size) count = await task_service.count_tasks(db, status, keyword) total_pages = (count + size - 1) // size if size > 0 else 0 return StandardResponse( code="0", message="success", data=PaginatedData( page=page, size=size, total_elements=count, total_pages=total_pages, content=tasks, ), ) @router.post( "", response_model=StandardResponse[CleaningTaskDto], summary="创建清洗任务", description="根据模板ID或算子列表创建清洗任务", tags=["mcp"], ) async def create_cleaning_task( request: CreateCleaningTaskRequest, db: AsyncSession = Depends(get_db), ): """Create cleaning task""" task_service = _get_task_service(db) task = await task_service.create_task(db, request) await db.commit() await task_service.execute_task(db, task.id) await db.commit() return StandardResponse(code="0", message="success", data=task) @router.get( "/{task_id}", response_model=StandardResponse[CleaningTaskDto], summary="获取清洗任务详情", description="根据ID获取清洗任务详细信息", ) async def get_cleaning_task( task_id: str, db: AsyncSession = Depends(get_db), ): """Get cleaning task by ID""" task_service = _get_task_service(db) task = await task_service.get_task(db, task_id) return StandardResponse(code="0", message="success", data=task) @router.delete( "/{task_id}", response_model=StandardResponse[str], summary="删除清洗任务", description="删除指定的清洗任务", ) async def delete_cleaning_task( task_id: str, db: AsyncSession = Depends(get_db), ): """Delete cleaning task""" task_service = _get_task_service(db) await task_service.delete_task(db, task_id) await db.commit() return StandardResponse(code="0", message="success", data=task_id) @router.post( "/{task_id}/stop", response_model=StandardResponse[str], summary="停止清洗任务", description="停止正在运行的清洗任务", ) async def stop_cleaning_task( task_id: str, db: AsyncSession = Depends(get_db), ): """Stop cleaning task""" task_service = _get_task_service(db) await task_service.stop_task(db, task_id) await db.commit() return StandardResponse(code="0", message="success", data=task_id) @router.post( "/{task_id}/execute", response_model=StandardResponse[str], summary="执行清洗任务", description="重新执行清洗任务", ) async def execute_cleaning_task( task_id: str, db: AsyncSession = Depends(get_db), ): """Execute cleaning task""" logger.info(f"🔥 [Route] 收到执行请求: task_id={task_id}") # 🔴 加这行 task_service = _get_task_service(db) await task_service.execute_task(db, task_id) await db.commit() return StandardResponse(code="0", message="success", data=task_id) @router.get( "/{task_id}/result", response_model=StandardResponse[list[CleaningResultDto]], summary="获取清洗任务结果", description="获取指定清洗任务的执行结果", ) async def get_cleaning_task_results( task_id: str, db: AsyncSession = Depends(get_db), ): """Get cleaning task results""" task_service = _get_task_service(db) results = await task_service.get_task_results(db, task_id) return StandardResponse(code="0", message="success", data=results) @router.get( "/{task_id}/result/download", summary="下载清洗任务结果文件压缩包", description="下载指定清洗任务的源文件和处理后文件压缩包", ) async def download_cleaning_result_files( task_id: str, db: AsyncSession = Depends(get_db), ): """Download cleaning task result files (both src and dest)""" import zipfile import io from pathlib import Path from fastapi.responses import StreamingResponse from app.core.exception import BusinessError, ErrorCodes task_service = _get_task_service(db) task = await task_service.get_task(db, task_id) results = await task_service.get_task_results(db, task_id) if not results: raise BusinessError(ErrorCodes.NOT_FOUND, f"Cleaning task {task_id} not found") # 基础路径保持不变 src_base_path = Path("/dataset") / task.src_dataset_id dest_base_path = Path("/dataset") / task.dest_dataset_id zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, "w") as zipf: for file_record in results: # 1. 修复路径拼接逻辑:基于 base_path 拼接,防止路径累积 if file_record.src_name: current_src_path = src_base_path / file_record.src_name if current_src_path.exists(): zipf.write(current_src_path, arcname=f"src/{file_record.src_name}") if file_record.dest_name: current_dest_path = dest_base_path / file_record.dest_name if current_dest_path.exists(): zipf.write(current_dest_path, arcname=f"dest/{file_record.dest_name}") zip_buffer.seek(0) # 2. 修复文件名生成:使用 task.name 更合理,避免使用循环变量 import re safe_name = re.sub(r'[^\w\u4e00-\u9fff\-]', '_', task.name or task_id) filename = f"cleaning_result_{safe_name}.zip" return StreamingResponse( zip_buffer, media_type="application/zip", headers={"Content-Disposition": f'attachment; filename="{filename}"'}, ) @router.get( "/{task_id}/log/stream", summary="流式获取清洗任务日志", description="通过SSE流式获取清洗任务日志", ) async def stream_cleaning_task_log( task_id: str, retry_count: int = Query(0, alias="retry_count"), db: AsyncSession = Depends(get_db) ): """流式输出任务日志 (支持启动等待与自动结束)""" task_service = _get_task_service(db) # 1. 尝试解析路径(此时可能还没生成) log_path = task_service._resolve_log_path(task_id, retry_count) if not log_path: # 2. 【新增】进入等待循环,给 Runtime 启动的时间 # 最多等待 60 秒。如果 60 秒还没生成,说明任务可能根本没启动 max_wait = 60 waited = 0 while waited < max_wait: await asyncio.sleep(1) log_path = task_service._resolve_log_path(task_id, retry_count) if log_path: break waited += 1 if not log_path: # 超时仍未生成,发送结束信号 async def empty_gen(): yield "data: " + json.dumps({"level": "WARN", "message": "[WAITING_TIMEOUT]"}) + "\n\n" return StreamingResponse(empty_gen(), media_type="text/event-stream") # 防止路径穿越 flow_root = Path("/flow").resolve() if flow_root not in log_path.resolve().parents: return StreamingResponse( iter(["data: " + json.dumps({"level": "ERROR", "message": "PATH_ERROR"}) + "\n\n"]), media_type="text/event-stream" ) # 3. 文件已就绪,开始 Tail 读取 async def log_generator(): with open(log_path, "r", encoding="utf-8") as f: # 跳到文件末尾 f.seek(0, 2) while True: line = f.readline() if line: # 解析日志级别并发送 level = "INFO" if "ERROR" in line: level = "ERROR" elif "WARNING" in line: level = "WARNING" yield "data: " + json.dumps({"level": level, "message": line.rstrip()}) + "\n\n" else: # 没有新内容,检查文件是否已很久未更新(任务结束迹象) # 简单策略:休眠后继续 await asyncio.sleep(0.5) # 注:复杂的结束逻辑通常由前端在 TaskStatus 变更为 COMPLETED 时主动断开 SSE 来处理 return StreamingResponse(log_generator(), media_type="text/event-stream") @router.get( "/{task_id}/log/{retry_count}", response_model=StandardResponse[list[CleaningTaskLog]], summary="获取清洗任务日志", description="获取指定清洗任务的执行日志", ) async def get_cleaning_task_log( task_id: str, retry_count: int, db: AsyncSession = Depends(get_db), ): """Get cleaning task log""" task_service = _get_task_service(db) logs = await task_service.get_task_log(db, task_id, retry_count) return StandardResponse(code="0", message="success", data=logs) @router.get( "/{task_id}/log/{retry_count}/download", summary="下载清洗任务日志文件", description="下载指定清洗任务的日志文件", ) async def download_cleaning_task_log( task_id: str, retry_count: int, db: AsyncSession = Depends(get_db), ): """Download cleaning task log file""" from pathlib import Path from fastapi.responses import FileResponse flow_base = Path("/flow").resolve() # 校验 task_id 并获取净化后的值(CodeQL 认可此返回值为安全) try: safe_task_id = _sanitize_task_id(task_id) except ValueError: logger.warning(f"Invalid task_id in download_log: task_id={task_id}") from app.core.exception import BusinessError, ErrorCodes raise BusinessError( ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND, f"Invalid task_id: {task_id}", ) task_service = _get_task_service(db) task = await task_service.task_repo.find_task_by_id(db, safe_task_id) if not task: from app.core.exception import BusinessError, ErrorCodes raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id) # 校验并净化 retry_count(CodeQL 认可此返回值为安全) safe_retry_count = _sanitize_retry_count(retry_count) log_filename = "output.log" if safe_retry_count == 0 else f"output.log.{safe_retry_count}" log_path = (flow_base / safe_task_id / log_filename).resolve() # 防止路径穿越:parents 校验仍在 flow_base 下 if flow_base not in log_path.parents: logger.warning(f"Path traversal attempt in download_log: task_id={task_id}") from app.core.exception import BusinessError, ErrorCodes raise BusinessError( ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND, f"Invalid task_id: {task_id}", ) if not log_path.exists(): from app.core.exception import BusinessError, ErrorCodes raise BusinessError( ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND, f"Log file not found for task {task_id}, retry {retry_count}", ) # Generate filename with task name and retry count import re task_name = task.name or "未命名任务" safe_task_name = re.sub( r"[^\w\u4e00-\u9fff\-]", "_", task_name ) # Keep alphanumeric, Chinese, and hyphens run_number = retry_count + 1 # retry_count is 0-indexed, so add 1 for display filename = f"{safe_task_name}_第{run_number}次运行.log" return FileResponse( path=log_path, media_type="text/plain", filename=filename, ) # ========================================== # 新增:更新任务配置的路由 # ========================================== @router.put( "/{task_id}", response_model=StandardResponse[str], summary="更新任务配置", description="全量更新任务的名称、描述以及关联的算子列表和参数配置", ) async def update_task_config_route( task_id: str, data: dict, # 接收前端传来的完整 JSON 配置 db: AsyncSession = Depends(get_db), ): """Update task configuration and operator settings""" # 获取 Service 实例 # 注意:这里需要复用路由文件中获取 task_service 的逻辑,通常如下: from app.module.cleaning.interface.cleaning_task_routes import _get_task_service task_service = _get_task_service(db) # 调用 Service 层逻辑 await task_service.update_task_config(db, task_id, data) await db.commit() return StandardResponse(code="0", message="任务配置已更新", data=task_id)