deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,50 @@
|
||||
from .schema import (
|
||||
CleaningTaskStatus,
|
||||
OperatorInstanceDto,
|
||||
CleaningProcess,
|
||||
CleaningTaskDto,
|
||||
CreateCleaningTaskRequest,
|
||||
CleaningResultDto,
|
||||
CleaningTaskLog,
|
||||
CleaningTemplateDto,
|
||||
CreateCleaningTemplateRequest,
|
||||
UpdateCleaningTemplateRequest,
|
||||
)
|
||||
|
||||
from .repository import (
|
||||
CleaningTaskRepository,
|
||||
CleaningTemplateRepository,
|
||||
CleaningResultRepository,
|
||||
OperatorInstanceRepository,
|
||||
)
|
||||
|
||||
from .service import (
|
||||
CleaningTaskValidator,
|
||||
CleaningTaskScheduler,
|
||||
CleaningTemplateService,
|
||||
CleaningTaskService,
|
||||
)
|
||||
|
||||
from .runtime_client import RuntimeClient
|
||||
|
||||
__all__ = [
|
||||
"CleaningTaskStatus",
|
||||
"OperatorInstanceDto",
|
||||
"CleaningProcess",
|
||||
"CleaningTaskDto",
|
||||
"CreateCleaningTaskRequest",
|
||||
"CleaningResultDto",
|
||||
"CleaningTaskLog",
|
||||
"CleaningTemplateDto",
|
||||
"CreateCleaningTemplateRequest",
|
||||
"UpdateCleaningTemplateRequest",
|
||||
"CleaningTaskRepository",
|
||||
"CleaningTemplateRepository",
|
||||
"CleaningResultRepository",
|
||||
"OperatorInstanceRepository",
|
||||
"CleaningTaskValidator",
|
||||
"CleaningTaskScheduler",
|
||||
"CleaningTemplateService",
|
||||
"CleaningTaskService",
|
||||
"RuntimeClient",
|
||||
]
|
||||
@@ -0,0 +1,57 @@
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class CleaningException(Exception):
|
||||
"""Base exception for cleaning module"""
|
||||
def __init__(self, message: str, details: Optional[dict] = None):
|
||||
self.message = message
|
||||
self.details = details
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class CleaningNameDuplicationError(CleaningException):
|
||||
"""Exception raised when cleaning task name is duplicated"""
|
||||
def __init__(self, name: str):
|
||||
super().__init__(f"Cleaning task name '{name}' is duplicated")
|
||||
|
||||
|
||||
class CleaningTaskNotFoundError(CleaningException):
|
||||
"""Exception raised when cleaning task is not found"""
|
||||
def __init__(self, task_id: str):
|
||||
super().__init__(f"Cleaning task '{task_id}' not found")
|
||||
|
||||
|
||||
class CleaningTemplateNotFoundError(CleaningException):
|
||||
"""Exception raised when cleaning template is not found"""
|
||||
def __init__(self, template_id: str):
|
||||
super().__init__(f"Cleaning template '{template_id}' not found")
|
||||
|
||||
|
||||
class InvalidOperatorInputError(CleaningException):
|
||||
"""Exception raised when operator input/output types are invalid"""
|
||||
def __init__(self, message: str = "Invalid operator input/output types"):
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class ExecutorTypeError(CleaningException):
|
||||
"""Exception raised when executor type is invalid"""
|
||||
def __init__(self, message: str = "Invalid executor type"):
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class DatasetNotFoundError(CleaningException):
|
||||
"""Exception raised when dataset is not found"""
|
||||
def __init__(self, dataset_id: str):
|
||||
super().__init__(f"Dataset '{dataset_id}' not found")
|
||||
|
||||
|
||||
class FileSystemError(CleaningException):
|
||||
"""Exception raised when file system operations fail"""
|
||||
def __init__(self, message: str, details: Optional[dict] = None):
|
||||
super().__init__(f"File system error: {message}", details)
|
||||
|
||||
|
||||
class SettingsParseError(CleaningException):
|
||||
"""Exception raised when operator settings parsing fails"""
|
||||
def __init__(self, message: str, details: Optional[dict] = None):
|
||||
super().__init__(f"Settings parse error: {message}", details)
|
||||
@@ -0,0 +1,8 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from .cleaning_task_routes import router as task_router
|
||||
from .cleaning_template_routes import router as template_router
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(task_router)
|
||||
router.include_router(template_router)
|
||||
@@ -0,0 +1,521 @@
|
||||
from typing import Optional
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_db
|
||||
from app.module.cleaning.schema import (
|
||||
CleaningTaskDto,
|
||||
CreateCleaningTaskRequest,
|
||||
CleaningResultDto,
|
||||
CleaningTaskLog,
|
||||
)
|
||||
from app.module.cleaning.service import CleaningTaskService
|
||||
from app.module.shared.schema import StandardResponse, PaginatedData
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Module-level pattern for task_id sanitization (CodeQL sanitizer)
|
||||
_TASK_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
|
||||
|
||||
def _sanitize_task_id(task_id: str) -> str:
|
||||
"""Validate and return sanitized task_id for safe path construction.
|
||||
|
||||
CodeQL recognizes the return value of this function as sanitized,
|
||||
allowing safe use in path construction.
|
||||
"""
|
||||
if not _TASK_ID_PATTERN.fullmatch(task_id):
|
||||
raise ValueError(f"Invalid task_id: {task_id}")
|
||||
return task_id
|
||||
|
||||
|
||||
def _sanitize_retry_count(retry_count: int) -> int:
|
||||
"""Validate and return sanitized retry_count for safe path construction.
|
||||
|
||||
CodeQL recognizes the return value of this function as sanitized,
|
||||
allowing safe use in path construction.
|
||||
"""
|
||||
if retry_count < 0 or retry_count > 1000:
|
||||
raise ValueError(f"Invalid retry_count: {retry_count}")
|
||||
return retry_count
|
||||
|
||||
router = APIRouter(prefix="/cleaning/tasks", tags=["Cleaning Tasks"])
|
||||
|
||||
|
||||
def _get_operator_service():
|
||||
"""Get operator service"""
|
||||
from app.module.operator.service import OperatorService
|
||||
from app.module.operator.repository import (
|
||||
OperatorRepository,
|
||||
CategoryRelationRepository,
|
||||
OperatorReleaseRepository,
|
||||
)
|
||||
from app.module.operator.parsers import ParserHolder
|
||||
from app.module.shared.file_service import FileService
|
||||
from app.module.shared.chunk_upload_repository import ChunkUploadRepository
|
||||
|
||||
return OperatorService(
|
||||
operator_repo=OperatorRepository(None),
|
||||
category_relation_repo=CategoryRelationRepository(None),
|
||||
operator_release_repo=OperatorReleaseRepository(None),
|
||||
parser_holder=ParserHolder(),
|
||||
file_service=FileService(ChunkUploadRepository()),
|
||||
)
|
||||
|
||||
|
||||
def _get_task_service(db: AsyncSession) -> CleaningTaskService:
|
||||
"""Get cleaning task service instance"""
|
||||
from app.module.cleaning.service import (
|
||||
CleaningTaskScheduler,
|
||||
CleaningTaskValidator,
|
||||
)
|
||||
from app.module.cleaning.service.cleaning_task_scheduler import get_scheduler
|
||||
from app.module.cleaning.repository import (
|
||||
CleaningTaskRepository,
|
||||
CleaningResultRepository,
|
||||
OperatorInstanceRepository,
|
||||
)
|
||||
from app.module.dataset.service import DatasetManagementService
|
||||
from app.module.shared.common.lineage import LineageService
|
||||
|
||||
scheduler = get_scheduler() # 使用全局单例，保持轮询状态跨请求共享
|
||||
operator_service = _get_operator_service()
|
||||
dataset_service = DatasetManagementService(db)
|
||||
lineage_service = LineageService(db)
|
||||
|
||||
task_repo = CleaningTaskRepository(None)
|
||||
|
||||
return CleaningTaskService(
|
||||
task_repo=task_repo,
|
||||
result_repo=CleaningResultRepository(None),
|
||||
operator_instance_repo=OperatorInstanceRepository(None),
|
||||
operator_service=operator_service,
|
||||
scheduler=scheduler,
|
||||
validator=CleaningTaskValidator(task_repo=task_repo, template_repo=None),
|
||||
dataset_service=dataset_service,
|
||||
lineage_service=lineage_service,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=StandardResponse[PaginatedData[CleaningTaskDto]],
|
||||
summary="查询清洗任务列表",
|
||||
description="根据参数查询清洗任务列表（支持分页、状态过滤、关键词搜索）",
|
||||
tags=["mcp"],
|
||||
)
|
||||
async def get_cleaning_tasks(
|
||||
page: int = 0,
|
||||
size: int = 10,
|
||||
status: Optional[str] = None,
|
||||
keyword: Optional[str] = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Query cleaning tasks"""
|
||||
task_service = _get_task_service(db)
|
||||
|
||||
tasks = await task_service.get_tasks(db, status, keyword, page, size)
|
||||
count = await task_service.count_tasks(db, status, keyword)
|
||||
total_pages = (count + size - 1) // size if size > 0 else 0
|
||||
|
||||
return StandardResponse(
|
||||
code="0",
|
||||
message="success",
|
||||
data=PaginatedData(
|
||||
page=page,
|
||||
size=size,
|
||||
total_elements=count,
|
||||
total_pages=total_pages,
|
||||
content=tasks,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"",
|
||||
response_model=StandardResponse[CleaningTaskDto],
|
||||
summary="创建清洗任务",
|
||||
description="根据模板ID或算子列表创建清洗任务",
|
||||
tags=["mcp"],
|
||||
)
|
||||
async def create_cleaning_task(
|
||||
request: CreateCleaningTaskRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Create cleaning task"""
|
||||
task_service = _get_task_service(db)
|
||||
|
||||
task = await task_service.create_task(db, request)
|
||||
await db.commit()
|
||||
|
||||
await task_service.execute_task(db, task.id)
|
||||
await db.commit()
|
||||
|
||||
return StandardResponse(code="0", message="success", data=task)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{task_id}",
|
||||
response_model=StandardResponse[CleaningTaskDto],
|
||||
summary="获取清洗任务详情",
|
||||
description="根据ID获取清洗任务详细信息",
|
||||
)
|
||||
async def get_cleaning_task(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get cleaning task by ID"""
|
||||
task_service = _get_task_service(db)
|
||||
task = await task_service.get_task(db, task_id)
|
||||
return StandardResponse(code="0", message="success", data=task)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/{task_id}",
|
||||
response_model=StandardResponse[str],
|
||||
summary="删除清洗任务",
|
||||
description="删除指定的清洗任务",
|
||||
)
|
||||
async def delete_cleaning_task(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Delete cleaning task"""
|
||||
task_service = _get_task_service(db)
|
||||
await task_service.delete_task(db, task_id)
|
||||
await db.commit()
|
||||
return StandardResponse(code="0", message="success", data=task_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{task_id}/stop",
|
||||
response_model=StandardResponse[str],
|
||||
summary="停止清洗任务",
|
||||
description="停止正在运行的清洗任务",
|
||||
)
|
||||
async def stop_cleaning_task(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Stop cleaning task"""
|
||||
task_service = _get_task_service(db)
|
||||
await task_service.stop_task(db, task_id)
|
||||
await db.commit()
|
||||
return StandardResponse(code="0", message="success", data=task_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{task_id}/execute",
|
||||
response_model=StandardResponse[str],
|
||||
summary="执行清洗任务",
|
||||
description="重新执行清洗任务",
|
||||
)
|
||||
async def execute_cleaning_task(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Execute cleaning task"""
|
||||
task_service = _get_task_service(db)
|
||||
await task_service.execute_task(db, task_id)
|
||||
await db.commit()
|
||||
return StandardResponse(code="0", message="success", data=task_id)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{task_id}/result",
|
||||
response_model=StandardResponse[list[CleaningResultDto]],
|
||||
summary="获取清洗任务结果",
|
||||
description="获取指定清洗任务的执行结果",
|
||||
)
|
||||
async def get_cleaning_task_results(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get cleaning task results"""
|
||||
task_service = _get_task_service(db)
|
||||
results = await task_service.get_task_results(db, task_id)
|
||||
return StandardResponse(code="0", message="success", data=results)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{task_id}/result/download",
|
||||
summary="下载清洗任务结果文件压缩包",
|
||||
description="下载指定清洗任务的源文件和处理后文件压缩包",
|
||||
)
|
||||
async def download_cleaning_result_files(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Download cleaning task result files (both src and dest)"""
|
||||
import zipfile
|
||||
import io
|
||||
from pathlib import Path
|
||||
from fastapi.responses import StreamingResponse
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
|
||||
task_service = _get_task_service(db)
|
||||
|
||||
task = await task_service.get_task(db, task_id)
|
||||
|
||||
results = await task_service.get_task_results(db, task_id)
|
||||
if not results:
|
||||
raise BusinessError(ErrorCodes.NOT_FOUND, f"Cleaning task {task_id} not found")
|
||||
|
||||
src_path = Path("/dataset") / task.src_dataset_id
|
||||
|
||||
dest_path = Path("/dataset") / task.dest_dataset_id
|
||||
|
||||
zip_buffer = io.BytesIO()
|
||||
|
||||
with zipfile.ZipFile(zip_buffer, "w") as zipf:
|
||||
for file_record in results:
|
||||
if file_record.src_name:
|
||||
src_path = src_path / file_record.src_name
|
||||
if src_path.exists():
|
||||
zipf.write(src_path, arcname=f"src/{file_record.src_name}")
|
||||
|
||||
if file_record.dest_name:
|
||||
dest_path = dest_path / file_record.dest_name
|
||||
if dest_path.exists():
|
||||
zipf.write(dest_path, arcname=f"dest/{file_record.dest_name}")
|
||||
|
||||
zip_buffer.seek(0)
|
||||
|
||||
filename = f"task_{task_id}_{file_record.src_name}.zip"
|
||||
|
||||
return StreamingResponse(
|
||||
zip_buffer,
|
||||
media_type="application/zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{task_id}/log/stream",
|
||||
summary="流式获取清洗任务日志",
|
||||
description="通过SSE流式获取清洗任务日志",
|
||||
)
|
||||
async def stream_cleaning_task_log(
|
||||
task_id: str,
|
||||
retry_count: int = 0,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Stream cleaning task log via SSE"""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
flow_base = Path("/flow").resolve()
|
||||
|
||||
# 校验 task_id 并获取净化后的值（CodeQL 认可此返回值为安全）
|
||||
try:
|
||||
safe_task_id = _sanitize_task_id(task_id)
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid task_id in stream_log: task_id={task_id}")
|
||||
|
||||
async def invalid_error_generator():
|
||||
yield f"data: {json.dumps({'level': 'ERROR', 'message': 'Invalid task_id'}, ensure_ascii=False)}\n\n"
|
||||
|
||||
return StreamingResponse(invalid_error_generator(), media_type="text/event-stream")
|
||||
|
||||
task_service = _get_task_service(db)
|
||||
task = await task_service.task_repo.find_task_by_id(db, safe_task_id)
|
||||
|
||||
if not task:
|
||||
|
||||
async def error_generator():
|
||||
yield f"data: {json.dumps({'level': 'ERROR', 'message': 'Task not found'}, ensure_ascii=False)}\n\n"
|
||||
|
||||
return StreamingResponse(error_generator(), media_type="text/event-stream")
|
||||
|
||||
# 校验并净化 retry_count（CodeQL 认可此返回值为安全）
|
||||
safe_retry_count = _sanitize_retry_count(retry_count)
|
||||
log_filename = "output.log" if safe_retry_count == 0 else f"output.log.{safe_retry_count}"
|
||||
log_path = (flow_base / safe_task_id / log_filename).resolve()
|
||||
|
||||
# 防止路径穿越：parents 校验仍在 flow_base 下
|
||||
if flow_base not in log_path.parents:
|
||||
logger.warning(f"Path traversal attempt in stream_log: task_id={task_id}")
|
||||
|
||||
async def traversal_error_generator():
|
||||
yield f"data: {json.dumps({'level': 'ERROR', 'message': 'Invalid task_id'}, ensure_ascii=False)}\n\n"
|
||||
|
||||
return StreamingResponse(traversal_error_generator(), media_type="text/event-stream")
|
||||
|
||||
standard_level_pattern = re.compile(
|
||||
r"\b(DEBUG|Debug|INFO|Info|WARN|Warn|WARNING|Warning|ERROR|Error|FATAL|Fatal)\b"
|
||||
)
|
||||
exception_suffix_pattern = re.compile(r"\b\w+(Warning|Error|Exception)\b")
|
||||
|
||||
def get_log_level(line: str, default_level: str) -> str:
|
||||
if not line or not line.strip():
|
||||
return default_level
|
||||
std_match = standard_level_pattern.search(line)
|
||||
if std_match:
|
||||
return std_match.group(1).upper()
|
||||
ex_match = exception_suffix_pattern.search(line)
|
||||
if ex_match:
|
||||
match = ex_match.group(1).upper()
|
||||
if match == "WARNING":
|
||||
return "WARN"
|
||||
if match in ["ERROR", "EXCEPTION"]:
|
||||
return "ERROR"
|
||||
return default_level
|
||||
|
||||
async def log_generator():
|
||||
last_position = 0
|
||||
last_level = "INFO"
|
||||
heartbeat_counter = 0
|
||||
HEARTBEAT_INTERVAL = 10
|
||||
|
||||
while True:
|
||||
try:
|
||||
current_task = await task_service.task_repo.find_task_by_id(db, safe_task_id)
|
||||
is_task_finished = current_task and current_task.status in [
|
||||
"COMPLETED",
|
||||
"FAILED",
|
||||
"STOPPED",
|
||||
]
|
||||
|
||||
if log_path.exists():
|
||||
with open(log_path, "r", encoding="utf-8") as f:
|
||||
f.seek(last_position)
|
||||
new_content = f.read()
|
||||
if new_content:
|
||||
for line in new_content.splitlines(keepends=True):
|
||||
last_level = get_log_level(line, last_level)
|
||||
log_entry = json.dumps(
|
||||
{"level": last_level, "message": line.rstrip()},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
yield f"data: {log_entry}\n\n"
|
||||
last_position = f.tell()
|
||||
heartbeat_counter = 0
|
||||
else:
|
||||
heartbeat_counter += 1
|
||||
|
||||
f.seek(0, 2)
|
||||
current_size = f.tell()
|
||||
if current_size < last_position:
|
||||
last_position = 0
|
||||
else:
|
||||
heartbeat_counter += 1
|
||||
|
||||
if is_task_finished:
|
||||
yield f"data: {json.dumps({'level': 'INFO', 'message': '[END_OF_STREAM]'}, ensure_ascii=False)}\n\n"
|
||||
break
|
||||
|
||||
if heartbeat_counter >= HEARTBEAT_INTERVAL:
|
||||
yield f"data: {json.dumps({'level': 'DEBUG', 'message': '[HEARTBEAT]'}, ensure_ascii=False)}\n\n"
|
||||
heartbeat_counter = 0
|
||||
|
||||
await asyncio.sleep(1)
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading log file: {e}")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
return StreamingResponse(
|
||||
log_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{task_id}/log/{retry_count}",
|
||||
response_model=StandardResponse[list[CleaningTaskLog]],
|
||||
summary="获取清洗任务日志",
|
||||
description="获取指定清洗任务的执行日志",
|
||||
)
|
||||
async def get_cleaning_task_log(
|
||||
task_id: str,
|
||||
retry_count: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get cleaning task log"""
|
||||
task_service = _get_task_service(db)
|
||||
logs = await task_service.get_task_log(db, task_id, retry_count)
|
||||
return StandardResponse(code="0", message="success", data=logs)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{task_id}/log/{retry_count}/download",
|
||||
summary="下载清洗任务日志文件",
|
||||
description="下载指定清洗任务的日志文件",
|
||||
)
|
||||
async def download_cleaning_task_log(
|
||||
task_id: str,
|
||||
retry_count: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Download cleaning task log file"""
|
||||
from pathlib import Path
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
flow_base = Path("/flow").resolve()
|
||||
|
||||
# 校验 task_id 并获取净化后的值（CodeQL 认可此返回值为安全）
|
||||
try:
|
||||
safe_task_id = _sanitize_task_id(task_id)
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid task_id in download_log: task_id={task_id}")
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND,
|
||||
f"Invalid task_id: {task_id}",
|
||||
)
|
||||
|
||||
task_service = _get_task_service(db)
|
||||
task = await task_service.task_repo.find_task_by_id(db, safe_task_id)
|
||||
|
||||
if not task:
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
|
||||
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
|
||||
|
||||
# 校验并净化 retry_count（CodeQL 认可此返回值为安全）
|
||||
safe_retry_count = _sanitize_retry_count(retry_count)
|
||||
log_filename = "output.log" if safe_retry_count == 0 else f"output.log.{safe_retry_count}"
|
||||
log_path = (flow_base / safe_task_id / log_filename).resolve()
|
||||
|
||||
# 防止路径穿越：parents 校验仍在 flow_base 下
|
||||
if flow_base not in log_path.parents:
|
||||
logger.warning(f"Path traversal attempt in download_log: task_id={task_id}")
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND,
|
||||
f"Invalid task_id: {task_id}",
|
||||
)
|
||||
|
||||
if not log_path.exists():
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_LOG_NOT_FOUND,
|
||||
f"Log file not found for task {task_id}, retry {retry_count}",
|
||||
)
|
||||
|
||||
# Generate filename with task name and retry count
|
||||
import re
|
||||
|
||||
task_name = task.name or "未命名任务"
|
||||
safe_task_name = re.sub(
|
||||
r"[^\w\u4e00-\u9fff\-]", "_", task_name
|
||||
) # Keep alphanumeric, Chinese, and hyphens
|
||||
run_number = retry_count + 1 # retry_count is 0-indexed, so add 1 for display
|
||||
filename = f"{safe_task_name}_第{run_number}次运行.log"
|
||||
|
||||
return FileResponse(
|
||||
path=log_path,
|
||||
media_type="text/plain",
|
||||
filename=filename,
|
||||
)
|
||||
@@ -0,0 +1,166 @@
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_db
|
||||
from app.module.cleaning.schema import (
|
||||
CleaningTemplateDto,
|
||||
CreateCleaningTemplateRequest,
|
||||
UpdateCleaningTemplateRequest,
|
||||
)
|
||||
from app.module.cleaning.service import CleaningTemplateService
|
||||
from app.module.shared.schema import StandardResponse, PaginatedData
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/cleaning/templates", tags=["Cleaning Templates"])
|
||||
|
||||
|
||||
def _get_operator_service():
|
||||
"""Get operator service"""
|
||||
from app.module.operator.service import OperatorService
|
||||
from app.module.operator.repository import (
|
||||
OperatorRepository,
|
||||
CategoryRelationRepository,
|
||||
OperatorReleaseRepository,
|
||||
)
|
||||
from app.module.operator.parsers import ParserHolder
|
||||
from app.module.shared.file_service import FileService
|
||||
from app.module.shared.chunk_upload_repository import ChunkUploadRepository
|
||||
|
||||
return OperatorService(
|
||||
operator_repo=OperatorRepository(None),
|
||||
category_relation_repo=CategoryRelationRepository(None),
|
||||
operator_release_repo=OperatorReleaseRepository(None),
|
||||
parser_holder=ParserHolder(),
|
||||
file_service=FileService(ChunkUploadRepository()),
|
||||
)
|
||||
|
||||
|
||||
def _get_template_service(db: AsyncSession) -> CleaningTemplateService:
|
||||
"""Get cleaning template service instance"""
|
||||
from app.module.cleaning.service import CleaningTaskValidator
|
||||
from app.module.cleaning.repository import (
|
||||
CleaningTemplateRepository,
|
||||
OperatorInstanceRepository,
|
||||
)
|
||||
|
||||
operator_service = _get_operator_service()
|
||||
|
||||
template_repo = CleaningTemplateRepository(None)
|
||||
|
||||
return CleaningTemplateService(
|
||||
template_repo=template_repo,
|
||||
operator_instance_repo=OperatorInstanceRepository(None),
|
||||
operator_service=operator_service,
|
||||
validator=CleaningTaskValidator(task_repo=None, template_repo=template_repo),
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=StandardResponse[PaginatedData[CleaningTemplateDto]],
|
||||
summary="查询清洗模板列表",
|
||||
description="分页查询清洗模板",
|
||||
)
|
||||
async def get_cleaning_templates(
|
||||
page: int = Query(0, description="页码"),
|
||||
size: int = Query(20, description="每页数量"),
|
||||
keyword: Optional[str] = Query(None, description="关键词搜索"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Query cleaning templates with pagination"""
|
||||
template_service = _get_template_service(db)
|
||||
|
||||
items, total = await template_service.get_templates(db, keyword, page, size)
|
||||
|
||||
total_pages = math.ceil(total / size) if total > 0 else 0
|
||||
|
||||
return StandardResponse(
|
||||
code="0",
|
||||
message="success",
|
||||
data=PaginatedData(
|
||||
content=items,
|
||||
total_elements=total,
|
||||
total_pages=total_pages,
|
||||
page=page,
|
||||
size=size,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"",
|
||||
response_model=StandardResponse[CleaningTemplateDto],
|
||||
summary="创建清洗模板",
|
||||
description="创建新的清洗模板",
|
||||
)
|
||||
async def create_cleaning_template(
|
||||
request: CreateCleaningTemplateRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Create cleaning template"""
|
||||
template_service = _get_template_service(db)
|
||||
|
||||
template = await template_service.create_template(db, request)
|
||||
await db.commit()
|
||||
|
||||
return StandardResponse(code="0", message="success", data=template)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{template_id}",
|
||||
response_model=StandardResponse[CleaningTemplateDto],
|
||||
summary="获取清洗模板详情",
|
||||
description="根据ID获取清洗模板详细信息",
|
||||
)
|
||||
async def get_cleaning_template(
|
||||
template_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get cleaning template by ID"""
|
||||
template_service = _get_template_service(db)
|
||||
|
||||
template = await template_service.get_template(db, template_id)
|
||||
return StandardResponse(code="0", message="success", data=template)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/{template_id}",
|
||||
response_model=StandardResponse[CleaningTemplateDto],
|
||||
summary="更新清洗模板",
|
||||
description="更新清洗模板信息",
|
||||
)
|
||||
async def update_cleaning_template(
|
||||
template_id: str,
|
||||
request: UpdateCleaningTemplateRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update cleaning template"""
|
||||
template_service = _get_template_service(db)
|
||||
|
||||
template = await template_service.update_template(db, template_id, request)
|
||||
await db.commit()
|
||||
|
||||
return StandardResponse(code="0", message="success", data=template)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/{template_id}",
|
||||
response_model=StandardResponse[str],
|
||||
summary="删除清洗模板",
|
||||
description="删除指定的清洗模板",
|
||||
)
|
||||
async def delete_cleaning_template(
|
||||
template_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Delete cleaning template"""
|
||||
template_service = _get_template_service(db)
|
||||
await template_service.delete_template(db, template_id)
|
||||
await db.commit()
|
||||
|
||||
return StandardResponse(code="0", message="success", data=template_id)
|
||||
@@ -0,0 +1,11 @@
|
||||
from .cleaning_task_repository import CleaningTaskRepository
|
||||
from .cleaning_template_repository import CleaningTemplateRepository
|
||||
from .cleaning_result_repository import CleaningResultRepository
|
||||
from .operator_instance_repository import OperatorInstanceRepository
|
||||
|
||||
__all__ = [
|
||||
"CleaningTaskRepository",
|
||||
"CleaningTemplateRepository",
|
||||
"CleaningResultRepository",
|
||||
"OperatorInstanceRepository",
|
||||
]
|
||||
@@ -0,0 +1,110 @@
|
||||
from typing import List, Optional
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, delete, func, Integer
|
||||
from app.db.models.cleaning import CleaningResult
|
||||
from app.module.cleaning.schema import CleaningResultDto
|
||||
|
||||
|
||||
class CleaningResultRepository:
|
||||
"""Repository for cleaning result operations"""
|
||||
|
||||
def __init__(self, model=None):
|
||||
self.model = model if model else CleaningResult
|
||||
|
||||
async def find_by_instance_id(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_id: str,
|
||||
status: Optional[str] = None
|
||||
) -> List[CleaningResultDto]:
|
||||
"""Query results by instance ID"""
|
||||
query = select(self.model).where(self.model.instance_id == instance_id)
|
||||
|
||||
if status:
|
||||
query = query.where(self.model.status == status)
|
||||
|
||||
result = await db.execute(query)
|
||||
results = result.scalars().all()
|
||||
|
||||
return [
|
||||
CleaningResultDto(
|
||||
instance_id=res.instance_id,
|
||||
src_file_id=res.src_file_id,
|
||||
dest_file_id=res.dest_file_id,
|
||||
src_name=res.src_name,
|
||||
dest_name=res.dest_name,
|
||||
src_type=res.src_type,
|
||||
dest_type=res.dest_type,
|
||||
src_size=res.src_size,
|
||||
dest_size=res.dest_size,
|
||||
status=res.status,
|
||||
result=res.result
|
||||
)
|
||||
for res in results
|
||||
]
|
||||
|
||||
async def count_by_instance_id(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_id: str
|
||||
) -> tuple[int, int]:
|
||||
"""Count results by instance ID (completed, failed) using single SQL"""
|
||||
query = select(
|
||||
func.sum(func.cast(self.model.status == "COMPLETED", Integer)).label("completed"),
|
||||
func.sum(func.cast(self.model.status == "FAILED", Integer)).label("failed"),
|
||||
).where(self.model.instance_id == instance_id)
|
||||
result = await db.execute(query)
|
||||
row = result.one()
|
||||
return (row.completed or 0, row.failed or 0)
|
||||
|
||||
async def count_total_by_instance_id(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_id: str
|
||||
) -> int:
|
||||
"""Count total results by instance ID using efficient SQL COUNT"""
|
||||
query = select(func.count()).select_from(self.model).where(
|
||||
self.model.instance_id == instance_id
|
||||
)
|
||||
result = await db.scalar(query)
|
||||
return result or 0
|
||||
|
||||
async def batch_count_by_instance_ids(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_ids: list[str]
|
||||
) -> dict[str, tuple[int, int, int]]:
|
||||
"""Batch count completed/failed/total for multiple instance IDs using single SQL.
|
||||
|
||||
Returns dict: {instance_id: (completed, failed, total)}
|
||||
"""
|
||||
if not instance_ids:
|
||||
return {}
|
||||
|
||||
query = select(
|
||||
self.model.instance_id,
|
||||
func.sum(func.cast(self.model.status == "COMPLETED", Integer)).label("completed"),
|
||||
func.sum(func.cast(self.model.status == "FAILED", Integer)).label("failed"),
|
||||
func.count().label("total"),
|
||||
).where(self.model.instance_id.in_(instance_ids)).group_by(self.model.instance_id)
|
||||
|
||||
result = await db.execute(query)
|
||||
return {
|
||||
row.instance_id: (row.completed or 0, row.failed or 0, row.total or 0)
|
||||
for row in result.all()
|
||||
}
|
||||
|
||||
async def delete_by_instance_id(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_id: str,
|
||||
status: Optional[str] = None
|
||||
) -> None:
|
||||
"""Delete results by instance ID"""
|
||||
query = delete(self.model).where(self.model.instance_id == instance_id)
|
||||
|
||||
if status:
|
||||
query = query.where(self.model.status == status)
|
||||
|
||||
await db.execute(query)
|
||||
await db.flush()
|
||||
@@ -0,0 +1,161 @@
|
||||
from typing import List, Optional
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, delete, func
|
||||
from app.db.models.cleaning import CleaningTask
|
||||
from app.module.cleaning.schema import CleaningTaskDto
|
||||
|
||||
|
||||
class CleaningTaskRepository:
|
||||
"""Repository for cleaning task operations"""
|
||||
|
||||
def __init__(self, model=None):
|
||||
self.model = model if model else CleaningTask
|
||||
|
||||
async def find_tasks(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
status: Optional[str] = None,
|
||||
keyword: Optional[str] = None,
|
||||
page: Optional[int] = None,
|
||||
size: Optional[int] = None
|
||||
) -> List[CleaningTaskDto]:
|
||||
"""Query cleaning tasks"""
|
||||
query = select(self.model)
|
||||
|
||||
if status:
|
||||
query = query.where(self.model.status == status)
|
||||
|
||||
if keyword:
|
||||
keyword_pattern = f"%{keyword}%"
|
||||
query = query.where(
|
||||
self.model.name.ilike(keyword_pattern) | self.model.description.ilike(keyword_pattern)
|
||||
)
|
||||
|
||||
query = query.order_by(self.model.created_at.desc())
|
||||
|
||||
if page is not None and size is not None:
|
||||
offset = max(page * size, 0)
|
||||
query = query.offset(offset).limit(size)
|
||||
|
||||
result = await db.execute(query)
|
||||
tasks = result.scalars().all()
|
||||
|
||||
return [
|
||||
CleaningTaskDto(
|
||||
id=task.id,
|
||||
name=task.name,
|
||||
description=task.description,
|
||||
status=task.status,
|
||||
src_dataset_id=task.src_dataset_id,
|
||||
src_dataset_name=task.src_dataset_name,
|
||||
dest_dataset_id=task.dest_dataset_id,
|
||||
dest_dataset_name=task.dest_dataset_name,
|
||||
before_size=task.before_size,
|
||||
after_size=task.after_size,
|
||||
file_count=task.file_count,
|
||||
retry_count=task.retry_count,
|
||||
started_at=task.started_at,
|
||||
finished_at=task.finished_at,
|
||||
created_at=task.created_at
|
||||
)
|
||||
for task in tasks
|
||||
]
|
||||
|
||||
async def find_task_by_id(self, db: AsyncSession, task_id: str) -> Optional[CleaningTaskDto]:
|
||||
"""Query task by ID"""
|
||||
query = select(self.model).where(self.model.id == task_id)
|
||||
result = await db.execute(query)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if not task:
|
||||
return None
|
||||
|
||||
return CleaningTaskDto(
|
||||
id=task.id,
|
||||
name=task.name,
|
||||
description=task.description,
|
||||
status=task.status,
|
||||
src_dataset_id=task.src_dataset_id,
|
||||
src_dataset_name=task.src_dataset_name,
|
||||
dest_dataset_id=task.dest_dataset_id,
|
||||
dest_dataset_name=task.dest_dataset_name,
|
||||
before_size=task.before_size,
|
||||
after_size=task.after_size,
|
||||
file_count=task.file_count,
|
||||
retry_count=task.retry_count,
|
||||
started_at=task.started_at,
|
||||
finished_at=task.finished_at,
|
||||
created_at=task.created_at
|
||||
)
|
||||
|
||||
async def insert_task(self, db: AsyncSession, task: CleaningTaskDto) -> None:
|
||||
"""Insert new task"""
|
||||
from app.db.models.cleaning import CleaningTask as CleaningTaskModel
|
||||
|
||||
db_task = CleaningTaskModel(
|
||||
id=task.id,
|
||||
name=task.name,
|
||||
description=task.description,
|
||||
status=task.status,
|
||||
src_dataset_id=task.src_dataset_id,
|
||||
src_dataset_name=task.src_dataset_name,
|
||||
dest_dataset_id=task.dest_dataset_id,
|
||||
dest_dataset_name=task.dest_dataset_name,
|
||||
before_size=task.before_size,
|
||||
after_size=task.after_size,
|
||||
file_count=task.file_count,
|
||||
retry_count=task.retry_count
|
||||
)
|
||||
db.add(db_task)
|
||||
await db.flush()
|
||||
|
||||
async def update_task(self, db: AsyncSession, task: CleaningTaskDto) -> None:
|
||||
"""Update task"""
|
||||
query = select(CleaningTask).where(CleaningTask.id == task.id)
|
||||
result = await db.execute(query)
|
||||
db_task = result.scalar_one_or_none()
|
||||
|
||||
if db_task:
|
||||
if task.status:
|
||||
db_task.status = task.status
|
||||
if task.started_at:
|
||||
db_task.started_at = task.started_at
|
||||
if task.finished_at:
|
||||
db_task.finished_at = task.finished_at
|
||||
if task.retry_count is not None:
|
||||
db_task.retry_count = task.retry_count
|
||||
|
||||
await db.flush()
|
||||
|
||||
async def delete_task_by_id(self, db: AsyncSession, task_id: str) -> None:
|
||||
"""Delete task by ID"""
|
||||
query = delete(self.model).where(self.model.id == task_id)
|
||||
await db.execute(query)
|
||||
await db.flush()
|
||||
|
||||
async def count_tasks(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
status: Optional[str] = None,
|
||||
keyword: Optional[str] = None
|
||||
) -> int:
|
||||
"""Count cleaning tasks using SQL COUNT"""
|
||||
query = select(func.count()).select_from(self.model)
|
||||
|
||||
if status:
|
||||
query = query.where(self.model.status == status)
|
||||
|
||||
if keyword:
|
||||
keyword_pattern = f"%{keyword}%"
|
||||
query = query.where(
|
||||
self.model.name.ilike(keyword_pattern) | self.model.description.ilike(keyword_pattern)
|
||||
)
|
||||
|
||||
result = await db.scalar(query)
|
||||
return result or 0
|
||||
|
||||
async def is_name_exist(self, db: AsyncSession, name: str) -> bool:
|
||||
"""Check if task name exists"""
|
||||
query = select(func.count()).select_from(self.model).where(self.model.name == name)
|
||||
result = await db.execute(query)
|
||||
return result.scalar_one() > 0 if result else False
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
from typing import List, Optional
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, delete, func
|
||||
from app.db.models.cleaning import CleaningTemplate
|
||||
|
||||
|
||||
class CleaningTemplateRepository:
|
||||
"""Repository for cleaning template operations"""
|
||||
|
||||
def __init__(self, model=None):
|
||||
self.model = model if model else CleaningTemplate
|
||||
|
||||
async def find_all_templates(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
keyword: Optional[str] = None,
|
||||
page: Optional[int] = None,
|
||||
size: Optional[int] = None,
|
||||
) -> List[CleaningTemplate]:
|
||||
"""Query all templates with optional pagination"""
|
||||
query = select(self.model)
|
||||
|
||||
if keyword:
|
||||
keyword_pattern = f"%{keyword}%"
|
||||
query = query.where(
|
||||
self.model.name.ilike(keyword_pattern)
|
||||
| self.model.description.ilike(keyword_pattern)
|
||||
)
|
||||
|
||||
query = query.order_by(self.model.created_at.desc())
|
||||
|
||||
if page is not None and size is not None:
|
||||
offset = page * size
|
||||
query = query.offset(offset).limit(size)
|
||||
|
||||
result = await db.execute(query)
|
||||
return result.scalars().all()
|
||||
|
||||
async def count_templates(
|
||||
self, db: AsyncSession, keyword: Optional[str] = None
|
||||
) -> int:
|
||||
"""Count templates"""
|
||||
query = select(func.count()).select_from(self.model)
|
||||
|
||||
if keyword:
|
||||
keyword_pattern = f"%{keyword}%"
|
||||
query = query.where(
|
||||
self.model.name.ilike(keyword_pattern)
|
||||
| self.model.description.ilike(keyword_pattern)
|
||||
)
|
||||
|
||||
result = await db.execute(query)
|
||||
return result.scalar_one()
|
||||
|
||||
async def find_template_by_id(
|
||||
self, db: AsyncSession, template_id: str
|
||||
) -> Optional[CleaningTemplate]:
|
||||
"""Query template by ID"""
|
||||
query = select(self.model).where(self.model.id == template_id)
|
||||
result = await db.execute(query)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def insert_template(
|
||||
self, db: AsyncSession, template: CleaningTemplate
|
||||
) -> None:
|
||||
"""Insert new template"""
|
||||
db.add(template)
|
||||
await db.flush()
|
||||
|
||||
async def update_template(
|
||||
self, db: AsyncSession, template: CleaningTemplate
|
||||
) -> None:
|
||||
"""Update template"""
|
||||
query = select(self.model).where(self.model.id == template.id)
|
||||
result = await db.execute(query)
|
||||
db_template = result.scalar_one_or_none()
|
||||
|
||||
if db_template:
|
||||
db_template.name = template.name
|
||||
db_template.description = template.description
|
||||
await db.flush()
|
||||
|
||||
async def delete_template(self, db: AsyncSession, template_id: str) -> None:
|
||||
"""Delete template"""
|
||||
query = delete(self.model).where(self.model.id == template_id)
|
||||
await db.execute(query)
|
||||
await db.flush()
|
||||
|
||||
async def is_name_exist(self, db: AsyncSession, name: str) -> bool:
|
||||
"""Check if template name exists"""
|
||||
query = (
|
||||
select(func.count()).select_from(self.model).where(self.model.name == name)
|
||||
)
|
||||
result = await db.execute(query)
|
||||
return result.scalar_one() > 0 if result else False
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
import json
|
||||
from typing import List
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, delete
|
||||
from app.db.models.cleaning import OperatorInstance
|
||||
|
||||
|
||||
class OperatorInstanceRepository:
|
||||
"""Repository for operator instance operations"""
|
||||
|
||||
def __init__(self, model=None):
|
||||
self.model = model if model else OperatorInstance
|
||||
|
||||
async def find_operator_by_instance_id(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_id: str
|
||||
) -> List[OperatorInstance]:
|
||||
"""Query operator instances by instance ID"""
|
||||
query = select(self.model).where(self.model.instance_id == instance_id)
|
||||
query = query.order_by(self.model.op_index.asc())
|
||||
result = await db.execute(query)
|
||||
return result.scalars().all()
|
||||
|
||||
async def find_instance_by_instance_id(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_id: str
|
||||
) -> List[OperatorInstance]:
|
||||
"""Query instances for template (same as find_operator_by_instance_id)"""
|
||||
return await self.find_operator_by_instance_id(db, instance_id)
|
||||
|
||||
async def insert_instance(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
instance_id: str,
|
||||
instances: List
|
||||
) -> None:
|
||||
"""Insert operator instances"""
|
||||
from app.db.models.cleaning import OperatorInstance as OperatorInstanceModel
|
||||
|
||||
for idx, instance in enumerate(instances):
|
||||
db_instance = OperatorInstanceModel(
|
||||
instance_id=instance_id,
|
||||
operator_id=instance.id,
|
||||
op_index=idx,
|
||||
settings_override=json.dumps(instance.overrides),
|
||||
)
|
||||
db.add(db_instance)
|
||||
await db.flush()
|
||||
|
||||
async def delete_by_instance_id(self, db: AsyncSession, instance_id: str) -> None:
|
||||
"""Delete instances by instance ID"""
|
||||
query = delete(self.model).where(self.model.instance_id == instance_id)
|
||||
await db.execute(query)
|
||||
await db.flush()
|
||||
@@ -0,0 +1,61 @@
|
||||
import httpx
|
||||
from typing import Optional
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class RuntimeClient:
|
||||
"""HTTP client for communicating with runtime service"""
|
||||
|
||||
def __init__(self, base_url: str = "http://deepdata-runtime:8081"):
|
||||
self.base_url = base_url
|
||||
self.client = httpx.AsyncClient(timeout=60.0)
|
||||
|
||||
async def submit_task(self, task_id: str, retry_count: int = 0) -> bool:
|
||||
"""Submit cleaning task to runtime executor"""
|
||||
try:
|
||||
url = f"{self.base_url}/api/task/{task_id}/submit"
|
||||
response = await self.client.post(url, json={"retry_count": retry_count})
|
||||
response.raise_for_status()
|
||||
logger.info(f"Task {task_id} submitted successfully with retry_count={retry_count}")
|
||||
return True
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Failed to submit task {task_id}: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error submitting task {task_id}: {e}")
|
||||
return False
|
||||
|
||||
async def stop_task(self, task_id: str) -> bool:
|
||||
"""Stop running cleaning task"""
|
||||
try:
|
||||
url = f"{self.base_url}/api/task/{task_id}/stop"
|
||||
response = await self.client.post(url)
|
||||
response.raise_for_status()
|
||||
logger.info(f"Task {task_id} stopped successfully")
|
||||
return True
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Failed to stop task {task_id}: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error stopping task {task_id}: {e}")
|
||||
return False
|
||||
|
||||
async def get_task_status(self, task_id: str) -> Optional[dict]:
|
||||
"""Get task status from runtime"""
|
||||
try:
|
||||
url = f"{self.base_url}/api/task/{task_id}/status"
|
||||
response = await self.client.get(url)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Failed to get task status {task_id}: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error getting task status {task_id}: {e}")
|
||||
return None
|
||||
|
||||
async def close(self):
|
||||
"""Close HTTP client"""
|
||||
await self.client.aclose()
|
||||
@@ -0,0 +1,25 @@
|
||||
from .cleaning import (
|
||||
CleaningTaskStatus,
|
||||
OperatorInstanceDto,
|
||||
CleaningProcess,
|
||||
CleaningTaskDto,
|
||||
CreateCleaningTaskRequest,
|
||||
CleaningResultDto,
|
||||
CleaningTaskLog,
|
||||
CleaningTemplateDto,
|
||||
CreateCleaningTemplateRequest,
|
||||
UpdateCleaningTemplateRequest,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CleaningTaskStatus",
|
||||
"OperatorInstanceDto",
|
||||
"CleaningProcess",
|
||||
"CleaningTaskDto",
|
||||
"CreateCleaningTaskRequest",
|
||||
"CleaningResultDto",
|
||||
"CleaningTaskLog",
|
||||
"CleaningTemplateDto",
|
||||
"CreateCleaningTemplateRequest",
|
||||
"UpdateCleaningTemplateRequest",
|
||||
]
|
||||
@@ -0,0 +1,148 @@
|
||||
from typing import Optional, List, Dict, Any
|
||||
from pydantic import BaseModel, Field
|
||||
from datetime import datetime
|
||||
from app.module.shared.schema.common import BaseResponseModel
|
||||
|
||||
|
||||
class CleaningTaskStatus:
|
||||
PENDING = "PENDING"
|
||||
RUNNING = "RUNNING"
|
||||
COMPLETED = "COMPLETED"
|
||||
PARTIAL_SUCCESS = "PARTIAL_SUCCESS"
|
||||
STOPPED = "STOPPED"
|
||||
FAILED = "FAILED"
|
||||
|
||||
|
||||
class OperatorInstanceDto(BaseResponseModel):
|
||||
"""Operator instance DTO for task or template"""
|
||||
id: str = Field(..., description="Operator ID")
|
||||
name: Optional[str] = Field(None, description="Operator name")
|
||||
description: Optional[str] = Field(None, description="Operator description")
|
||||
inputs: Optional[str] = Field(None, description="Input types: text/image/audio/video/multimodal")
|
||||
outputs: Optional[str] = Field(None, description="Output types: text/image/audio/video/multimodal")
|
||||
categories: Optional[List[str]] = Field(None, description="Category IDs")
|
||||
settings: Optional[str] = Field(None, description="算子设置（JSON）")
|
||||
overrides: Dict[str, Any] = Field(default_factory=dict, description="Operator parameter overrides")
|
||||
|
||||
|
||||
class CleaningProcess(BaseResponseModel):
|
||||
"""Task progress information"""
|
||||
process: float = Field(..., description="Progress percentage (based on successful files)")
|
||||
successRate: float = Field(..., description="Success rate percentage")
|
||||
totalFileNum: int = Field(..., description="Total file count")
|
||||
succeedFileNum: int = Field(..., description="Succeeded file count")
|
||||
failedFileNum: int = Field(..., description="Failed file count")
|
||||
finishedFileNum: int = Field(..., description="Successfully processed file count (excludes failed)")
|
||||
|
||||
@classmethod
|
||||
def of(cls, total: int, succeed: int, failed: int) -> 'CleaningProcess':
|
||||
"""Create progress info
|
||||
|
||||
- finishedFileNum: only counts successfully processed files (excludes failed)
|
||||
- process: progress percentage based on successful files only
|
||||
- successRate: percentage of finished files that succeeded
|
||||
"""
|
||||
# finished_file_num only counts successfully processed files
|
||||
finished_file_num = succeed
|
||||
|
||||
if total == 0:
|
||||
process = 0.0
|
||||
else:
|
||||
# Progress only counts successful files
|
||||
process = round(succeed * 100.0 / total, 2)
|
||||
|
||||
# Calculate total processed (succeed + failed) for success rate
|
||||
total_processed = succeed + failed
|
||||
if total_processed == 0:
|
||||
success_rate = 0.0
|
||||
else:
|
||||
success_rate = round(succeed * 100.0 / total_processed, 2)
|
||||
|
||||
return cls(
|
||||
process=process,
|
||||
successRate=success_rate,
|
||||
totalFileNum=total,
|
||||
succeedFileNum=succeed,
|
||||
failedFileNum=failed,
|
||||
finishedFileNum=finished_file_num,
|
||||
)
|
||||
|
||||
|
||||
class CleaningTaskDto(BaseResponseModel):
|
||||
"""Cleaning task DTO"""
|
||||
id: Optional[str] = Field(None, description="Task ID")
|
||||
name: Optional[str] = Field(None, description="Task name")
|
||||
description: Optional[str] = Field(None, description="Task description")
|
||||
src_dataset_id: Optional[str] = Field(None, description="Source dataset ID")
|
||||
src_dataset_name: Optional[str] = Field(None, description="Source dataset name")
|
||||
dest_dataset_id: Optional[str] = Field(None, description="Destination dataset ID")
|
||||
dest_dataset_name: Optional[str] = Field(None, description="Destination dataset name")
|
||||
before_size: Optional[int] = Field(None, description="Data size before cleaning")
|
||||
after_size: Optional[int] = Field(None, description="Data size after cleaning")
|
||||
file_count: Optional[int] = Field(None, description="Total file count")
|
||||
retry_count: Optional[int] = Field(None, description="Retry count")
|
||||
status: Optional[str] = Field(None, description="Task status")
|
||||
template_id: Optional[str] = Field(None, description="Template ID if created from template")
|
||||
instance: Optional[List[OperatorInstanceDto]] = Field(None, description="Operator instances")
|
||||
progress: Optional[CleaningProcess] = Field(None, description="Task progress")
|
||||
created_at: Optional[datetime] = Field(None, description="Creation time")
|
||||
started_at: Optional[datetime] = Field(None, description="Start time")
|
||||
finished_at: Optional[datetime] = Field(None, description="Finish time")
|
||||
|
||||
|
||||
class CreateCleaningTaskRequest(BaseResponseModel):
|
||||
"""Request to create cleaning task"""
|
||||
name: str = Field(..., description="Cleaning task name")
|
||||
description: str = Field(..., description="Cleaning task description")
|
||||
src_dataset_id: str = Field(..., description="Source dataset ID")
|
||||
src_dataset_name: str = Field(..., description="Source dataset name")
|
||||
dest_dataset_id: Optional[str] = Field(None, description="Destination dataset ID")
|
||||
dest_dataset_name: str = Field(..., description="Destination dataset name, creates new dataset if destDatasetId is empty")
|
||||
dest_dataset_type: str = Field(..., description="Destination dataset type: TEXT/IMAGE/VIDEO/AUDIO/OTHER")
|
||||
template_id: Optional[str] = Field(None, description="Template ID (alternative to instance)")
|
||||
instance: List[OperatorInstanceDto] = Field(default_factory=list, description="Operator list (alternative to templateId)")
|
||||
|
||||
|
||||
class CleaningResultDto(BaseResponseModel):
|
||||
"""Cleaning result DTO"""
|
||||
instance_id: Optional[str] = Field(None, description="Instance ID")
|
||||
src_file_id: Optional[str] = Field(None, description="Source file ID")
|
||||
dest_file_id: Optional[str] = Field(None, description="Destination file ID")
|
||||
src_name: Optional[str] = Field(None, description="Source file name")
|
||||
dest_name: Optional[str] = Field(None, description="Destination file name")
|
||||
src_type: Optional[str] = Field(None, description="Source file type")
|
||||
dest_type: Optional[str] = Field(None, description="Destination file type")
|
||||
src_size: Optional[int] = Field(None, description="Source file size")
|
||||
dest_size: Optional[int] = Field(None, description="Destination file size")
|
||||
status: Optional[str] = Field(None, description="Cleaning status")
|
||||
result: Optional[str] = Field(None, description="Cleaning result message")
|
||||
|
||||
|
||||
class CleaningTaskLog(BaseResponseModel):
|
||||
"""Task log entry"""
|
||||
level: str = Field(..., description="Log level: INFO, WARN, ERROR")
|
||||
message: str = Field(..., description="Log message")
|
||||
|
||||
|
||||
class CleaningTemplateDto(BaseResponseModel):
|
||||
"""Cleaning template DTO"""
|
||||
id: Optional[str] = Field(None, description="Template ID")
|
||||
name: Optional[str] = Field(None, description="Template name")
|
||||
description: Optional[str] = Field(None, description="Template description")
|
||||
instance: List[OperatorInstanceDto] = Field(default_factory=list, description="Operator instances")
|
||||
created_at: Optional[datetime] = Field(None, description="Creation time")
|
||||
updated_at: Optional[datetime] = Field(None, description="Update time")
|
||||
|
||||
|
||||
class CreateCleaningTemplateRequest(BaseResponseModel):
|
||||
"""Request to create cleaning template"""
|
||||
name: str = Field(..., description="Template name")
|
||||
description: str = Field(..., description="Template description")
|
||||
instance: List[OperatorInstanceDto] = Field(..., description="Operator instances")
|
||||
|
||||
|
||||
class UpdateCleaningTemplateRequest(BaseResponseModel):
|
||||
"""Request to update cleaning template"""
|
||||
name: str = Field(..., description="Template name")
|
||||
description: str = Field(..., description="Template description")
|
||||
instance: List[OperatorInstanceDto] = Field(..., description="Operator instances")
|
||||
@@ -0,0 +1,11 @@
|
||||
from .cleaning_task_validator import CleaningTaskValidator
|
||||
from .cleaning_task_scheduler import CleaningTaskScheduler
|
||||
from .cleaning_template_service import CleaningTemplateService
|
||||
from .cleaning_task_service import CleaningTaskService
|
||||
|
||||
__all__ = [
|
||||
"CleaningTaskValidator",
|
||||
"CleaningTaskScheduler",
|
||||
"CleaningTemplateService",
|
||||
"CleaningTaskService",
|
||||
]
|
||||
@@ -0,0 +1,164 @@
|
||||
import asyncio
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.core.logging import get_logger
|
||||
from app.module.cleaning.repository import CleaningTaskRepository
|
||||
from app.module.cleaning.runtime_client import RuntimeClient
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# 模块级单例：进程内所有请求共享同一个调度器
|
||||
_scheduler: "CleaningTaskScheduler | None" = None
|
||||
|
||||
|
||||
def get_scheduler() -> "CleaningTaskScheduler":
|
||||
"""获取全局调度器单例，不存在则创建"""
|
||||
global _scheduler
|
||||
if _scheduler is None:
|
||||
_scheduler = CleaningTaskScheduler(
|
||||
CleaningTaskRepository(None),
|
||||
RuntimeClient()
|
||||
)
|
||||
return _scheduler
|
||||
|
||||
|
||||
class CleaningTaskScheduler:
|
||||
"""Scheduler for executing cleaning tasks"""
|
||||
|
||||
def __init__(self, task_repo: CleaningTaskRepository, runtime_client: RuntimeClient):
|
||||
self.task_repo = task_repo
|
||||
self.runtime_client = runtime_client
|
||||
self._polling_task_ids: set[str] = set() # 待轮询的任务 ID 集合
|
||||
self._polling_started: bool = False
|
||||
self._poll_failure_count: dict[str, int] = {} # runtime 不可达连续失败计数
|
||||
self._MAX_POLL_FAILURES = 1 # 一次超时(60s) → 标记 FAILED
|
||||
|
||||
async def execute_task(self, db: AsyncSession, task_id: str, retry_count: int) -> bool:
|
||||
"""Execute cleaning task"""
|
||||
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
||||
from datetime import datetime
|
||||
|
||||
task = CleaningTaskDto()
|
||||
task.id = task_id
|
||||
task.status = CleaningTaskStatus.RUNNING
|
||||
task.started_at = datetime.now()
|
||||
task.retry_count = retry_count
|
||||
|
||||
submitted = await self.runtime_client.submit_task(task_id, retry_count)
|
||||
|
||||
if submitted:
|
||||
await self.task_repo.update_task(db, task)
|
||||
self._polling_task_ids.add(task_id)
|
||||
return submitted
|
||||
|
||||
task.status = CleaningTaskStatus.FAILED
|
||||
task.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, task)
|
||||
|
||||
return submitted
|
||||
|
||||
async def stop_task(self, db: AsyncSession, task_id: str) -> bool:
|
||||
"""Stop cleaning task"""
|
||||
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
||||
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
|
||||
await self.runtime_client.stop_task(task_id)
|
||||
|
||||
task = CleaningTaskDto()
|
||||
task.id = task_id
|
||||
task.status = CleaningTaskStatus.STOPPED
|
||||
|
||||
await self.task_repo.update_task(db, task)
|
||||
return True
|
||||
|
||||
async def startup(self):
|
||||
"""进程启动时调用：恢复未完成任务的轮询 + 启动全局轮询协程"""
|
||||
from app.module.cleaning.schema import CleaningTaskStatus
|
||||
from app.db.session import AsyncSessionLocal
|
||||
|
||||
# 从数据库恢复所有 RUNNING 状态的任务
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
tasks = await self.task_repo.find_tasks(db, status=CleaningTaskStatus.RUNNING)
|
||||
for task in tasks:
|
||||
logger.info(f"[Polling] Recovered RUNNING task from DB: {task.id}")
|
||||
self._polling_task_ids.add(task.id)
|
||||
except Exception as e:
|
||||
logger.error(f"[Polling] Failed to recover tasks from DB: {e}")
|
||||
|
||||
# 启动全局轮询协程
|
||||
if not self._polling_started:
|
||||
self._polling_started = True
|
||||
asyncio.create_task(self._poll_all_tasks())
|
||||
logger.info("[Polling] Global polling loop started")
|
||||
|
||||
async def _poll_all_tasks(self):
|
||||
"""全局轮询协程：每 2 秒轮询所有 RUNNING 任务的 runtime 状态"""
|
||||
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
||||
from app.db.session import AsyncSessionLocal
|
||||
from datetime import datetime
|
||||
|
||||
logger.info("[Polling] Global status polling loop started")
|
||||
terminal_statuses = {"completed", "failed", "cancelled", "stopped", "not_found"}
|
||||
|
||||
while True:
|
||||
task_ids = list(self._polling_task_ids)
|
||||
if not task_ids:
|
||||
# 没有待轮询任务，休眠后继续等
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
|
||||
for task_id in task_ids:
|
||||
try:
|
||||
status_data = await self.runtime_client.get_task_status(task_id)
|
||||
if status_data is None:
|
||||
# runtime 不可达，累计失败计数
|
||||
count = self._poll_failure_count.get(task_id, 0) + 1
|
||||
self._poll_failure_count[task_id] = count
|
||||
logger.warning(
|
||||
f"[Polling] Task {task_id} unreachable ({count}/{self._MAX_POLL_FAILURES})"
|
||||
)
|
||||
if count >= self._MAX_POLL_FAILURES:
|
||||
logger.error(
|
||||
f"[Polling] Task {task_id} marked FAILED: "
|
||||
f"runtime unreachable after {count} attempts"
|
||||
)
|
||||
async with AsyncSessionLocal() as db:
|
||||
task_dto = CleaningTaskDto()
|
||||
task_dto.id = task_id
|
||||
task_dto.status = CleaningTaskStatus.FAILED
|
||||
task_dto.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, task_dto)
|
||||
await db.commit()
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
continue
|
||||
|
||||
# runtime 可达，重置失败计数
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
|
||||
current_status = (status_data.get("status", "") or "").lower()
|
||||
logger.debug(f"[Polling] Task {task_id} status: {current_status}")
|
||||
|
||||
if current_status in terminal_statuses:
|
||||
async with AsyncSessionLocal() as db:
|
||||
task = CleaningTaskDto()
|
||||
task.id = task_id
|
||||
if current_status == "completed":
|
||||
task.status = CleaningTaskStatus.COMPLETED
|
||||
else:
|
||||
task.status = CleaningTaskStatus.FAILED
|
||||
task.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, task)
|
||||
await db.commit()
|
||||
logger.info(
|
||||
f"[Polling] Task {task_id} finished: {current_status}"
|
||||
)
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[Polling] Error polling task {task_id}: {e}")
|
||||
|
||||
await asyncio.sleep(2)
|
||||
@@ -0,0 +1,578 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Set
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models.base_entity import LineageNode, LineageEdge
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
from app.module.cleaning.repository import (
|
||||
CleaningTaskRepository,
|
||||
CleaningResultRepository,
|
||||
OperatorInstanceRepository,
|
||||
)
|
||||
from app.module.cleaning.schema import (
|
||||
CleaningTaskDto,
|
||||
CreateCleaningTaskRequest,
|
||||
CleaningResultDto,
|
||||
CleaningTaskLog,
|
||||
OperatorInstanceDto,
|
||||
CleaningProcess,
|
||||
CleaningTaskStatus,
|
||||
)
|
||||
from app.module.cleaning.service.cleaning_task_validator import CleaningTaskValidator
|
||||
from app.module.cleaning.service.cleaning_task_scheduler import CleaningTaskScheduler
|
||||
from app.module.shared.common.lineage import LineageService
|
||||
from app.module.shared.schema.lineage import NodeType, EdgeType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Module-level sanitizers (CodeQL recognizes return values of module-level
|
||||
# functions as sanitized; instance-method calls are not tracked through)
|
||||
_TASK_ID_PATTERN = re.compile(
|
||||
r'^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$'
|
||||
)
|
||||
|
||||
|
||||
def _sanitize_task_id(task_id: str) -> str:
|
||||
"""Validate and return sanitized task_id for safe path construction."""
|
||||
if not task_id:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TASK_ID_REQUIRED)
|
||||
if not _TASK_ID_PATTERN.match(task_id):
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_ID_REQUIRED,
|
||||
f"Invalid task_id format: {task_id}",
|
||||
)
|
||||
return task_id
|
||||
|
||||
|
||||
def _sanitize_retry_count(retry_count: int) -> int:
|
||||
"""Validate and return sanitized retry_count for safe path construction."""
|
||||
if retry_count < 0 or retry_count > 1000:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_ID_REQUIRED,
|
||||
f"Invalid retry_count: {retry_count}",
|
||||
)
|
||||
return retry_count
|
||||
|
||||
DATASET_PATH = "/dataset"
|
||||
FLOW_PATH = "/flow"
|
||||
|
||||
|
||||
class CleaningTaskService:
|
||||
"""Service for managing cleaning tasks"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task_repo: CleaningTaskRepository,
|
||||
result_repo: CleaningResultRepository,
|
||||
operator_instance_repo: OperatorInstanceRepository,
|
||||
operator_service,
|
||||
scheduler: CleaningTaskScheduler,
|
||||
validator: CleaningTaskValidator,
|
||||
dataset_service,
|
||||
lineage_service: LineageService,
|
||||
):
|
||||
self.task_repo = task_repo
|
||||
self.result_repo = result_repo
|
||||
self.operator_instance_repo = operator_instance_repo
|
||||
self.operator_service = operator_service
|
||||
self.scheduler = scheduler
|
||||
self.validator = validator
|
||||
self.dataset_service = dataset_service
|
||||
self.lineage_service = lineage_service
|
||||
|
||||
async def get_tasks(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
status: str | None = None,
|
||||
keyword: str | None = None,
|
||||
page: int | None = None,
|
||||
size: int | None = None,
|
||||
) -> List[CleaningTaskDto]:
|
||||
"""Get cleaning tasks"""
|
||||
tasks = await self.task_repo.find_tasks(db, status, keyword, page, size)
|
||||
|
||||
if not tasks:
|
||||
return tasks
|
||||
|
||||
# Batch query progress for all tasks in a single SQL (avoids N+1)
|
||||
task_ids = [task.id for task in tasks]
|
||||
progress_map = await self.result_repo.batch_count_by_instance_ids(db, task_ids)
|
||||
|
||||
for task in tasks:
|
||||
completed, failed, actual_total = progress_map.get(task.id, (0, 0, 0))
|
||||
total = max(actual_total, task.file_count or 0)
|
||||
task.progress = CleaningProcess.of(total, completed, failed)
|
||||
|
||||
return tasks
|
||||
|
||||
async def _set_process(self, db: AsyncSession, task: CleaningTaskDto) -> None:
|
||||
"""Set task progress using actual results from database"""
|
||||
completed, failed = await self.result_repo.count_by_instance_id(db, task.id)
|
||||
# Use actual total from database (t_clean_result table), fallback to task.file_count
|
||||
actual_total = await self.result_repo.count_total_by_instance_id(db, task.id)
|
||||
total = max(actual_total, task.file_count or 0)
|
||||
task.progress = CleaningProcess.of(total, completed, failed)
|
||||
|
||||
async def count_tasks(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
status: str | None = None,
|
||||
keyword: str | None = None,
|
||||
) -> int:
|
||||
"""Count cleaning tasks using SQL COUNT"""
|
||||
return await self.task_repo.count_tasks(db, status, keyword)
|
||||
|
||||
async def get_task(self, db: AsyncSession, task_id: str) -> CleaningTaskDto:
|
||||
"""Get task by ID"""
|
||||
task = await self.task_repo.find_task_by_id(db, task_id)
|
||||
if not task:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
|
||||
|
||||
await self._set_process(db, task)
|
||||
|
||||
instances = await self.operator_instance_repo.find_operator_by_instance_id(
|
||||
db, task_id
|
||||
)
|
||||
|
||||
# Batch query operators
|
||||
all_operators = await self.operator_service.get_operators(
|
||||
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
|
||||
)
|
||||
operator_map = {op.id: op for op in all_operators}
|
||||
|
||||
task.instance = []
|
||||
for inst in instances:
|
||||
operator = operator_map.get(inst.operator_id)
|
||||
if operator:
|
||||
task.instance.append(
|
||||
OperatorInstanceDto(
|
||||
id=operator.id,
|
||||
name=operator.name,
|
||||
description=operator.description,
|
||||
inputs=operator.inputs,
|
||||
outputs=operator.outputs,
|
||||
settings=operator.settings,
|
||||
categories=operator.categories,
|
||||
)
|
||||
)
|
||||
else:
|
||||
task.instance.append(OperatorInstanceDto(id=inst.operator_id))
|
||||
|
||||
return task
|
||||
|
||||
async def create_task(
|
||||
self, db: AsyncSession, request: CreateCleaningTaskRequest
|
||||
) -> CleaningTaskDto:
|
||||
"""Create new cleaning task"""
|
||||
if request.instance and request.template_id:
|
||||
instances = await self.get_instance_by_template_id(db, request.template_id)
|
||||
request.instance = instances
|
||||
|
||||
await self.validator.check_task_name_duplication(db, request.name)
|
||||
self.validator.check_input_and_output(request.instance)
|
||||
executor_type = self.validator.check_and_get_executor_type(request.instance)
|
||||
|
||||
task_id = str(uuid.uuid4())
|
||||
|
||||
dest_dataset_id = request.dest_dataset_id
|
||||
dest_dataset_name = request.dest_dataset_name
|
||||
|
||||
if not dest_dataset_id:
|
||||
logger.info(
|
||||
f"Creating new dataset: {dest_dataset_name}, type: {request.dest_dataset_type}"
|
||||
)
|
||||
dest_dataset_response = await self.dataset_service.create_dataset(
|
||||
name=dest_dataset_name,
|
||||
dataset_type=request.dest_dataset_type,
|
||||
description="",
|
||||
status="ACTIVE",
|
||||
)
|
||||
dest_dataset_id = dest_dataset_response.id
|
||||
logger.info(f"Successfully created dataset: {dest_dataset_id}")
|
||||
else:
|
||||
logger.info(f"Using existing dataset: {dest_dataset_id}")
|
||||
dest_dataset_response = await self.dataset_service.get_dataset(
|
||||
dest_dataset_id
|
||||
)
|
||||
|
||||
src_dataset = await self.dataset_service.get_dataset(request.src_dataset_id)
|
||||
if not src_dataset:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_DATASET_NOT_FOUND, request.src_dataset_id
|
||||
)
|
||||
|
||||
task_dto = CleaningTaskDto(
|
||||
id=task_id,
|
||||
name=request.name,
|
||||
description=request.description,
|
||||
status=CleaningTaskStatus.PENDING,
|
||||
src_dataset_id=request.src_dataset_id,
|
||||
src_dataset_name=request.src_dataset_name,
|
||||
dest_dataset_id=dest_dataset_id,
|
||||
dest_dataset_name=dest_dataset_name,
|
||||
before_size=src_dataset.totalSize,
|
||||
file_count=src_dataset.fileCount,
|
||||
retry_count=-1,
|
||||
)
|
||||
|
||||
await self.task_repo.insert_task(db, task_dto)
|
||||
|
||||
await self._add_cleaning_to_graph(src_dataset, task_dto, dest_dataset_response)
|
||||
|
||||
await self.operator_instance_repo.insert_instance(db, task_id, request.instance)
|
||||
|
||||
# Increment operator usage count
|
||||
operator_ids = [inst.id for inst in request.instance if inst.id]
|
||||
if operator_ids:
|
||||
await self.operator_service.increment_usage_count(operator_ids, db)
|
||||
|
||||
all_operators = await self.operator_service.get_operators(
|
||||
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
|
||||
)
|
||||
operator_map = {op.id: op for op in all_operators}
|
||||
|
||||
await self.prepare_task(
|
||||
dest_dataset_id, task_id, request.instance, operator_map, executor_type
|
||||
)
|
||||
|
||||
return await self.get_task(db, task_id)
|
||||
|
||||
async def _add_cleaning_to_graph(
|
||||
self,
|
||||
src_dataset,
|
||||
task: CleaningTaskDto,
|
||||
dest_dataset,
|
||||
) -> None:
|
||||
"""
|
||||
添加清洗任务到血缘图
|
||||
"""
|
||||
from_node = LineageNode(
|
||||
id=src_dataset.id,
|
||||
node_type=NodeType.DATASET.value,
|
||||
name=src_dataset.name,
|
||||
description=src_dataset.description or "",
|
||||
)
|
||||
|
||||
to_node = LineageNode(
|
||||
id=dest_dataset.id,
|
||||
node_type=NodeType.DATASET.value,
|
||||
name=dest_dataset.name,
|
||||
description=dest_dataset.description or "",
|
||||
)
|
||||
|
||||
edge = LineageEdge(
|
||||
process_id=task.id,
|
||||
name=task.name or "",
|
||||
description=task.description or "",
|
||||
edge_type=EdgeType.DATA_CLEANING.value,
|
||||
from_node_id=from_node.id,
|
||||
to_node_id=to_node.id,
|
||||
)
|
||||
|
||||
await self.lineage_service.generate_graph(from_node, edge, to_node)
|
||||
|
||||
async def prepare_task(
|
||||
self,
|
||||
dataset_id: str,
|
||||
task_id: str,
|
||||
instances: List[OperatorInstanceDto],
|
||||
operator_map: dict,
|
||||
executor_type: str,
|
||||
) -> None:
|
||||
"""Prepare task configuration file"""
|
||||
process_config = {
|
||||
"dataset_id": dataset_id,
|
||||
"instance_id": task_id,
|
||||
"dataset_path": f"{FLOW_PATH}/{task_id}/dataset.jsonl",
|
||||
"export_path": f"{DATASET_PATH}/{dataset_id}",
|
||||
"executor_type": executor_type,
|
||||
"process": [],
|
||||
}
|
||||
|
||||
for instance in instances:
|
||||
operator = operator_map.get(instance.id)
|
||||
if not operator:
|
||||
continue
|
||||
|
||||
operator_config = self._get_default_values(operator)
|
||||
operator_config.update(instance.overrides)
|
||||
|
||||
runtime_config = self._get_runtime_config(operator)
|
||||
operator_config.update(runtime_config)
|
||||
|
||||
process_config["process"].append({instance.id: operator_config})
|
||||
|
||||
config_file_path = Path(f"{FLOW_PATH}/{task_id}/process.yaml")
|
||||
config_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
with open(config_file_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(
|
||||
process_config, f, default_flow_style=False, allow_unicode=True
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to write process.yaml: {e}")
|
||||
raise BusinessError(ErrorCodes.CLEANING_FILE_SYSTEM_ERROR, str(e))
|
||||
|
||||
def _get_default_values(self, operator) -> Dict[str, Any]:
|
||||
"""Get default values from operator settings"""
|
||||
if not operator.settings:
|
||||
return {}
|
||||
|
||||
try:
|
||||
settings = json.loads(operator.settings)
|
||||
defaults = {}
|
||||
|
||||
for key, value in settings.items():
|
||||
setting_type = value.get("type")
|
||||
if "defaultVal" in value:
|
||||
defaults[key] = value["defaultVal"]
|
||||
|
||||
return defaults
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse settings: {e}")
|
||||
return {}
|
||||
|
||||
def _get_runtime_config(self, operator) -> Dict[str, Any]:
|
||||
"""Get runtime configuration from operator"""
|
||||
if not operator.runtime:
|
||||
return {}
|
||||
|
||||
try:
|
||||
return json.loads(operator.runtime)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse runtime config: {e}")
|
||||
return {}
|
||||
|
||||
async def scan_dataset(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
task_id: str,
|
||||
src_dataset_id: str,
|
||||
succeed_files: Set[str] | None = None,
|
||||
) -> None:
|
||||
"""Scan source dataset and create dataset.jsonl"""
|
||||
target_file_path = Path(f"{FLOW_PATH}/{task_id}/dataset.jsonl")
|
||||
target_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
query = text("""
|
||||
SELECT id, file_name, file_path, file_type, file_size, metadata
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = :dataset_id
|
||||
ORDER BY created_at
|
||||
""")
|
||||
|
||||
result = await db.execute(query, {"dataset_id": src_dataset_id})
|
||||
files = result.fetchall()
|
||||
|
||||
with open(target_file_path, "w", encoding="utf-8") as f:
|
||||
for file in files:
|
||||
if succeed_files and file.id in succeed_files:
|
||||
continue
|
||||
|
||||
metadata_dict = {}
|
||||
if file.metadata:
|
||||
try:
|
||||
parsed = json.loads(file.metadata)
|
||||
if isinstance(parsed, dict):
|
||||
metadata_dict = parsed
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
file_info = {
|
||||
"fileId": file.id,
|
||||
"fileName": file.file_name,
|
||||
"filePath": file.file_path,
|
||||
"fileType": file.file_type,
|
||||
"fileSize": file.file_size,
|
||||
"metadata": metadata_dict,
|
||||
}
|
||||
f.write(json.dumps(file_info, ensure_ascii=False) + "\n")
|
||||
|
||||
async def get_task_results(
|
||||
self, db: AsyncSession, task_id: str
|
||||
) -> List[CleaningResultDto]:
|
||||
"""Get task results"""
|
||||
return await self.result_repo.find_by_instance_id(db, task_id)
|
||||
|
||||
async def get_task_log(
|
||||
self, db: AsyncSession, task_id: str, retry_count: int
|
||||
) -> List[CleaningTaskLog]:
|
||||
"""Get task log"""
|
||||
safe_task_id = _sanitize_task_id(task_id)
|
||||
safe_retry_count = _sanitize_retry_count(retry_count)
|
||||
|
||||
flow_root = Path(FLOW_PATH).resolve()
|
||||
log_path = flow_root / safe_task_id / "output.log"
|
||||
if safe_retry_count > 0:
|
||||
log_path = flow_root / safe_task_id / f"output.log.{safe_retry_count}"
|
||||
|
||||
# 防止路径穿越：规范化后校验仍在 FLOW_PATH 下
|
||||
resolved_log_path = log_path.resolve()
|
||||
if flow_root not in resolved_log_path.parents:
|
||||
logger.warning(f"Path traversal attempt detected: task_id={task_id}")
|
||||
return []
|
||||
|
||||
if not resolved_log_path.exists():
|
||||
return []
|
||||
|
||||
logs = []
|
||||
last_level = "INFO"
|
||||
|
||||
standard_level_pattern = re.compile(
|
||||
r"\b(DEBUG|Debug|INFO|Info|WARN|Warn|WARNING|Warning|ERROR|Error|FATAL|Fatal)\b"
|
||||
)
|
||||
exception_suffix_pattern = re.compile(r"\b\w+(Warning|Error|Exception)\b")
|
||||
|
||||
with open(resolved_log_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
last_level = self._get_log_level(
|
||||
line, last_level, standard_level_pattern, exception_suffix_pattern
|
||||
)
|
||||
logs.append(CleaningTaskLog(level=last_level, message=line.rstrip()))
|
||||
|
||||
return logs
|
||||
|
||||
def _get_log_level(
|
||||
self, line: str, default_level: str, std_pattern, ex_pattern
|
||||
) -> str:
|
||||
"""Extract log level from log line"""
|
||||
if not line or not line.strip():
|
||||
return default_level
|
||||
|
||||
std_match = std_pattern.search(line)
|
||||
if std_match:
|
||||
return std_match.group(1).upper()
|
||||
|
||||
ex_match = ex_pattern.search(line)
|
||||
if ex_match:
|
||||
match = ex_match.group(1).upper()
|
||||
if match == "WARNING":
|
||||
return "WARN"
|
||||
if match in ["ERROR", "EXCEPTION"]:
|
||||
return "ERROR"
|
||||
|
||||
return default_level
|
||||
|
||||
async def delete_task(self, db: AsyncSession, task_id: str) -> None:
|
||||
"""Delete task"""
|
||||
safe_task_id = _sanitize_task_id(task_id)
|
||||
|
||||
task = await self.task_repo.find_task_by_id(db, safe_task_id)
|
||||
if not task:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
|
||||
|
||||
# 运行中的任务无法删除
|
||||
if task.status == CleaningTaskStatus.RUNNING:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_STATUS_INVALID,
|
||||
"Task is running, cannot be deleted. Please stop the task first."
|
||||
)
|
||||
|
||||
await self.task_repo.delete_task_by_id(db, safe_task_id)
|
||||
await self.operator_instance_repo.delete_by_instance_id(db, safe_task_id)
|
||||
await self.result_repo.delete_by_instance_id(db, safe_task_id)
|
||||
|
||||
# 删除任务相关文件
|
||||
flow_root = Path(FLOW_PATH).resolve()
|
||||
task_path = (flow_root / safe_task_id).resolve()
|
||||
# 防止路径穿越：parents 校验目标路径仍在 flow_root 下
|
||||
if flow_root not in task_path.parents:
|
||||
logger.warning(f"Path traversal attempt in delete_task: task_id={task_id}")
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_NOT_FOUND,
|
||||
f"Invalid task_id: {task_id}",
|
||||
)
|
||||
|
||||
if task_path.exists():
|
||||
try:
|
||||
shutil.rmtree(task_path)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete task path {task_id}: {e}")
|
||||
|
||||
async def execute_task(self, db: AsyncSession, task_id: str) -> bool:
|
||||
"""Execute task"""
|
||||
succeeded = await self.result_repo.find_by_instance_id(db, task_id, "COMPLETED")
|
||||
succeed_set = {res.src_file_id for res in succeeded}
|
||||
|
||||
task = await self.task_repo.find_task_by_id(db, task_id)
|
||||
if not task:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TASK_NOT_FOUND, task_id)
|
||||
|
||||
src_dataset = await self.dataset_service.get_dataset(task.src_dataset_id)
|
||||
if src_dataset:
|
||||
task.before_size = src_dataset.totalSize
|
||||
task.file_count = src_dataset.fileCount
|
||||
await self.task_repo.update_task(db, task)
|
||||
|
||||
await self.scan_dataset(db, task_id, task.src_dataset_id, succeed_set)
|
||||
await self.result_repo.delete_by_instance_id(db, task_id, "FAILED")
|
||||
|
||||
try:
|
||||
return await self.scheduler.execute_task(
|
||||
db, task_id, (task.retry_count or 0) + 1
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"execute_task failed, task_id={task_id}, retry_count={(task.retry_count or 0) + 1}: {e}"
|
||||
)
|
||||
|
||||
task = CleaningTaskDto()
|
||||
task.id = task_id
|
||||
task.status = CleaningTaskStatus.FAILED
|
||||
task.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, task)
|
||||
return False
|
||||
|
||||
async def stop_task(self, db: AsyncSession, task_id: str) -> bool:
|
||||
"""Stop task"""
|
||||
return await self.scheduler.stop_task(db, task_id)
|
||||
|
||||
async def get_instance_by_template_id(
|
||||
self, db: AsyncSession, template_id: str
|
||||
) -> List[OperatorInstanceDto]:
|
||||
"""Get instances by template ID (delegated to template service)"""
|
||||
instances = await self.operator_instance_repo.find_operator_by_instance_id(
|
||||
db, template_id
|
||||
)
|
||||
|
||||
# Batch query operators
|
||||
all_operators = await self.operator_service.get_operators(
|
||||
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
|
||||
)
|
||||
operator_map = {op.id: op for op in all_operators}
|
||||
|
||||
result = []
|
||||
for inst in instances:
|
||||
operator = operator_map.get(inst.operator_id)
|
||||
if operator:
|
||||
operator_dto = OperatorInstanceDto(
|
||||
id=operator.id,
|
||||
name=operator.name,
|
||||
description=operator.description,
|
||||
inputs=operator.inputs,
|
||||
outputs=operator.outputs,
|
||||
settings=operator.settings,
|
||||
categories=operator.categories,
|
||||
)
|
||||
if inst.settings_override:
|
||||
try:
|
||||
operator_dto.overrides = json.loads(inst.settings_override)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(
|
||||
f"Failed to parse settings for {inst.operator_id}: {e}"
|
||||
)
|
||||
result.append(operator_dto)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,136 @@
|
||||
import re
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
from app.module.cleaning.schema import OperatorInstanceDto
|
||||
from app.module.operator.constants import CATEGORY_DATA_JUICER_ID, CATEGORY_DEEPDATA_ID
|
||||
|
||||
# UUID pattern for task_id validation (prevents path traversal)
|
||||
_TASK_ID_PATTERN = re.compile(
|
||||
r'^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$'
|
||||
)
|
||||
|
||||
|
||||
class CleaningTaskValidator:
|
||||
"""Validator for cleaning tasks and templates"""
|
||||
|
||||
def __init__(self, task_repo=None, template_repo=None):
|
||||
self.task_repo = task_repo
|
||||
self.template_repo = template_repo
|
||||
|
||||
async def check_task_name_duplication(self, db: AsyncSession, name: str) -> None:
|
||||
"""Check if task name is duplicated"""
|
||||
if not name:
|
||||
raise BusinessError(ErrorCodes.CLEANING_NAME_DUPLICATED)
|
||||
if await self.task_repo.is_name_exist(db, name):
|
||||
raise BusinessError(ErrorCodes.CLEANING_NAME_DUPLICATED)
|
||||
|
||||
async def check_template_name_duplication(self, db: AsyncSession, name: str) -> None:
|
||||
"""Check if template name is duplicated"""
|
||||
if not name:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TEMPLATE_NAME_DUPLICATED)
|
||||
if await self.template_repo.is_name_exist(db, name):
|
||||
raise BusinessError(ErrorCodes.CLEANING_TEMPLATE_NAME_DUPLICATED)
|
||||
|
||||
@staticmethod
|
||||
def check_input_and_output(instances: list[OperatorInstanceDto]) -> None:
|
||||
"""Validate that operator input/output types are compatible.
|
||||
|
||||
Rules:
|
||||
- multimodal is compatible with any type (acts as wildcard)
|
||||
- for other types (text, image, audio, video), upstream outputs
|
||||
must match downstream inputs exactly
|
||||
"""
|
||||
if not instances:
|
||||
return
|
||||
|
||||
for i in range(len(instances) - 1):
|
||||
current = instances[i]
|
||||
next_op = instances[i + 1]
|
||||
|
||||
if not current.outputs:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_INVALID_OPERATOR_INPUT,
|
||||
f"Operator {current.id} has no outputs defined"
|
||||
)
|
||||
|
||||
if not next_op.inputs:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_INVALID_OPERATOR_INPUT,
|
||||
f"Operator {next_op.id} has no inputs defined"
|
||||
)
|
||||
|
||||
current_outputs = current.outputs.lower().strip()
|
||||
next_inputs = next_op.inputs.lower().strip()
|
||||
|
||||
# multimodal is compatible with everything
|
||||
if "multimodal" in current_outputs or "multimodal" in next_inputs:
|
||||
continue
|
||||
|
||||
# non-multimodal: types must match exactly
|
||||
if current_outputs != next_inputs:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_INVALID_OPERATOR_INPUT,
|
||||
f"Operator '{current.id}' outputs '{current.outputs}', "
|
||||
f"but operator '{next_op.id}' requires '{next_op.inputs}'"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def check_and_get_executor_type(instances: list[OperatorInstanceDto]) -> str:
|
||||
"""Check operator categories and determine executor type (deepdata/datajuicer)"""
|
||||
if not instances:
|
||||
return "deepdata"
|
||||
|
||||
executor_types = set()
|
||||
|
||||
for instance in instances:
|
||||
if instance.categories:
|
||||
for category in instance.categories:
|
||||
if CATEGORY_DATA_JUICER_ID in category.lower():
|
||||
executor_types.add("default")
|
||||
elif CATEGORY_DEEPDATA_ID in category.lower():
|
||||
executor_types.add("deepdata")
|
||||
|
||||
if len(executor_types) > 1:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_INVALID_EXECUTOR_TYPE,
|
||||
"Cannot mix DeepData and DataJuicer operators in same task"
|
||||
)
|
||||
|
||||
return executor_types.pop() if executor_types else "deepdata"
|
||||
|
||||
@staticmethod
|
||||
def check_task_id(task_id: str) -> None:
|
||||
"""Validate task ID — rejects non-UUID and path traversal patterns"""
|
||||
CleaningTaskValidator.sanitize_task_id(task_id)
|
||||
|
||||
@staticmethod
|
||||
def sanitize_task_id(task_id: str) -> str:
|
||||
"""Validate and return sanitized task_id for safe path construction.
|
||||
|
||||
CodeQL recognizes the return value of this function as sanitized,
|
||||
allowing safe use in path construction.
|
||||
"""
|
||||
if not task_id:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TASK_ID_REQUIRED)
|
||||
if not _TASK_ID_PATTERN.match(task_id):
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_ID_REQUIRED,
|
||||
f"Invalid task_id format: {task_id}",
|
||||
)
|
||||
return task_id
|
||||
|
||||
@staticmethod
|
||||
def sanitize_retry_count(retry_count: int) -> int:
|
||||
"""Validate and return sanitized retry_count for safe path construction.
|
||||
|
||||
CodeQL recognizes the return value of this function as sanitized,
|
||||
allowing safe use in path construction.
|
||||
"""
|
||||
if retry_count < 0 or retry_count > 1000:
|
||||
raise BusinessError(
|
||||
ErrorCodes.CLEANING_TASK_ID_REQUIRED,
|
||||
f"Invalid retry_count: {retry_count}",
|
||||
)
|
||||
return retry_count
|
||||
@@ -0,0 +1,249 @@
|
||||
import json
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.exception import BusinessError, ErrorCodes
|
||||
from app.core.logging import get_logger
|
||||
from app.module.cleaning import UpdateCleaningTemplateRequest
|
||||
from app.module.cleaning.repository import (
|
||||
CleaningTemplateRepository,
|
||||
OperatorInstanceRepository,
|
||||
)
|
||||
from app.module.cleaning.schema import (
|
||||
CleaningTemplateDto,
|
||||
CreateCleaningTemplateRequest,
|
||||
OperatorInstanceDto,
|
||||
)
|
||||
from app.module.cleaning.service.cleaning_task_validator import CleaningTaskValidator
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
PRESET_TEMPLATE_IDS = {
|
||||
"550e8400-e29b-41d4-a716-446655440001",
|
||||
"661f9500-f3ac-52e5-b827-557766550002",
|
||||
"772a0611-a4bd-63f6-c938-668877660003",
|
||||
"883b1722-b5ce-7407-d049-779988770004",
|
||||
"994c2833-c6df-8518-e150-880099880005",
|
||||
}
|
||||
|
||||
|
||||
class CleaningTemplateService:
|
||||
"""Service for managing cleaning templates"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
template_repo: CleaningTemplateRepository,
|
||||
operator_instance_repo: OperatorInstanceRepository,
|
||||
operator_service,
|
||||
validator: CleaningTaskValidator,
|
||||
):
|
||||
self.template_repo = template_repo
|
||||
self.operator_instance_repo = operator_instance_repo
|
||||
self.operator_service = operator_service
|
||||
self.validator = validator
|
||||
|
||||
async def get_templates(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
keyword: str | None = None,
|
||||
page: int | None = None,
|
||||
size: int | None = None,
|
||||
) -> tuple[List[CleaningTemplateDto], int]:
|
||||
"""Get templates with pagination"""
|
||||
total = await self.template_repo.count_templates(db, keyword)
|
||||
templates = await self.template_repo.find_all_templates(db, keyword, page, size)
|
||||
|
||||
# Collect all operator IDs
|
||||
template_instances_map = {}
|
||||
for template in templates:
|
||||
instances = await self.operator_instance_repo.find_operator_by_instance_id(
|
||||
db, template.id
|
||||
)
|
||||
template_instances_map[template.id] = instances
|
||||
|
||||
# Batch query all operators
|
||||
all_operators = await self.operator_service.get_operators(
|
||||
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
|
||||
)
|
||||
operator_map = {op.id: op for op in all_operators}
|
||||
|
||||
# Build result
|
||||
result = []
|
||||
for template in templates:
|
||||
template_dto = CleaningTemplateDto(
|
||||
id=template.id,
|
||||
name=template.name,
|
||||
description=template.description,
|
||||
instance=[],
|
||||
created_at=template.created_at,
|
||||
updated_at=template.updated_at,
|
||||
)
|
||||
|
||||
instances = template_instances_map.get(template.id, [])
|
||||
for inst in instances:
|
||||
operator = operator_map.get(inst.operator_id)
|
||||
if operator:
|
||||
operator_dto = OperatorInstanceDto(
|
||||
id=operator.id,
|
||||
name=operator.name,
|
||||
description=operator.description,
|
||||
inputs=operator.inputs,
|
||||
outputs=operator.outputs,
|
||||
settings=operator.settings,
|
||||
categories=operator.categories,
|
||||
)
|
||||
if inst.settings_override:
|
||||
try:
|
||||
operator_dto.overrides = json.loads(inst.settings_override)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(
|
||||
f"Failed to parse settings for {inst.operator_id}: {e}"
|
||||
)
|
||||
template_dto.instance.append(operator_dto)
|
||||
|
||||
result.append(template_dto)
|
||||
|
||||
return result, total
|
||||
|
||||
async def get_template(
|
||||
self, db: AsyncSession, template_id: str
|
||||
) -> CleaningTemplateDto:
|
||||
"""Get template by ID"""
|
||||
template = await self.template_repo.find_template_by_id(db, template_id)
|
||||
if not template:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TEMPLATE_NOT_FOUND, template_id)
|
||||
|
||||
template_dto = CleaningTemplateDto(
|
||||
id=template.id,
|
||||
name=template.name,
|
||||
description=template.description,
|
||||
instance=[],
|
||||
created_at=template.created_at,
|
||||
updated_at=template.updated_at,
|
||||
)
|
||||
|
||||
instances = await self.operator_instance_repo.find_operator_by_instance_id(
|
||||
db, template_id
|
||||
)
|
||||
|
||||
# Batch query operators
|
||||
all_operators = await self.operator_service.get_operators(
|
||||
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
|
||||
)
|
||||
operator_map = {op.id: op for op in all_operators}
|
||||
|
||||
for inst in instances:
|
||||
operator = operator_map.get(inst.operator_id)
|
||||
if operator:
|
||||
operator_dto = OperatorInstanceDto(
|
||||
id=operator.id,
|
||||
name=operator.name,
|
||||
description=operator.description,
|
||||
inputs=operator.inputs,
|
||||
outputs=operator.outputs,
|
||||
settings=operator.settings,
|
||||
categories=operator.categories,
|
||||
)
|
||||
if inst.settings_override:
|
||||
try:
|
||||
operator_dto.overrides = json.loads(inst.settings_override)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(
|
||||
f"Failed to parse settings for {inst.operator_id}: {e}"
|
||||
)
|
||||
template_dto.instance.append(operator_dto)
|
||||
|
||||
return template_dto
|
||||
|
||||
async def create_template(
|
||||
self, db: AsyncSession, request: CreateCleaningTemplateRequest
|
||||
) -> CleaningTemplateDto:
|
||||
"""Create new template"""
|
||||
from app.db.models.cleaning import CleaningTemplate
|
||||
|
||||
await self.validator.check_template_name_duplication(db, request.name)
|
||||
self.validator.check_input_and_output(request.instance)
|
||||
self.validator.check_and_get_executor_type(request.instance)
|
||||
|
||||
template_id = str(uuid.uuid4())
|
||||
template = CleaningTemplate(
|
||||
id=template_id,
|
||||
name=request.name,
|
||||
description=request.description,
|
||||
)
|
||||
|
||||
await self.template_repo.insert_template(db, template)
|
||||
|
||||
await self.operator_instance_repo.insert_instance(
|
||||
db, template_id, request.instance
|
||||
)
|
||||
|
||||
return await self.get_template(db, template_id)
|
||||
|
||||
async def update_template(
|
||||
self, db: AsyncSession, template_id: str, request: UpdateCleaningTemplateRequest
|
||||
) -> CleaningTemplateDto:
|
||||
"""Update template"""
|
||||
|
||||
template = await self.template_repo.find_template_by_id(db, template_id)
|
||||
if not template:
|
||||
raise BusinessError(ErrorCodes.CLEANING_TEMPLATE_NOT_FOUND, template_id)
|
||||
|
||||
template.name = request.name
|
||||
template.description = request.description
|
||||
|
||||
await self.template_repo.update_template(db, template)
|
||||
await self.operator_instance_repo.delete_by_instance_id(db, template_id)
|
||||
|
||||
await self.operator_instance_repo.insert_instance(
|
||||
db, template_id, request.instance
|
||||
)
|
||||
|
||||
return await self.get_template(db, template_id)
|
||||
|
||||
async def delete_template(self, db: AsyncSession, template_id: str) -> None:
|
||||
"""Delete template"""
|
||||
if template_id in PRESET_TEMPLATE_IDS:
|
||||
raise BusinessError(ErrorCodes.CLEANING_CANNOT_DELETE_PRESET_TEMPLATE)
|
||||
await self.template_repo.delete_template(db, template_id)
|
||||
await self.operator_instance_repo.delete_by_instance_id(db, template_id)
|
||||
|
||||
async def get_instance_by_template_id(
|
||||
self, db: AsyncSession, template_id: str
|
||||
) -> List[OperatorInstanceDto]:
|
||||
"""Get operator instances by template ID"""
|
||||
instances = await self.operator_instance_repo.find_operator_by_instance_id(
|
||||
db, template_id
|
||||
)
|
||||
|
||||
# Batch query operators
|
||||
all_operators = await self.operator_service.get_operators(
|
||||
db=db, page=0, size=1000, categories=[], keyword=None, is_star=None
|
||||
)
|
||||
operator_map = {op.id: op for op in all_operators}
|
||||
|
||||
result = []
|
||||
for inst in instances:
|
||||
operator = operator_map.get(inst.operator_id)
|
||||
if operator:
|
||||
operator_dto = OperatorInstanceDto(
|
||||
id=operator.id,
|
||||
name=operator.name,
|
||||
description=operator.description,
|
||||
inputs=operator.inputs,
|
||||
outputs=operator.outputs,
|
||||
settings=operator.settings,
|
||||
categories=operator.categories,
|
||||
)
|
||||
if inst.settings_override:
|
||||
try:
|
||||
operator_dto.overrides = json.loads(inst.settings_override)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(
|
||||
f"Failed to parse settings for {inst.operator_id}: {e}"
|
||||
)
|
||||
result.append(operator_dto)
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user