deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,251 @@
|
||||
import json
|
||||
import threading
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.security import mask_sensitive_info
|
||||
from app.db.models.data_collection import CollectionTask, TaskExecution, CollectionTemplate
|
||||
from app.module.collection.schema.collection import CollectionConfig, SyncMode
|
||||
from app.module.shared.schema import TaskStatus
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
class DataxClient:
|
||||
def __init__(self, task: CollectionTask, execution: TaskExecution, template: CollectionTemplate):
|
||||
self.execution = execution
|
||||
self.task = task
|
||||
self.template = template
|
||||
# ✅ 确保路径在共享存储 /flow 下，供 Runtime 和 Ray Worker 访问
|
||||
self.config_file_path = f"/flow/data-collection/{task.id}/config.json"
|
||||
self.python_path = "python"
|
||||
self.datax_main = "/opt/datax/bin/datax.py"
|
||||
Path(self.config_file_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def validate_json_string(self) -> Dict[str, Any]:
|
||||
"""
|
||||
验证 JSON 字符串
|
||||
|
||||
Returns:
|
||||
解析后的配置字典
|
||||
"""
|
||||
try:
|
||||
config = json.loads(self.task.config)
|
||||
|
||||
# 基本验证
|
||||
if 'job' not in config:
|
||||
raise ValueError("JSON 必须包含 'job' 字段")
|
||||
|
||||
if 'content' not in config.get('job', {}):
|
||||
raise ValueError("job 必须包含 'content' 字段")
|
||||
|
||||
logger.info("JSON 配置验证通过")
|
||||
return config
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"JSON 格式错误: {e}")
|
||||
except Exception as e:
|
||||
raise ValueError(f"配置验证失败: {e}")
|
||||
|
||||
@staticmethod
|
||||
def generate_datx_config(task_config: CollectionConfig, template: CollectionTemplate, target_path: str):
|
||||
# 校验参数
|
||||
dest_path_target = {"nfswriter", "s3writer", "glusterfswriter"}
|
||||
reader_parameter = {
|
||||
**(task_config.parameter if task_config.parameter else {}),
|
||||
**(task_config.reader if task_config.reader else {})
|
||||
}
|
||||
dest_parameter = {}
|
||||
if template.target_type == "txtfilewriter":
|
||||
dest_parameter = {
|
||||
"path": target_path,
|
||||
"fileName": "collection_result",
|
||||
"writeMode": "truncate",
|
||||
"fileFormat": "csv"
|
||||
}
|
||||
elif dest_path_target.__contains__(template.target_type):
|
||||
dest_parameter = {
|
||||
"destPath": target_path
|
||||
}
|
||||
writer_parameter = {
|
||||
**(task_config.parameter if task_config.parameter else {}),
|
||||
**(task_config.writer if task_config.writer else {}),
|
||||
**dest_parameter
|
||||
}
|
||||
# 生成任务运行配置
|
||||
job_config = {
|
||||
"content": [
|
||||
{
|
||||
"reader": {
|
||||
"name": template.source_type,
|
||||
"parameter": reader_parameter
|
||||
},
|
||||
"writer": {
|
||||
"name": template.target_type,
|
||||
"parameter": writer_parameter
|
||||
}
|
||||
}
|
||||
],
|
||||
"setting": {
|
||||
"speed": {
|
||||
"channel": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
task_config.job = job_config
|
||||
|
||||
def create__config_file(self) -> str:
|
||||
"""
|
||||
创建配置文件
|
||||
|
||||
Returns:
|
||||
临时文件路径
|
||||
"""
|
||||
# 验证 JSON
|
||||
config = self.validate_json_string()
|
||||
|
||||
# 写入临时文件
|
||||
with open(self.config_file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(config, f, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.debug(f"创建配置文件: {self.config_file_path}")
|
||||
return self.config_file_path
|
||||
|
||||
def generate_config(self) -> str:
|
||||
"""
|
||||
✅ 新增：仅生成 DataX JSON 配置，不执行任务。
|
||||
用于 Runtime/Ray 模式：Backend 生成配置后提交给集群执行。
|
||||
|
||||
Returns:
|
||||
配置文件的绝对路径
|
||||
"""
|
||||
return self.create__config_file()
|
||||
|
||||
def run_datax_job(self):
|
||||
"""
|
||||
启动 DataX 任务 (Local/Fallback 模式)
|
||||
|
||||
Returns:
|
||||
执行结果字典
|
||||
"""
|
||||
if not self.execution.started_at:
|
||||
self.execution.started_at = datetime.now()
|
||||
|
||||
try:
|
||||
# ✅ 复用 generate_config 生成文件
|
||||
config_path = self.generate_config()
|
||||
|
||||
cmd = [self.python_path, str(self.datax_main), str(config_path)]
|
||||
cmd_str = ' '.join(cmd)
|
||||
logger.info(f"执行命令: {cmd_str}")
|
||||
|
||||
with open(self.execution.log_path, 'w', encoding='utf-8') as log_f:
|
||||
self.write_header_log(cmd_str, log_f)
|
||||
exit_code = self._run_process(cmd, log_f)
|
||||
|
||||
self.execution.completed_at = datetime.now()
|
||||
self.execution.duration_seconds = (self.execution.completed_at - self.execution.started_at).total_seconds()
|
||||
self.write_tail_log(exit_code, log_f)
|
||||
|
||||
if exit_code == 0:
|
||||
logger.info(f"DataX 任务执行成功: {self.execution.id}")
|
||||
logger.info(f"执行耗时: {self.execution.duration_seconds:.2f} 秒")
|
||||
self.execution.status = TaskStatus.COMPLETED.name
|
||||
self.rename_collection_result()
|
||||
else:
|
||||
self.execution.error_message = self.execution.error_message or f"DataX 任务执行失败，退出码: {exit_code}"
|
||||
self.execution.status = TaskStatus.FAILED.name
|
||||
logger.error(self.execution.error_message)
|
||||
except Exception as e:
|
||||
self.execution.completed_at = datetime.now()
|
||||
if self.execution.started_at:
|
||||
self.execution.duration_seconds = (self.execution.completed_at - self.execution.started_at).total_seconds()
|
||||
self.execution.error_message = f"执行异常: {e}"
|
||||
self.execution.status = TaskStatus.FAILED.name
|
||||
logger.error(f"执行异常: {e}", exc_info=True)
|
||||
with open(self.execution.log_path, 'w', encoding='utf-8') as log_f:
|
||||
log_f.write(f"任务执行失败: {e}\n")
|
||||
|
||||
if self.task.sync_mode == SyncMode.ONCE:
|
||||
self.task.status = self.execution.status
|
||||
else:
|
||||
self.task.status = TaskStatus.PENDING.name
|
||||
|
||||
def rename_collection_result(self):
|
||||
if self.template.target_type != "txtfilewriter":
|
||||
return
|
||||
target_path = Path(self.task.target_path)
|
||||
if not target_path.exists():
|
||||
logger.warning(f"Target path does not exist: {target_path}")
|
||||
return
|
||||
# If it's a directory, find all files without extensions
|
||||
for file_path in target_path.iterdir():
|
||||
if file_path.is_file() and not file_path.suffix:
|
||||
new_path = file_path.with_suffix('.csv')
|
||||
try:
|
||||
file_path.rename(new_path)
|
||||
logger.info(f"Renamed {file_path} to {new_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to rename {file_path} to {new_path}: {str(e)}")
|
||||
|
||||
def _run_process(self, cmd: list[str], log_f) -> int:
|
||||
# 启动进程
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding='utf-8',
|
||||
bufsize=1,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
# 创建读取线程
|
||||
stdout_thread = threading.Thread(target=lambda stream=process.stdout: self.read_stream(stream, log_f))
|
||||
stderr_thread = threading.Thread(target=lambda stream=process.stderr: self.read_stream(stream, log_f))
|
||||
|
||||
stdout_thread.start()
|
||||
stderr_thread.start()
|
||||
|
||||
# 等待进程完成
|
||||
try:
|
||||
exit_code = process.wait(timeout=self.task.timeout_seconds)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
exit_code = -1
|
||||
self.execution.error_message = f"任务执行超时（{self.task.timeout_seconds}秒）"
|
||||
logger.error(f"任务执行超时（{self.task.timeout_seconds}秒）")
|
||||
|
||||
# 等待线程完成
|
||||
stdout_thread.join(timeout=5)
|
||||
stderr_thread.join(timeout=5)
|
||||
return exit_code
|
||||
|
||||
def write_tail_log(self, exit_code: int, log_f):
|
||||
log_f.write("\n" + "=" * 100 + "\n")
|
||||
log_f.write(f"End Time: {self.execution.completed_at}\n")
|
||||
log_f.write(f"Execution Time: {self.execution.duration_seconds:.2f} seconds\n")
|
||||
log_f.write(f"Exit Code: {exit_code}\n")
|
||||
log_f.write(f"Status: {'SUCCESS' if exit_code == 0 else 'FAILED'}\n")
|
||||
|
||||
def write_header_log(self, cmd: str, log_f):
|
||||
log_f.write(f"DataX Task Execution Log\n")
|
||||
log_f.write(f"Job ID: {self.execution.id}\n")
|
||||
log_f.write(f"Start Time: {self.execution.started_at}\n")
|
||||
log_f.write(f"Config Source: JSON String\n")
|
||||
log_f.write(f"Command: {cmd}\n")
|
||||
log_f.write("=" * 100 + "\n\n")
|
||||
|
||||
@staticmethod
|
||||
def read_stream(stream, log_f):
|
||||
"""读取输出流并屏蔽敏感信息"""
|
||||
for line in stream:
|
||||
line = line.rstrip('\n')
|
||||
if line:
|
||||
# Mask sensitive information before writing to log
|
||||
masked_line = mask_sensitive_info(line)
|
||||
# 写入日志文件
|
||||
log_f.write(f"{masked_line}\n")
|
||||
log_f.flush()
|
||||
@@ -0,0 +1,15 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/data-collection",
|
||||
tags = ["data-collection"]
|
||||
)
|
||||
|
||||
# Include sub-routers
|
||||
from .collection import router as collection_router
|
||||
from .execution import router as execution_router
|
||||
from .template import router as template_router
|
||||
|
||||
router.include_router(collection_router)
|
||||
router.include_router(execution_router)
|
||||
router.include_router(template_router)
|
||||
@@ -0,0 +1,398 @@
|
||||
import json
|
||||
import math
|
||||
import uuid
|
||||
import shutil
|
||||
import os
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.exception import ErrorCodes, BusinessError, SuccessResponse, transaction
|
||||
from app.core.logging import get_logger
|
||||
from app.core.security import preserve_sensitive_values
|
||||
from app.db.models import Dataset, DatasetFiles
|
||||
from app.db.models.data_collection import CollectionTask, TaskExecution, CollectionTemplate
|
||||
from app.db.session import get_db
|
||||
from app.module.collection.client.datax_client import DataxClient
|
||||
from app.module.collection.schema.collection import CollectionTaskBase, CollectionTaskCreate, CollectionTaskUpdate, CollectionConfig, converter_to_response, \
|
||||
convert_for_create, SyncMode
|
||||
from app.module.collection.schedule import schedule_collection_task, remove_collection_task
|
||||
from app.module.collection.service.collection import CollectionTaskService
|
||||
from app.module.shared.schema import StandardResponse, PaginatedData, TaskStatus
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/tasks",
|
||||
tags=["data-collection/tasks"],
|
||||
)
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
async def is_hard_link(file_path: str) -> bool:
|
||||
"""检查文件是否是硬链接"""
|
||||
try:
|
||||
stat_info = await asyncio.to_thread(os.stat, file_path)
|
||||
# 如果链接数大于1，说明是硬链接
|
||||
return stat_info.st_nlink > 1
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
async def convert_hardlink_to_real_file(file_path: str) -> bool:
|
||||
"""
|
||||
将硬链接转换为实体文件
|
||||
通过读取并重新写入文件内容，创建一个独立的副本
|
||||
"""
|
||||
try:
|
||||
# 创建临时文件
|
||||
temp_path = f"{file_path}.tmp"
|
||||
# 使用 shutil.copy2 创建副本（保留元数据）
|
||||
await asyncio.to_thread(shutil.copy2, file_path, temp_path)
|
||||
# 删除原文件（硬链接）
|
||||
await asyncio.to_thread(os.unlink, file_path)
|
||||
# 重命名临时文件为原文件名
|
||||
await asyncio.to_thread(os.replace, temp_path, file_path)
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.warning(f"Failed to convert hard link to real file {file_path}: {e}")
|
||||
# 清理临时文件（如果存在）
|
||||
if os.path.exists(f"{file_path}.tmp"):
|
||||
try:
|
||||
await asyncio.to_thread(os.remove, f"{file_path}.tmp")
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
async def convert_dataset_hardlinks_before_delete(task_id: str, db: AsyncSession) -> int:
|
||||
"""
|
||||
删除归集任务前，将数据集中的硬链接文件转换为实体文件
|
||||
|
||||
Args:
|
||||
task_id: 归集任务ID
|
||||
db: 数据库会话
|
||||
|
||||
Returns:
|
||||
转换成功的文件数量
|
||||
"""
|
||||
try:
|
||||
# 查找所有数据集文件（通过文件路径匹配任务ID）
|
||||
# 注意：归集任务的源文件路径是 tmp/dataset/{task_id}/
|
||||
# 我们需要找到数据集中所有以这个路径为源的文件
|
||||
source_prefix = f"tmp/dataset/{task_id}/"
|
||||
|
||||
# 查询所有可能相关的数据集文件
|
||||
result = await db.execute(
|
||||
select(DatasetFiles).where(
|
||||
DatasetFiles.file_path.like(f"%/dataset/%"),
|
||||
DatasetFiles.status == "ACTIVE"
|
||||
)
|
||||
)
|
||||
dataset_files = result.scalars().all()
|
||||
|
||||
converted_count = 0
|
||||
for dataset_file in dataset_files:
|
||||
file_path = dataset_file.file_path
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
# 检查文件是否是硬链接
|
||||
if await is_hard_link(file_path):
|
||||
logger.info(f"Converting hard link to real file: {file_path}")
|
||||
success = await convert_hardlink_to_real_file(file_path)
|
||||
if success:
|
||||
converted_count += 1
|
||||
else:
|
||||
logger.warning(f"Failed to convert hard link: {file_path}")
|
||||
|
||||
if converted_count > 0:
|
||||
logger.info(f"Converted {converted_count} hard link(s) to real file(s) for task {task_id}")
|
||||
|
||||
return converted_count
|
||||
except Exception as e:
|
||||
logger.error(f"Error converting hard links for task {task_id}: {e}", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
@router.post("", response_model=StandardResponse[CollectionTaskBase], operation_id="create_collect_task", tags=["mcp"])
|
||||
async def create_task(
|
||||
request: CollectionTaskCreate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""创建归集任务"""
|
||||
template = await db.execute(select(CollectionTemplate).where(CollectionTemplate.id == request.template_id))
|
||||
template = template.scalar_one_or_none()
|
||||
if not template:
|
||||
raise BusinessError(ErrorCodes.COLLECTION_TEMPLATE_NOT_FOUND, data={"template_id": request.template_id})
|
||||
|
||||
task_id = str(uuid.uuid4())
|
||||
DataxClient.generate_datx_config(request.config, template, f"/dataset/local/{task_id}")
|
||||
task = convert_for_create(request, task_id)
|
||||
task.template_name = template.name
|
||||
dataset = None
|
||||
|
||||
async with transaction(db):
|
||||
if request.dataset_name:
|
||||
target_dataset_id = uuid.uuid4()
|
||||
dataset = Dataset(
|
||||
id=str(target_dataset_id),
|
||||
name=request.dataset_name,
|
||||
description="",
|
||||
dataset_type=request.dataset_type.name,
|
||||
status="DRAFT",
|
||||
path=f"/dataset/{target_dataset_id}",
|
||||
)
|
||||
db.add(dataset)
|
||||
|
||||
task_service = CollectionTaskService(db)
|
||||
task = await task_service.create_task(task, dataset)
|
||||
|
||||
task = await db.execute(select(CollectionTask).where(CollectionTask.id == task.id))
|
||||
task = task.scalar_one_or_none()
|
||||
|
||||
# 事务已提交，执行调度
|
||||
if task and task.sync_mode == SyncMode.SCHEDULED.value and task.schedule_expression:
|
||||
schedule_collection_task(task.id, task.schedule_expression)
|
||||
|
||||
return SuccessResponse(data=converter_to_response(task))
|
||||
|
||||
|
||||
@router.get("", response_model=StandardResponse[PaginatedData[CollectionTaskBase]])
|
||||
async def list_tasks(
|
||||
page: int = 1,
|
||||
size: int = 20,
|
||||
name: Optional[str] = Query(None, description="Fuzzy search by task name"),
|
||||
status: Optional[str] = Query(None, description="Filter by task status"),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""分页查询归集任务"""
|
||||
# 构建查询条件
|
||||
page = page if page > 0 else 1
|
||||
size = size if size > 0 else 20
|
||||
query = select(CollectionTask)
|
||||
|
||||
if name:
|
||||
query = query.where(CollectionTask.name.ilike(f"%{name}%"))
|
||||
|
||||
if status:
|
||||
query = query.where(CollectionTask.status == status)
|
||||
|
||||
# 获取总数
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = (await db.execute(count_query)).scalar_one()
|
||||
|
||||
# 分页查询
|
||||
offset = (page - 1) * size
|
||||
tasks = (await db.execute(
|
||||
query.order_by(CollectionTask.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(size)
|
||||
)).scalars().all()
|
||||
|
||||
# 转换为响应模型
|
||||
items = [converter_to_response(task) for task in tasks]
|
||||
total_pages = math.ceil(total / size) if total > 0 else 0
|
||||
|
||||
return SuccessResponse(
|
||||
data=PaginatedData(
|
||||
content=items,
|
||||
total_elements=total,
|
||||
total_pages=total_pages,
|
||||
page=page,
|
||||
size=size,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/execute", response_model=StandardResponse[str], operation_id="execute_task")
|
||||
async def execute_task(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""执行归集任务"""
|
||||
from app.module.collection.service.collection import CollectionTaskService
|
||||
|
||||
# 验证任务是否存在
|
||||
task = await db.execute(select(CollectionTask).where(CollectionTask.id == task_id))
|
||||
task = task.scalar_one_or_none()
|
||||
if not task:
|
||||
raise BusinessError(ErrorCodes.COLLECTION_TASK_NOT_FOUND, data={"task_id": task_id})
|
||||
|
||||
# 调用服务执行任务
|
||||
await CollectionTaskService.run_async(task_id)
|
||||
|
||||
return SuccessResponse(
|
||||
data=task_id
|
||||
)
|
||||
|
||||
|
||||
@router.put("/{task_id}", response_model=StandardResponse[CollectionTaskBase], operation_id="update_task")
|
||||
async def update_task(
|
||||
task_id: str,
|
||||
request: CollectionTaskUpdate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
更新归集任务
|
||||
|
||||
支持修改的字段：
|
||||
- description: 任务描述
|
||||
- schedule_expression: 定时表达式（仅 SCHEDULED 模式）
|
||||
- timeout_seconds: 超时时间
|
||||
- config: 任务配置
|
||||
|
||||
状态限制：
|
||||
- DRAFT/PENDING: 可以修改所有字段
|
||||
- RUNNING: 不允许修改
|
||||
- FAILED: 可以修改，修改后可重新执行
|
||||
- COMPLETED: ONCE 模式不允许修改，SCHEDULED 模式可以修改
|
||||
|
||||
Args:
|
||||
task_id: 任务ID
|
||||
request: 更新请求
|
||||
db: 数据库会话
|
||||
|
||||
Returns:
|
||||
更新后的任务信息
|
||||
"""
|
||||
# 查询任务
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if not task:
|
||||
raise BusinessError(ErrorCodes.COLLECTION_TASK_NOT_FOUND, data={"task_id": task_id})
|
||||
|
||||
# 检查任务状态，RUNNING 状态不允许修改
|
||||
if task.status == TaskStatus.RUNNING.name:
|
||||
raise BusinessError(
|
||||
ErrorCodes.VALIDATION_ERROR,
|
||||
data={"task_id": task_id, "current_status": task.status}
|
||||
)
|
||||
|
||||
# 检查任务同步模式，ONCE 模式已完成的不允许修改
|
||||
if task.sync_mode == SyncMode.ONCE.name and task.status == TaskStatus.COMPLETED.name:
|
||||
raise BusinessError(
|
||||
ErrorCodes.VALIDATION_ERROR,
|
||||
data={"task_id": task_id, "current_status": task.status}
|
||||
)
|
||||
|
||||
# 验证 schedule_expression 只能由 SCHEDULED 模式修改
|
||||
if request.schedule_expression is not None and task.sync_mode != SyncMode.SCHEDULED.name:
|
||||
raise BusinessError(
|
||||
ErrorCodes.VALIDATION_ERROR,
|
||||
data={"sync_mode": task.sync_mode}
|
||||
)
|
||||
|
||||
# 应用更新
|
||||
update_data = request.model_dump(exclude_unset=True)
|
||||
|
||||
if 'description' in update_data:
|
||||
task.description = update_data['description']
|
||||
|
||||
if 'timeout_seconds' in update_data:
|
||||
task.timeout_seconds = update_data['timeout_seconds']
|
||||
|
||||
if 'schedule_expression' in update_data:
|
||||
old_schedule_expression = task.schedule_expression
|
||||
task.schedule_expression = update_data['schedule_expression']
|
||||
# 如果调度表达式发生变化，需要重新调度任务
|
||||
if old_schedule_expression != task.schedule_expression:
|
||||
from app.module.collection.schedule import reschedule_collection_task
|
||||
reschedule_collection_task(task_id, task.schedule_expression)
|
||||
|
||||
if 'config' in update_data:
|
||||
# Get original config from database
|
||||
original_config = json.loads(task.config) if task.config else {}
|
||||
|
||||
# Preserve sensitive values if masked pattern detected
|
||||
preserved_config = preserve_sensitive_values(
|
||||
request.config.dict(),
|
||||
original_config
|
||||
)
|
||||
|
||||
# Regenerate task config file with preserved sensitive values
|
||||
template = await db.execute(select(CollectionTemplate).where(CollectionTemplate.id == task.template_id))
|
||||
template = template.scalar_one_or_none()
|
||||
if template:
|
||||
# Use preserved config to generate DataX config
|
||||
config_obj = CollectionConfig(**preserved_config)
|
||||
DataxClient.generate_datx_config(
|
||||
config_obj,
|
||||
template,
|
||||
task.target_path
|
||||
)
|
||||
# Save the modified config (with regenerated job) to database
|
||||
task.config = json.dumps(config_obj.model_dump())
|
||||
|
||||
# 如果任务处于 FAILED 状态，修改后重置为 PENDING，允许重新执行
|
||||
if task.status == TaskStatus.FAILED.name:
|
||||
task.status = TaskStatus.PENDING.name
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
|
||||
return SuccessResponse(
|
||||
data=converter_to_response(task)
|
||||
)
|
||||
|
||||
|
||||
@router.delete("", response_model=StandardResponse[str], status_code=200)
|
||||
async def delete_collection_tasks(
|
||||
ids: list[str] = Query(..., description="List of task IDs to delete"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
删除归集任务
|
||||
|
||||
Args:
|
||||
ids: 任务ID
|
||||
db: 数据库会话
|
||||
|
||||
Returns:
|
||||
StandardResponse[str]: 删除结果
|
||||
"""
|
||||
# 检查任务是否存在
|
||||
task_id = ids[0]
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if not task:
|
||||
raise BusinessError(ErrorCodes.COLLECTION_TASK_NOT_FOUND, data={"task_id": task_id})
|
||||
|
||||
# 删除任务执行记录（在事务内）
|
||||
async with transaction(db):
|
||||
await db.execute(
|
||||
TaskExecution.__table__.delete()
|
||||
.where(TaskExecution.task_id == task_id)
|
||||
)
|
||||
|
||||
# 删除任务
|
||||
await db.delete(task)
|
||||
|
||||
# 事务提交后，先转换硬链接，再删除文件系统和调度
|
||||
logger.info(f"Converting hard links before deleting task {task_id}")
|
||||
await convert_dataset_hardlinks_before_delete(task_id, db)
|
||||
|
||||
remove_collection_task(task_id)
|
||||
|
||||
target_path = f"/dataset/local/{task_id}"
|
||||
if os.path.exists(target_path):
|
||||
shutil.rmtree(target_path)
|
||||
job_path = f"/flow/data-collection/{task_id}"
|
||||
if os.path.exists(job_path):
|
||||
shutil.rmtree(job_path)
|
||||
|
||||
return SuccessResponse(
|
||||
data="success"
|
||||
)
|
||||
|
||||
@router.get("/{task_id}", response_model=StandardResponse[CollectionTaskBase])
|
||||
async def get_task(
|
||||
task_id: str,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""获取归集任务详情"""
|
||||
# 根据ID查询任务
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if not task:
|
||||
raise BusinessError(ErrorCodes.COLLECTION_TASK_NOT_FOUND, data={"task_id": task_id})
|
||||
|
||||
return SuccessResponse(data=converter_to_response(task))
|
||||
@@ -0,0 +1,124 @@
|
||||
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models.data_collection import TaskExecution
|
||||
from app.db.session import get_db
|
||||
from app.module.collection.schema.collection import TaskExecutionBase, converter_execution_to_response
|
||||
from app.module.shared.schema import StandardResponse, PaginatedData
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/executions",
|
||||
tags=["data-collection/executions"],
|
||||
)
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@router.get("", response_model=StandardResponse[PaginatedData[TaskExecutionBase]])
|
||||
async def list_executions(
|
||||
page: int = 1,
|
||||
size: int = 20,
|
||||
task_id: Optional[str] = Query(None, description="Task ID"),
|
||||
task_name: Optional[str] = Query(None, description="Fuzzy search by task name"),
|
||||
status: Optional[str] = Query(None, description="Filter by execution status"),
|
||||
start_time: Optional[datetime] = Query(None, description="Start time range from (started_at >= start_time)"),
|
||||
end_time: Optional[datetime] = Query(None, description="Start time range to (started_at <= end_time)"),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""分页查询归集任务执行记录"""
|
||||
try:
|
||||
query = select(TaskExecution)
|
||||
|
||||
if task_id:
|
||||
query = query.where(TaskExecution.task_id == task_id)
|
||||
|
||||
if task_name:
|
||||
query = query.where(TaskExecution.task_name.ilike(f"%{task_name}%"))
|
||||
|
||||
if status:
|
||||
query = query.where(TaskExecution.status == status)
|
||||
|
||||
if start_time:
|
||||
query = query.where(TaskExecution.started_at >= start_time)
|
||||
|
||||
if end_time:
|
||||
query = query.where(TaskExecution.started_at <= end_time)
|
||||
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = (await db.execute(count_query)).scalar_one()
|
||||
|
||||
offset = (page - 1) * size
|
||||
executions = (await db.execute(
|
||||
query.order_by(TaskExecution.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(size)
|
||||
)).scalars().all()
|
||||
|
||||
items = [converter_execution_to_response(exe) for exe in executions]
|
||||
total_pages = math.ceil(total / size) if total > 0 else 0
|
||||
|
||||
return StandardResponse(
|
||||
code="0",
|
||||
message="Success",
|
||||
data=PaginatedData(
|
||||
content=items,
|
||||
total_elements=total,
|
||||
total_pages=total_pages,
|
||||
page=page,
|
||||
size=size,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list task executions: {str(e)}", e)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.get("/{execution_id}/log")
|
||||
async def get_execution_log(
|
||||
execution_id: str,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Get log file content for execution record"""
|
||||
try:
|
||||
execution = await db.get(TaskExecution, execution_id)
|
||||
if not execution:
|
||||
raise HTTPException(status_code=404, detail="Execution record not found")
|
||||
|
||||
log_path = getattr(execution, "log_path", None)
|
||||
if not log_path:
|
||||
raise HTTPException(status_code=404, detail="Log path not found")
|
||||
|
||||
path = Path(str(log_path))
|
||||
if not path.is_absolute():
|
||||
path = Path(os.getcwd()) / path
|
||||
path = path.resolve()
|
||||
|
||||
if not path.exists() or not path.is_file():
|
||||
raise HTTPException(status_code=404, detail="Log file not found")
|
||||
|
||||
filename = path.name
|
||||
headers = {
|
||||
"Content-Disposition": f'inline; filename={filename}'
|
||||
}
|
||||
return FileResponse(
|
||||
path=str(path),
|
||||
media_type="text/plain; charset=utf-8",
|
||||
filename=filename,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get execution log: {str(e)}", e)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
@@ -0,0 +1,67 @@
|
||||
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models.data_collection import CollectionTemplate
|
||||
from app.db.session import get_db
|
||||
from app.module.collection.schema.collection import CollectionTemplateBase, converter_template_to_response
|
||||
from app.module.shared.schema import StandardResponse, PaginatedData
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/templates",
|
||||
tags=["data-collection/templates"],
|
||||
)
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@router.get("", response_model=StandardResponse[PaginatedData[CollectionTemplateBase]])
|
||||
async def list_templates(
|
||||
page: int = 1,
|
||||
size: int = 20,
|
||||
name: Optional[str] = Query(None, description="Fuzzy search by template name"),
|
||||
built_in: Optional[bool] = Query(None, description="Filter by system built-in template"),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""分页查询归集任务模板"""
|
||||
try:
|
||||
query = select(CollectionTemplate)
|
||||
|
||||
if name:
|
||||
query = query.where(CollectionTemplate.name.ilike(f"%{name}%"))
|
||||
|
||||
if built_in is not None:
|
||||
query = query.where(CollectionTemplate.built_in == built_in)
|
||||
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = (await db.execute(count_query)).scalar_one()
|
||||
|
||||
offset = (page - 1) * size
|
||||
templates = (await db.execute(
|
||||
query.order_by(CollectionTemplate.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(size)
|
||||
)).scalars().all()
|
||||
|
||||
items = [converter_template_to_response(tpl) for tpl in templates]
|
||||
total_pages = math.ceil(total / size) if total > 0 else 0
|
||||
|
||||
return StandardResponse(
|
||||
code="0",
|
||||
message="Success",
|
||||
data=PaginatedData(
|
||||
content=items,
|
||||
total_elements=total,
|
||||
total_pages=total_pages,
|
||||
page=page,
|
||||
size=size,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list collection templates: {str(e)}", e)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models.data_collection import CollectionTask
|
||||
from app.db.session import AsyncSessionLocal
|
||||
from app.module.collection.schema.collection import SyncMode
|
||||
from app.module.shared.schedule import Scheduler
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_scheduler: Optional[Scheduler] = None
|
||||
|
||||
|
||||
def set_collection_scheduler(scheduler: Scheduler) -> None:
|
||||
global _scheduler
|
||||
_scheduler = scheduler
|
||||
|
||||
|
||||
def _get_scheduler() -> Scheduler:
|
||||
if _scheduler is None:
|
||||
raise RuntimeError("Collection scheduler not initialized")
|
||||
return _scheduler
|
||||
|
||||
|
||||
def schedule_collection_task(task_id: str, schedule_expression: str, dataset_id: Optional[str] = None) -> None:
|
||||
scheduler = _get_scheduler()
|
||||
from app.module.collection.service.collection import CollectionTaskService
|
||||
|
||||
scheduler.add_cron_job(
|
||||
job_id=f"collection:{task_id}",
|
||||
cron_expression=schedule_expression,
|
||||
func=CollectionTaskService.run_async,
|
||||
args=[task_id, dataset_id],
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
misfire_grace_time=60,
|
||||
)
|
||||
logger.info(f"Scheduled collection task {task_id} with cron {schedule_expression}")
|
||||
|
||||
|
||||
def remove_collection_task(task_id: str) -> None:
|
||||
scheduler = _get_scheduler()
|
||||
job_id = f"collection:{task_id}"
|
||||
if scheduler.has_job(job_id):
|
||||
scheduler.remove_job(job_id)
|
||||
logger.info(f"Removed scheduled collection task {task_id}")
|
||||
|
||||
|
||||
def reschedule_collection_task(task_id: str, schedule_expression: str, dataset_id: Optional[str] = None) -> None:
|
||||
remove_collection_task(task_id)
|
||||
schedule_collection_task(task_id, schedule_expression, dataset_id)
|
||||
|
||||
|
||||
def validate_schedule_expression(schedule_expression: str) -> None:
|
||||
Scheduler.validate_cron_expression(schedule_expression)
|
||||
|
||||
|
||||
async def load_scheduled_collection_tasks() -> None:
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(CollectionTask).where(
|
||||
CollectionTask.sync_mode == SyncMode.SCHEDULED.value,
|
||||
CollectionTask.schedule_expression.isnot(None),
|
||||
)
|
||||
)
|
||||
tasks = result.scalars().all()
|
||||
|
||||
for task in tasks:
|
||||
if not task.schedule_expression:
|
||||
continue
|
||||
try:
|
||||
schedule_collection_task(task.id, task.schedule_expression)
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to schedule task {task.id}: {exc}")
|
||||
@@ -0,0 +1,226 @@
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from pydantic import BaseModel, Field, validator, ConfigDict, field_validator, field_serializer
|
||||
from pydantic.alias_generators import to_camel
|
||||
|
||||
from app.core.security import mask_sensitive_dict
|
||||
from app.db.models.data_collection import CollectionTask, TaskExecution, CollectionTemplate
|
||||
from app.module.dataset.schema import DatasetTypeResponse
|
||||
from app.module.dataset.schema.dataset import DatasetType
|
||||
from app.module.shared.schema import TaskStatus
|
||||
|
||||
|
||||
class SyncMode(str, Enum):
|
||||
ONCE = "ONCE"
|
||||
SCHEDULED = "SCHEDULED"
|
||||
|
||||
class CollectionConfig(BaseModel):
|
||||
parameter: Optional[dict] = Field(None, description="模板参数")
|
||||
reader: Optional[dict] = Field(None, description="reader参数")
|
||||
writer: Optional[dict] = Field(None, description="writer参数")
|
||||
job: Optional[dict] = Field(None, description="任务配置")
|
||||
|
||||
@field_serializer('parameter', 'reader', 'writer', 'job', when_used='json')
|
||||
def mask_sensitive_fields(self, value: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
"""Mask sensitive fields when serializing to JSON"""
|
||||
if value is None:
|
||||
return value
|
||||
return mask_sensitive_dict(value)
|
||||
|
||||
class CollectionTaskBase(BaseModel):
|
||||
id: str = Field(..., description="任务id")
|
||||
name: str = Field(..., description="任务名称")
|
||||
description: Optional[str] = Field(None, description="任务描述")
|
||||
target_path: str = Field(..., description="目标存放路径")
|
||||
config: CollectionConfig = Field(..., description="任务配置")
|
||||
template_id: str = Field(..., description="模板ID")
|
||||
template_name: Optional[str] = Field(None, description="模板名称")
|
||||
status: TaskStatus = Field(..., description="任务状态")
|
||||
sync_mode: SyncMode = Field(default=SyncMode.ONCE, description="同步方式")
|
||||
schedule_expression: Optional[str] = Field(None, description="调度表达式（cron）")
|
||||
retry_count: int = Field(default=3, description="重试次数")
|
||||
timeout_seconds: int = Field(default=3600, description="超时时间")
|
||||
last_execution_id: Optional[str] = Field(None, description="最后执行id")
|
||||
created_at: Optional[datetime] = Field(None, description="创建时间")
|
||||
updated_at: Optional[datetime] = Field(None, description="更新时间")
|
||||
created_by: Optional[str] = Field(None, description="创建人")
|
||||
updated_by: Optional[str] = Field(None, description="更新人")
|
||||
|
||||
model_config = ConfigDict(
|
||||
alias_generator=to_camel,
|
||||
populate_by_name=True
|
||||
)
|
||||
|
||||
class CollectionTaskCreate(BaseModel):
|
||||
name: str = Field(..., description="任务名称")
|
||||
description: Optional[str] = Field(None, description="任务描述")
|
||||
sync_mode: SyncMode = Field(default=SyncMode.ONCE, description="同步方式")
|
||||
schedule_expression: Optional[str] = Field(None, description="调度表达式（cron）")
|
||||
config: CollectionConfig = Field(..., description="任务配置")
|
||||
template_id: str = Field(..., description="模板ID")
|
||||
dataset_name: Optional[str] = Field(None, description="数据集名称")
|
||||
dataset_type: Optional[DatasetType] = Field(DatasetType.TEXT, description="数据集类型")
|
||||
|
||||
model_config = ConfigDict(
|
||||
alias_generator=to_camel,
|
||||
populate_by_name=True
|
||||
)
|
||||
|
||||
class CollectionTaskUpdate(BaseModel):
|
||||
"""任务更新模型 - 只包含可更新的字段"""
|
||||
description: Optional[str] = Field(None, description="任务描述")
|
||||
schedule_expression: Optional[str] = Field(None, description="调度表达式（cron）- 仅SCHEDULED模式可修改")
|
||||
timeout_seconds: Optional[int] = Field(None, description="超时时间（秒）")
|
||||
config: Optional[CollectionConfig] = Field(None, description="任务配置")
|
||||
|
||||
@field_validator('schedule_expression')
|
||||
@classmethod
|
||||
def validate_schedule_expression(cls, v, info):
|
||||
"""验证调度表达式"""
|
||||
if v is not None:
|
||||
# SCHEDULED 模式必须有调度表达式
|
||||
if not v or not v.strip():
|
||||
raise ValueError("schedule_expression is required for SCHEDULED mode")
|
||||
return v
|
||||
|
||||
@field_validator('timeout_seconds')
|
||||
@classmethod
|
||||
def validate_timeout(cls, v):
|
||||
"""验证超时时间"""
|
||||
if v is not None and v <= 0:
|
||||
raise ValueError("timeout_seconds must be greater than 0")
|
||||
return v
|
||||
|
||||
model_config = ConfigDict(
|
||||
alias_generator=to_camel,
|
||||
populate_by_name=True
|
||||
)
|
||||
|
||||
def converter_to_response(task: CollectionTask) -> CollectionTaskBase:
|
||||
config_dict = json.loads(task.config)
|
||||
return CollectionTaskBase(
|
||||
id=task.id,
|
||||
name=task.name,
|
||||
description=task.description,
|
||||
sync_mode=task.sync_mode,
|
||||
template_id=task.template_id,
|
||||
template_name=task.template_name,
|
||||
target_path=task.target_path,
|
||||
config=CollectionConfig(**config_dict),
|
||||
schedule_expression=task.schedule_expression,
|
||||
status=task.status,
|
||||
retry_count=task.retry_count,
|
||||
timeout_seconds=task.timeout_seconds,
|
||||
last_execution_id=task.last_execution_id,
|
||||
created_at=task.created_at,
|
||||
updated_at=task.updated_at,
|
||||
created_by=task.created_by,
|
||||
updated_by=task.updated_by,
|
||||
)
|
||||
|
||||
def convert_for_create(task: CollectionTaskCreate, task_id: str) -> CollectionTask:
|
||||
schedule_expression = task.schedule_expression if task.sync_mode == SyncMode.SCHEDULED else None
|
||||
return CollectionTask(
|
||||
id=task_id,
|
||||
name=task.name,
|
||||
description=task.description,
|
||||
sync_mode=task.sync_mode,
|
||||
template_id=task.template_id,
|
||||
target_path=f"/dataset/local/{task_id}",
|
||||
config=json.dumps(task.config.dict()),
|
||||
schedule_expression=schedule_expression,
|
||||
status=TaskStatus.PENDING.name
|
||||
)
|
||||
|
||||
def create_execute_record(task: CollectionTask) -> TaskExecution:
|
||||
execution_id = str(uuid.uuid4())
|
||||
return TaskExecution(
|
||||
id=execution_id,
|
||||
task_id=task.id,
|
||||
task_name=task.name,
|
||||
status=TaskStatus.RUNNING.name,
|
||||
started_at=datetime.now(),
|
||||
log_path=f"/flow/data-collection/{task.id}/{execution_id}.log"
|
||||
)
|
||||
|
||||
|
||||
class TaskExecutionBase(BaseModel):
|
||||
id: str = Field(..., description="执行记录ID")
|
||||
task_id: str = Field(..., description="任务ID")
|
||||
task_name: str = Field(..., description="任务名称")
|
||||
status: Optional[str] = Field(None, description="执行状态")
|
||||
log_path: Optional[str] = Field(None, description="日志文件路径")
|
||||
started_at: Optional[datetime] = Field(None, description="开始时间")
|
||||
completed_at: Optional[datetime] = Field(None, description="完成时间")
|
||||
duration_seconds: Optional[int] = Field(None, description="执行时长（秒）")
|
||||
error_message: Optional[str] = Field(None, description="错误信息")
|
||||
created_at: Optional[datetime] = Field(None, description="创建时间")
|
||||
updated_at: Optional[datetime] = Field(None, description="更新时间")
|
||||
created_by: Optional[str] = Field(None, description="创建者")
|
||||
updated_by: Optional[str] = Field(None, description="更新者")
|
||||
|
||||
model_config = ConfigDict(
|
||||
alias_generator=to_camel,
|
||||
populate_by_name=True
|
||||
)
|
||||
|
||||
|
||||
def converter_execution_to_response(execution: TaskExecution) -> TaskExecutionBase:
|
||||
return TaskExecutionBase(
|
||||
id=execution.id,
|
||||
task_id=execution.task_id,
|
||||
task_name=execution.task_name,
|
||||
status=execution.status,
|
||||
log_path=execution.log_path,
|
||||
started_at=execution.started_at,
|
||||
completed_at=execution.completed_at,
|
||||
duration_seconds=execution.duration_seconds,
|
||||
error_message=execution.error_message,
|
||||
created_at=execution.created_at,
|
||||
updated_at=execution.updated_at,
|
||||
created_by=execution.created_by,
|
||||
updated_by=execution.updated_by,
|
||||
)
|
||||
|
||||
|
||||
class CollectionTemplateBase(BaseModel):
|
||||
id: str = Field(..., description="模板ID")
|
||||
name: str = Field(..., description="模板名称")
|
||||
description: Optional[str] = Field(None, description="模板描述")
|
||||
source_type: str = Field(..., description="源数据源类型")
|
||||
source_name: str = Field(..., description="源数据源名称")
|
||||
target_type: str = Field(..., description="目标数据源类型")
|
||||
target_name: str = Field(..., description="目标数据源名称")
|
||||
template_content: dict = Field(..., description="模板内容")
|
||||
built_in: Optional[bool] = Field(None, description="是否系统内置模板")
|
||||
created_at: Optional[datetime] = Field(None, description="创建时间")
|
||||
updated_at: Optional[datetime] = Field(None, description="更新时间")
|
||||
created_by: Optional[str] = Field(None, description="创建者")
|
||||
updated_by: Optional[str] = Field(None, description="更新者")
|
||||
|
||||
model_config = ConfigDict(
|
||||
alias_generator=to_camel,
|
||||
populate_by_name=True
|
||||
)
|
||||
|
||||
|
||||
def converter_template_to_response(template: CollectionTemplate) -> CollectionTemplateBase:
|
||||
return CollectionTemplateBase(
|
||||
id=template.id,
|
||||
name=template.name,
|
||||
description=template.description,
|
||||
source_type=template.source_type,
|
||||
source_name=template.source_name,
|
||||
target_type=template.target_type,
|
||||
target_name=template.target_name,
|
||||
template_content=template.template_content,
|
||||
built_in=template.built_in,
|
||||
created_at=template.created_at,
|
||||
updated_at=template.updated_at,
|
||||
created_by=template.created_by,
|
||||
updated_by=template.updated_by,
|
||||
)
|
||||
@@ -0,0 +1,281 @@
|
||||
import os
|
||||
import yaml
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models import Dataset
|
||||
# ✅ 确保引入 TaskExecution
|
||||
from app.db.models.data_collection import CollectionTask, CollectionTemplate, TaskExecution
|
||||
from app.db.session import AsyncSessionLocal
|
||||
from app.module.collection.client.datax_client import DataxClient
|
||||
from app.module.collection.schema.collection import SyncMode, create_execute_record
|
||||
from app.module.collection.schedule import validate_schedule_expression
|
||||
from app.module.dataset.service.service import Service
|
||||
from app.module.shared.schema import TaskStatus, NodeType, EdgeType
|
||||
from app.module.shared.common.lineage import LineageService
|
||||
from app.db.models.base_entity import LineageNode, LineageEdge
|
||||
# ✅ 确保引入 RuntimeClient
|
||||
from app.module.shared.runtime_client import RuntimeClient
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# 执行模式开关：'local' (默认) | 'runtime'
|
||||
COLLECTION_EXECUTION_MODE = os.getenv("COLLECTION_EXECUTION_MODE", "local")
|
||||
|
||||
class _TempExecution:
|
||||
"""用于生成配置时的临时占位符"""
|
||||
def __init__(self, task_id):
|
||||
self.id = task_id
|
||||
|
||||
class CollectionTaskService:
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
async def create_task(self, task: CollectionTask, dataset: Dataset) -> CollectionTask:
|
||||
self.db.add(task)
|
||||
if task.sync_mode == SyncMode.ONCE:
|
||||
task.status = TaskStatus.RUNNING.name
|
||||
await self.db.commit()
|
||||
asyncio.create_task(CollectionTaskService.run_async(task.id, dataset.id if dataset else None))
|
||||
elif task.sync_mode == SyncMode.SCHEDULED:
|
||||
if not task.schedule_expression:
|
||||
raise ValueError("schedule_expression is required for scheduled tasks")
|
||||
validate_schedule_expression(task.schedule_expression)
|
||||
return task
|
||||
|
||||
@staticmethod
|
||||
async def run_async(task_id: str, dataset_id: str = None):
|
||||
"""任务执行入口"""
|
||||
logger.info(f"Start to execute task {task_id}")
|
||||
|
||||
# 1. 获取任务基础信息
|
||||
async with AsyncSessionLocal() as session:
|
||||
task_result = await session.execute(select(CollectionTask).where(CollectionTask.id == task_id))
|
||||
task = task_result.scalar_one_or_none()
|
||||
if not task:
|
||||
logger.error(f"task {task_id} not exist")
|
||||
return
|
||||
|
||||
template_result = await session.execute(select(CollectionTemplate).where(CollectionTemplate.id == task.template_id))
|
||||
template = template_result.scalar_one_or_none()
|
||||
if not template:
|
||||
logger.error(f"template {task.template_name} not exist")
|
||||
return
|
||||
|
||||
# 2. 创建执行记录 (此时 task_execution.log_path 已经根据 schema 规则被设置好)
|
||||
task_execution = create_execute_record(task)
|
||||
session.add(task_execution)
|
||||
await session.commit()
|
||||
await session.refresh(task_execution)
|
||||
exec_id = task_execution.id
|
||||
|
||||
# 3. 根据模式分发
|
||||
try:
|
||||
if COLLECTION_EXECUTION_MODE == "runtime":
|
||||
# ✅ 关键修改：将 task_execution 对象传入，以便提取 execution_id
|
||||
await CollectionTaskService._execute_via_runtime(
|
||||
task_id, dataset_id, task, template, task_execution
|
||||
)
|
||||
|
||||
# 提交成功后更新状态为 SUBMITTED
|
||||
async with AsyncSessionLocal() as update_session:
|
||||
exec_record = await update_session.get(TaskExecution, exec_id)
|
||||
if exec_record:
|
||||
exec_record.status = "SUBMITTED"
|
||||
await update_session.commit()
|
||||
|
||||
logger.info(f"Task {task_id} submitted to Runtime. Starting background poller.")
|
||||
# 启动后台轮询
|
||||
asyncio.create_task(
|
||||
CollectionTaskService._poll_runtime_status(task_id, dataset_id, exec_id)
|
||||
)
|
||||
else:
|
||||
# Local 模式
|
||||
await CollectionTaskService._execute_locally(task_id, dataset_id, task, template, exec_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Task execution failed for {task_id}: {e}")
|
||||
async with AsyncSessionLocal() as fail_session:
|
||||
exec_record = await fail_session.get(TaskExecution, exec_id)
|
||||
if exec_record:
|
||||
exec_record.status = TaskStatus.FAILED.name
|
||||
exec_record.error_message = str(e)
|
||||
await fail_session.commit()
|
||||
|
||||
# ==========================================
|
||||
# 🟢 模式 A：本地执行逻辑
|
||||
# ==========================================
|
||||
@staticmethod
|
||||
async def _execute_locally(task_id, dataset_id, task, template, exec_id):
|
||||
"""本地直接运行 DataX"""
|
||||
logger.info(f"[Local Mode] Executing DataX for task {task_id}")
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
exec_record = await session.get(TaskExecution, exec_id)
|
||||
# DataX 运行是阻塞的，放在线程池中运行
|
||||
datax_client = DataxClient(task=task, execution=exec_record, template=template)
|
||||
await asyncio.to_thread(datax_client.run_datax_job)
|
||||
|
||||
# 同步 Task 状态
|
||||
task.status = exec_record.status
|
||||
session.add(task)
|
||||
|
||||
if exec_record.status == TaskStatus.COMPLETED.name:
|
||||
await CollectionTaskService._finalize_task(session, task_id, dataset_id, task)
|
||||
|
||||
await session.commit()
|
||||
|
||||
# ==========================================
|
||||
# 🔵 模式 B：Runtime -> Ray 执行逻辑
|
||||
# ==========================================
|
||||
@staticmethod
|
||||
async def _execute_via_runtime(task_id, dataset_id, task, template, task_execution):
|
||||
"""
|
||||
生成配置并提交给 Runtime。
|
||||
不需要 session，因为 task_execution 对象已包含所有必要信息。
|
||||
"""
|
||||
logger.info(f"[Runtime Mode] Submitting task {task_id} to deepdata-runtime")
|
||||
|
||||
# ✅ 关键步骤：提取 execution_id，这是日志文件名的关键
|
||||
execution_id = task_execution.id
|
||||
log_path = task_execution.log_path
|
||||
|
||||
# 1. 生成 DataX JSON 配置
|
||||
datax_client = DataxClient(task=task, execution=_TempExecution(task_id), template=template)
|
||||
datax_json_path = datax_client.generate_config()
|
||||
|
||||
# 2. 构造 process.yaml 指令文件
|
||||
flow_dir = Path(f"/flow/{task_id}")
|
||||
flow_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
process_cfg = {
|
||||
"executor_type": "deepdata",
|
||||
"instance_id": task_id,
|
||||
"execution_id": execution_id,
|
||||
"log_path": log_path,
|
||||
"dataset_id": dataset_id,
|
||||
"cli_config": {
|
||||
"command": "python",
|
||||
"args": ["/opt/datax/bin/datax.py", datax_json_path],
|
||||
"timeout": 7200
|
||||
}
|
||||
}
|
||||
|
||||
# 写入指令文件
|
||||
with open(flow_dir / "process.yaml", "w", encoding="utf-8") as f:
|
||||
yaml.safe_dump(process_cfg, f)
|
||||
|
||||
# 3. 提交给 Runtime
|
||||
runtime_client = RuntimeClient()
|
||||
success = await runtime_client.submit_task(task_id)
|
||||
if not success:
|
||||
raise RuntimeError("Runtime submission returned False")
|
||||
|
||||
@staticmethod
|
||||
async def _poll_runtime_status(task_id, dataset_id, exec_id):
|
||||
"""后台轮询 Runtime 状态"""
|
||||
runtime_client = RuntimeClient()
|
||||
|
||||
for _ in range(2000):
|
||||
try:
|
||||
status_info = await runtime_client.get_task_status(task_id)
|
||||
|
||||
if status_info:
|
||||
current_status = (status_info.get("status") or "").upper()
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
exec_record = await session.get(TaskExecution, exec_id)
|
||||
if not exec_record: return
|
||||
|
||||
task_result = await session.execute(select(CollectionTask).where(CollectionTask.id == task_id))
|
||||
task = task_result.scalar_one_or_none()
|
||||
|
||||
if current_status in ["COMPLETED", "SUCCEEDED", "SUCCESS"]:
|
||||
exec_record.status = TaskStatus.COMPLETED.name
|
||||
await session.commit()
|
||||
logger.info(f"Task {task_id} completed in Runtime. Finalizing.")
|
||||
if task:
|
||||
await CollectionTaskService._finalize_task(session, task_id, dataset_id, task)
|
||||
return
|
||||
|
||||
elif current_status in ["FAILED", "ERROR"]:
|
||||
exec_record.status = TaskStatus.FAILED.name
|
||||
exec_record.error_message = status_info.get("error", "Runtime task failed")
|
||||
await session.commit()
|
||||
logger.error(f"Task {task_id} failed in Runtime: {exec_record.error_message}")
|
||||
return
|
||||
|
||||
await asyncio.sleep(5)
|
||||
except Exception as e:
|
||||
logger.warning(f"Polling error for {task_id}: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
# ==========================================
|
||||
# 🛠️ 公共收尾逻辑
|
||||
# ==========================================
|
||||
@staticmethod
|
||||
async def _finalize_task(session, task_id, dataset_id, task):
|
||||
"""任务执行成功后的后续处理"""
|
||||
if not dataset_id: return
|
||||
|
||||
# 🐛 修复：对于 S3/远端写入器，本地 target_path 可能不存在
|
||||
# 应当检查 task.target_path 是否为本地路径，或者跳过本地扫描逻辑
|
||||
target_path = Path(task.target_path)
|
||||
if target_path.exists() and target_path.is_dir():
|
||||
service = Service(db=session)
|
||||
source_paths = []
|
||||
for file_path in target_path.rglob('*'):
|
||||
if file_path.is_file():
|
||||
source_paths.append(str(file_path.absolute()))
|
||||
|
||||
if source_paths:
|
||||
await service.add_files_to_dataset(dataset_id=dataset_id, source_paths=source_paths)
|
||||
await CollectionTaskService._add_dataset_to_graph(session, dataset_id, task)
|
||||
else:
|
||||
logger.warning(f"[Finalize] Target path {target_path} does not exist locally. "
|
||||
f"Skipping dataset import. (This is expected for s3writer/nfswriter)")
|
||||
|
||||
@staticmethod
|
||||
async def _add_dataset_to_graph(session, dataset_id, task):
|
||||
"""将数据集加入血缘图"""
|
||||
try:
|
||||
dataset = await session.get(Dataset, dataset_id)
|
||||
if not dataset:
|
||||
logger.warning(f"dataset {dataset_id} not found when building lineage graph")
|
||||
return
|
||||
|
||||
dataset_node = LineageNode(
|
||||
id=dataset.id,
|
||||
node_type=NodeType.DATASET.value,
|
||||
name=dataset.name,
|
||||
description=dataset.description
|
||||
)
|
||||
|
||||
collection_node = LineageNode(
|
||||
id=task.id,
|
||||
node_type=NodeType.DATASOURCE.value,
|
||||
name=task.name,
|
||||
description=task.description
|
||||
)
|
||||
|
||||
collection_edge = LineageEdge(
|
||||
process_id=task.id,
|
||||
name=task.name,
|
||||
edge_type=EdgeType.DATA_COLLECTION.value,
|
||||
description=dataset.description,
|
||||
from_node_id=task.id,
|
||||
to_node_id=dataset.id
|
||||
)
|
||||
|
||||
lineage_service = LineageService(db=session)
|
||||
await lineage_service.generate_graph(collection_node, collection_edge, dataset_node)
|
||||
await session.commit()
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to add dataset lineage graph: {exc}")
|
||||
await session.rollback()
|
||||
Reference in New Issue
Block a user