deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,104 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>services</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>data-management-service</artifactId>
|
||||
<name>Data Management Service</name>
|
||||
<description>数据管理服务</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>domain-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-redis</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-openfeign</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.openapitools</groupId>
|
||||
<artifactId>jackson-databind-nullable</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>jakarta.validation</groupId>
|
||||
<artifactId>jakarta.validation-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.data</groupId>
|
||||
<artifactId>spring-data-commons</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<arguments>true</arguments>
|
||||
<classifier>exec</classifier>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<configuration>
|
||||
<source>${maven.compiler.source}</source>
|
||||
<target>${maven.compiler.target}</target>
|
||||
<annotationProcessorPaths>
|
||||
<!-- 顺序很重要 -->
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>${lombok.version}</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok-mapstruct-binding</artifactId>
|
||||
<version>${lombok-mapstruct-binding.version}</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct-processor</artifactId>
|
||||
<version>${mapstruct.version}</version>
|
||||
</path>
|
||||
</annotationProcessorPaths>
|
||||
<compilerArgs>
|
||||
<arg>-parameters</arg>
|
||||
<arg>-Amapstruct.defaultComponentModel=spring</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.deepdata.datamanagement;
|
||||
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.EnableAsync;
|
||||
|
||||
/**
|
||||
* Data Management Service Configuration
|
||||
* 数据管理服务配置类 - 多源接入、元数据、血缘治理
|
||||
*/
|
||||
@Configuration
|
||||
@EnableAsync
|
||||
@ComponentScan(basePackages = {
|
||||
"com.deepdata.datamanagement",
|
||||
"com.deepdata.shared"
|
||||
})
|
||||
public class DataManagementServiceConfiguration {
|
||||
// Service configuration class for JAR packaging
|
||||
// 作为jar包形式提供服务的配置类
|
||||
}
|
||||
+352
@@ -0,0 +1,352 @@
|
||||
package com.deepdata.datamanagement.application;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.deepdata.common.domain.enums.EdgeType;
|
||||
import com.deepdata.common.domain.enums.NodeType;
|
||||
import com.deepdata.common.domain.model.LineageEdge;
|
||||
import com.deepdata.common.domain.model.LineageNode;
|
||||
import com.deepdata.common.domain.service.LineageService;
|
||||
import com.deepdata.common.domain.utils.ChunksSaver;
|
||||
import com.deepdata.common.setting.application.SysParamApplicationService;
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import com.deepdata.common.infrastructure.exception.BusinessAssert;
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.infrastructure.client.CollectionTaskClient;
|
||||
import com.deepdata.datamanagement.infrastructure.client.dto.CollectionTaskDetailResponse;
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.TagMapper;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
||||
import com.deepdata.datamanagement.interfaces.converter.DatasetConverter;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* 数据集应用服务（对齐 DB schema，使用 UUID 字符串主键）
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@Transactional
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetApplicationService {
|
||||
private static final String DATASET_PVC_NAME = "sys.management.dataset.pvc.name";
|
||||
private final DatasetRepository datasetRepository;
|
||||
private final TagMapper tagMapper;
|
||||
private final DatasetFileRepository datasetFileRepository;
|
||||
private final CollectionTaskClient collectionTaskClient;
|
||||
private final DatasetFileApplicationService datasetFileApplicationService;
|
||||
private final SysParamApplicationService sysParamService;
|
||||
private final LineageService lineageService;
|
||||
|
||||
@Value("${deepdata.data-management.base-path:/dataset}")
|
||||
private String datasetBasePath;
|
||||
|
||||
/**
|
||||
* 创建数据集
|
||||
*/
|
||||
@Transactional
|
||||
public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
|
||||
BusinessAssert.isTrue(datasetRepository.findByName(createDatasetRequest.getName()) == null, DataManagementErrorCode.DATASET_ALREADY_EXISTS);
|
||||
// 创建数据集对象
|
||||
Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
|
||||
dataset.initCreateParam(datasetBasePath);
|
||||
|
||||
// xuhuanqing
|
||||
// ================= 新增代码 START =================
|
||||
// 防止后续调用归集任务接口时因 createdBy 为 null 报错
|
||||
// 实际项目中应从当前登录用户的 Session/Token 获取，这里先设为默认值
|
||||
dataset.setCreatedBy("admin");
|
||||
dataset.setUpdatedBy("admin");
|
||||
// ================= 新增代码 END =================
|
||||
|
||||
// 处理标签
|
||||
if (CollectionUtils.isNotEmpty(createDatasetRequest.getTags())) {
|
||||
dataset.setTags(processTagNames(createDatasetRequest.getTags()));
|
||||
}
|
||||
datasetRepository.save(dataset);
|
||||
// 记录血缘关系
|
||||
addDatasetToGraph(dataset, null);
|
||||
|
||||
//todo 需要解耦这块逻辑
|
||||
if (StringUtils.hasText(createDatasetRequest.getDataSource())) {
|
||||
// 数据源id不为空，使用异步线程进行文件扫盘落库
|
||||
processDataSourceAsync(dataset.getId(), createDatasetRequest.getDataSource());
|
||||
}
|
||||
return dataset;
|
||||
}
|
||||
|
||||
private void addDatasetToGraph(Dataset dataset, CollectionTaskDetailResponse collection) {
|
||||
LineageNode datasetNode = new LineageNode();
|
||||
datasetNode.setId(dataset.getId());
|
||||
datasetNode.setNodeType(NodeType.DATASET);
|
||||
datasetNode.setName(dataset.getName());
|
||||
datasetNode.setDescription(dataset.getDescription());
|
||||
|
||||
LineageNode collectionNode = null;
|
||||
LineageEdge collectionEdge = null;
|
||||
if(Objects.nonNull(collection)) {
|
||||
collectionNode = new LineageNode();
|
||||
collectionNode.setId(collection.getId());
|
||||
collectionNode.setName(collection.getName());
|
||||
collectionNode.setDescription(collection.getDescription());
|
||||
collectionNode.setNodeType(NodeType.DATASOURCE);
|
||||
|
||||
collectionEdge = new LineageEdge();
|
||||
collectionEdge.setProcessId(collection.getId());
|
||||
collectionEdge.setName(collection.getName());
|
||||
collectionEdge.setEdgeType(EdgeType.DATA_COLLECTION);
|
||||
collectionEdge.setDescription(dataset.getDescription());
|
||||
collectionEdge.setFromNodeId(collectionNode.getId());
|
||||
collectionEdge.setToNodeId(datasetNode.getId());
|
||||
lineageService.generateGraph(collectionNode, collectionEdge, datasetNode);
|
||||
} else {
|
||||
lineageService.generateGraph(datasetNode, null, null);
|
||||
}
|
||||
}
|
||||
|
||||
public DatasetLineage getDatasetLineage(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
if (Objects.isNull(dataset)) {
|
||||
return new DatasetLineage();
|
||||
}
|
||||
LineageNode datasetNode = lineageService.getNodeById(datasetId);
|
||||
String graphId = datasetNode.getGraphId();
|
||||
return new DatasetLineage(lineageService.getNodesByGraphId(graphId), lineageService.getEdgesByGraphId(graphId));
|
||||
}
|
||||
|
||||
public String getDatasetPvcName() {
|
||||
return sysParamService.getParamByKey(DATASET_PVC_NAME);
|
||||
}
|
||||
|
||||
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||
if (StringUtils.hasText(updateDatasetRequest.getName())) {
|
||||
dataset.setName(updateDatasetRequest.getName());
|
||||
}
|
||||
if (StringUtils.hasText(updateDatasetRequest.getDescription())) {
|
||||
dataset.setDescription(updateDatasetRequest.getDescription());
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(updateDatasetRequest.getTags())) {
|
||||
dataset.setTags(processTagNames(updateDatasetRequest.getTags()));
|
||||
}
|
||||
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
|
||||
dataset.setStatus(updateDatasetRequest.getStatus());
|
||||
}
|
||||
datasetRepository.updateById(dataset);
|
||||
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
|
||||
// 数据源id不为空，使用异步线程进行文件扫盘落库
|
||||
processDataSourceAsync(dataset.getId(), updateDatasetRequest.getDataSource());
|
||||
}
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除数据集
|
||||
*/
|
||||
@Transactional
|
||||
public void deleteDataset(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
datasetRepository.removeById(datasetId);
|
||||
if (dataset != null) {
|
||||
ChunksSaver.deleteFolder(dataset.getPath());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取数据集详情
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Dataset getDataset(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||
List<DatasetFile> datasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
||||
dataset.setFiles(datasetFiles);
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页查询数据集
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
|
||||
IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
|
||||
page = datasetRepository.findByCriteria(page, query);
|
||||
String datasetPvcName = getDatasetPvcName();
|
||||
List<DatasetResponse> datasetResponses = DatasetConverter.INSTANCE.convertToResponse(page.getRecords());
|
||||
datasetResponses.forEach(dataset -> dataset.setPvcName(datasetPvcName));
|
||||
return PagedResponse.of(page.getCurrent(), page.getSize(), page.getTotal(), page.getPages(), datasetResponses);
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理标签名称，创建或获取标签
|
||||
*/
|
||||
private List<Tag> processTagNames(List<String> tagNames) {
|
||||
Set<Tag> tags = new HashSet<>();
|
||||
for (String tagName : tagNames) {
|
||||
Tag tag = tagMapper.findByName(tagName);
|
||||
if (tag == null) {
|
||||
Tag newTag = new Tag(tagName, null, null, "#007bff");
|
||||
newTag.setUsageCount(0L);
|
||||
newTag.setId(UUID.randomUUID().toString());
|
||||
tagMapper.insert(newTag);
|
||||
tag = newTag;
|
||||
}
|
||||
tag.setUsageCount(tag.getUsageCount() == null ? 1L : tag.getUsageCount() + 1);
|
||||
tagMapper.updateUsageCount(tag.getId(), tag.getUsageCount());
|
||||
tags.add(tag);
|
||||
}
|
||||
return tags.stream().toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取数据集统计信息
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Map<String, Object> getDatasetStatistics(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
if (dataset == null) {
|
||||
throw new IllegalArgumentException("Dataset not found: " + datasetId);
|
||||
}
|
||||
|
||||
Map<String, Object> statistics = new HashMap<>();
|
||||
|
||||
// 基础统计
|
||||
Long totalFiles = datasetFileRepository.countByDatasetId(datasetId);
|
||||
Long completedFiles = datasetFileRepository.countCompletedByDatasetId(datasetId);
|
||||
Long totalSize = datasetFileRepository.sumSizeByDatasetId(datasetId);
|
||||
|
||||
statistics.put("totalFiles", totalFiles != null ? totalFiles.intValue() : 0);
|
||||
statistics.put("completedFiles", completedFiles != null ? completedFiles.intValue() : 0);
|
||||
statistics.put("totalSize", totalSize != null ? totalSize : 0L);
|
||||
|
||||
// 完成率计算
|
||||
float completionRate = 0.0f;
|
||||
if (totalFiles != null && totalFiles > 0) {
|
||||
completionRate = (completedFiles != null ? completedFiles.floatValue() : 0.0f) / totalFiles.floatValue() * 100.0f;
|
||||
}
|
||||
statistics.put("completionRate", completionRate);
|
||||
|
||||
// 文件类型分布统计
|
||||
Map<String, Integer> fileTypeDistribution = new HashMap<>();
|
||||
List<DatasetFile> allFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
||||
if (allFiles != null) {
|
||||
for (DatasetFile file : allFiles) {
|
||||
String fileType = file.getFileType() != null ? file.getFileType() : "unknown";
|
||||
fileTypeDistribution.put(fileType, fileTypeDistribution.getOrDefault(fileType, 0) + 1);
|
||||
}
|
||||
}
|
||||
statistics.put("fileTypeDistribution", fileTypeDistribution);
|
||||
|
||||
// 状态分布统计
|
||||
Map<String, Integer> statusDistribution = new HashMap<>();
|
||||
if (allFiles != null) {
|
||||
for (DatasetFile file : allFiles) {
|
||||
String status = file.getStatus() != null ? file.getStatus() : "unknown";
|
||||
statusDistribution.put(status, statusDistribution.getOrDefault(status, 0) + 1);
|
||||
}
|
||||
}
|
||||
statistics.put("statusDistribution", statusDistribution);
|
||||
|
||||
return statistics;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有数据集的汇总统计信息
|
||||
*/
|
||||
public AllDatasetStatisticsResponse getAllDatasetStatistics() {
|
||||
return datasetRepository.getAllDatasetStatistics();
|
||||
}
|
||||
|
||||
/**
|
||||
* 异步处理数据源文件扫描
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param dataSourceId 数据源ID（归集任务ID）
|
||||
*/
|
||||
@Async
|
||||
public void processDataSourceAsync(String datasetId, String dataSourceId) {
|
||||
try {
|
||||
log.info("Initiating data source file scanning, dataset ID: {}, collection task ID: {}", datasetId, dataSourceId);
|
||||
List<String> filePaths = getFilePaths(dataSourceId, datasetRepository.getById(datasetId));
|
||||
if (CollectionUtils.isEmpty(filePaths)) {
|
||||
return;
|
||||
}
|
||||
// 创建请求并启用硬链接（softAdd=true）
|
||||
AddFilesRequest addFilesRequest = new AddFilesRequest(filePaths);
|
||||
addFilesRequest.setSoftAdd(true);
|
||||
datasetFileApplicationService.addFilesToDataset(datasetId, addFilesRequest);
|
||||
log.info("Success file scan, total files: {}", filePaths.size());
|
||||
} catch (Exception e) {
|
||||
log.error("处理数据源文件扫描失败，数据集ID: {}, 数据源ID: {}", datasetId, dataSourceId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> getFilePaths(String dataSourceId, Dataset dataset) {
|
||||
// xuhuanqing
|
||||
// ================= 修改代码 START =================
|
||||
// 防御性编程：如果 createdBy 为空，使用 "system" 代替，防止 Feign Header 报错
|
||||
String user = "system";
|
||||
if (dataset != null && dataset.getCreatedBy() != null) {
|
||||
user = dataset.getCreatedBy();
|
||||
}
|
||||
|
||||
// 将原来的 dataset.getCreatedBy() 替换为变量 user
|
||||
Response<CollectionTaskDetailResponse> response = collectionTaskClient.getTaskDetail(dataSourceId, user);
|
||||
|
||||
// 检查响应状态
|
||||
if (response == null || response.getData() == null) {
|
||||
log.warn("Fail to get collection task detail from Feign client, response is empty");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
CollectionTaskDetailResponse taskDetail = response.getData();
|
||||
// ================= 修改代码 END =================
|
||||
|
||||
// CollectionTaskDetailResponse taskDetail = collectionTaskClient.getTaskDetail(dataSourceId, dataset.getCreatedBy()).getData();
|
||||
if (taskDetail == null) {
|
||||
log.warn("Fail to get collection task detail, task ID: {}", dataSourceId);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
// 记录血缘关系
|
||||
addDatasetToGraph(dataset, taskDetail);
|
||||
Path targetPath = Paths.get(taskDetail.getTargetPath());
|
||||
if (!Files.exists(targetPath) || !Files.isDirectory(targetPath)) {
|
||||
log.warn("Target path not exists or is not a directory: {}", taskDetail.getTargetPath());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
try (Stream<Path> paths = Files.walk(targetPath, 1)) {
|
||||
return paths
|
||||
.filter(Files::isRegularFile) // 只保留文件，排除目录
|
||||
.map(Path::toString) // 转换为字符串路径
|
||||
.collect(Collectors.toList());
|
||||
} catch (IOException e) {
|
||||
log.error("Fail to scan directory: {}", targetPath, e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
}
|
||||
+1265
File diff suppressed because it is too large
Load Diff
+118
@@ -0,0 +1,118 @@
|
||||
package com.deepdata.datamanagement.application;
|
||||
|
||||
import com.deepdata.common.infrastructure.exception.BusinessException;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.TagMapper;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* 标签应用服务（UUID 主键）
|
||||
*/
|
||||
@Service
|
||||
@Transactional
|
||||
public class TagApplicationService {
|
||||
|
||||
private final TagMapper tagMapper;
|
||||
|
||||
@Autowired
|
||||
public TagApplicationService(TagMapper tagMapper) {
|
||||
this.tagMapper = tagMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建标签
|
||||
*/
|
||||
public Tag createTag(String name, String color, String description) {
|
||||
// 检查名称是否已存在
|
||||
if (tagMapper.findByName(name) != null) {
|
||||
throw BusinessException.of(DataManagementErrorCode.TAG_NAME_DUPLICATE);
|
||||
}
|
||||
|
||||
Tag tag = new Tag(name, description, null, color);
|
||||
tag.setUsageCount(0L);
|
||||
tag.setId(UUID.randomUUID().toString());
|
||||
tagMapper.insert(tag);
|
||||
return tagMapper.findById(tag.getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新标签
|
||||
*
|
||||
* @param tag 待更新的标签实体，必须包含有效的 ID
|
||||
* @return 更新结果
|
||||
*/
|
||||
@Transactional
|
||||
public Tag updateTag(Tag tag) {
|
||||
Tag existingTag = tagMapper.findById(tag.getId());
|
||||
if (existingTag == null) {
|
||||
throw new IllegalArgumentException("Tag not found: " + tag.getId());
|
||||
}
|
||||
existingTag.setName(tag.getName());
|
||||
existingTag.setColor(tag.getColor());
|
||||
existingTag.setDescription(tag.getDescription());
|
||||
tagMapper.update(existingTag);
|
||||
return tagMapper.findById(existingTag.getId());
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTag(List<String> tagIds) {
|
||||
List<Tag> tags = tagMapper.findByIdIn(tagIds);
|
||||
if (tags.stream().anyMatch(tag -> tag.getUsageCount() > 0)) {
|
||||
throw new IllegalArgumentException("Cannot delete tags that are in use");
|
||||
}
|
||||
if (CollectionUtils.isEmpty(tags)) {
|
||||
return;
|
||||
}
|
||||
tagMapper.deleteTagsById(tags.stream().map(Tag::getId).toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有标签
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public List<Tag> getAllTags() {
|
||||
return tagMapper.findAllByOrderByUsageCountDesc();
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据关键词搜索标签
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public List<Tag> searchTags(String keyword) {
|
||||
if (keyword == null || keyword.trim().isEmpty()) {
|
||||
return getAllTags();
|
||||
}
|
||||
return tagMapper.findByKeyword(keyword.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取标签详情
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Tag getTag(String tagId) {
|
||||
Tag tag = tagMapper.findById(tagId);
|
||||
if (tag == null) {
|
||||
throw new IllegalArgumentException("Tag not found: " + tagId);
|
||||
}
|
||||
return tag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据名称获取标签
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Tag getTagByName(String name) {
|
||||
Tag tag = tagMapper.findByName(name);
|
||||
if (tag == null) {
|
||||
throw new IllegalArgumentException("Tag not found: " + name);
|
||||
}
|
||||
return tag;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package com.deepdata.datamanagement.common.enums;
|
||||
|
||||
/**
|
||||
* 数据集状态类型
|
||||
* <p>数据集可以处于以下几种状态：
|
||||
* <p>草稿（DRAFT）：数据集正在创建中，尚未完成。
|
||||
* <p>活动（ACTIVE）：数据集处于活动状态, 可以被查询和使用，也可以被更新和删除。
|
||||
* <p>处理中（PROCESSING）：数据集正在处理中，可能需要一些时间，处理完成后会变成活动状态。
|
||||
* <p>已归档（ARCHIVED）：数据集已被归档，不可以更新文件，可以解锁变成活动状态。
|
||||
* <p>已发布（PUBLISHED）：数据集已被发布，可供外部使用，外部用户可以查询和使用数据集。
|
||||
* <p>已弃用（DEPRECATED）：数据集已被弃用，不建议再使用。
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-17
|
||||
*/
|
||||
public enum DatasetStatusType {
|
||||
/**
|
||||
* 草稿状态
|
||||
*/
|
||||
DRAFT,
|
||||
/**
|
||||
* 活动状态
|
||||
*/
|
||||
ACTIVE,
|
||||
/**
|
||||
* 处理中状态
|
||||
*/
|
||||
PROCESSING,
|
||||
/**
|
||||
* 已归档状态
|
||||
*/
|
||||
ARCHIVED,
|
||||
/**
|
||||
* 已发布状态
|
||||
*/
|
||||
PUBLISHED,
|
||||
/**
|
||||
* 已弃用状态
|
||||
*/
|
||||
DEPRECATED
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.common.enums;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* 数据集类型值对象
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
public enum DatasetType {
|
||||
TEXT("text", "文本数据集"),
|
||||
IMAGE("image", "图像数据集"),
|
||||
AUDIO("audio", "音频数据集"),
|
||||
VIDEO("video", "视频数据集"),
|
||||
OTHER("other", "其他数据集");
|
||||
|
||||
@Getter
|
||||
private final String code;
|
||||
|
||||
@Getter
|
||||
private final String description;
|
||||
|
||||
DatasetType(String code, String description) {
|
||||
this.code = code;
|
||||
this.description = description;
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package com.deepdata.datamanagement.common.enums;
|
||||
|
||||
/**
|
||||
* 文件重名时的处理方式
|
||||
*
|
||||
* @since 2025/11/18
|
||||
*/
|
||||
public enum DuplicateMethod {
|
||||
ERROR,
|
||||
COVER
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package com.deepdata.datamanagement.domain.contants;
|
||||
|
||||
/**
|
||||
* 数据集常量
|
||||
*/
|
||||
public interface DatasetConstant {
|
||||
/**
|
||||
* 服务ID
|
||||
*/
|
||||
String SERVICE_ID = "DATA_MANAGEMENT";
|
||||
}
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.deepdata.common.domain.model.base.BaseEntity;
|
||||
import com.deepdata.common.infrastructure.config.PgJsonTypeHandler;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetType;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.io.File;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 数据集实体（与数据库表 t_dm_datasets 对齐）
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@TableName(value = "t_dm_datasets", autoResultMap = true)
|
||||
public class Dataset extends BaseEntity<String> {
|
||||
/**
|
||||
* 数据集名称
|
||||
*/
|
||||
private String name;
|
||||
/**
|
||||
* 数据集描述
|
||||
*/
|
||||
private String description;
|
||||
/**
|
||||
* 数据集类型
|
||||
*/
|
||||
private DatasetType datasetType;
|
||||
/**
|
||||
* 数据集分类
|
||||
*/
|
||||
private String category;
|
||||
/**
|
||||
* 数据集路径
|
||||
*/
|
||||
private String path;
|
||||
/**
|
||||
* 数据集格式
|
||||
*/
|
||||
private String format;
|
||||
/**
|
||||
* 数据集模式信息，JSON格式, 用于解析当前数据集的文件结构
|
||||
*/
|
||||
private String schemaInfo;
|
||||
/**
|
||||
* 数据集大小（字节）
|
||||
*/
|
||||
private Long sizeBytes = 0L;
|
||||
/**
|
||||
* 文件数量
|
||||
*/
|
||||
private Long fileCount = 0L;
|
||||
/**
|
||||
* 记录数量
|
||||
*/
|
||||
private Long recordCount = 0L;
|
||||
/**
|
||||
* 数据集保留天数
|
||||
*/
|
||||
private Integer retentionDays = 0;
|
||||
/**
|
||||
* 标签列表, JSON格式
|
||||
*/
|
||||
@TableField(typeHandler = PgJsonTypeHandler.class)
|
||||
private List<Tag> tags;
|
||||
/**
|
||||
* 额外元数据，JSON格式
|
||||
*/
|
||||
@TableField(typeHandler = PgJsonTypeHandler.class)
|
||||
private String metadata;
|
||||
/**
|
||||
* 数据集状态
|
||||
*/
|
||||
private DatasetStatusType status;
|
||||
/**
|
||||
* 是否为公共数据集
|
||||
*/
|
||||
private Boolean isPublic = false;
|
||||
/**
|
||||
* 是否为精选数据集
|
||||
*/
|
||||
private Boolean isFeatured = false;
|
||||
/**
|
||||
* 数据集版本号
|
||||
*/
|
||||
private Long version = 0L;
|
||||
|
||||
@TableField(exist = false)
|
||||
private List<DatasetFile> files = new ArrayList<>();
|
||||
|
||||
public Dataset() {
|
||||
}
|
||||
|
||||
public Dataset(String name, String description, DatasetType datasetType, String category, String path,
|
||||
String format, DatasetStatusType status, String createdBy) {
|
||||
this.name = name;
|
||||
this.description = description;
|
||||
this.datasetType = datasetType;
|
||||
this.category = category;
|
||||
this.path = path;
|
||||
this.format = format;
|
||||
this.status = status;
|
||||
this.createdBy = createdBy;
|
||||
this.createdAt = LocalDateTime.now();
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void initCreateParam(String datasetBasePath) {
|
||||
this.id = UUID.randomUUID().toString();
|
||||
this.path = datasetBasePath + File.separator + this.id;
|
||||
if (this.status == null) {
|
||||
this.status = DatasetStatusType.DRAFT;
|
||||
}
|
||||
}
|
||||
|
||||
public void updateBasicInfo(String name, String description, String category) {
|
||||
if (name != null && !name.isEmpty()) this.name = name;
|
||||
if (description != null) this.description = description;
|
||||
if (category != null) this.category = category;
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void updateStatus(DatasetStatusType status, String updatedBy) {
|
||||
this.status = status;
|
||||
this.updatedBy = updatedBy;
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void addFile(DatasetFile file) {
|
||||
this.files.add(file);
|
||||
this.fileCount = this.fileCount + 1;
|
||||
this.sizeBytes = this.sizeBytes + (file.getFileSize() != null ? file.getFileSize() : 0L);
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void removeFile(DatasetFile file) {
|
||||
if (this.files.remove(file)) {
|
||||
this.fileCount = Math.max(0, this.fileCount - 1);
|
||||
this.sizeBytes = Math.max(0, this.sizeBytes - (file.getFileSize() != null ? file.getFileSize() : 0L));
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
}
|
||||
|
||||
public void active() {
|
||||
if (this.status == DatasetStatusType.DRAFT) {
|
||||
this.status = DatasetStatusType.ACTIVE;
|
||||
}
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.deepdata.common.infrastructure.config.PgJsonTypeHandler;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.*;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件实体（与数据库表 t_dm_dataset_files 对齐）
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@Slf4j
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@TableName("t_dm_dataset_files")
|
||||
public class DatasetFile {
|
||||
@TableId
|
||||
private String id; // UUID
|
||||
private String datasetId; // UUID
|
||||
private String fileName;
|
||||
private String filePath;
|
||||
private String fileType; // JPG/PNG/DCM/TXT
|
||||
private Long fileSize; // bytes
|
||||
private String checkSum;
|
||||
private String tags;
|
||||
@TableField(typeHandler = PgJsonTypeHandler.class)
|
||||
private String metadata;
|
||||
private String status; // UPLOADED, PROCESSING, COMPLETED, ERROR
|
||||
private LocalDateTime uploadTime;
|
||||
private LocalDateTime lastAccessTime;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
|
||||
/** 标记是否为目录（非持久化字段） */
|
||||
@TableField(exist = false)
|
||||
private Boolean directory;
|
||||
|
||||
/** 目录包含的文件数量（非持久化字段） */
|
||||
@TableField(exist = false)
|
||||
private Long fileCount;
|
||||
|
||||
/**
|
||||
* 解析标签
|
||||
*
|
||||
* @return 标签列表
|
||||
*/
|
||||
public List<FileTag> analyzeTag() {
|
||||
try {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper.readValue(tags, new TypeReference<List<FileTag>>() {});
|
||||
} catch (Exception e) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 数据集文件上传检查信息
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DatasetFileUploadCheckInfo {
|
||||
/** 数据集id */
|
||||
private String datasetId;
|
||||
|
||||
/** 是否为压缩包上传 */
|
||||
private boolean hasArchive;
|
||||
|
||||
/** 目标子目录前缀，例如 "images/"，为空表示数据集根目录 */
|
||||
private String prefix;
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
public class FileTag {
|
||||
private Map<String, Object> values;
|
||||
|
||||
private String type;
|
||||
|
||||
private String id;
|
||||
|
||||
private String fromName;
|
||||
|
||||
public List<String> getTagValues() {
|
||||
List<String> tags = new ArrayList<>();
|
||||
Object tagValues = values.get(type);
|
||||
if (tagValues instanceof List) {
|
||||
for (Object tag : (List<?>) tagValues) {
|
||||
if (tag instanceof String) {
|
||||
tags.add((String) tag);
|
||||
}
|
||||
}
|
||||
} else if (tagValues instanceof String) {
|
||||
tags.add((String) tagValues);
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.deepdata.common.domain.model.base.BaseEntity;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 标签实体（与数据库表 t_dm_tags 对齐）
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Tag extends BaseEntity<String> {
|
||||
private String name;
|
||||
private String description;
|
||||
private String category;
|
||||
private String color;
|
||||
private Long usageCount = 0L;
|
||||
|
||||
public Tag(String name, String description, String category, String color) {
|
||||
this.name = name;
|
||||
this.description = description;
|
||||
this.category = category;
|
||||
this.color = color;
|
||||
}
|
||||
|
||||
public void decrementUsage() {
|
||||
if (this.usageCount != null && this.usageCount > 0) this.usageCount--;
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.deepdata.datamanagement.infrastructure.client;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.datamanagement.infrastructure.client.dto.CollectionTaskDetailResponse;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
|
||||
/**
|
||||
* 数据归集服务 Feign Client
|
||||
*/
|
||||
@FeignClient(name = "collection-service", url = "${collection.service.url:http://deepdata-backend-python:18000}")
|
||||
public interface CollectionTaskClient {
|
||||
|
||||
/**
|
||||
* 获取归集任务详情
|
||||
* @param taskId 任务ID
|
||||
* @return 任务详情
|
||||
*/
|
||||
@GetMapping("/api/data-collection/tasks/{id}")
|
||||
Response<CollectionTaskDetailResponse> getTaskDetail(@PathVariable("id") String taskId, @RequestHeader("User") String authorization);
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.datamanagement.infrastructure.client.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 归集任务详情响应
|
||||
*/
|
||||
@Data
|
||||
public class CollectionTaskDetailResponse {
|
||||
private String id;
|
||||
private String name;
|
||||
private String description;
|
||||
private String targetPath;
|
||||
private Map<String, Object> config;
|
||||
private String status;
|
||||
private String syncMode;
|
||||
private String scheduleExpression;
|
||||
private String lastExecutionId;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.deepdata.datamanagement.infrastructure.client.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 本地归集任务配置
|
||||
*/
|
||||
@Data
|
||||
public class LocalCollectionConfig {
|
||||
/**
|
||||
* 归集类型
|
||||
*/
|
||||
private String type;
|
||||
|
||||
/**
|
||||
* 文件路径列表
|
||||
*/
|
||||
private List<String> filePaths;
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package com.deepdata.datamanagement.infrastructure.config;
|
||||
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cache.CacheManager;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
import org.springframework.web.multipart.support.StandardServletMultipartResolver;
|
||||
|
||||
/**
|
||||
* 数据管理服务配置
|
||||
*/
|
||||
@Configuration
|
||||
@EnableTransactionManagement
|
||||
@EnableCaching
|
||||
@EnableConfigurationProperties(DataManagementProperties.class)
|
||||
public class DataManagementConfig {
|
||||
|
||||
/**
|
||||
* 缓存管理器
|
||||
*/
|
||||
@Bean
|
||||
public CacheManager cacheManager() {
|
||||
return new ConcurrentMapCacheManager("datasets", "datasetFiles", "tags");
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件上传解析器
|
||||
*/
|
||||
@Bean
|
||||
public StandardServletMultipartResolver multipartResolver() {
|
||||
StandardServletMultipartResolver resolver = new StandardServletMultipartResolver();
|
||||
return resolver;
|
||||
}
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.deepdata.datamanagement.infrastructure.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据管理服务配置属性
|
||||
*/
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "datamanagement")
|
||||
public class DataManagementProperties {
|
||||
|
||||
private FileStorage fileStorage = new FileStorage();
|
||||
private Cache cache = new Cache();
|
||||
|
||||
public FileStorage getFileStorage() {
|
||||
return fileStorage;
|
||||
}
|
||||
|
||||
public void setFileStorage(FileStorage fileStorage) {
|
||||
this.fileStorage = fileStorage;
|
||||
}
|
||||
|
||||
public Cache getCache() {
|
||||
return cache;
|
||||
}
|
||||
|
||||
public void setCache(Cache cache) {
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
public static class FileStorage {
|
||||
private String uploadDir = "./uploads";
|
||||
private long maxFileSize = 10485760; // 10MB
|
||||
private long maxRequestSize = 52428800; // 50MB
|
||||
private List<String> allowedSourceDirs = new ArrayList<>();
|
||||
|
||||
public String getUploadDir() {
|
||||
return uploadDir;
|
||||
}
|
||||
|
||||
public void setUploadDir(String uploadDir) {
|
||||
this.uploadDir = uploadDir;
|
||||
}
|
||||
|
||||
public long getMaxFileSize() {
|
||||
return maxFileSize;
|
||||
}
|
||||
|
||||
public void setMaxFileSize(long maxFileSize) {
|
||||
this.maxFileSize = maxFileSize;
|
||||
}
|
||||
|
||||
public long getMaxRequestSize() {
|
||||
return maxRequestSize;
|
||||
}
|
||||
|
||||
public void setMaxRequestSize(long maxRequestSize) {
|
||||
this.maxRequestSize = maxRequestSize;
|
||||
}
|
||||
|
||||
public List<String> getAllowedSourceDirs() {
|
||||
return allowedSourceDirs;
|
||||
}
|
||||
|
||||
public void setAllowedSourceDirs(List<String> allowedSourceDirs) {
|
||||
this.allowedSourceDirs = allowedSourceDirs;
|
||||
}
|
||||
}
|
||||
|
||||
public static class Cache {
|
||||
private int ttl = 3600; // 1 hour
|
||||
private int maxSize = 1000;
|
||||
|
||||
public int getTtl() {
|
||||
return ttl;
|
||||
}
|
||||
|
||||
public void setTtl(int ttl) {
|
||||
this.ttl = ttl;
|
||||
}
|
||||
|
||||
public int getMaxSize() {
|
||||
return maxSize;
|
||||
}
|
||||
|
||||
public void setMaxSize(int maxSize) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package com.deepdata.datamanagement.infrastructure.exception;
|
||||
|
||||
import com.deepdata.common.infrastructure.exception.ErrorCode;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* 数据管理模块错误码
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-20
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum DataManagementErrorCode implements ErrorCode {
|
||||
/**
|
||||
* 数据集不存在
|
||||
*/
|
||||
DATASET_NOT_FOUND("data_management.0001", "数据集不存在"),
|
||||
/**
|
||||
* 数据集已存在
|
||||
*/
|
||||
DATASET_ALREADY_EXISTS("data_management.0002", "数据集已存在"),
|
||||
/**
|
||||
* 数据集状态错误
|
||||
*/
|
||||
DATASET_STATUS_ERROR("data_management.0003", "数据集状态错误"),
|
||||
/**
|
||||
* 数据集标签不存在
|
||||
*/
|
||||
DATASET_TAG_NOT_FOUND("data_management.0004", "数据集标签不存在"),
|
||||
/**
|
||||
* 数据集标签已存在
|
||||
*/
|
||||
DATASET_TAG_ALREADY_EXISTS("data_management.0005", "数据集标签已存在"),
|
||||
/**
|
||||
* 数据集文件已存在
|
||||
*/
|
||||
DATASET_FILE_ALREADY_EXISTS("data_management.0006", "数据集文件已存在"),
|
||||
/**
|
||||
* 目录不存在
|
||||
*/
|
||||
DIRECTORY_NOT_FOUND("data_management.0007", "目录不存在"),
|
||||
/**
|
||||
* 数据集名称长度不能超过255个字符
|
||||
*/
|
||||
DATASET_NAME_TOO_LONG("data_management.0008", "数据集名称长度不能超过255个字符"),
|
||||
/**
|
||||
* 数据集类型不合法
|
||||
*/
|
||||
DATASET_TYPE_INVALID("data_management.0009", "数据集类型不合法"),
|
||||
/**
|
||||
* 数据集状态不合法
|
||||
*/
|
||||
DATASET_STATUS_INVALID("data_management.0010", "数据集状态不合法"),
|
||||
/**
|
||||
* 标签名称长度不能超过100个字符
|
||||
*/
|
||||
TAG_NAME_TOO_LONG("data_management.0011", "标签名称长度不能超过100个字符"),
|
||||
/**
|
||||
* 标签名称已存在
|
||||
*/
|
||||
TAG_NAME_DUPLICATE("data_management.0012", "标签名称已存在"),
|
||||
/**
|
||||
* 标签颜色格式不正确，应为十六进制颜色代码
|
||||
*/
|
||||
TAG_COLOR_INVALID("data_management.0013", "标签颜色格式不正确，应为十六进制颜色代码"),
|
||||
/**
|
||||
* 保留天数必须为非负整数
|
||||
*/
|
||||
RETENTION_DAYS_INVALID("data_management.0014", "保留天数必须为非负整数"),
|
||||
/**
|
||||
* 路径长度不能超过限制
|
||||
*/
|
||||
PATH_TOO_LONG("data_management.0015", "路径长度不能超过限制"),
|
||||
/**
|
||||
* 文件名包含非法字符
|
||||
*/
|
||||
FILE_NAME_INVALID("data_management.0016", "文件名包含非法字符"),
|
||||
/**
|
||||
* 路径前缀不能以点开头
|
||||
*/
|
||||
PREFIX_INVALID("data_management.0017", "路径前缀不能以点开头");
|
||||
|
||||
private final String code;
|
||||
private final String message;
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.deepdata.common.infrastructure.config.IgnoreDataScopeAnnotation;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
@IgnoreDataScopeAnnotation
|
||||
public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
|
||||
DatasetFile findById(@Param("id") String id);
|
||||
List<DatasetFile> findByDatasetId(@Param("datasetId") String datasetId, RowBounds rowBounds);
|
||||
List<DatasetFile> findByDatasetIdAndStatus(@Param("datasetId") String datasetId, @Param("status") String status, RowBounds rowBounds);
|
||||
List<DatasetFile> findByDatasetIdAndFileType(@Param("datasetId") String datasetId, @Param("fileType") String fileType, RowBounds rowBounds);
|
||||
Long countByDatasetId(@Param("datasetId") String datasetId);
|
||||
Long countCompletedByDatasetId(@Param("datasetId") String datasetId);
|
||||
Long sumSizeByDatasetId(@Param("datasetId") String datasetId);
|
||||
DatasetFile findByDatasetIdAndFileName(@Param("datasetId") String datasetId, @Param("fileName") String fileName);
|
||||
List<DatasetFile> findAllByDatasetId(@Param("datasetId") String datasetId);
|
||||
List<DatasetFile> findByCriteria(@Param("datasetId") String datasetId,
|
||||
@Param("fileType") String fileType,
|
||||
@Param("status") String status,
|
||||
RowBounds rowBounds);
|
||||
|
||||
int update(DatasetFile file);
|
||||
int deleteById(@Param("id") String id);
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface DatasetMapper extends BaseMapper<Dataset> {
|
||||
Dataset findById(@Param("id") String id);
|
||||
|
||||
/**
|
||||
* 使用悲观锁查询数据集（FOR UPDATE）
|
||||
*/
|
||||
@Select("SELECT * FROM t_dm_datasets WHERE id = #{id} FOR UPDATE")
|
||||
Dataset findByIdWithLock(@Param("id") String id);
|
||||
Dataset findByName(@Param("name") String name);
|
||||
List<Dataset> findByStatus(@Param("status") String status);
|
||||
List<Dataset> findByCreatedBy(@Param("createdBy") String createdBy, RowBounds rowBounds);
|
||||
List<Dataset> findByTypeCode(@Param("typeCode") String typeCode, RowBounds rowBounds);
|
||||
List<Dataset> findByTagNames(@Param("tagNames") List<String> tagNames, RowBounds rowBounds);
|
||||
List<Dataset> findByKeyword(@Param("keyword") String keyword, RowBounds rowBounds);
|
||||
List<Dataset> findByCriteria(@Param("typeCode") String typeCode,
|
||||
@Param("status") String status,
|
||||
@Param("keyword") String keyword,
|
||||
@Param("tagNames") List<String> tagNames,
|
||||
RowBounds rowBounds);
|
||||
long countByCriteria(@Param("typeCode") String typeCode,
|
||||
@Param("status") String status,
|
||||
@Param("keyword") String keyword,
|
||||
@Param("tagNames") List<String> tagNames);
|
||||
|
||||
int deleteById(@Param("id") String id);
|
||||
AllDatasetStatisticsResponse getAllDatasetStatistics();
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.mapper;
|
||||
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface TagMapper {
|
||||
Tag findById(@Param("id") String id);
|
||||
Tag findByName(@Param("name") String name);
|
||||
List<Tag> findByNameIn(@Param("list") List<String> names);
|
||||
List<Tag> findByIdIn(@Param("ids") List<String> ids);
|
||||
List<Tag> findByKeyword(@Param("keyword") String keyword);
|
||||
List<Tag> findAllByOrderByUsageCountDesc();
|
||||
|
||||
int insert(Tag tag);
|
||||
int update(Tag tag);
|
||||
int updateUsageCount(@Param("id") String id, @Param("usageCount") Long usageCount);
|
||||
|
||||
// Relations with dataset
|
||||
int insertDatasetTag(@Param("datasetId") String datasetId, @Param("tagId") String tagId);
|
||||
int deleteDatasetTagsByDatasetId(@Param("datasetId") String datasetId);
|
||||
List<Tag> findByDatasetId(@Param("datasetId") String datasetId);
|
||||
void deleteTagsById(@Param("ids") List<String> ids);
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件仓储接口
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
public interface DatasetFileRepository extends IRepository<DatasetFile> {
|
||||
Long countByDatasetId(String datasetId);
|
||||
|
||||
Long countCompletedByDatasetId(String datasetId);
|
||||
|
||||
Long sumSizeByDatasetId(String datasetId);
|
||||
|
||||
List<DatasetFile> findAllByDatasetId(String datasetId);
|
||||
|
||||
DatasetFile findByDatasetIdAndFileName(String datasetId, String fileName);
|
||||
|
||||
IPage<DatasetFile> findByCriteria(String datasetId, String fileType, String status, String name,
|
||||
IPage<DatasetFile> page);
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetPagingQuery;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/**
|
||||
* 数据集仓储层
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
public interface DatasetRepository extends IRepository<Dataset> {
|
||||
Dataset findByName(String name);
|
||||
|
||||
/**
|
||||
* 使用悲观锁获取数据集（用于更新操作，防止并发冲突）
|
||||
*/
|
||||
Dataset getByIdWithLock(String id);
|
||||
|
||||
List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList, RowBounds bounds);
|
||||
|
||||
long countByCriteria(String type, String status, String keyword, List<String> tagList);
|
||||
|
||||
AllDatasetStatisticsResponse getAllDatasetStatistics();
|
||||
|
||||
IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query);
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件仓储实现类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper, DatasetFile> implements DatasetFileRepository {
|
||||
private final DatasetFileMapper datasetFileMapper;
|
||||
|
||||
@Override
|
||||
public Long countByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.selectCount(new LambdaQueryWrapper<DatasetFile>().eq(DatasetFile::getDatasetId, datasetId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long countCompletedByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.countCompletedByDatasetId(datasetId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long sumSizeByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.sumSizeByDatasetId(datasetId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatasetFile> findAllByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.findAllByDatasetId(datasetId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatasetFile findByDatasetIdAndFileName(String datasetId, String fileName) {
|
||||
return datasetFileMapper.findByDatasetIdAndFileName(datasetId, fileName);
|
||||
}
|
||||
|
||||
public IPage<DatasetFile> findByCriteria(String datasetId, String fileType, String status, String name,
|
||||
IPage<DatasetFile> page) {
|
||||
return datasetFileMapper.selectPage(page, new LambdaQueryWrapper<DatasetFile>()
|
||||
.eq(DatasetFile::getDatasetId, datasetId)
|
||||
.eq(StringUtils.hasText(fileType), DatasetFile::getFileType, fileType)
|
||||
.eq(StringUtils.hasText(status), DatasetFile::getStatus, status)
|
||||
.like(StringUtils.hasText(name), DatasetFile::getFileName, name));
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetMapper;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetPagingQuery;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集仓储层实现类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetRepositoryImpl extends CrudRepository<DatasetMapper, Dataset> implements DatasetRepository {
|
||||
private final DatasetMapper datasetMapper;
|
||||
|
||||
@Override
|
||||
public Dataset findByName(String name) {
|
||||
return datasetMapper.selectOne(new LambdaQueryWrapper<Dataset>().eq(Dataset::getName, name));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Dataset getByIdWithLock(String id) {
|
||||
return datasetMapper.findByIdWithLock(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList,
|
||||
RowBounds bounds) {
|
||||
return datasetMapper.findByCriteria(type, status, keyword, tagList, bounds);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long countByCriteria(String type, String status, String keyword, List<String> tagList) {
|
||||
return datasetMapper.countByCriteria(type, status, keyword, tagList);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AllDatasetStatisticsResponse getAllDatasetStatistics() {
|
||||
return datasetMapper.getAllDatasetStatistics();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query) {
|
||||
LambdaQueryWrapper<Dataset> wrapper = new LambdaQueryWrapper<Dataset>()
|
||||
.eq(query.getType() != null, Dataset::getDatasetType, query.getType())
|
||||
.eq(query.getStatus() != null, Dataset::getStatus, query.getStatus());
|
||||
|
||||
if (StringUtils.isNotBlank(query.getKeyword())) {
|
||||
wrapper.and(w ->
|
||||
w.like(Dataset::getName, query.getKeyword()).or()
|
||||
.like(Dataset::getDescription, query.getKeyword()));
|
||||
}
|
||||
|
||||
/*
|
||||
标签过滤 {@link Tag}
|
||||
*/
|
||||
for (String tagName : query.getTags()) {
|
||||
wrapper.and(w ->
|
||||
w.apply("EXISTS ( " +
|
||||
"SELECT 1 FROM jsonb_array_elements( " +
|
||||
"CASE WHEN jsonb_typeof(tags::jsonb) = 'array' THEN tags::jsonb ELSE '[]'::jsonb END " +
|
||||
") AS tag " +
|
||||
"WHERE tag->>'name' = {0} " +
|
||||
")", tagName)
|
||||
);
|
||||
}
|
||||
wrapper.orderByDesc(Dataset::getCreatedAt);
|
||||
return datasetMapper.selectPage(page, wrapper);
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package com.deepdata.datamanagement.interfaces.converter;
|
||||
|
||||
import com.deepdata.common.domain.model.ChunkUploadRequest;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.FileTag;
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.Mapping;
|
||||
import org.mapstruct.Named;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 数据集文件转换器
|
||||
*/
|
||||
@Mapper
|
||||
public interface DatasetConverter {
|
||||
/** 单例实例 */
|
||||
DatasetConverter INSTANCE = Mappers.getMapper(DatasetConverter.class);
|
||||
|
||||
/**
|
||||
* 将数据集转换为响应
|
||||
*/
|
||||
@Mapping(source = "sizeBytes", target = "totalSize")
|
||||
@Mapping(source = "path", target = "targetLocation")
|
||||
@Mapping(source = "files", target = "distribution", qualifiedByName = "getDistribution")
|
||||
DatasetResponse convertToResponse(Dataset dataset);
|
||||
|
||||
/**
|
||||
* 将数据集转换为响应
|
||||
*/
|
||||
@Mapping(target = "tags", ignore = true)
|
||||
Dataset convertToDataset(CreateDatasetRequest createDatasetRequest);
|
||||
|
||||
/**
|
||||
* 将上传文件请求转换为分片上传请求
|
||||
*/
|
||||
ChunkUploadRequest toChunkUploadRequest(UploadFileRequest uploadFileRequest);
|
||||
|
||||
/**
|
||||
* 将数据集转换为响应
|
||||
*/
|
||||
List<DatasetResponse> convertToResponse(List<Dataset> datasets);
|
||||
|
||||
/**
|
||||
*
|
||||
* 将数据集文件转换为响应
|
||||
*/
|
||||
DatasetFileResponse convertToResponse(DatasetFile datasetFile);
|
||||
|
||||
|
||||
/**
|
||||
* 将数据集文件列表转换为响应
|
||||
*/
|
||||
List<DatasetFileResponse> convertToResponseList(List<DatasetFile> datasetFiles);
|
||||
|
||||
|
||||
/**
|
||||
* 获取数据文件的标签分布
|
||||
*
|
||||
* @param datasetFiles 数据集文件
|
||||
* @return 标签分布
|
||||
*/
|
||||
@Named("getDistribution")
|
||||
default Map<String, Map<String, Long>> getDistribution(List<DatasetFile> datasetFiles) {
|
||||
Map<String, Map<String, Long>> distribution = new HashMap<>();
|
||||
if (CollectionUtils.isEmpty(datasetFiles)) {
|
||||
return distribution;
|
||||
}
|
||||
for (DatasetFile datasetFile : datasetFiles) {
|
||||
List<FileTag> tags = datasetFile.analyzeTag();
|
||||
if (CollectionUtils.isEmpty(tags)) {
|
||||
continue;
|
||||
}
|
||||
for (FileTag tag : tags) {
|
||||
Map<String, Long> tagValueMap = distribution.getOrDefault(tag.getFromName(), new HashMap<>());
|
||||
tag.getTagValues().forEach(tagValue -> tagValueMap.put(tagValue, tagValueMap.getOrDefault(tagValue, 0L) + 1));
|
||||
distribution.put(tag.getFromName(), tagValueMap);
|
||||
}
|
||||
}
|
||||
return distribution;
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.deepdata.datamanagement.interfaces.converter;
|
||||
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.interfaces.dto.TagResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UpdateTagRequest;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
/**
|
||||
* 标签转换器
|
||||
*/
|
||||
@Mapper
|
||||
public interface TagConverter {
|
||||
/** 单例实例 */
|
||||
TagConverter INSTANCE = Mappers.getMapper(TagConverter.class);
|
||||
|
||||
/**
|
||||
* 将 UpdateTagRequest 转换为 Tag 实体
|
||||
* @param request 更新标签请求DTO
|
||||
* @return 标签实体
|
||||
*/
|
||||
Tag updateRequestToTag(UpdateTagRequest request);
|
||||
|
||||
/**
|
||||
* 将 Tag 实体转换为 TagResponse DTO
|
||||
* @param tag 标签实体
|
||||
* @return 标签响应DTO
|
||||
*/
|
||||
TagResponse convertToResponse(Tag tag);
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* AddFilesRequest
|
||||
*
|
||||
* @since 2026/2/9
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AddFilesRequest {
|
||||
private boolean softAdd;
|
||||
|
||||
@ValidPath
|
||||
private String prefix = "";
|
||||
|
||||
@NotEmpty(message = "文件列表不能为空")
|
||||
@Size(max = 1000, message = "文件数量不能超过1000个")
|
||||
@Valid
|
||||
private List<FileRequest> files;
|
||||
|
||||
public AddFilesRequest(List<String> paths) {
|
||||
if (CollectionUtils.isEmpty(paths)) {
|
||||
return;
|
||||
}
|
||||
this.files = paths.stream().map(path -> {
|
||||
FileRequest file = new FileRequest();
|
||||
file.setFilePath(path);
|
||||
return file;
|
||||
}).toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文件的前缀，优先使用文件自身的 prefix，为空时回退到请求级别的 prefix
|
||||
*/
|
||||
public String getEffectivePrefix(FileRequest file) {
|
||||
return (file.getPrefix() != null && !file.getPrefix().isEmpty()) ? file.getPrefix() : prefix;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class FileRequest {
|
||||
@NotBlank(message = "文件路径不能为空")
|
||||
@Size(max = 1000, message = "文件路径长度不能超过1000个字符")
|
||||
private String filePath;
|
||||
|
||||
@ValidPath
|
||||
private String prefix = "";
|
||||
|
||||
private Map<String, Object> metadata = new HashMap<>();
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 所有数据集统计信息响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class AllDatasetStatisticsResponse {
|
||||
/** 总数据集数 */
|
||||
private Integer totalDatasets = 0;
|
||||
|
||||
/** 总文件数 */
|
||||
private Long totalSize = 0L;
|
||||
|
||||
/** 总大小（字节） */
|
||||
private Long totalFiles = 0L;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 批量删除文件请求
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class BatchDeleteFilesRequest {
|
||||
|
||||
/**
|
||||
* 要删除的文件ID列表
|
||||
*/
|
||||
@NotEmpty(message = "文件ID列表不能为空")
|
||||
private List<String> fileIds;
|
||||
|
||||
/**
|
||||
* 文件路径前缀（用于处理子目录中的文件）
|
||||
*/
|
||||
private String prefix = "";
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.Size;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 复制文件请求DTO
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-11-13
|
||||
*/
|
||||
public record CopyFilesRequest(
|
||||
@NotEmpty(message = "源文件路径列表不能为空")
|
||||
@Size(max = 1000, message = "文件数量不能超过1000个")
|
||||
List<String> sourcePaths) {
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetType;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.springaicommunity.mcp.annotation.McpToolParam;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 创建数据集请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class CreateDatasetRequest {
|
||||
/** 数据集名称 */
|
||||
@Size(min = 1, max = 255, message = "数据集名称长度不能超过255个字符")
|
||||
@NotBlank(message = "数据集名称不能为空")
|
||||
@McpToolParam(description = "数据集名称")
|
||||
private String name;
|
||||
/** 数据集描述 */
|
||||
@Size(max = 500, message = "数据集描述长度不能超过500个字符")
|
||||
@McpToolParam(description = "数据集描述", required = false)
|
||||
private String description;
|
||||
/** 数据集类型 */
|
||||
@NotNull(message = "数据集类型不能为空")
|
||||
@McpToolParam(description = "数据集类型，取值范围为TEXT/IMAGE/VIDEO/AUDIO/OTHER")
|
||||
private DatasetType datasetType;
|
||||
/** 标签列表 */
|
||||
@Size(max = 20, message = "标签数量不能超过20个")
|
||||
@McpToolParam(description = "标签列表", required = false)
|
||||
private List<String> tags;
|
||||
/** 数据源 */
|
||||
@Size(max = 255, message = "数据源长度不能超过255个字符")
|
||||
@McpToolParam(description = "数据源", required = false)
|
||||
private String dataSource;
|
||||
/** 保留天数 */
|
||||
@Min(value = 0, message = "保留天数必须为非负整数")
|
||||
@McpToolParam(description = "保留天数", required = false)
|
||||
private Integer retentionDays;
|
||||
/** 数据集状态 */
|
||||
@McpToolParam(description = "数据集状态", required = false)
|
||||
private DatasetStatusType status;
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFileName;
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 创建数据集子目录请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class CreateDirectoryRequest {
|
||||
|
||||
/** 父级前缀路径，例如 "images/"，为空表示数据集根目录 */
|
||||
@ValidPath()
|
||||
private String parentPrefix;
|
||||
|
||||
/** 新建目录名称 */
|
||||
@NotBlank(message = "目录名称不能为空")
|
||||
@ValidFileName
|
||||
@Size(max = 255, message = "目录名称长度不能超过255个字符")
|
||||
private String directoryName;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidHexColor;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 创建标签请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class CreateTagRequest {
|
||||
/** 标签名称 */
|
||||
@NotBlank(message = "标签名称不能为空")
|
||||
@Size(min = 1, max = 100, message = "标签名称长度不能超过100个字符")
|
||||
private String name;
|
||||
/** 标签颜色 */
|
||||
@ValidHexColor
|
||||
private String color;
|
||||
/** 标签描述 */
|
||||
@Size(max = 500, message = "标签描述长度不能超过500个字符")
|
||||
private String description;
|
||||
/** 标签分类 */
|
||||
@Size(max = 50, message = "标签分类长度不能超过50个字符")
|
||||
private String category;
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 数据集文件响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetFileResponse {
|
||||
/** 文件ID */
|
||||
private String id;
|
||||
/** 文件名 */
|
||||
private String fileName;
|
||||
/** 原始文件名 */
|
||||
private String originalName;
|
||||
/** 文件类型 */
|
||||
private String fileType;
|
||||
/** 文件大小（字节） */
|
||||
private Long fileSize;
|
||||
/** 文件状态 */
|
||||
private String status;
|
||||
/** 文件描述 */
|
||||
private String description;
|
||||
/** 文件路径 */
|
||||
private String filePath;
|
||||
/** 文件标签（JSON 字符串） */
|
||||
private String tags;
|
||||
/** 标签更新时间 */
|
||||
private LocalDateTime tagsUpdatedAt;
|
||||
/** 文件元数据（包含标注信息等，JSON 字符串） */
|
||||
private String metadata;
|
||||
/** 上传时间 */
|
||||
private LocalDateTime uploadTime;
|
||||
/** 最后更新时间 */
|
||||
private LocalDateTime lastAccessTime;
|
||||
/** 上传者 */
|
||||
private String uploadedBy;
|
||||
/** 是否为目录 */
|
||||
private Boolean directory;
|
||||
/** 目录文件数量 */
|
||||
private Long fileCount;
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.common.domain.model.LineageEdge;
|
||||
import com.deepdata.common.domain.model.LineageNode;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集血缘
|
||||
*
|
||||
* @since 2026/1/23
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DatasetLineage {
|
||||
/**
|
||||
* 节点列表
|
||||
*/
|
||||
private List<LineageNode> lineageNodes;
|
||||
/**
|
||||
* 边列表
|
||||
*/
|
||||
private List<LineageEdge> lineageEdges;
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.common.interfaces.PagingQuery;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetType;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.springaicommunity.mcp.annotation.McpToolParam;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集分页查询请求
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DatasetPagingQuery extends PagingQuery {
|
||||
/**
|
||||
* 数据集类型过滤
|
||||
*/
|
||||
@McpToolParam(description = "数据集类型过滤，取值范围为text/image/video/audio/other", required = false)
|
||||
private DatasetType type;
|
||||
|
||||
/**
|
||||
* 标签名过滤
|
||||
*/
|
||||
@Size(max = 10, message = "过滤标签数量不能超过10个")
|
||||
@McpToolParam(description = "标签名过滤", required = false)
|
||||
private List<String> tags = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 关键词搜索（名称或描述）
|
||||
*/
|
||||
@Size(max = 100, message = "关键词长度不能超过100个字符")
|
||||
@McpToolParam(description = "关键词搜索（名称或描述）", required = false)
|
||||
private String keyword;
|
||||
|
||||
/**
|
||||
* 状态过滤
|
||||
*/
|
||||
@McpToolParam(description = "状态过滤", required = false)
|
||||
private DatasetStatusType status;
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据集响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetResponse {
|
||||
/** 数据集ID */
|
||||
private String id;
|
||||
/** 数据集名称 */
|
||||
private String name;
|
||||
/** 数据集描述 */
|
||||
private String description;
|
||||
/** 数据集类型 */
|
||||
private String datasetType;
|
||||
/** 数据集状态 */
|
||||
private String status;
|
||||
/** 标签列表 */
|
||||
private List<TagResponse> tags;
|
||||
/** 数据集保留天数 */
|
||||
private Integer retentionDays;
|
||||
/** 数据源 */
|
||||
private String dataSource;
|
||||
/** 目标位置 */
|
||||
private String targetLocation;
|
||||
/** 文件数量 */
|
||||
private Integer fileCount;
|
||||
/** 总大小（字节） */
|
||||
private Long totalSize;
|
||||
/** 完成率（0-100） */
|
||||
private Float completionRate;
|
||||
/** 创建时间 */
|
||||
private LocalDateTime createdAt;
|
||||
/** 更新时间 */
|
||||
private LocalDateTime updatedAt;
|
||||
/** 创建者 */
|
||||
private String createdBy;
|
||||
/** 更新者 */
|
||||
private String updatedBy;
|
||||
/** 分布 */
|
||||
private Map<String, Map<String, Long>> distribution;
|
||||
/** 数据集pvc名称 */
|
||||
private String pvcName;
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据集统计信息响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetStatisticsResponse {
|
||||
/** 总文件数 */
|
||||
private Integer totalFiles;
|
||||
/** 已完成文件数 */
|
||||
private Integer completedFiles;
|
||||
/** 总大小（字节） */
|
||||
private Long totalSize;
|
||||
/** 完成率（0-100） */
|
||||
private Float completionRate;
|
||||
/** 文件类型分布 */
|
||||
private Map<String, Integer> fileTypeDistribution;
|
||||
/** 状态分布 */
|
||||
private Map<String, Integer> statusDistribution;
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集类型响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetTypeResponse {
|
||||
/** 类型编码 */
|
||||
private String code;
|
||||
/** 类型名称 */
|
||||
private String name;
|
||||
/** 类型描述 */
|
||||
private String description;
|
||||
/** 支持的文件格式 */
|
||||
private List<String> supportedFormats;
|
||||
/** 图标 */
|
||||
private String icon;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件分页响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class PagedDatasetFileResponse {
|
||||
/** 文件内容列表 */
|
||||
private List<DatasetFileResponse> content;
|
||||
/** 当前页码 */
|
||||
private Integer page;
|
||||
/** 每页大小 */
|
||||
private Integer size;
|
||||
/** 总元素数 */
|
||||
private Integer totalElements;
|
||||
/** 总页数 */
|
||||
private Integer totalPages;
|
||||
/** 是否为第一页 */
|
||||
private Boolean first;
|
||||
/** 是否为最后一页 */
|
||||
private Boolean last;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集分页响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class PagedDatasetResponse {
|
||||
/** 数据集内容列表 */
|
||||
private List<DatasetResponse> content;
|
||||
/** 当前页码 */
|
||||
private Integer page;
|
||||
/** 每页大小 */
|
||||
private Integer size;
|
||||
/** 总元素数 */
|
||||
private Integer totalElements;
|
||||
/** 总页数 */
|
||||
private Integer totalPages;
|
||||
/** 是否为第一页 */
|
||||
private Boolean first;
|
||||
/** 是否为最后一页 */
|
||||
private Boolean last;
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFileName;
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 重命名数据集目录请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class RenameDirectoryRequest {
|
||||
|
||||
/** 目录前缀，例如 "images/"，与列表/删除目录接口保持一致 */
|
||||
@NotBlank(message = "目录前缀不能为空")
|
||||
@ValidPath(maxLength = 500)
|
||||
private String prefix;
|
||||
|
||||
/** 新的目录名称 */
|
||||
@NotBlank(message = "新目录名称不能为空")
|
||||
@ValidFileName
|
||||
@Size(max = 255, message = "目录名称长度不能超过255个字符")
|
||||
private String newName;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFileName;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 重命名数据集文件请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class RenameFileRequest {
|
||||
|
||||
/** 新的文件名称（不包含后缀） */
|
||||
@NotBlank(message = "新文件名不能为空")
|
||||
@ValidFileName
|
||||
@Size(max = 255, message = "文件名称长度不能超过255个字符")
|
||||
private String newName;
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 标签响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class TagResponse {
|
||||
/** 标签ID */
|
||||
private String id;
|
||||
/** 标签名称 */
|
||||
private String name;
|
||||
/** 标签颜色 */
|
||||
private String color;
|
||||
/** 标签描述 */
|
||||
private String description;
|
||||
/** 使用次数 */
|
||||
private Integer usageCount;
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 更新数据集请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UpdateDatasetRequest {
|
||||
/** 数据集名称 */
|
||||
@Size(min = 1, max = 255, message = "数据集名称长度不能超过255个字符")
|
||||
@NotBlank(message = "数据集名称不能为空")
|
||||
private String name;
|
||||
/** 数据集描述 */
|
||||
@Size(max = 500, message = "数据集描述长度不能超过500个字符")
|
||||
private String description;
|
||||
/** 归集任务id */
|
||||
@Size(max = 255, message = "数据源长度不能超过255个字符")
|
||||
private String dataSource;
|
||||
/** 标签列表 */
|
||||
@Size(max = 20, message = "标签数量不能超过20个")
|
||||
private List<String> tags;
|
||||
/** 数据集状态 */
|
||||
private DatasetStatusType status;
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidHexColor;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 更新标签请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UpdateTagRequest {
|
||||
/** 标签 ID */
|
||||
@NotBlank(message = "标签ID不能为空")
|
||||
private String id;
|
||||
/** 标签名称 */
|
||||
@Size(max = 100, message = "标签名称长度不能超过100个字符")
|
||||
private String name;
|
||||
/** 标签颜色 */
|
||||
@ValidHexColor
|
||||
private String color;
|
||||
/** 标签描述 */
|
||||
@Size(max = 500, message = "标签描述长度不能超过500个字符")
|
||||
private String description;
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFilePath;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
/**
|
||||
* 上传文件请求
|
||||
* 用于分块上传文件时的请求参数封装，支持大文件分片上传功能
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UploadFileRequest {
|
||||
/** 预上传返回的id，用来确认同一个任务 */
|
||||
@NotBlank(message = "请求ID不能为空")
|
||||
private String reqId;
|
||||
|
||||
/** 文件编号，用于标识批量上传中的第几个文件 */
|
||||
@Min(value = 0, message = "文件编号必须为非负整数")
|
||||
private int fileNo;
|
||||
|
||||
/** 文件名称（支持相对路径，用于文件夹上传） */
|
||||
@NotBlank(message = "文件名称不能为空")
|
||||
@ValidFilePath
|
||||
@Size(max = 255, message = "文件名称长度不能超过255个字符")
|
||||
private String fileName;
|
||||
|
||||
/** 文件总分块数量 */
|
||||
@Min(value = 1, message = "总分块数量必须大于0")
|
||||
private int totalChunkNum;
|
||||
|
||||
/** 当前分块编号，从1开始 */
|
||||
@Min(value = 1, message = "分块编号必须大于0")
|
||||
private int chunkNo;
|
||||
|
||||
/** 上传的文件分块内容 */
|
||||
@NotNull(message = "文件内容不能为空")
|
||||
private MultipartFile file;
|
||||
|
||||
/** 文件分块的校验和（十六进制字符串），用于验证文件完整性 */
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{64}$", message = "校验和格式不正确，应为64位十六进制字符串")
|
||||
private String checkSumHex;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 切片上传预上传请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UploadFilesPreRequest {
|
||||
/** 是否为压缩包上传 */
|
||||
private boolean hasArchive;
|
||||
|
||||
/** 总文件数量 */
|
||||
@Min(value = 1, message = "总文件数量必须大于0")
|
||||
private int totalFileNum;
|
||||
|
||||
/** 总文件大小 */
|
||||
@Min(value = 1, message = "总文件大小必须大于0")
|
||||
private long totalSize;
|
||||
|
||||
/** 目标子目录前缀，例如 "images/"，为空表示数据集根目录 */
|
||||
@ValidPath(maxLength = 500)
|
||||
private String prefix;
|
||||
}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.datamanagement.application.DatasetApplicationService;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.interfaces.converter.DatasetConverter;
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springaicommunity.mcp.annotation.McpTool;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据集 REST 控制器
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/data-management/datasets")
|
||||
@Validated
|
||||
public class DatasetController {
|
||||
private final DatasetApplicationService datasetApplicationService;
|
||||
|
||||
/**
|
||||
* 获取数据集列表
|
||||
*
|
||||
* @param query 分页查询参数
|
||||
* @return 分页的数据集列表
|
||||
*/
|
||||
@GetMapping
|
||||
@McpTool(name = "query_datasets", description = "根据参数查询满足条件的数据集列表")
|
||||
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
|
||||
return datasetApplicationService.getDatasets(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建数据集
|
||||
*
|
||||
* @param createDatasetRequest 创建数据集请求参数
|
||||
* @return 创建的数据集响应
|
||||
*/
|
||||
@PostMapping
|
||||
@McpTool(name = "create_dataset", description = "创建数据集")
|
||||
public DatasetResponse createDataset(@RequestBody @Valid CreateDatasetRequest createDatasetRequest) {
|
||||
Dataset dataset = datasetApplicationService.createDataset(createDatasetRequest);
|
||||
return DatasetConverter.INSTANCE.convertToResponse(dataset);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据ID获取数据集详情
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @return 数据集响应
|
||||
*/
|
||||
@GetMapping("/{datasetId}")
|
||||
public DatasetResponse getDatasetById(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
DatasetResponse dataset = DatasetConverter.INSTANCE.convertToResponse(datasetApplicationService.getDataset(datasetId));
|
||||
dataset.setPvcName(datasetApplicationService.getDatasetPvcName());
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据ID更新数据集
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param updateDatasetRequest 更新数据集请求参数
|
||||
* @return 更新后的数据集响应
|
||||
*/
|
||||
@PutMapping("/{datasetId}")
|
||||
public DatasetResponse updateDataset(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId,
|
||||
@RequestBody UpdateDatasetRequest updateDatasetRequest) {
|
||||
Dataset dataset = datasetApplicationService.updateDataset(datasetId, updateDatasetRequest);
|
||||
return DatasetConverter.INSTANCE.convertToResponse(dataset);
|
||||
}
|
||||
|
||||
@GetMapping("/{datasetId}/lineage")
|
||||
public DatasetLineage getDatasetLineage(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
return datasetApplicationService.getDatasetLineage(datasetId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据ID删除数据集
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
*/
|
||||
@DeleteMapping("/{datasetId}")
|
||||
public void deleteDataset(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
datasetApplicationService.deleteDataset(datasetId);
|
||||
}
|
||||
|
||||
@GetMapping("/{datasetId}/statistics")
|
||||
public ResponseEntity<Response<DatasetStatisticsResponse>> getDatasetStatistics(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
try {
|
||||
Map<String, Object> stats = datasetApplicationService.getDatasetStatistics(datasetId);
|
||||
|
||||
DatasetStatisticsResponse response = new DatasetStatisticsResponse();
|
||||
response.setTotalFiles((Integer) stats.get("totalFiles"));
|
||||
response.setCompletedFiles((Integer) stats.get("completedFiles"));
|
||||
response.setTotalSize((Long) stats.get("totalSize"));
|
||||
response.setCompletionRate((Float) stats.get("completionRate"));
|
||||
response.setFileTypeDistribution((Map<String, Integer>) stats.get("fileTypeDistribution"));
|
||||
response.setStatusDistribution((Map<String, Integer>) stats.get("statusDistribution"));
|
||||
|
||||
return ResponseEntity.ok(Response.ok(response));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
} catch (Exception e) {
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/statistics")
|
||||
public ResponseEntity<Response<AllDatasetStatisticsResponse>> getAllStatistics() {
|
||||
return ResponseEntity.ok(Response.ok(datasetApplicationService.getAllDatasetStatistics()));
|
||||
}
|
||||
}
|
||||
+245
@@ -0,0 +1,245 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.IgnoreResponseWrap;
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.common.interfaces.PagingQuery;
|
||||
import com.deepdata.datamanagement.application.DatasetApplicationService;
|
||||
import com.deepdata.datamanagement.application.DatasetFileApplicationService;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.interfaces.converter.DatasetConverter;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AddFilesRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.BatchDeleteFilesRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.CopyFilesRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.CreateDirectoryRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetFileResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UploadFileRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UploadFilesPreRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.RenameFileRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.RenameDirectoryRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件 REST 控制器（UUID 模式）
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/data-management/datasets/{datasetId}/files")
|
||||
public class DatasetFileController {
|
||||
|
||||
private final DatasetFileApplicationService datasetFileApplicationService;
|
||||
|
||||
private final DatasetApplicationService datasetApplicationService;
|
||||
|
||||
@GetMapping
|
||||
public Response<PagedResponse<DatasetFile>> getDatasetFiles(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@RequestParam(value = "isWithDirectory", required = false) boolean isWithDirectory,
|
||||
@RequestParam(value = "page", required = false, defaultValue = "0") Integer page,
|
||||
@RequestParam(value = "size", required = false, defaultValue = "20") Integer size,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
PagingQuery pagingQuery = new PagingQuery(page, size);
|
||||
PagedResponse<DatasetFile> filesPage;
|
||||
if (isWithDirectory) {
|
||||
filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(datasetId, prefix, pagingQuery);
|
||||
} else {
|
||||
filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, null, null, pagingQuery);
|
||||
}
|
||||
return Response.ok(filesPage);
|
||||
}
|
||||
|
||||
@GetMapping("/{fileId}")
|
||||
public ResponseEntity<Response<DatasetFileResponse>> getDatasetFileById(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
try {
|
||||
Dataset dataset = datasetApplicationService.getDataset(datasetId);
|
||||
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId, prefix);
|
||||
return ResponseEntity.ok(Response.ok(DatasetConverter.INSTANCE.convertToResponse(datasetFile)));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
@DeleteMapping("/{fileId}")
|
||||
public ResponseEntity<Response<Void>> deleteDatasetFile(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
try {
|
||||
datasetFileApplicationService.deleteDatasetFile(datasetId, fileId, prefix);
|
||||
return ResponseEntity.ok().build();
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量删除文件
|
||||
*/
|
||||
@DeleteMapping("/batch")
|
||||
public ResponseEntity<Response<Void>> batchDeleteFiles(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid BatchDeleteFilesRequest request) {
|
||||
try {
|
||||
datasetFileApplicationService.batchDeleteFiles(datasetId, request);
|
||||
return ResponseEntity.ok().build();
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
@IgnoreResponseWrap
|
||||
@GetMapping(value = "/{fileId}/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE + ";charset=UTF-8")
|
||||
public ResponseEntity<Resource> downloadDatasetFileById(@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
try {
|
||||
log.info("downloadDatasetFileById datasetId:{}, fileId:{}, prefix:{}", datasetId, fileId, prefix);
|
||||
Dataset dataset = datasetApplicationService.getDataset(datasetId);
|
||||
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId, prefix);
|
||||
Resource resource = datasetFileApplicationService.downloadFile(datasetFile);
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"" + datasetFile.getFileName() + "\"")
|
||||
.body(resource);
|
||||
} catch (IllegalArgumentException e) {
|
||||
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
|
||||
} catch (Exception e) {
|
||||
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
||||
}
|
||||
}
|
||||
|
||||
@IgnoreResponseWrap
|
||||
@GetMapping(value = "/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
|
||||
public void downloadDatasetFileAsZip(@PathVariable("datasetId") String datasetId, HttpServletResponse response) {
|
||||
datasetFileApplicationService.downloadDatasetFileAsZip(datasetId, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件上传请求
|
||||
*
|
||||
* @param request 批量文件上传请求
|
||||
* @return 批量上传请求id
|
||||
*/
|
||||
@PostMapping("/upload/pre-upload")
|
||||
public ResponseEntity<Response<String>> preUpload(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid UploadFilesPreRequest request) {
|
||||
return ResponseEntity.ok(Response.ok(datasetFileApplicationService.preUpload(request, datasetId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分块上传
|
||||
*
|
||||
* @param uploadFileRequest 上传文件请求
|
||||
*/
|
||||
@PostMapping("/upload/chunk")
|
||||
public ResponseEntity<Void> chunkUpload(@PathVariable("datasetId") String datasetId,
|
||||
@Valid UploadFileRequest uploadFileRequest) {
|
||||
log.info("file upload reqId:{}, fileNo:{}, total chunk num:{}, current chunkNo:{}",
|
||||
uploadFileRequest.getReqId(), uploadFileRequest.getFileNo(), uploadFileRequest.getTotalChunkNum(),
|
||||
uploadFileRequest.getChunkNo());
|
||||
datasetFileApplicationService.chunkUpload(datasetId, uploadFileRequest);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 将指定路径中的文件拷贝到数据集目录下
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param req 源文件路径列表
|
||||
* @return 数据集文件响应DTO列表
|
||||
*/
|
||||
@PostMapping("/upload/copy")
|
||||
public List<DatasetFileResponse> copyFilesToDatasetDir(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid CopyFilesRequest req) {
|
||||
AddFilesRequest addFilesRequest = new AddFilesRequest(req.sourcePaths());
|
||||
List<DatasetFile> datasetFiles = datasetFileApplicationService.addFilesToDataset(datasetId, addFilesRequest);
|
||||
return DatasetConverter.INSTANCE.convertToResponseList(datasetFiles);
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加文件到数据集（仅创建数据库记录，不执行文件系统操作）
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param req 添加文件请求（包含源文件路径列表和softAdd标志）
|
||||
* @return 数据集文件响应DTO列表
|
||||
*/
|
||||
@PostMapping("/upload/add")
|
||||
public List<DatasetFileResponse> addFilesToDataset(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid AddFilesRequest req) {
|
||||
List<DatasetFile> datasetFiles = datasetFileApplicationService.addFilesToDataset(datasetId, req);
|
||||
return DatasetConverter.INSTANCE.convertToResponseList(datasetFiles);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在数据集下创建子目录
|
||||
*/
|
||||
@PostMapping("/directories")
|
||||
public ResponseEntity<Void> createDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid CreateDirectoryRequest req) {
|
||||
datasetFileApplicationService.createDirectory(datasetId, req);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载目录（压缩为 ZIP）
|
||||
*/
|
||||
@IgnoreResponseWrap
|
||||
@GetMapping(value = "/directories/download", produces = "application/zip")
|
||||
public void downloadDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix,
|
||||
HttpServletResponse response) {
|
||||
datasetFileApplicationService.downloadDirectory(datasetId, prefix, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除目录及其所有内容
|
||||
*/
|
||||
@DeleteMapping("/directories")
|
||||
public ResponseEntity<Void> deleteDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
datasetFileApplicationService.deleteDirectory(datasetId, prefix);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 重命名文件
|
||||
*/
|
||||
@PutMapping("/{fileId}/rename")
|
||||
public ResponseEntity<Void> renameFile(@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestBody @Valid RenameFileRequest request) {
|
||||
datasetFileApplicationService.renameFile(datasetId, fileId, request);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 重命名目录
|
||||
*/
|
||||
@PutMapping("/directories/rename")
|
||||
public ResponseEntity<Void> renameDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid RenameDirectoryRequest request) {
|
||||
datasetFileApplicationService.renameDirectory(datasetId, request);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetTypeResponse;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集类型 REST 控制器
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/data-management/dataset-types")
|
||||
public class DatasetTypeController {
|
||||
|
||||
/**
|
||||
* 获取所有支持的数据集类型
|
||||
* @return 数据集类型列表
|
||||
*/
|
||||
@GetMapping
|
||||
public List<DatasetTypeResponse> getDatasetTypes() {
|
||||
return Arrays.asList(
|
||||
createDatasetType("IMAGE", "图像数据集", "用于机器学习的图像数据集", Arrays.asList("jpg", "jpeg", "png", "bmp", "gif")),
|
||||
createDatasetType("TEXT", "文本数据集", "用于文本分析的文本数据集", Arrays.asList("txt", "csv", "json", "xml")),
|
||||
createDatasetType("AUDIO", "音频数据集", "用于音频处理的音频数据集", Arrays.asList("wav", "mp3", "flac", "aac")),
|
||||
createDatasetType("VIDEO", "视频数据集", "用于视频分析的视频数据集", Arrays.asList("mp4", "avi", "mov", "mkv")),
|
||||
createDatasetType("MULTIMODAL", "多模态数据集", "包含多种数据类型的数据集", List.of("*"))
|
||||
);
|
||||
}
|
||||
|
||||
private DatasetTypeResponse createDatasetType(String code, String name, String description, List<String> supportedFormats) {
|
||||
DatasetTypeResponse response = new DatasetTypeResponse();
|
||||
response.setCode(code);
|
||||
response.setName(name);
|
||||
response.setDescription(description);
|
||||
response.setSupportedFormats(supportedFormats);
|
||||
response.setIcon(getIconForType(code));
|
||||
return response;
|
||||
}
|
||||
|
||||
private String getIconForType(String typeCode) {
|
||||
return switch (typeCode) {
|
||||
case "IMAGE" -> "🖼️";
|
||||
case "TEXT" -> "📄";
|
||||
case "AUDIO" -> "🎵";
|
||||
case "VIDEO" -> "🎬";
|
||||
case "MULTIMODAL" -> "📊";
|
||||
default -> "📁";
|
||||
};
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.deepdata.datamanagement.application.TagApplicationService;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.interfaces.converter.TagConverter;
|
||||
import com.deepdata.datamanagement.interfaces.dto.CreateTagRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.TagResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UpdateTagRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 标签 REST 控制器（UUID 模式）
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/data-management/tags")
|
||||
public class TagController {
|
||||
|
||||
private final TagApplicationService tagApplicationService;
|
||||
|
||||
@Autowired
|
||||
public TagController(TagApplicationService tagApplicationService) {
|
||||
this.tagApplicationService = tagApplicationService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询标签列表
|
||||
*/
|
||||
@GetMapping
|
||||
public ResponseEntity<Response<List<TagResponse>>> getTags(@RequestParam(name = "keyword", required = false) String keyword) {
|
||||
List<Tag> tags = tagApplicationService.searchTags(keyword);
|
||||
List<TagResponse> response = tags.stream()
|
||||
.map(TagConverter.INSTANCE::convertToResponse)
|
||||
.collect(Collectors.toList());
|
||||
return ResponseEntity.ok(Response.ok(response));
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建标签
|
||||
*/
|
||||
@PostMapping
|
||||
public ResponseEntity<Response<TagResponse>> createTag(@RequestBody CreateTagRequest createTagRequest) {
|
||||
try {
|
||||
Tag tag = tagApplicationService.createTag(
|
||||
createTagRequest.getName(),
|
||||
createTagRequest.getColor(),
|
||||
createTagRequest.getDescription()
|
||||
);
|
||||
return ResponseEntity.ok(Response.ok(TagConverter.INSTANCE.convertToResponse(tag)));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.badRequest().body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新标签
|
||||
*
|
||||
* @param updateTagRequest 更新参数
|
||||
* @return 更新结果
|
||||
*/
|
||||
@PutMapping
|
||||
public ResponseEntity<Response<TagResponse>> updateTag(@RequestBody @Valid UpdateTagRequest updateTagRequest) {
|
||||
Tag tag = tagApplicationService.updateTag(TagConverter.INSTANCE.updateRequestToTag(updateTagRequest));
|
||||
return ResponseEntity.ok(Response.ok(TagConverter.INSTANCE.convertToResponse(tag)));
|
||||
}
|
||||
|
||||
@DeleteMapping
|
||||
public ResponseEntity<Response<Void>> deleteTag(@RequestParam(value = "ids") @Valid @Size(max = 10) List<String> ids) {
|
||||
try {
|
||||
tagApplicationService.deleteTag(ids.stream().filter(StringUtils::isNoneBlank).distinct().toList());
|
||||
return ResponseEntity.ok(Response.ok(null));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.badRequest().body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.deepdata.datamanagement.interfaces.scheduler;
|
||||
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.datamanagement.application.DatasetApplicationService;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetPagingQuery;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetResponse;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 数据集定时任务触发
|
||||
*
|
||||
* @since 2025/10/24
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetScheduler {
|
||||
private final DatasetApplicationService datasetApplicationService;
|
||||
|
||||
/**
|
||||
* 每天凌晨 00:00 扫描并删除超出保留期的数据集
|
||||
*/
|
||||
@Scheduled(cron = "0 0 0 * * ?")
|
||||
public void cleanupExpiredDatasets() {
|
||||
int pageNo = 1;
|
||||
int pageSize = 500;
|
||||
|
||||
while (true) {
|
||||
DatasetPagingQuery datasetPagingQuery = new DatasetPagingQuery();
|
||||
datasetPagingQuery.setPage(pageNo);
|
||||
datasetPagingQuery.setSize(pageSize);
|
||||
PagedResponse<DatasetResponse> datasets = datasetApplicationService.getDatasets(datasetPagingQuery);
|
||||
if (CollectionUtils.isEmpty(datasets.getContent())) {
|
||||
break;
|
||||
}
|
||||
|
||||
datasets.getContent().forEach(dataset -> {
|
||||
Integer retentionDays = dataset.getRetentionDays();
|
||||
LocalDateTime createdAt = dataset.getCreatedAt();
|
||||
if (retentionDays != null && retentionDays > 0 && createdAt != null) {
|
||||
LocalDateTime expireAt = createdAt.plusDays(retentionDays);
|
||||
if (expireAt.isBefore(LocalDateTime.now())) {
|
||||
try {
|
||||
log.info("Deleting dataset {}, expired at {} (retentionDays={})", dataset.getId(), expireAt, retentionDays);
|
||||
datasetApplicationService.deleteDataset(dataset.getId());
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to delete expired dataset {}: {}", dataset.getId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (datasets.getPage() >= datasets.getTotalPages()) {
|
||||
break;
|
||||
}
|
||||
pageNo++;
|
||||
}
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 文件名校验注解
|
||||
* 验证文件名不包含非法字符
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidFileNameValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidFileName {
|
||||
|
||||
String message() default "文件名包含非法字符";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 文件名校验器
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
public class ValidFileNameValidator implements ConstraintValidator<ValidFileName, String> {
|
||||
|
||||
/**
|
||||
* 文件名正则表达式
|
||||
* 不允许包含特殊字符: / \ : * ? " < > | \0
|
||||
* 允许字母、数字、中文、常见符号（- _ . space）
|
||||
*/
|
||||
private static final Pattern FILE_NAME_PATTERN = Pattern.compile(
|
||||
"^[^/\\\\:*?\"<>|\\x00]+$"
|
||||
);
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null || value.isEmpty()) {
|
||||
return true; // 空值由 @NotBlank 等其他注解处理
|
||||
}
|
||||
|
||||
boolean isValid = FILE_NAME_PATTERN.matcher(value).matches();
|
||||
|
||||
if (!isValid) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.FILE_NAME_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
}
|
||||
|
||||
return isValid;
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 文件路径校验注解
|
||||
* 验证文件路径不包含非法字符（允许 / 用于支持文件夹上传）
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/03/12
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidFilePathValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidFilePath {
|
||||
|
||||
String message() default "文件路径包含非法字符";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 文件路径校验器
|
||||
* 允许路径分隔符 / 用于支持文件夹上传
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/03/12
|
||||
*/
|
||||
public class ValidFilePathValidator implements ConstraintValidator<ValidFilePath, String> {
|
||||
|
||||
/**
|
||||
* 文件路径正则表达式
|
||||
* 不允许包含特殊字符: \ : * ? " < > | \0
|
||||
* 允许字母、数字、中文、常见符号（- _ . space /）
|
||||
* 注意：允许 / 是为了支持文件夹上传的相对路径
|
||||
*/
|
||||
private static final Pattern FILE_PATH_PATTERN = Pattern.compile(
|
||||
"^[^\\\\:*?\"<>|\\x00]+$"
|
||||
);
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null || value.isEmpty()) {
|
||||
return true; // 空值由 @NotBlank 等其他注解处理
|
||||
}
|
||||
|
||||
// 检查是否包含路径遍历序列 ..
|
||||
if (value.contains("..")) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
"文件路径不允许包含 '..'"
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean isValid = FILE_PATH_PATTERN.matcher(value).matches();
|
||||
|
||||
if (!isValid) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.FILE_NAME_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
}
|
||||
|
||||
return isValid;
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 十六进制颜色格式校验注解
|
||||
* 验证字符串是否符合十六进制颜色格式 (#RRGGBB 或 #RGB)
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidHexColorValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidHexColor {
|
||||
|
||||
String message() default "标签颜色格式不正确，应为十六进制颜色代码";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import com.deepdata.common.infrastructure.exception.ErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
import org.hibernate.validator.internal.constraintvalidators.bv.size.SizeValidatorForCharSequence;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 十六进制颜色格式校验器
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
public class ValidHexColorValidator implements ConstraintValidator<ValidHexColor, String> {
|
||||
|
||||
/**
|
||||
* 十六进制颜色正则表达式
|
||||
* 支持 #RGB 和 #RRGGBB 格式
|
||||
*/
|
||||
private static final Pattern HEX_COLOR_PATTERN = Pattern.compile("^#[0-9a-fA-F]{6}$");
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null) {
|
||||
return true; // null 值由 @NotBlank 等其他注解处理
|
||||
}
|
||||
|
||||
boolean isValid = HEX_COLOR_PATTERN.matcher(value).matches();
|
||||
|
||||
if (!isValid) {
|
||||
// 自定义错误消息和错误码
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.TAG_COLOR_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
}
|
||||
|
||||
return isValid;
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 路径格式校验注解
|
||||
* 验证路径格式和长度限制
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidPathValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidPath {
|
||||
|
||||
String message() default "路径格式不正确";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
|
||||
/**
|
||||
* 最大路径长度，默认500
|
||||
*/
|
||||
int maxLength() default 500;
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 路径格式校验器
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
public class ValidPathValidator implements ConstraintValidator<ValidPath, String> {
|
||||
|
||||
/**
|
||||
* 路径正则表达式
|
||||
* 不允许以点开头（隐藏文件/目录）
|
||||
* 不允许包含特殊字符如 \0, <, >, :, ", |, ?, *
|
||||
*/
|
||||
private static final Pattern PATH_PATTERN = Pattern.compile(
|
||||
"^[^<>:\"|?*\\x00]+" // 不以点开头，不包含特殊字符
|
||||
);
|
||||
|
||||
private int maxLength = 500;
|
||||
|
||||
@Override
|
||||
public void initialize(ValidPath constraintAnnotation) {
|
||||
this.maxLength = constraintAnnotation.maxLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null || value.isEmpty()) {
|
||||
return true; // 空值由其他注解处理
|
||||
}
|
||||
|
||||
// 检查长度
|
||||
if (value.length() > maxLength) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.PATH_TOO_LONG.getMessage()
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否以点开头
|
||||
if (value.startsWith(".")) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.PREFIX_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否包含路径遍历序列 ..
|
||||
if (value.contains("..")) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
"路径不允许包含 '..'"
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否包含非法字符
|
||||
if (!PATH_PATTERN.matcher(value).matches()) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate("路径包含非法字符").addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
deepdata:
|
||||
datamanagement:
|
||||
file-storage:
|
||||
upload-dir: ${FILE_UPLOAD_DIR:./uploads}
|
||||
max-file-size: 10485760 # 10MB
|
||||
max-request-size: 52428800 # 50MB
|
||||
allowed-source-dirs:
|
||||
- ${DATASET_BASE_PATH:/dataset}
|
||||
- ${FILE_UPLOAD_DIR:./uploads}
|
||||
cache:
|
||||
ttl: 3600
|
||||
max-size: 1000
|
||||
# MyBatis is configured centrally in main-application (mapper-locations & aliases)
|
||||
# to avoid list overriding issues when importing multiple module configs.
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper">
|
||||
<sql id="Base_Column_List">
|
||||
id, dataset_id, file_name, file_path, file_type, file_size, check_sum, tags, tags_updated_at, metadata, status,
|
||||
upload_time, last_access_time, created_at, updated_at
|
||||
</sql>
|
||||
|
||||
<select id="findById" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetId" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetIdAndStatus" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
AND status = #{status}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetIdAndFileType" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
AND file_type = #{fileType}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="countByDatasetId" parameterType="string" resultType="long">
|
||||
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
|
||||
</select>
|
||||
|
||||
<select id="countCompletedByDatasetId" parameterType="string" resultType="long">
|
||||
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId} AND status = 'COMPLETED'
|
||||
</select>
|
||||
|
||||
<select id="sumSizeByDatasetId" parameterType="string" resultType="long">
|
||||
SELECT COALESCE(SUM(file_size), 0) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetIdAndFileName" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId} AND file_name = #{fileName}
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
<select id="findAllByDatasetId" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="findByCriteria" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
<!-- Replace invalid XML '&&' with 'and' for MyBatis OGNL -->
|
||||
<if test="fileType != null and fileType != ''">
|
||||
AND file_type = #{fileType}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND status = #{status}
|
||||
</if>
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
|
||||
<update id="update" parameterType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
UPDATE t_dm_dataset_files
|
||||
SET file_name = #{fileName},
|
||||
file_path = #{filePath},
|
||||
file_type = #{fileType},
|
||||
file_size = #{fileSize},
|
||||
upload_time = #{uploadTime},
|
||||
last_access_time = #{lastAccessTime},
|
||||
status = #{status}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<delete id="deleteById" parameterType="string">
|
||||
DELETE FROM t_dm_dataset_files WHERE id = #{id}
|
||||
</delete>
|
||||
</mapper>
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetMapper">
|
||||
|
||||
<sql id="Base_Column_List">
|
||||
id, name, description, dataset_type, category, path, format, schema_info, size_bytes, file_count, record_count,
|
||||
retention_days, tags, metadata, status, is_public, is_featured, version, created_at, updated_at, created_by, updated_by
|
||||
</sql>
|
||||
|
||||
<sql id="Alias_D_Column_List">
|
||||
d.id AS id,
|
||||
d.name AS name,
|
||||
d.description AS description,
|
||||
d.dataset_type AS dataset_type,
|
||||
d.category AS category,
|
||||
d.path AS path,
|
||||
d.format AS format,
|
||||
d.schema_info AS schema_info,
|
||||
d.size_bytes AS size_bytes,
|
||||
d.file_count AS file_count,
|
||||
d.record_count AS record_count,
|
||||
d.retention_days AS retention_days,
|
||||
d.tags AS tags,
|
||||
d.metadata AS metadata,
|
||||
d.status AS status,
|
||||
d.is_public AS is_public,
|
||||
d.is_featured AS is_featured,
|
||||
d.version AS version,
|
||||
d.created_at AS created_at,
|
||||
d.updated_at AS updated_at,
|
||||
d.created_by AS created_by,
|
||||
d.updated_by AS updated_by
|
||||
</sql>
|
||||
|
||||
<select id="findById" parameterType="string" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<select id="findByName" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE name = #{name}
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
<select id="findByStatus" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE status = #{status}
|
||||
ORDER BY updated_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByCreatedBy" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE created_by = #{createdBy}
|
||||
ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByTypeCode" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE dataset_type = #{typeCode}
|
||||
ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByTagNames" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT DISTINCT <include refid="Alias_D_Column_List"/>
|
||||
FROM t_dm_datasets d
|
||||
JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
|
||||
JOIN t_dm_tags t ON t.id = dt.tag_id
|
||||
WHERE t.name IN
|
||||
<foreach collection="tagNames" item="name" open="(" separator="," close=")">
|
||||
#{name}
|
||||
</foreach>
|
||||
ORDER BY d.created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByKeyword" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE name LIKE CONCAT('%', #{keyword}, '%')
|
||||
OR description LIKE CONCAT('%', #{keyword}, '%')
|
||||
ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByCriteria" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT DISTINCT <include refid="Alias_D_Column_List"/>
|
||||
FROM t_dm_datasets d
|
||||
LEFT JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
|
||||
LEFT JOIN t_dm_tags t ON t.id = dt.tag_id
|
||||
<where>
|
||||
<if test="typeCode != null and typeCode != ''">
|
||||
AND d.dataset_type = #{typeCode}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND d.status = #{status}
|
||||
</if>
|
||||
<if test="keyword != null and keyword != ''">
|
||||
AND (d.name LIKE CONCAT('%', #{keyword}, '%') OR d.description LIKE CONCAT('%', #{keyword}, '%'))
|
||||
</if>
|
||||
<if test="tagNames != null and tagNames.size > 0">
|
||||
AND t.name IN
|
||||
<foreach collection="tagNames" item="n" open="(" separator="," close=")">
|
||||
#{n}
|
||||
</foreach>
|
||||
</if>
|
||||
</where>
|
||||
ORDER BY d.created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="countByCriteria" resultType="long">
|
||||
SELECT COUNT(DISTINCT d.id)
|
||||
FROM t_dm_datasets d
|
||||
LEFT JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
|
||||
LEFT JOIN t_dm_tags t ON t.id = dt.tag_id
|
||||
<where>
|
||||
<if test="typeCode != null and typeCode != ''">
|
||||
AND d.dataset_type = #{typeCode}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND d.status = #{status}
|
||||
</if>
|
||||
<if test="keyword != null and keyword != ''">
|
||||
AND (d.name LIKE CONCAT('%', #{keyword}, '%') OR d.description LIKE CONCAT('%', #{keyword}, '%'))
|
||||
</if>
|
||||
<if test="tagNames != null and tagNames.size > 0">
|
||||
AND t.name IN
|
||||
<foreach collection="tagNames" item="n" open="(" separator="," close=")">
|
||||
#{n}
|
||||
</foreach>
|
||||
</if>
|
||||
</where>
|
||||
</select>
|
||||
|
||||
<delete id="deleteById" parameterType="string">
|
||||
DELETE FROM t_dm_datasets WHERE id = #{id}
|
||||
</delete>
|
||||
|
||||
<select id="getAllDatasetStatistics" resultType="com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse">
|
||||
SELECT
|
||||
COUNT(*) AS total_datasets,
|
||||
SUM(size_bytes) AS total_size,
|
||||
SUM(file_count) AS total_files
|
||||
FROM t_dm_datasets;
|
||||
</select>
|
||||
</mapper>
|
||||
@@ -0,0 +1,111 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.deepdata.datamanagement.infrastructure.persistence.mapper.TagMapper">
|
||||
|
||||
<resultMap id="TagResultMap" type="com.deepdata.datamanagement.domain.model.dataset.Tag">
|
||||
<id column="id" property="id"/>
|
||||
<result column="name" property="name"/>
|
||||
<result column="description" property="description"/>
|
||||
<result column="category" property="category"/>
|
||||
<result column="color" property="color"/>
|
||||
<result column="usage_count" property="usageCount"/>
|
||||
<result column="created_at" property="createdAt"/>
|
||||
<result column="updated_at" property="updatedAt"/>
|
||||
</resultMap>
|
||||
|
||||
<sql id="Base_Column_List">
|
||||
id, name, description, category, color, usage_count, created_at, updated_at
|
||||
</sql>
|
||||
|
||||
<select id="findById" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<select id="findByName" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE name = #{name}
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
<select id="findByNameIn" parameterType="list" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE name IN
|
||||
<foreach collection="list" item="n" open="(" separator="," close=")">
|
||||
#{n}
|
||||
</foreach>
|
||||
</select>
|
||||
|
||||
<select id="findByKeyword" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE name LIKE CONCAT('%', #{keyword}, '%')
|
||||
ORDER BY usage_count DESC, name ASC
|
||||
</select>
|
||||
|
||||
<select id="findAllByOrderByUsageCountDesc" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
ORDER BY usage_count DESC, name ASC
|
||||
</select>
|
||||
|
||||
<insert id="insert" parameterType="com.deepdata.datamanagement.domain.model.dataset.Tag">
|
||||
INSERT INTO t_dm_tags (id, name, description, category, color, usage_count)
|
||||
VALUES (#{id}, #{name}, #{description}, #{category}, #{color}, #{usageCount})
|
||||
</insert>
|
||||
|
||||
<update id="update" parameterType="com.deepdata.datamanagement.domain.model.dataset.Tag">
|
||||
UPDATE t_dm_tags
|
||||
SET name = #{name},
|
||||
description = #{description},
|
||||
category = #{category},
|
||||
color = #{color},
|
||||
usage_count = #{usageCount}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<update id="updateUsageCount">
|
||||
UPDATE t_dm_tags
|
||||
SET usage_count = #{usageCount}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Dataset & Tag relations -->
|
||||
<insert id="insertDatasetTag">
|
||||
INSERT INTO t_dm_dataset_tags (dataset_id, tag_id)
|
||||
VALUES (#{datasetId}, #{tagId})
|
||||
</insert>
|
||||
|
||||
<delete id="deleteDatasetTagsByDatasetId">
|
||||
DELETE FROM t_dm_dataset_tags WHERE dataset_id = #{datasetId}
|
||||
</delete>
|
||||
|
||||
<select id="findByDatasetId" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT t.id, t.name, t.description, t.category, t.color, t.usage_count, t.created_at, t.updated_at
|
||||
FROM t_dm_tags t
|
||||
JOIN t_dm_dataset_tags dt ON dt.tag_id = t.id
|
||||
WHERE dt.dataset_id = #{datasetId}
|
||||
ORDER BY t.usage_count DESC, t.name ASC
|
||||
</select>
|
||||
|
||||
<delete id="deleteTagsById">
|
||||
DELETE FROM t_dm_tags WHERE
|
||||
id IN
|
||||
<foreach collection="ids" item="id" open="(" separator="," close=")">
|
||||
#{id}
|
||||
</foreach>
|
||||
</delete>
|
||||
|
||||
<select id="findByIdIn" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE id IN
|
||||
<foreach collection="ids" item="id" open="(" separator="," close=")">
|
||||
#{id}
|
||||
</foreach>
|
||||
</select>
|
||||
</mapper>
|
||||
Reference in New Issue
Block a user