deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,104 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>services</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>data-management-service</artifactId>
|
||||
<name>Data Management Service</name>
|
||||
<description>数据管理服务</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>domain-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-redis</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-openfeign</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.openapitools</groupId>
|
||||
<artifactId>jackson-databind-nullable</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>jakarta.validation</groupId>
|
||||
<artifactId>jakarta.validation-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.data</groupId>
|
||||
<artifactId>spring-data-commons</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<arguments>true</arguments>
|
||||
<classifier>exec</classifier>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<configuration>
|
||||
<source>${maven.compiler.source}</source>
|
||||
<target>${maven.compiler.target}</target>
|
||||
<annotationProcessorPaths>
|
||||
<!-- 顺序很重要 -->
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>${lombok.version}</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok-mapstruct-binding</artifactId>
|
||||
<version>${lombok-mapstruct-binding.version}</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct-processor</artifactId>
|
||||
<version>${mapstruct.version}</version>
|
||||
</path>
|
||||
</annotationProcessorPaths>
|
||||
<compilerArgs>
|
||||
<arg>-parameters</arg>
|
||||
<arg>-Amapstruct.defaultComponentModel=spring</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.deepdata.datamanagement;
|
||||
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.EnableAsync;
|
||||
|
||||
/**
|
||||
* Data Management Service Configuration
|
||||
* 数据管理服务配置类 - 多源接入、元数据、血缘治理
|
||||
*/
|
||||
@Configuration
|
||||
@EnableAsync
|
||||
@ComponentScan(basePackages = {
|
||||
"com.deepdata.datamanagement",
|
||||
"com.deepdata.shared"
|
||||
})
|
||||
public class DataManagementServiceConfiguration {
|
||||
// Service configuration class for JAR packaging
|
||||
// 作为jar包形式提供服务的配置类
|
||||
}
|
||||
+352
@@ -0,0 +1,352 @@
|
||||
package com.deepdata.datamanagement.application;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.deepdata.common.domain.enums.EdgeType;
|
||||
import com.deepdata.common.domain.enums.NodeType;
|
||||
import com.deepdata.common.domain.model.LineageEdge;
|
||||
import com.deepdata.common.domain.model.LineageNode;
|
||||
import com.deepdata.common.domain.service.LineageService;
|
||||
import com.deepdata.common.domain.utils.ChunksSaver;
|
||||
import com.deepdata.common.setting.application.SysParamApplicationService;
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import com.deepdata.common.infrastructure.exception.BusinessAssert;
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.infrastructure.client.CollectionTaskClient;
|
||||
import com.deepdata.datamanagement.infrastructure.client.dto.CollectionTaskDetailResponse;
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.TagMapper;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
||||
import com.deepdata.datamanagement.interfaces.converter.DatasetConverter;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* 数据集应用服务（对齐 DB schema，使用 UUID 字符串主键）
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@Transactional
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetApplicationService {
|
||||
private static final String DATASET_PVC_NAME = "sys.management.dataset.pvc.name";
|
||||
private final DatasetRepository datasetRepository;
|
||||
private final TagMapper tagMapper;
|
||||
private final DatasetFileRepository datasetFileRepository;
|
||||
private final CollectionTaskClient collectionTaskClient;
|
||||
private final DatasetFileApplicationService datasetFileApplicationService;
|
||||
private final SysParamApplicationService sysParamService;
|
||||
private final LineageService lineageService;
|
||||
|
||||
@Value("${deepdata.data-management.base-path:/dataset}")
|
||||
private String datasetBasePath;
|
||||
|
||||
/**
|
||||
* 创建数据集
|
||||
*/
|
||||
@Transactional
|
||||
public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
|
||||
BusinessAssert.isTrue(datasetRepository.findByName(createDatasetRequest.getName()) == null, DataManagementErrorCode.DATASET_ALREADY_EXISTS);
|
||||
// 创建数据集对象
|
||||
Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
|
||||
dataset.initCreateParam(datasetBasePath);
|
||||
|
||||
// xuhuanqing
|
||||
// ================= 新增代码 START =================
|
||||
// 防止后续调用归集任务接口时因 createdBy 为 null 报错
|
||||
// 实际项目中应从当前登录用户的 Session/Token 获取，这里先设为默认值
|
||||
dataset.setCreatedBy("admin");
|
||||
dataset.setUpdatedBy("admin");
|
||||
// ================= 新增代码 END =================
|
||||
|
||||
// 处理标签
|
||||
if (CollectionUtils.isNotEmpty(createDatasetRequest.getTags())) {
|
||||
dataset.setTags(processTagNames(createDatasetRequest.getTags()));
|
||||
}
|
||||
datasetRepository.save(dataset);
|
||||
// 记录血缘关系
|
||||
addDatasetToGraph(dataset, null);
|
||||
|
||||
//todo 需要解耦这块逻辑
|
||||
if (StringUtils.hasText(createDatasetRequest.getDataSource())) {
|
||||
// 数据源id不为空，使用异步线程进行文件扫盘落库
|
||||
processDataSourceAsync(dataset.getId(), createDatasetRequest.getDataSource());
|
||||
}
|
||||
return dataset;
|
||||
}
|
||||
|
||||
private void addDatasetToGraph(Dataset dataset, CollectionTaskDetailResponse collection) {
|
||||
LineageNode datasetNode = new LineageNode();
|
||||
datasetNode.setId(dataset.getId());
|
||||
datasetNode.setNodeType(NodeType.DATASET);
|
||||
datasetNode.setName(dataset.getName());
|
||||
datasetNode.setDescription(dataset.getDescription());
|
||||
|
||||
LineageNode collectionNode = null;
|
||||
LineageEdge collectionEdge = null;
|
||||
if(Objects.nonNull(collection)) {
|
||||
collectionNode = new LineageNode();
|
||||
collectionNode.setId(collection.getId());
|
||||
collectionNode.setName(collection.getName());
|
||||
collectionNode.setDescription(collection.getDescription());
|
||||
collectionNode.setNodeType(NodeType.DATASOURCE);
|
||||
|
||||
collectionEdge = new LineageEdge();
|
||||
collectionEdge.setProcessId(collection.getId());
|
||||
collectionEdge.setName(collection.getName());
|
||||
collectionEdge.setEdgeType(EdgeType.DATA_COLLECTION);
|
||||
collectionEdge.setDescription(dataset.getDescription());
|
||||
collectionEdge.setFromNodeId(collectionNode.getId());
|
||||
collectionEdge.setToNodeId(datasetNode.getId());
|
||||
lineageService.generateGraph(collectionNode, collectionEdge, datasetNode);
|
||||
} else {
|
||||
lineageService.generateGraph(datasetNode, null, null);
|
||||
}
|
||||
}
|
||||
|
||||
public DatasetLineage getDatasetLineage(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
if (Objects.isNull(dataset)) {
|
||||
return new DatasetLineage();
|
||||
}
|
||||
LineageNode datasetNode = lineageService.getNodeById(datasetId);
|
||||
String graphId = datasetNode.getGraphId();
|
||||
return new DatasetLineage(lineageService.getNodesByGraphId(graphId), lineageService.getEdgesByGraphId(graphId));
|
||||
}
|
||||
|
||||
public String getDatasetPvcName() {
|
||||
return sysParamService.getParamByKey(DATASET_PVC_NAME);
|
||||
}
|
||||
|
||||
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||
if (StringUtils.hasText(updateDatasetRequest.getName())) {
|
||||
dataset.setName(updateDatasetRequest.getName());
|
||||
}
|
||||
if (StringUtils.hasText(updateDatasetRequest.getDescription())) {
|
||||
dataset.setDescription(updateDatasetRequest.getDescription());
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(updateDatasetRequest.getTags())) {
|
||||
dataset.setTags(processTagNames(updateDatasetRequest.getTags()));
|
||||
}
|
||||
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
|
||||
dataset.setStatus(updateDatasetRequest.getStatus());
|
||||
}
|
||||
datasetRepository.updateById(dataset);
|
||||
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
|
||||
// 数据源id不为空，使用异步线程进行文件扫盘落库
|
||||
processDataSourceAsync(dataset.getId(), updateDatasetRequest.getDataSource());
|
||||
}
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除数据集
|
||||
*/
|
||||
@Transactional
|
||||
public void deleteDataset(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
datasetRepository.removeById(datasetId);
|
||||
if (dataset != null) {
|
||||
ChunksSaver.deleteFolder(dataset.getPath());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取数据集详情
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Dataset getDataset(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||
List<DatasetFile> datasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
||||
dataset.setFiles(datasetFiles);
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页查询数据集
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
|
||||
IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
|
||||
page = datasetRepository.findByCriteria(page, query);
|
||||
String datasetPvcName = getDatasetPvcName();
|
||||
List<DatasetResponse> datasetResponses = DatasetConverter.INSTANCE.convertToResponse(page.getRecords());
|
||||
datasetResponses.forEach(dataset -> dataset.setPvcName(datasetPvcName));
|
||||
return PagedResponse.of(page.getCurrent(), page.getSize(), page.getTotal(), page.getPages(), datasetResponses);
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理标签名称，创建或获取标签
|
||||
*/
|
||||
private List<Tag> processTagNames(List<String> tagNames) {
|
||||
Set<Tag> tags = new HashSet<>();
|
||||
for (String tagName : tagNames) {
|
||||
Tag tag = tagMapper.findByName(tagName);
|
||||
if (tag == null) {
|
||||
Tag newTag = new Tag(tagName, null, null, "#007bff");
|
||||
newTag.setUsageCount(0L);
|
||||
newTag.setId(UUID.randomUUID().toString());
|
||||
tagMapper.insert(newTag);
|
||||
tag = newTag;
|
||||
}
|
||||
tag.setUsageCount(tag.getUsageCount() == null ? 1L : tag.getUsageCount() + 1);
|
||||
tagMapper.updateUsageCount(tag.getId(), tag.getUsageCount());
|
||||
tags.add(tag);
|
||||
}
|
||||
return tags.stream().toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取数据集统计信息
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Map<String, Object> getDatasetStatistics(String datasetId) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
if (dataset == null) {
|
||||
throw new IllegalArgumentException("Dataset not found: " + datasetId);
|
||||
}
|
||||
|
||||
Map<String, Object> statistics = new HashMap<>();
|
||||
|
||||
// 基础统计
|
||||
Long totalFiles = datasetFileRepository.countByDatasetId(datasetId);
|
||||
Long completedFiles = datasetFileRepository.countCompletedByDatasetId(datasetId);
|
||||
Long totalSize = datasetFileRepository.sumSizeByDatasetId(datasetId);
|
||||
|
||||
statistics.put("totalFiles", totalFiles != null ? totalFiles.intValue() : 0);
|
||||
statistics.put("completedFiles", completedFiles != null ? completedFiles.intValue() : 0);
|
||||
statistics.put("totalSize", totalSize != null ? totalSize : 0L);
|
||||
|
||||
// 完成率计算
|
||||
float completionRate = 0.0f;
|
||||
if (totalFiles != null && totalFiles > 0) {
|
||||
completionRate = (completedFiles != null ? completedFiles.floatValue() : 0.0f) / totalFiles.floatValue() * 100.0f;
|
||||
}
|
||||
statistics.put("completionRate", completionRate);
|
||||
|
||||
// 文件类型分布统计
|
||||
Map<String, Integer> fileTypeDistribution = new HashMap<>();
|
||||
List<DatasetFile> allFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
||||
if (allFiles != null) {
|
||||
for (DatasetFile file : allFiles) {
|
||||
String fileType = file.getFileType() != null ? file.getFileType() : "unknown";
|
||||
fileTypeDistribution.put(fileType, fileTypeDistribution.getOrDefault(fileType, 0) + 1);
|
||||
}
|
||||
}
|
||||
statistics.put("fileTypeDistribution", fileTypeDistribution);
|
||||
|
||||
// 状态分布统计
|
||||
Map<String, Integer> statusDistribution = new HashMap<>();
|
||||
if (allFiles != null) {
|
||||
for (DatasetFile file : allFiles) {
|
||||
String status = file.getStatus() != null ? file.getStatus() : "unknown";
|
||||
statusDistribution.put(status, statusDistribution.getOrDefault(status, 0) + 1);
|
||||
}
|
||||
}
|
||||
statistics.put("statusDistribution", statusDistribution);
|
||||
|
||||
return statistics;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有数据集的汇总统计信息
|
||||
*/
|
||||
public AllDatasetStatisticsResponse getAllDatasetStatistics() {
|
||||
return datasetRepository.getAllDatasetStatistics();
|
||||
}
|
||||
|
||||
/**
|
||||
* 异步处理数据源文件扫描
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param dataSourceId 数据源ID（归集任务ID）
|
||||
*/
|
||||
@Async
|
||||
public void processDataSourceAsync(String datasetId, String dataSourceId) {
|
||||
try {
|
||||
log.info("Initiating data source file scanning, dataset ID: {}, collection task ID: {}", datasetId, dataSourceId);
|
||||
List<String> filePaths = getFilePaths(dataSourceId, datasetRepository.getById(datasetId));
|
||||
if (CollectionUtils.isEmpty(filePaths)) {
|
||||
return;
|
||||
}
|
||||
// 创建请求并启用硬链接（softAdd=true）
|
||||
AddFilesRequest addFilesRequest = new AddFilesRequest(filePaths);
|
||||
addFilesRequest.setSoftAdd(true);
|
||||
datasetFileApplicationService.addFilesToDataset(datasetId, addFilesRequest);
|
||||
log.info("Success file scan, total files: {}", filePaths.size());
|
||||
} catch (Exception e) {
|
||||
log.error("处理数据源文件扫描失败，数据集ID: {}, 数据源ID: {}", datasetId, dataSourceId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> getFilePaths(String dataSourceId, Dataset dataset) {
|
||||
// xuhuanqing
|
||||
// ================= 修改代码 START =================
|
||||
// 防御性编程：如果 createdBy 为空，使用 "system" 代替，防止 Feign Header 报错
|
||||
String user = "system";
|
||||
if (dataset != null && dataset.getCreatedBy() != null) {
|
||||
user = dataset.getCreatedBy();
|
||||
}
|
||||
|
||||
// 将原来的 dataset.getCreatedBy() 替换为变量 user
|
||||
Response<CollectionTaskDetailResponse> response = collectionTaskClient.getTaskDetail(dataSourceId, user);
|
||||
|
||||
// 检查响应状态
|
||||
if (response == null || response.getData() == null) {
|
||||
log.warn("Fail to get collection task detail from Feign client, response is empty");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
CollectionTaskDetailResponse taskDetail = response.getData();
|
||||
// ================= 修改代码 END =================
|
||||
|
||||
// CollectionTaskDetailResponse taskDetail = collectionTaskClient.getTaskDetail(dataSourceId, dataset.getCreatedBy()).getData();
|
||||
if (taskDetail == null) {
|
||||
log.warn("Fail to get collection task detail, task ID: {}", dataSourceId);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
// 记录血缘关系
|
||||
addDatasetToGraph(dataset, taskDetail);
|
||||
Path targetPath = Paths.get(taskDetail.getTargetPath());
|
||||
if (!Files.exists(targetPath) || !Files.isDirectory(targetPath)) {
|
||||
log.warn("Target path not exists or is not a directory: {}", taskDetail.getTargetPath());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
try (Stream<Path> paths = Files.walk(targetPath, 1)) {
|
||||
return paths
|
||||
.filter(Files::isRegularFile) // 只保留文件，排除目录
|
||||
.map(Path::toString) // 转换为字符串路径
|
||||
.collect(Collectors.toList());
|
||||
} catch (IOException e) {
|
||||
log.error("Fail to scan directory: {}", targetPath, e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
}
|
||||
+1265
File diff suppressed because it is too large
Load Diff
+118
@@ -0,0 +1,118 @@
|
||||
package com.deepdata.datamanagement.application;
|
||||
|
||||
import com.deepdata.common.infrastructure.exception.BusinessException;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.TagMapper;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* 标签应用服务（UUID 主键）
|
||||
*/
|
||||
@Service
|
||||
@Transactional
|
||||
public class TagApplicationService {
|
||||
|
||||
private final TagMapper tagMapper;
|
||||
|
||||
@Autowired
|
||||
public TagApplicationService(TagMapper tagMapper) {
|
||||
this.tagMapper = tagMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建标签
|
||||
*/
|
||||
public Tag createTag(String name, String color, String description) {
|
||||
// 检查名称是否已存在
|
||||
if (tagMapper.findByName(name) != null) {
|
||||
throw BusinessException.of(DataManagementErrorCode.TAG_NAME_DUPLICATE);
|
||||
}
|
||||
|
||||
Tag tag = new Tag(name, description, null, color);
|
||||
tag.setUsageCount(0L);
|
||||
tag.setId(UUID.randomUUID().toString());
|
||||
tagMapper.insert(tag);
|
||||
return tagMapper.findById(tag.getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新标签
|
||||
*
|
||||
* @param tag 待更新的标签实体，必须包含有效的 ID
|
||||
* @return 更新结果
|
||||
*/
|
||||
@Transactional
|
||||
public Tag updateTag(Tag tag) {
|
||||
Tag existingTag = tagMapper.findById(tag.getId());
|
||||
if (existingTag == null) {
|
||||
throw new IllegalArgumentException("Tag not found: " + tag.getId());
|
||||
}
|
||||
existingTag.setName(tag.getName());
|
||||
existingTag.setColor(tag.getColor());
|
||||
existingTag.setDescription(tag.getDescription());
|
||||
tagMapper.update(existingTag);
|
||||
return tagMapper.findById(existingTag.getId());
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTag(List<String> tagIds) {
|
||||
List<Tag> tags = tagMapper.findByIdIn(tagIds);
|
||||
if (tags.stream().anyMatch(tag -> tag.getUsageCount() > 0)) {
|
||||
throw new IllegalArgumentException("Cannot delete tags that are in use");
|
||||
}
|
||||
if (CollectionUtils.isEmpty(tags)) {
|
||||
return;
|
||||
}
|
||||
tagMapper.deleteTagsById(tags.stream().map(Tag::getId).toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有标签
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public List<Tag> getAllTags() {
|
||||
return tagMapper.findAllByOrderByUsageCountDesc();
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据关键词搜索标签
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public List<Tag> searchTags(String keyword) {
|
||||
if (keyword == null || keyword.trim().isEmpty()) {
|
||||
return getAllTags();
|
||||
}
|
||||
return tagMapper.findByKeyword(keyword.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取标签详情
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Tag getTag(String tagId) {
|
||||
Tag tag = tagMapper.findById(tagId);
|
||||
if (tag == null) {
|
||||
throw new IllegalArgumentException("Tag not found: " + tagId);
|
||||
}
|
||||
return tag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据名称获取标签
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Tag getTagByName(String name) {
|
||||
Tag tag = tagMapper.findByName(name);
|
||||
if (tag == null) {
|
||||
throw new IllegalArgumentException("Tag not found: " + name);
|
||||
}
|
||||
return tag;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package com.deepdata.datamanagement.common.enums;
|
||||
|
||||
/**
|
||||
* 数据集状态类型
|
||||
* <p>数据集可以处于以下几种状态：
|
||||
* <p>草稿（DRAFT）：数据集正在创建中，尚未完成。
|
||||
* <p>活动（ACTIVE）：数据集处于活动状态, 可以被查询和使用，也可以被更新和删除。
|
||||
* <p>处理中（PROCESSING）：数据集正在处理中，可能需要一些时间，处理完成后会变成活动状态。
|
||||
* <p>已归档（ARCHIVED）：数据集已被归档，不可以更新文件，可以解锁变成活动状态。
|
||||
* <p>已发布（PUBLISHED）：数据集已被发布，可供外部使用，外部用户可以查询和使用数据集。
|
||||
* <p>已弃用（DEPRECATED）：数据集已被弃用，不建议再使用。
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-17
|
||||
*/
|
||||
public enum DatasetStatusType {
|
||||
/**
|
||||
* 草稿状态
|
||||
*/
|
||||
DRAFT,
|
||||
/**
|
||||
* 活动状态
|
||||
*/
|
||||
ACTIVE,
|
||||
/**
|
||||
* 处理中状态
|
||||
*/
|
||||
PROCESSING,
|
||||
/**
|
||||
* 已归档状态
|
||||
*/
|
||||
ARCHIVED,
|
||||
/**
|
||||
* 已发布状态
|
||||
*/
|
||||
PUBLISHED,
|
||||
/**
|
||||
* 已弃用状态
|
||||
*/
|
||||
DEPRECATED
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.common.enums;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* 数据集类型值对象
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
public enum DatasetType {
|
||||
TEXT("text", "文本数据集"),
|
||||
IMAGE("image", "图像数据集"),
|
||||
AUDIO("audio", "音频数据集"),
|
||||
VIDEO("video", "视频数据集"),
|
||||
OTHER("other", "其他数据集");
|
||||
|
||||
@Getter
|
||||
private final String code;
|
||||
|
||||
@Getter
|
||||
private final String description;
|
||||
|
||||
DatasetType(String code, String description) {
|
||||
this.code = code;
|
||||
this.description = description;
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package com.deepdata.datamanagement.common.enums;
|
||||
|
||||
/**
|
||||
* 文件重名时的处理方式
|
||||
*
|
||||
* @since 2025/11/18
|
||||
*/
|
||||
public enum DuplicateMethod {
|
||||
ERROR,
|
||||
COVER
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package com.deepdata.datamanagement.domain.contants;
|
||||
|
||||
/**
|
||||
* 数据集常量
|
||||
*/
|
||||
public interface DatasetConstant {
|
||||
/**
|
||||
* 服务ID
|
||||
*/
|
||||
String SERVICE_ID = "DATA_MANAGEMENT";
|
||||
}
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.deepdata.common.domain.model.base.BaseEntity;
|
||||
import com.deepdata.common.infrastructure.config.PgJsonTypeHandler;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetType;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.io.File;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 数据集实体（与数据库表 t_dm_datasets 对齐）
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@TableName(value = "t_dm_datasets", autoResultMap = true)
|
||||
public class Dataset extends BaseEntity<String> {
|
||||
/**
|
||||
* 数据集名称
|
||||
*/
|
||||
private String name;
|
||||
/**
|
||||
* 数据集描述
|
||||
*/
|
||||
private String description;
|
||||
/**
|
||||
* 数据集类型
|
||||
*/
|
||||
private DatasetType datasetType;
|
||||
/**
|
||||
* 数据集分类
|
||||
*/
|
||||
private String category;
|
||||
/**
|
||||
* 数据集路径
|
||||
*/
|
||||
private String path;
|
||||
/**
|
||||
* 数据集格式
|
||||
*/
|
||||
private String format;
|
||||
/**
|
||||
* 数据集模式信息，JSON格式, 用于解析当前数据集的文件结构
|
||||
*/
|
||||
private String schemaInfo;
|
||||
/**
|
||||
* 数据集大小（字节）
|
||||
*/
|
||||
private Long sizeBytes = 0L;
|
||||
/**
|
||||
* 文件数量
|
||||
*/
|
||||
private Long fileCount = 0L;
|
||||
/**
|
||||
* 记录数量
|
||||
*/
|
||||
private Long recordCount = 0L;
|
||||
/**
|
||||
* 数据集保留天数
|
||||
*/
|
||||
private Integer retentionDays = 0;
|
||||
/**
|
||||
* 标签列表, JSON格式
|
||||
*/
|
||||
@TableField(typeHandler = PgJsonTypeHandler.class)
|
||||
private List<Tag> tags;
|
||||
/**
|
||||
* 额外元数据，JSON格式
|
||||
*/
|
||||
@TableField(typeHandler = PgJsonTypeHandler.class)
|
||||
private String metadata;
|
||||
/**
|
||||
* 数据集状态
|
||||
*/
|
||||
private DatasetStatusType status;
|
||||
/**
|
||||
* 是否为公共数据集
|
||||
*/
|
||||
private Boolean isPublic = false;
|
||||
/**
|
||||
* 是否为精选数据集
|
||||
*/
|
||||
private Boolean isFeatured = false;
|
||||
/**
|
||||
* 数据集版本号
|
||||
*/
|
||||
private Long version = 0L;
|
||||
|
||||
@TableField(exist = false)
|
||||
private List<DatasetFile> files = new ArrayList<>();
|
||||
|
||||
public Dataset() {
|
||||
}
|
||||
|
||||
public Dataset(String name, String description, DatasetType datasetType, String category, String path,
|
||||
String format, DatasetStatusType status, String createdBy) {
|
||||
this.name = name;
|
||||
this.description = description;
|
||||
this.datasetType = datasetType;
|
||||
this.category = category;
|
||||
this.path = path;
|
||||
this.format = format;
|
||||
this.status = status;
|
||||
this.createdBy = createdBy;
|
||||
this.createdAt = LocalDateTime.now();
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void initCreateParam(String datasetBasePath) {
|
||||
this.id = UUID.randomUUID().toString();
|
||||
this.path = datasetBasePath + File.separator + this.id;
|
||||
if (this.status == null) {
|
||||
this.status = DatasetStatusType.DRAFT;
|
||||
}
|
||||
}
|
||||
|
||||
public void updateBasicInfo(String name, String description, String category) {
|
||||
if (name != null && !name.isEmpty()) this.name = name;
|
||||
if (description != null) this.description = description;
|
||||
if (category != null) this.category = category;
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void updateStatus(DatasetStatusType status, String updatedBy) {
|
||||
this.status = status;
|
||||
this.updatedBy = updatedBy;
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void addFile(DatasetFile file) {
|
||||
this.files.add(file);
|
||||
this.fileCount = this.fileCount + 1;
|
||||
this.sizeBytes = this.sizeBytes + (file.getFileSize() != null ? file.getFileSize() : 0L);
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
public void removeFile(DatasetFile file) {
|
||||
if (this.files.remove(file)) {
|
||||
this.fileCount = Math.max(0, this.fileCount - 1);
|
||||
this.sizeBytes = Math.max(0, this.sizeBytes - (file.getFileSize() != null ? file.getFileSize() : 0L));
|
||||
this.updatedAt = LocalDateTime.now();
|
||||
}
|
||||
}
|
||||
|
||||
public void active() {
|
||||
if (this.status == DatasetStatusType.DRAFT) {
|
||||
this.status = DatasetStatusType.ACTIVE;
|
||||
}
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.deepdata.common.infrastructure.config.PgJsonTypeHandler;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.*;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件实体（与数据库表 t_dm_dataset_files 对齐）
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@Slf4j
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@TableName("t_dm_dataset_files")
|
||||
public class DatasetFile {
|
||||
@TableId
|
||||
private String id; // UUID
|
||||
private String datasetId; // UUID
|
||||
private String fileName;
|
||||
private String filePath;
|
||||
private String fileType; // JPG/PNG/DCM/TXT
|
||||
private Long fileSize; // bytes
|
||||
private String checkSum;
|
||||
private String tags;
|
||||
@TableField(typeHandler = PgJsonTypeHandler.class)
|
||||
private String metadata;
|
||||
private String status; // UPLOADED, PROCESSING, COMPLETED, ERROR
|
||||
private LocalDateTime uploadTime;
|
||||
private LocalDateTime lastAccessTime;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
|
||||
/** 标记是否为目录（非持久化字段） */
|
||||
@TableField(exist = false)
|
||||
private Boolean directory;
|
||||
|
||||
/** 目录包含的文件数量（非持久化字段） */
|
||||
@TableField(exist = false)
|
||||
private Long fileCount;
|
||||
|
||||
/**
|
||||
* 解析标签
|
||||
*
|
||||
* @return 标签列表
|
||||
*/
|
||||
public List<FileTag> analyzeTag() {
|
||||
try {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper.readValue(tags, new TypeReference<List<FileTag>>() {});
|
||||
} catch (Exception e) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 数据集文件上传检查信息
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DatasetFileUploadCheckInfo {
|
||||
/** 数据集id */
|
||||
private String datasetId;
|
||||
|
||||
/** 是否为压缩包上传 */
|
||||
private boolean hasArchive;
|
||||
|
||||
/** 目标子目录前缀，例如 "images/"，为空表示数据集根目录 */
|
||||
private String prefix;
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
public class FileTag {
|
||||
private Map<String, Object> values;
|
||||
|
||||
private String type;
|
||||
|
||||
private String id;
|
||||
|
||||
private String fromName;
|
||||
|
||||
public List<String> getTagValues() {
|
||||
List<String> tags = new ArrayList<>();
|
||||
Object tagValues = values.get(type);
|
||||
if (tagValues instanceof List) {
|
||||
for (Object tag : (List<?>) tagValues) {
|
||||
if (tag instanceof String) {
|
||||
tags.add((String) tag);
|
||||
}
|
||||
}
|
||||
} else if (tagValues instanceof String) {
|
||||
tags.add((String) tagValues);
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package com.deepdata.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.deepdata.common.domain.model.base.BaseEntity;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 标签实体（与数据库表 t_dm_tags 对齐）
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Tag extends BaseEntity<String> {
|
||||
private String name;
|
||||
private String description;
|
||||
private String category;
|
||||
private String color;
|
||||
private Long usageCount = 0L;
|
||||
|
||||
public Tag(String name, String description, String category, String color) {
|
||||
this.name = name;
|
||||
this.description = description;
|
||||
this.category = category;
|
||||
this.color = color;
|
||||
}
|
||||
|
||||
public void decrementUsage() {
|
||||
if (this.usageCount != null && this.usageCount > 0) this.usageCount--;
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.deepdata.datamanagement.infrastructure.client;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.datamanagement.infrastructure.client.dto.CollectionTaskDetailResponse;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
|
||||
/**
|
||||
* 数据归集服务 Feign Client
|
||||
*/
|
||||
@FeignClient(name = "collection-service", url = "${collection.service.url:http://deepdata-backend-python:18000}")
|
||||
public interface CollectionTaskClient {
|
||||
|
||||
/**
|
||||
* 获取归集任务详情
|
||||
* @param taskId 任务ID
|
||||
* @return 任务详情
|
||||
*/
|
||||
@GetMapping("/api/data-collection/tasks/{id}")
|
||||
Response<CollectionTaskDetailResponse> getTaskDetail(@PathVariable("id") String taskId, @RequestHeader("User") String authorization);
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.datamanagement.infrastructure.client.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 归集任务详情响应
|
||||
*/
|
||||
@Data
|
||||
public class CollectionTaskDetailResponse {
|
||||
private String id;
|
||||
private String name;
|
||||
private String description;
|
||||
private String targetPath;
|
||||
private Map<String, Object> config;
|
||||
private String status;
|
||||
private String syncMode;
|
||||
private String scheduleExpression;
|
||||
private String lastExecutionId;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.deepdata.datamanagement.infrastructure.client.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 本地归集任务配置
|
||||
*/
|
||||
@Data
|
||||
public class LocalCollectionConfig {
|
||||
/**
|
||||
* 归集类型
|
||||
*/
|
||||
private String type;
|
||||
|
||||
/**
|
||||
* 文件路径列表
|
||||
*/
|
||||
private List<String> filePaths;
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package com.deepdata.datamanagement.infrastructure.config;
|
||||
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cache.CacheManager;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
import org.springframework.web.multipart.support.StandardServletMultipartResolver;
|
||||
|
||||
/**
|
||||
* 数据管理服务配置
|
||||
*/
|
||||
@Configuration
|
||||
@EnableTransactionManagement
|
||||
@EnableCaching
|
||||
@EnableConfigurationProperties(DataManagementProperties.class)
|
||||
public class DataManagementConfig {
|
||||
|
||||
/**
|
||||
* 缓存管理器
|
||||
*/
|
||||
@Bean
|
||||
public CacheManager cacheManager() {
|
||||
return new ConcurrentMapCacheManager("datasets", "datasetFiles", "tags");
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件上传解析器
|
||||
*/
|
||||
@Bean
|
||||
public StandardServletMultipartResolver multipartResolver() {
|
||||
StandardServletMultipartResolver resolver = new StandardServletMultipartResolver();
|
||||
return resolver;
|
||||
}
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.deepdata.datamanagement.infrastructure.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据管理服务配置属性
|
||||
*/
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "datamanagement")
|
||||
public class DataManagementProperties {
|
||||
|
||||
private FileStorage fileStorage = new FileStorage();
|
||||
private Cache cache = new Cache();
|
||||
|
||||
public FileStorage getFileStorage() {
|
||||
return fileStorage;
|
||||
}
|
||||
|
||||
public void setFileStorage(FileStorage fileStorage) {
|
||||
this.fileStorage = fileStorage;
|
||||
}
|
||||
|
||||
public Cache getCache() {
|
||||
return cache;
|
||||
}
|
||||
|
||||
public void setCache(Cache cache) {
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
public static class FileStorage {
|
||||
private String uploadDir = "./uploads";
|
||||
private long maxFileSize = 10485760; // 10MB
|
||||
private long maxRequestSize = 52428800; // 50MB
|
||||
private List<String> allowedSourceDirs = new ArrayList<>();
|
||||
|
||||
public String getUploadDir() {
|
||||
return uploadDir;
|
||||
}
|
||||
|
||||
public void setUploadDir(String uploadDir) {
|
||||
this.uploadDir = uploadDir;
|
||||
}
|
||||
|
||||
public long getMaxFileSize() {
|
||||
return maxFileSize;
|
||||
}
|
||||
|
||||
public void setMaxFileSize(long maxFileSize) {
|
||||
this.maxFileSize = maxFileSize;
|
||||
}
|
||||
|
||||
public long getMaxRequestSize() {
|
||||
return maxRequestSize;
|
||||
}
|
||||
|
||||
public void setMaxRequestSize(long maxRequestSize) {
|
||||
this.maxRequestSize = maxRequestSize;
|
||||
}
|
||||
|
||||
public List<String> getAllowedSourceDirs() {
|
||||
return allowedSourceDirs;
|
||||
}
|
||||
|
||||
public void setAllowedSourceDirs(List<String> allowedSourceDirs) {
|
||||
this.allowedSourceDirs = allowedSourceDirs;
|
||||
}
|
||||
}
|
||||
|
||||
public static class Cache {
|
||||
private int ttl = 3600; // 1 hour
|
||||
private int maxSize = 1000;
|
||||
|
||||
public int getTtl() {
|
||||
return ttl;
|
||||
}
|
||||
|
||||
public void setTtl(int ttl) {
|
||||
this.ttl = ttl;
|
||||
}
|
||||
|
||||
public int getMaxSize() {
|
||||
return maxSize;
|
||||
}
|
||||
|
||||
public void setMaxSize(int maxSize) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package com.deepdata.datamanagement.infrastructure.exception;
|
||||
|
||||
import com.deepdata.common.infrastructure.exception.ErrorCode;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* 数据管理模块错误码
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-20
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum DataManagementErrorCode implements ErrorCode {
|
||||
/**
|
||||
* 数据集不存在
|
||||
*/
|
||||
DATASET_NOT_FOUND("data_management.0001", "数据集不存在"),
|
||||
/**
|
||||
* 数据集已存在
|
||||
*/
|
||||
DATASET_ALREADY_EXISTS("data_management.0002", "数据集已存在"),
|
||||
/**
|
||||
* 数据集状态错误
|
||||
*/
|
||||
DATASET_STATUS_ERROR("data_management.0003", "数据集状态错误"),
|
||||
/**
|
||||
* 数据集标签不存在
|
||||
*/
|
||||
DATASET_TAG_NOT_FOUND("data_management.0004", "数据集标签不存在"),
|
||||
/**
|
||||
* 数据集标签已存在
|
||||
*/
|
||||
DATASET_TAG_ALREADY_EXISTS("data_management.0005", "数据集标签已存在"),
|
||||
/**
|
||||
* 数据集文件已存在
|
||||
*/
|
||||
DATASET_FILE_ALREADY_EXISTS("data_management.0006", "数据集文件已存在"),
|
||||
/**
|
||||
* 目录不存在
|
||||
*/
|
||||
DIRECTORY_NOT_FOUND("data_management.0007", "目录不存在"),
|
||||
/**
|
||||
* 数据集名称长度不能超过255个字符
|
||||
*/
|
||||
DATASET_NAME_TOO_LONG("data_management.0008", "数据集名称长度不能超过255个字符"),
|
||||
/**
|
||||
* 数据集类型不合法
|
||||
*/
|
||||
DATASET_TYPE_INVALID("data_management.0009", "数据集类型不合法"),
|
||||
/**
|
||||
* 数据集状态不合法
|
||||
*/
|
||||
DATASET_STATUS_INVALID("data_management.0010", "数据集状态不合法"),
|
||||
/**
|
||||
* 标签名称长度不能超过100个字符
|
||||
*/
|
||||
TAG_NAME_TOO_LONG("data_management.0011", "标签名称长度不能超过100个字符"),
|
||||
/**
|
||||
* 标签名称已存在
|
||||
*/
|
||||
TAG_NAME_DUPLICATE("data_management.0012", "标签名称已存在"),
|
||||
/**
|
||||
* 标签颜色格式不正确，应为十六进制颜色代码
|
||||
*/
|
||||
TAG_COLOR_INVALID("data_management.0013", "标签颜色格式不正确，应为十六进制颜色代码"),
|
||||
/**
|
||||
* 保留天数必须为非负整数
|
||||
*/
|
||||
RETENTION_DAYS_INVALID("data_management.0014", "保留天数必须为非负整数"),
|
||||
/**
|
||||
* 路径长度不能超过限制
|
||||
*/
|
||||
PATH_TOO_LONG("data_management.0015", "路径长度不能超过限制"),
|
||||
/**
|
||||
* 文件名包含非法字符
|
||||
*/
|
||||
FILE_NAME_INVALID("data_management.0016", "文件名包含非法字符"),
|
||||
/**
|
||||
* 路径前缀不能以点开头
|
||||
*/
|
||||
PREFIX_INVALID("data_management.0017", "路径前缀不能以点开头");
|
||||
|
||||
private final String code;
|
||||
private final String message;
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.deepdata.common.infrastructure.config.IgnoreDataScopeAnnotation;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
@IgnoreDataScopeAnnotation
|
||||
public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
|
||||
DatasetFile findById(@Param("id") String id);
|
||||
List<DatasetFile> findByDatasetId(@Param("datasetId") String datasetId, RowBounds rowBounds);
|
||||
List<DatasetFile> findByDatasetIdAndStatus(@Param("datasetId") String datasetId, @Param("status") String status, RowBounds rowBounds);
|
||||
List<DatasetFile> findByDatasetIdAndFileType(@Param("datasetId") String datasetId, @Param("fileType") String fileType, RowBounds rowBounds);
|
||||
Long countByDatasetId(@Param("datasetId") String datasetId);
|
||||
Long countCompletedByDatasetId(@Param("datasetId") String datasetId);
|
||||
Long sumSizeByDatasetId(@Param("datasetId") String datasetId);
|
||||
DatasetFile findByDatasetIdAndFileName(@Param("datasetId") String datasetId, @Param("fileName") String fileName);
|
||||
List<DatasetFile> findAllByDatasetId(@Param("datasetId") String datasetId);
|
||||
List<DatasetFile> findByCriteria(@Param("datasetId") String datasetId,
|
||||
@Param("fileType") String fileType,
|
||||
@Param("status") String status,
|
||||
RowBounds rowBounds);
|
||||
|
||||
int update(DatasetFile file);
|
||||
int deleteById(@Param("id") String id);
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface DatasetMapper extends BaseMapper<Dataset> {
|
||||
Dataset findById(@Param("id") String id);
|
||||
|
||||
/**
|
||||
* 使用悲观锁查询数据集（FOR UPDATE）
|
||||
*/
|
||||
@Select("SELECT * FROM t_dm_datasets WHERE id = #{id} FOR UPDATE")
|
||||
Dataset findByIdWithLock(@Param("id") String id);
|
||||
Dataset findByName(@Param("name") String name);
|
||||
List<Dataset> findByStatus(@Param("status") String status);
|
||||
List<Dataset> findByCreatedBy(@Param("createdBy") String createdBy, RowBounds rowBounds);
|
||||
List<Dataset> findByTypeCode(@Param("typeCode") String typeCode, RowBounds rowBounds);
|
||||
List<Dataset> findByTagNames(@Param("tagNames") List<String> tagNames, RowBounds rowBounds);
|
||||
List<Dataset> findByKeyword(@Param("keyword") String keyword, RowBounds rowBounds);
|
||||
List<Dataset> findByCriteria(@Param("typeCode") String typeCode,
|
||||
@Param("status") String status,
|
||||
@Param("keyword") String keyword,
|
||||
@Param("tagNames") List<String> tagNames,
|
||||
RowBounds rowBounds);
|
||||
long countByCriteria(@Param("typeCode") String typeCode,
|
||||
@Param("status") String status,
|
||||
@Param("keyword") String keyword,
|
||||
@Param("tagNames") List<String> tagNames);
|
||||
|
||||
int deleteById(@Param("id") String id);
|
||||
AllDatasetStatisticsResponse getAllDatasetStatistics();
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.mapper;
|
||||
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface TagMapper {
|
||||
Tag findById(@Param("id") String id);
|
||||
Tag findByName(@Param("name") String name);
|
||||
List<Tag> findByNameIn(@Param("list") List<String> names);
|
||||
List<Tag> findByIdIn(@Param("ids") List<String> ids);
|
||||
List<Tag> findByKeyword(@Param("keyword") String keyword);
|
||||
List<Tag> findAllByOrderByUsageCountDesc();
|
||||
|
||||
int insert(Tag tag);
|
||||
int update(Tag tag);
|
||||
int updateUsageCount(@Param("id") String id, @Param("usageCount") Long usageCount);
|
||||
|
||||
// Relations with dataset
|
||||
int insertDatasetTag(@Param("datasetId") String datasetId, @Param("tagId") String tagId);
|
||||
int deleteDatasetTagsByDatasetId(@Param("datasetId") String datasetId);
|
||||
List<Tag> findByDatasetId(@Param("datasetId") String datasetId);
|
||||
void deleteTagsById(@Param("ids") List<String> ids);
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件仓储接口
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
public interface DatasetFileRepository extends IRepository<DatasetFile> {
|
||||
Long countByDatasetId(String datasetId);
|
||||
|
||||
Long countCompletedByDatasetId(String datasetId);
|
||||
|
||||
Long sumSizeByDatasetId(String datasetId);
|
||||
|
||||
List<DatasetFile> findAllByDatasetId(String datasetId);
|
||||
|
||||
DatasetFile findByDatasetIdAndFileName(String datasetId, String fileName);
|
||||
|
||||
IPage<DatasetFile> findByCriteria(String datasetId, String fileType, String status, String name,
|
||||
IPage<DatasetFile> page);
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetPagingQuery;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/**
|
||||
* 数据集仓储层
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
public interface DatasetRepository extends IRepository<Dataset> {
|
||||
Dataset findByName(String name);
|
||||
|
||||
/**
|
||||
* 使用悲观锁获取数据集（用于更新操作，防止并发冲突）
|
||||
*/
|
||||
Dataset getByIdWithLock(String id);
|
||||
|
||||
List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList, RowBounds bounds);
|
||||
|
||||
long countByCriteria(String type, String status, String keyword, List<String> tagList);
|
||||
|
||||
AllDatasetStatisticsResponse getAllDatasetStatistics();
|
||||
|
||||
IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query);
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件仓储实现类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper, DatasetFile> implements DatasetFileRepository {
|
||||
private final DatasetFileMapper datasetFileMapper;
|
||||
|
||||
@Override
|
||||
public Long countByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.selectCount(new LambdaQueryWrapper<DatasetFile>().eq(DatasetFile::getDatasetId, datasetId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long countCompletedByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.countCompletedByDatasetId(datasetId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long sumSizeByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.sumSizeByDatasetId(datasetId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatasetFile> findAllByDatasetId(String datasetId) {
|
||||
return datasetFileMapper.findAllByDatasetId(datasetId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatasetFile findByDatasetIdAndFileName(String datasetId, String fileName) {
|
||||
return datasetFileMapper.findByDatasetIdAndFileName(datasetId, fileName);
|
||||
}
|
||||
|
||||
public IPage<DatasetFile> findByCriteria(String datasetId, String fileType, String status, String name,
|
||||
IPage<DatasetFile> page) {
|
||||
return datasetFileMapper.selectPage(page, new LambdaQueryWrapper<DatasetFile>()
|
||||
.eq(DatasetFile::getDatasetId, datasetId)
|
||||
.eq(StringUtils.hasText(fileType), DatasetFile::getFileType, fileType)
|
||||
.eq(StringUtils.hasText(status), DatasetFile::getStatus, status)
|
||||
.like(StringUtils.hasText(name), DatasetFile::getFileName, name));
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.deepdata.datamanagement.infrastructure.persistence.repository.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetMapper;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetPagingQuery;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.ibatis.session.RowBounds;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集仓储层实现类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-15
|
||||
*/
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetRepositoryImpl extends CrudRepository<DatasetMapper, Dataset> implements DatasetRepository {
|
||||
private final DatasetMapper datasetMapper;
|
||||
|
||||
@Override
|
||||
public Dataset findByName(String name) {
|
||||
return datasetMapper.selectOne(new LambdaQueryWrapper<Dataset>().eq(Dataset::getName, name));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Dataset getByIdWithLock(String id) {
|
||||
return datasetMapper.findByIdWithLock(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList,
|
||||
RowBounds bounds) {
|
||||
return datasetMapper.findByCriteria(type, status, keyword, tagList, bounds);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long countByCriteria(String type, String status, String keyword, List<String> tagList) {
|
||||
return datasetMapper.countByCriteria(type, status, keyword, tagList);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AllDatasetStatisticsResponse getAllDatasetStatistics() {
|
||||
return datasetMapper.getAllDatasetStatistics();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query) {
|
||||
LambdaQueryWrapper<Dataset> wrapper = new LambdaQueryWrapper<Dataset>()
|
||||
.eq(query.getType() != null, Dataset::getDatasetType, query.getType())
|
||||
.eq(query.getStatus() != null, Dataset::getStatus, query.getStatus());
|
||||
|
||||
if (StringUtils.isNotBlank(query.getKeyword())) {
|
||||
wrapper.and(w ->
|
||||
w.like(Dataset::getName, query.getKeyword()).or()
|
||||
.like(Dataset::getDescription, query.getKeyword()));
|
||||
}
|
||||
|
||||
/*
|
||||
标签过滤 {@link Tag}
|
||||
*/
|
||||
for (String tagName : query.getTags()) {
|
||||
wrapper.and(w ->
|
||||
w.apply("EXISTS ( " +
|
||||
"SELECT 1 FROM jsonb_array_elements( " +
|
||||
"CASE WHEN jsonb_typeof(tags::jsonb) = 'array' THEN tags::jsonb ELSE '[]'::jsonb END " +
|
||||
") AS tag " +
|
||||
"WHERE tag->>'name' = {0} " +
|
||||
")", tagName)
|
||||
);
|
||||
}
|
||||
wrapper.orderByDesc(Dataset::getCreatedAt);
|
||||
return datasetMapper.selectPage(page, wrapper);
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package com.deepdata.datamanagement.interfaces.converter;
|
||||
|
||||
import com.deepdata.common.domain.model.ChunkUploadRequest;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.FileTag;
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.Mapping;
|
||||
import org.mapstruct.Named;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 数据集文件转换器
|
||||
*/
|
||||
@Mapper
|
||||
public interface DatasetConverter {
|
||||
/** 单例实例 */
|
||||
DatasetConverter INSTANCE = Mappers.getMapper(DatasetConverter.class);
|
||||
|
||||
/**
|
||||
* 将数据集转换为响应
|
||||
*/
|
||||
@Mapping(source = "sizeBytes", target = "totalSize")
|
||||
@Mapping(source = "path", target = "targetLocation")
|
||||
@Mapping(source = "files", target = "distribution", qualifiedByName = "getDistribution")
|
||||
DatasetResponse convertToResponse(Dataset dataset);
|
||||
|
||||
/**
|
||||
* 将数据集转换为响应
|
||||
*/
|
||||
@Mapping(target = "tags", ignore = true)
|
||||
Dataset convertToDataset(CreateDatasetRequest createDatasetRequest);
|
||||
|
||||
/**
|
||||
* 将上传文件请求转换为分片上传请求
|
||||
*/
|
||||
ChunkUploadRequest toChunkUploadRequest(UploadFileRequest uploadFileRequest);
|
||||
|
||||
/**
|
||||
* 将数据集转换为响应
|
||||
*/
|
||||
List<DatasetResponse> convertToResponse(List<Dataset> datasets);
|
||||
|
||||
/**
|
||||
*
|
||||
* 将数据集文件转换为响应
|
||||
*/
|
||||
DatasetFileResponse convertToResponse(DatasetFile datasetFile);
|
||||
|
||||
|
||||
/**
|
||||
* 将数据集文件列表转换为响应
|
||||
*/
|
||||
List<DatasetFileResponse> convertToResponseList(List<DatasetFile> datasetFiles);
|
||||
|
||||
|
||||
/**
|
||||
* 获取数据文件的标签分布
|
||||
*
|
||||
* @param datasetFiles 数据集文件
|
||||
* @return 标签分布
|
||||
*/
|
||||
@Named("getDistribution")
|
||||
default Map<String, Map<String, Long>> getDistribution(List<DatasetFile> datasetFiles) {
|
||||
Map<String, Map<String, Long>> distribution = new HashMap<>();
|
||||
if (CollectionUtils.isEmpty(datasetFiles)) {
|
||||
return distribution;
|
||||
}
|
||||
for (DatasetFile datasetFile : datasetFiles) {
|
||||
List<FileTag> tags = datasetFile.analyzeTag();
|
||||
if (CollectionUtils.isEmpty(tags)) {
|
||||
continue;
|
||||
}
|
||||
for (FileTag tag : tags) {
|
||||
Map<String, Long> tagValueMap = distribution.getOrDefault(tag.getFromName(), new HashMap<>());
|
||||
tag.getTagValues().forEach(tagValue -> tagValueMap.put(tagValue, tagValueMap.getOrDefault(tagValue, 0L) + 1));
|
||||
distribution.put(tag.getFromName(), tagValueMap);
|
||||
}
|
||||
}
|
||||
return distribution;
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.deepdata.datamanagement.interfaces.converter;
|
||||
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.interfaces.dto.TagResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UpdateTagRequest;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
/**
|
||||
* 标签转换器
|
||||
*/
|
||||
@Mapper
|
||||
public interface TagConverter {
|
||||
/** 单例实例 */
|
||||
TagConverter INSTANCE = Mappers.getMapper(TagConverter.class);
|
||||
|
||||
/**
|
||||
* 将 UpdateTagRequest 转换为 Tag 实体
|
||||
* @param request 更新标签请求DTO
|
||||
* @return 标签实体
|
||||
*/
|
||||
Tag updateRequestToTag(UpdateTagRequest request);
|
||||
|
||||
/**
|
||||
* 将 Tag 实体转换为 TagResponse DTO
|
||||
* @param tag 标签实体
|
||||
* @return 标签响应DTO
|
||||
*/
|
||||
TagResponse convertToResponse(Tag tag);
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* AddFilesRequest
|
||||
*
|
||||
* @since 2026/2/9
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AddFilesRequest {
|
||||
private boolean softAdd;
|
||||
|
||||
@ValidPath
|
||||
private String prefix = "";
|
||||
|
||||
@NotEmpty(message = "文件列表不能为空")
|
||||
@Size(max = 1000, message = "文件数量不能超过1000个")
|
||||
@Valid
|
||||
private List<FileRequest> files;
|
||||
|
||||
public AddFilesRequest(List<String> paths) {
|
||||
if (CollectionUtils.isEmpty(paths)) {
|
||||
return;
|
||||
}
|
||||
this.files = paths.stream().map(path -> {
|
||||
FileRequest file = new FileRequest();
|
||||
file.setFilePath(path);
|
||||
return file;
|
||||
}).toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文件的前缀，优先使用文件自身的 prefix，为空时回退到请求级别的 prefix
|
||||
*/
|
||||
public String getEffectivePrefix(FileRequest file) {
|
||||
return (file.getPrefix() != null && !file.getPrefix().isEmpty()) ? file.getPrefix() : prefix;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class FileRequest {
|
||||
@NotBlank(message = "文件路径不能为空")
|
||||
@Size(max = 1000, message = "文件路径长度不能超过1000个字符")
|
||||
private String filePath;
|
||||
|
||||
@ValidPath
|
||||
private String prefix = "";
|
||||
|
||||
private Map<String, Object> metadata = new HashMap<>();
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 所有数据集统计信息响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class AllDatasetStatisticsResponse {
|
||||
/** 总数据集数 */
|
||||
private Integer totalDatasets = 0;
|
||||
|
||||
/** 总文件数 */
|
||||
private Long totalSize = 0L;
|
||||
|
||||
/** 总大小（字节） */
|
||||
private Long totalFiles = 0L;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 批量删除文件请求
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class BatchDeleteFilesRequest {
|
||||
|
||||
/**
|
||||
* 要删除的文件ID列表
|
||||
*/
|
||||
@NotEmpty(message = "文件ID列表不能为空")
|
||||
private List<String> fileIds;
|
||||
|
||||
/**
|
||||
* 文件路径前缀（用于处理子目录中的文件）
|
||||
*/
|
||||
private String prefix = "";
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.Size;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 复制文件请求DTO
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-11-13
|
||||
*/
|
||||
public record CopyFilesRequest(
|
||||
@NotEmpty(message = "源文件路径列表不能为空")
|
||||
@Size(max = 1000, message = "文件数量不能超过1000个")
|
||||
List<String> sourcePaths) {
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetType;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.springaicommunity.mcp.annotation.McpToolParam;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 创建数据集请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class CreateDatasetRequest {
|
||||
/** 数据集名称 */
|
||||
@Size(min = 1, max = 255, message = "数据集名称长度不能超过255个字符")
|
||||
@NotBlank(message = "数据集名称不能为空")
|
||||
@McpToolParam(description = "数据集名称")
|
||||
private String name;
|
||||
/** 数据集描述 */
|
||||
@Size(max = 500, message = "数据集描述长度不能超过500个字符")
|
||||
@McpToolParam(description = "数据集描述", required = false)
|
||||
private String description;
|
||||
/** 数据集类型 */
|
||||
@NotNull(message = "数据集类型不能为空")
|
||||
@McpToolParam(description = "数据集类型，取值范围为TEXT/IMAGE/VIDEO/AUDIO/OTHER")
|
||||
private DatasetType datasetType;
|
||||
/** 标签列表 */
|
||||
@Size(max = 20, message = "标签数量不能超过20个")
|
||||
@McpToolParam(description = "标签列表", required = false)
|
||||
private List<String> tags;
|
||||
/** 数据源 */
|
||||
@Size(max = 255, message = "数据源长度不能超过255个字符")
|
||||
@McpToolParam(description = "数据源", required = false)
|
||||
private String dataSource;
|
||||
/** 保留天数 */
|
||||
@Min(value = 0, message = "保留天数必须为非负整数")
|
||||
@McpToolParam(description = "保留天数", required = false)
|
||||
private Integer retentionDays;
|
||||
/** 数据集状态 */
|
||||
@McpToolParam(description = "数据集状态", required = false)
|
||||
private DatasetStatusType status;
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFileName;
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 创建数据集子目录请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class CreateDirectoryRequest {
|
||||
|
||||
/** 父级前缀路径，例如 "images/"，为空表示数据集根目录 */
|
||||
@ValidPath()
|
||||
private String parentPrefix;
|
||||
|
||||
/** 新建目录名称 */
|
||||
@NotBlank(message = "目录名称不能为空")
|
||||
@ValidFileName
|
||||
@Size(max = 255, message = "目录名称长度不能超过255个字符")
|
||||
private String directoryName;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidHexColor;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 创建标签请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class CreateTagRequest {
|
||||
/** 标签名称 */
|
||||
@NotBlank(message = "标签名称不能为空")
|
||||
@Size(min = 1, max = 100, message = "标签名称长度不能超过100个字符")
|
||||
private String name;
|
||||
/** 标签颜色 */
|
||||
@ValidHexColor
|
||||
private String color;
|
||||
/** 标签描述 */
|
||||
@Size(max = 500, message = "标签描述长度不能超过500个字符")
|
||||
private String description;
|
||||
/** 标签分类 */
|
||||
@Size(max = 50, message = "标签分类长度不能超过50个字符")
|
||||
private String category;
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 数据集文件响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetFileResponse {
|
||||
/** 文件ID */
|
||||
private String id;
|
||||
/** 文件名 */
|
||||
private String fileName;
|
||||
/** 原始文件名 */
|
||||
private String originalName;
|
||||
/** 文件类型 */
|
||||
private String fileType;
|
||||
/** 文件大小（字节） */
|
||||
private Long fileSize;
|
||||
/** 文件状态 */
|
||||
private String status;
|
||||
/** 文件描述 */
|
||||
private String description;
|
||||
/** 文件路径 */
|
||||
private String filePath;
|
||||
/** 文件标签（JSON 字符串） */
|
||||
private String tags;
|
||||
/** 标签更新时间 */
|
||||
private LocalDateTime tagsUpdatedAt;
|
||||
/** 文件元数据（包含标注信息等，JSON 字符串） */
|
||||
private String metadata;
|
||||
/** 上传时间 */
|
||||
private LocalDateTime uploadTime;
|
||||
/** 最后更新时间 */
|
||||
private LocalDateTime lastAccessTime;
|
||||
/** 上传者 */
|
||||
private String uploadedBy;
|
||||
/** 是否为目录 */
|
||||
private Boolean directory;
|
||||
/** 目录文件数量 */
|
||||
private Long fileCount;
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.common.domain.model.LineageEdge;
|
||||
import com.deepdata.common.domain.model.LineageNode;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集血缘
|
||||
*
|
||||
* @since 2026/1/23
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DatasetLineage {
|
||||
/**
|
||||
* 节点列表
|
||||
*/
|
||||
private List<LineageNode> lineageNodes;
|
||||
/**
|
||||
* 边列表
|
||||
*/
|
||||
private List<LineageEdge> lineageEdges;
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.common.interfaces.PagingQuery;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import com.deepdata.datamanagement.common.enums.DatasetType;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.springaicommunity.mcp.annotation.McpToolParam;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集分页查询请求
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DatasetPagingQuery extends PagingQuery {
|
||||
/**
|
||||
* 数据集类型过滤
|
||||
*/
|
||||
@McpToolParam(description = "数据集类型过滤，取值范围为text/image/video/audio/other", required = false)
|
||||
private DatasetType type;
|
||||
|
||||
/**
|
||||
* 标签名过滤
|
||||
*/
|
||||
@Size(max = 10, message = "过滤标签数量不能超过10个")
|
||||
@McpToolParam(description = "标签名过滤", required = false)
|
||||
private List<String> tags = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 关键词搜索（名称或描述）
|
||||
*/
|
||||
@Size(max = 100, message = "关键词长度不能超过100个字符")
|
||||
@McpToolParam(description = "关键词搜索（名称或描述）", required = false)
|
||||
private String keyword;
|
||||
|
||||
/**
|
||||
* 状态过滤
|
||||
*/
|
||||
@McpToolParam(description = "状态过滤", required = false)
|
||||
private DatasetStatusType status;
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据集响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetResponse {
|
||||
/** 数据集ID */
|
||||
private String id;
|
||||
/** 数据集名称 */
|
||||
private String name;
|
||||
/** 数据集描述 */
|
||||
private String description;
|
||||
/** 数据集类型 */
|
||||
private String datasetType;
|
||||
/** 数据集状态 */
|
||||
private String status;
|
||||
/** 标签列表 */
|
||||
private List<TagResponse> tags;
|
||||
/** 数据集保留天数 */
|
||||
private Integer retentionDays;
|
||||
/** 数据源 */
|
||||
private String dataSource;
|
||||
/** 目标位置 */
|
||||
private String targetLocation;
|
||||
/** 文件数量 */
|
||||
private Integer fileCount;
|
||||
/** 总大小（字节） */
|
||||
private Long totalSize;
|
||||
/** 完成率（0-100） */
|
||||
private Float completionRate;
|
||||
/** 创建时间 */
|
||||
private LocalDateTime createdAt;
|
||||
/** 更新时间 */
|
||||
private LocalDateTime updatedAt;
|
||||
/** 创建者 */
|
||||
private String createdBy;
|
||||
/** 更新者 */
|
||||
private String updatedBy;
|
||||
/** 分布 */
|
||||
private Map<String, Map<String, Long>> distribution;
|
||||
/** 数据集pvc名称 */
|
||||
private String pvcName;
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据集统计信息响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetStatisticsResponse {
|
||||
/** 总文件数 */
|
||||
private Integer totalFiles;
|
||||
/** 已完成文件数 */
|
||||
private Integer completedFiles;
|
||||
/** 总大小（字节） */
|
||||
private Long totalSize;
|
||||
/** 完成率（0-100） */
|
||||
private Float completionRate;
|
||||
/** 文件类型分布 */
|
||||
private Map<String, Integer> fileTypeDistribution;
|
||||
/** 状态分布 */
|
||||
private Map<String, Integer> statusDistribution;
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集类型响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class DatasetTypeResponse {
|
||||
/** 类型编码 */
|
||||
private String code;
|
||||
/** 类型名称 */
|
||||
private String name;
|
||||
/** 类型描述 */
|
||||
private String description;
|
||||
/** 支持的文件格式 */
|
||||
private List<String> supportedFormats;
|
||||
/** 图标 */
|
||||
private String icon;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件分页响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class PagedDatasetFileResponse {
|
||||
/** 文件内容列表 */
|
||||
private List<DatasetFileResponse> content;
|
||||
/** 当前页码 */
|
||||
private Integer page;
|
||||
/** 每页大小 */
|
||||
private Integer size;
|
||||
/** 总元素数 */
|
||||
private Integer totalElements;
|
||||
/** 总页数 */
|
||||
private Integer totalPages;
|
||||
/** 是否为第一页 */
|
||||
private Boolean first;
|
||||
/** 是否为最后一页 */
|
||||
private Boolean last;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集分页响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class PagedDatasetResponse {
|
||||
/** 数据集内容列表 */
|
||||
private List<DatasetResponse> content;
|
||||
/** 当前页码 */
|
||||
private Integer page;
|
||||
/** 每页大小 */
|
||||
private Integer size;
|
||||
/** 总元素数 */
|
||||
private Integer totalElements;
|
||||
/** 总页数 */
|
||||
private Integer totalPages;
|
||||
/** 是否为第一页 */
|
||||
private Boolean first;
|
||||
/** 是否为最后一页 */
|
||||
private Boolean last;
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFileName;
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 重命名数据集目录请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class RenameDirectoryRequest {
|
||||
|
||||
/** 目录前缀，例如 "images/"，与列表/删除目录接口保持一致 */
|
||||
@NotBlank(message = "目录前缀不能为空")
|
||||
@ValidPath(maxLength = 500)
|
||||
private String prefix;
|
||||
|
||||
/** 新的目录名称 */
|
||||
@NotBlank(message = "新目录名称不能为空")
|
||||
@ValidFileName
|
||||
@Size(max = 255, message = "目录名称长度不能超过255个字符")
|
||||
private String newName;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFileName;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 重命名数据集文件请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class RenameFileRequest {
|
||||
|
||||
/** 新的文件名称（不包含后缀） */
|
||||
@NotBlank(message = "新文件名不能为空")
|
||||
@ValidFileName
|
||||
@Size(max = 255, message = "文件名称长度不能超过255个字符")
|
||||
private String newName;
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 标签响应DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class TagResponse {
|
||||
/** 标签ID */
|
||||
private String id;
|
||||
/** 标签名称 */
|
||||
private String name;
|
||||
/** 标签颜色 */
|
||||
private String color;
|
||||
/** 标签描述 */
|
||||
private String description;
|
||||
/** 使用次数 */
|
||||
private Integer usageCount;
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.common.enums.DatasetStatusType;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 更新数据集请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UpdateDatasetRequest {
|
||||
/** 数据集名称 */
|
||||
@Size(min = 1, max = 255, message = "数据集名称长度不能超过255个字符")
|
||||
@NotBlank(message = "数据集名称不能为空")
|
||||
private String name;
|
||||
/** 数据集描述 */
|
||||
@Size(max = 500, message = "数据集描述长度不能超过500个字符")
|
||||
private String description;
|
||||
/** 归集任务id */
|
||||
@Size(max = 255, message = "数据源长度不能超过255个字符")
|
||||
private String dataSource;
|
||||
/** 标签列表 */
|
||||
@Size(max = 20, message = "标签数量不能超过20个")
|
||||
private List<String> tags;
|
||||
/** 数据集状态 */
|
||||
private DatasetStatusType status;
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidHexColor;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 更新标签请求DTO
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UpdateTagRequest {
|
||||
/** 标签 ID */
|
||||
@NotBlank(message = "标签ID不能为空")
|
||||
private String id;
|
||||
/** 标签名称 */
|
||||
@Size(max = 100, message = "标签名称长度不能超过100个字符")
|
||||
private String name;
|
||||
/** 标签颜色 */
|
||||
@ValidHexColor
|
||||
private String color;
|
||||
/** 标签描述 */
|
||||
@Size(max = 500, message = "标签描述长度不能超过500个字符")
|
||||
private String description;
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidFilePath;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
/**
|
||||
* 上传文件请求
|
||||
* 用于分块上传文件时的请求参数封装，支持大文件分片上传功能
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UploadFileRequest {
|
||||
/** 预上传返回的id，用来确认同一个任务 */
|
||||
@NotBlank(message = "请求ID不能为空")
|
||||
private String reqId;
|
||||
|
||||
/** 文件编号，用于标识批量上传中的第几个文件 */
|
||||
@Min(value = 0, message = "文件编号必须为非负整数")
|
||||
private int fileNo;
|
||||
|
||||
/** 文件名称（支持相对路径，用于文件夹上传） */
|
||||
@NotBlank(message = "文件名称不能为空")
|
||||
@ValidFilePath
|
||||
@Size(max = 255, message = "文件名称长度不能超过255个字符")
|
||||
private String fileName;
|
||||
|
||||
/** 文件总分块数量 */
|
||||
@Min(value = 1, message = "总分块数量必须大于0")
|
||||
private int totalChunkNum;
|
||||
|
||||
/** 当前分块编号，从1开始 */
|
||||
@Min(value = 1, message = "分块编号必须大于0")
|
||||
private int chunkNo;
|
||||
|
||||
/** 上传的文件分块内容 */
|
||||
@NotNull(message = "文件内容不能为空")
|
||||
private MultipartFile file;
|
||||
|
||||
/** 文件分块的校验和（十六进制字符串），用于验证文件完整性 */
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{64}$", message = "校验和格式不正确，应为64位十六进制字符串")
|
||||
private String checkSumHex;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.deepdata.datamanagement.interfaces.dto;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.validation.ValidPath;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 切片上传预上传请求
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class UploadFilesPreRequest {
|
||||
/** 是否为压缩包上传 */
|
||||
private boolean hasArchive;
|
||||
|
||||
/** 总文件数量 */
|
||||
@Min(value = 1, message = "总文件数量必须大于0")
|
||||
private int totalFileNum;
|
||||
|
||||
/** 总文件大小 */
|
||||
@Min(value = 1, message = "总文件大小必须大于0")
|
||||
private long totalSize;
|
||||
|
||||
/** 目标子目录前缀，例如 "images/"，为空表示数据集根目录 */
|
||||
@ValidPath(maxLength = 500)
|
||||
private String prefix;
|
||||
}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.datamanagement.application.DatasetApplicationService;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.interfaces.converter.DatasetConverter;
|
||||
import com.deepdata.datamanagement.interfaces.dto.*;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springaicommunity.mcp.annotation.McpTool;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据集 REST 控制器
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/data-management/datasets")
|
||||
@Validated
|
||||
public class DatasetController {
|
||||
private final DatasetApplicationService datasetApplicationService;
|
||||
|
||||
/**
|
||||
* 获取数据集列表
|
||||
*
|
||||
* @param query 分页查询参数
|
||||
* @return 分页的数据集列表
|
||||
*/
|
||||
@GetMapping
|
||||
@McpTool(name = "query_datasets", description = "根据参数查询满足条件的数据集列表")
|
||||
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
|
||||
return datasetApplicationService.getDatasets(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建数据集
|
||||
*
|
||||
* @param createDatasetRequest 创建数据集请求参数
|
||||
* @return 创建的数据集响应
|
||||
*/
|
||||
@PostMapping
|
||||
@McpTool(name = "create_dataset", description = "创建数据集")
|
||||
public DatasetResponse createDataset(@RequestBody @Valid CreateDatasetRequest createDatasetRequest) {
|
||||
Dataset dataset = datasetApplicationService.createDataset(createDatasetRequest);
|
||||
return DatasetConverter.INSTANCE.convertToResponse(dataset);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据ID获取数据集详情
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @return 数据集响应
|
||||
*/
|
||||
@GetMapping("/{datasetId}")
|
||||
public DatasetResponse getDatasetById(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
DatasetResponse dataset = DatasetConverter.INSTANCE.convertToResponse(datasetApplicationService.getDataset(datasetId));
|
||||
dataset.setPvcName(datasetApplicationService.getDatasetPvcName());
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据ID更新数据集
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param updateDatasetRequest 更新数据集请求参数
|
||||
* @return 更新后的数据集响应
|
||||
*/
|
||||
@PutMapping("/{datasetId}")
|
||||
public DatasetResponse updateDataset(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId,
|
||||
@RequestBody UpdateDatasetRequest updateDatasetRequest) {
|
||||
Dataset dataset = datasetApplicationService.updateDataset(datasetId, updateDatasetRequest);
|
||||
return DatasetConverter.INSTANCE.convertToResponse(dataset);
|
||||
}
|
||||
|
||||
@GetMapping("/{datasetId}/lineage")
|
||||
public DatasetLineage getDatasetLineage(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
return datasetApplicationService.getDatasetLineage(datasetId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据ID删除数据集
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
*/
|
||||
@DeleteMapping("/{datasetId}")
|
||||
public void deleteDataset(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
datasetApplicationService.deleteDataset(datasetId);
|
||||
}
|
||||
|
||||
@GetMapping("/{datasetId}/statistics")
|
||||
public ResponseEntity<Response<DatasetStatisticsResponse>> getDatasetStatistics(
|
||||
@PathVariable("datasetId")
|
||||
@Pattern(regexp = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", message = "数据集ID格式不正确，应为UUID格式")
|
||||
String datasetId) {
|
||||
try {
|
||||
Map<String, Object> stats = datasetApplicationService.getDatasetStatistics(datasetId);
|
||||
|
||||
DatasetStatisticsResponse response = new DatasetStatisticsResponse();
|
||||
response.setTotalFiles((Integer) stats.get("totalFiles"));
|
||||
response.setCompletedFiles((Integer) stats.get("completedFiles"));
|
||||
response.setTotalSize((Long) stats.get("totalSize"));
|
||||
response.setCompletionRate((Float) stats.get("completionRate"));
|
||||
response.setFileTypeDistribution((Map<String, Integer>) stats.get("fileTypeDistribution"));
|
||||
response.setStatusDistribution((Map<String, Integer>) stats.get("statusDistribution"));
|
||||
|
||||
return ResponseEntity.ok(Response.ok(response));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
} catch (Exception e) {
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/statistics")
|
||||
public ResponseEntity<Response<AllDatasetStatisticsResponse>> getAllStatistics() {
|
||||
return ResponseEntity.ok(Response.ok(datasetApplicationService.getAllDatasetStatistics()));
|
||||
}
|
||||
}
|
||||
+245
@@ -0,0 +1,245 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.IgnoreResponseWrap;
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.common.interfaces.PagingQuery;
|
||||
import com.deepdata.datamanagement.application.DatasetApplicationService;
|
||||
import com.deepdata.datamanagement.application.DatasetFileApplicationService;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.interfaces.converter.DatasetConverter;
|
||||
import com.deepdata.datamanagement.interfaces.dto.AddFilesRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.BatchDeleteFilesRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.CopyFilesRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.CreateDirectoryRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetFileResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UploadFileRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UploadFilesPreRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.RenameFileRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.RenameDirectoryRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集文件 REST 控制器（UUID 模式）
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/data-management/datasets/{datasetId}/files")
|
||||
public class DatasetFileController {
|
||||
|
||||
private final DatasetFileApplicationService datasetFileApplicationService;
|
||||
|
||||
private final DatasetApplicationService datasetApplicationService;
|
||||
|
||||
@GetMapping
|
||||
public Response<PagedResponse<DatasetFile>> getDatasetFiles(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@RequestParam(value = "isWithDirectory", required = false) boolean isWithDirectory,
|
||||
@RequestParam(value = "page", required = false, defaultValue = "0") Integer page,
|
||||
@RequestParam(value = "size", required = false, defaultValue = "20") Integer size,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
PagingQuery pagingQuery = new PagingQuery(page, size);
|
||||
PagedResponse<DatasetFile> filesPage;
|
||||
if (isWithDirectory) {
|
||||
filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(datasetId, prefix, pagingQuery);
|
||||
} else {
|
||||
filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, null, null, pagingQuery);
|
||||
}
|
||||
return Response.ok(filesPage);
|
||||
}
|
||||
|
||||
@GetMapping("/{fileId}")
|
||||
public ResponseEntity<Response<DatasetFileResponse>> getDatasetFileById(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
try {
|
||||
Dataset dataset = datasetApplicationService.getDataset(datasetId);
|
||||
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId, prefix);
|
||||
return ResponseEntity.ok(Response.ok(DatasetConverter.INSTANCE.convertToResponse(datasetFile)));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
@DeleteMapping("/{fileId}")
|
||||
public ResponseEntity<Response<Void>> deleteDatasetFile(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
try {
|
||||
datasetFileApplicationService.deleteDatasetFile(datasetId, fileId, prefix);
|
||||
return ResponseEntity.ok().build();
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量删除文件
|
||||
*/
|
||||
@DeleteMapping("/batch")
|
||||
public ResponseEntity<Response<Void>> batchDeleteFiles(
|
||||
@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid BatchDeleteFilesRequest request) {
|
||||
try {
|
||||
datasetFileApplicationService.batchDeleteFiles(datasetId, request);
|
||||
return ResponseEntity.ok().build();
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
@IgnoreResponseWrap
|
||||
@GetMapping(value = "/{fileId}/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE + ";charset=UTF-8")
|
||||
public ResponseEntity<Resource> downloadDatasetFileById(@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
try {
|
||||
log.info("downloadDatasetFileById datasetId:{}, fileId:{}, prefix:{}", datasetId, fileId, prefix);
|
||||
Dataset dataset = datasetApplicationService.getDataset(datasetId);
|
||||
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId, prefix);
|
||||
Resource resource = datasetFileApplicationService.downloadFile(datasetFile);
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"" + datasetFile.getFileName() + "\"")
|
||||
.body(resource);
|
||||
} catch (IllegalArgumentException e) {
|
||||
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
|
||||
} catch (Exception e) {
|
||||
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
||||
}
|
||||
}
|
||||
|
||||
@IgnoreResponseWrap
|
||||
@GetMapping(value = "/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
|
||||
public void downloadDatasetFileAsZip(@PathVariable("datasetId") String datasetId, HttpServletResponse response) {
|
||||
datasetFileApplicationService.downloadDatasetFileAsZip(datasetId, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件上传请求
|
||||
*
|
||||
* @param request 批量文件上传请求
|
||||
* @return 批量上传请求id
|
||||
*/
|
||||
@PostMapping("/upload/pre-upload")
|
||||
public ResponseEntity<Response<String>> preUpload(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid UploadFilesPreRequest request) {
|
||||
return ResponseEntity.ok(Response.ok(datasetFileApplicationService.preUpload(request, datasetId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分块上传
|
||||
*
|
||||
* @param uploadFileRequest 上传文件请求
|
||||
*/
|
||||
@PostMapping("/upload/chunk")
|
||||
public ResponseEntity<Void> chunkUpload(@PathVariable("datasetId") String datasetId,
|
||||
@Valid UploadFileRequest uploadFileRequest) {
|
||||
log.info("file upload reqId:{}, fileNo:{}, total chunk num:{}, current chunkNo:{}",
|
||||
uploadFileRequest.getReqId(), uploadFileRequest.getFileNo(), uploadFileRequest.getTotalChunkNum(),
|
||||
uploadFileRequest.getChunkNo());
|
||||
datasetFileApplicationService.chunkUpload(datasetId, uploadFileRequest);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 将指定路径中的文件拷贝到数据集目录下
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param req 源文件路径列表
|
||||
* @return 数据集文件响应DTO列表
|
||||
*/
|
||||
@PostMapping("/upload/copy")
|
||||
public List<DatasetFileResponse> copyFilesToDatasetDir(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid CopyFilesRequest req) {
|
||||
AddFilesRequest addFilesRequest = new AddFilesRequest(req.sourcePaths());
|
||||
List<DatasetFile> datasetFiles = datasetFileApplicationService.addFilesToDataset(datasetId, addFilesRequest);
|
||||
return DatasetConverter.INSTANCE.convertToResponseList(datasetFiles);
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加文件到数据集（仅创建数据库记录，不执行文件系统操作）
|
||||
*
|
||||
* @param datasetId 数据集ID
|
||||
* @param req 添加文件请求（包含源文件路径列表和softAdd标志）
|
||||
* @return 数据集文件响应DTO列表
|
||||
*/
|
||||
@PostMapping("/upload/add")
|
||||
public List<DatasetFileResponse> addFilesToDataset(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid AddFilesRequest req) {
|
||||
List<DatasetFile> datasetFiles = datasetFileApplicationService.addFilesToDataset(datasetId, req);
|
||||
return DatasetConverter.INSTANCE.convertToResponseList(datasetFiles);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在数据集下创建子目录
|
||||
*/
|
||||
@PostMapping("/directories")
|
||||
public ResponseEntity<Void> createDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid CreateDirectoryRequest req) {
|
||||
datasetFileApplicationService.createDirectory(datasetId, req);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载目录（压缩为 ZIP）
|
||||
*/
|
||||
@IgnoreResponseWrap
|
||||
@GetMapping(value = "/directories/download", produces = "application/zip")
|
||||
public void downloadDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix,
|
||||
HttpServletResponse response) {
|
||||
datasetFileApplicationService.downloadDirectory(datasetId, prefix, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除目录及其所有内容
|
||||
*/
|
||||
@DeleteMapping("/directories")
|
||||
public ResponseEntity<Void> deleteDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) {
|
||||
datasetFileApplicationService.deleteDirectory(datasetId, prefix);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 重命名文件
|
||||
*/
|
||||
@PutMapping("/{fileId}/rename")
|
||||
public ResponseEntity<Void> renameFile(@PathVariable("datasetId") String datasetId,
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestBody @Valid RenameFileRequest request) {
|
||||
datasetFileApplicationService.renameFile(datasetId, fileId, request);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 重命名目录
|
||||
*/
|
||||
@PutMapping("/directories/rename")
|
||||
public ResponseEntity<Void> renameDirectory(@PathVariable("datasetId") String datasetId,
|
||||
@RequestBody @Valid RenameDirectoryRequest request) {
|
||||
datasetFileApplicationService.renameDirectory(datasetId, request);
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetTypeResponse;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据集类型 REST 控制器
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/data-management/dataset-types")
|
||||
public class DatasetTypeController {
|
||||
|
||||
/**
|
||||
* 获取所有支持的数据集类型
|
||||
* @return 数据集类型列表
|
||||
*/
|
||||
@GetMapping
|
||||
public List<DatasetTypeResponse> getDatasetTypes() {
|
||||
return Arrays.asList(
|
||||
createDatasetType("IMAGE", "图像数据集", "用于机器学习的图像数据集", Arrays.asList("jpg", "jpeg", "png", "bmp", "gif")),
|
||||
createDatasetType("TEXT", "文本数据集", "用于文本分析的文本数据集", Arrays.asList("txt", "csv", "json", "xml")),
|
||||
createDatasetType("AUDIO", "音频数据集", "用于音频处理的音频数据集", Arrays.asList("wav", "mp3", "flac", "aac")),
|
||||
createDatasetType("VIDEO", "视频数据集", "用于视频分析的视频数据集", Arrays.asList("mp4", "avi", "mov", "mkv")),
|
||||
createDatasetType("MULTIMODAL", "多模态数据集", "包含多种数据类型的数据集", List.of("*"))
|
||||
);
|
||||
}
|
||||
|
||||
private DatasetTypeResponse createDatasetType(String code, String name, String description, List<String> supportedFormats) {
|
||||
DatasetTypeResponse response = new DatasetTypeResponse();
|
||||
response.setCode(code);
|
||||
response.setName(name);
|
||||
response.setDescription(description);
|
||||
response.setSupportedFormats(supportedFormats);
|
||||
response.setIcon(getIconForType(code));
|
||||
return response;
|
||||
}
|
||||
|
||||
private String getIconForType(String typeCode) {
|
||||
return switch (typeCode) {
|
||||
case "IMAGE" -> "🖼️";
|
||||
case "TEXT" -> "📄";
|
||||
case "AUDIO" -> "🎵";
|
||||
case "VIDEO" -> "🎬";
|
||||
case "MULTIMODAL" -> "📊";
|
||||
default -> "📁";
|
||||
};
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.deepdata.datamanagement.interfaces.rest;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.deepdata.datamanagement.application.TagApplicationService;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Tag;
|
||||
import com.deepdata.datamanagement.interfaces.converter.TagConverter;
|
||||
import com.deepdata.datamanagement.interfaces.dto.CreateTagRequest;
|
||||
import com.deepdata.datamanagement.interfaces.dto.TagResponse;
|
||||
import com.deepdata.datamanagement.interfaces.dto.UpdateTagRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 标签 REST 控制器（UUID 模式）
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/data-management/tags")
|
||||
public class TagController {
|
||||
|
||||
private final TagApplicationService tagApplicationService;
|
||||
|
||||
@Autowired
|
||||
public TagController(TagApplicationService tagApplicationService) {
|
||||
this.tagApplicationService = tagApplicationService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询标签列表
|
||||
*/
|
||||
@GetMapping
|
||||
public ResponseEntity<Response<List<TagResponse>>> getTags(@RequestParam(name = "keyword", required = false) String keyword) {
|
||||
List<Tag> tags = tagApplicationService.searchTags(keyword);
|
||||
List<TagResponse> response = tags.stream()
|
||||
.map(TagConverter.INSTANCE::convertToResponse)
|
||||
.collect(Collectors.toList());
|
||||
return ResponseEntity.ok(Response.ok(response));
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建标签
|
||||
*/
|
||||
@PostMapping
|
||||
public ResponseEntity<Response<TagResponse>> createTag(@RequestBody CreateTagRequest createTagRequest) {
|
||||
try {
|
||||
Tag tag = tagApplicationService.createTag(
|
||||
createTagRequest.getName(),
|
||||
createTagRequest.getColor(),
|
||||
createTagRequest.getDescription()
|
||||
);
|
||||
return ResponseEntity.ok(Response.ok(TagConverter.INSTANCE.convertToResponse(tag)));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.badRequest().body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新标签
|
||||
*
|
||||
* @param updateTagRequest 更新参数
|
||||
* @return 更新结果
|
||||
*/
|
||||
@PutMapping
|
||||
public ResponseEntity<Response<TagResponse>> updateTag(@RequestBody @Valid UpdateTagRequest updateTagRequest) {
|
||||
Tag tag = tagApplicationService.updateTag(TagConverter.INSTANCE.updateRequestToTag(updateTagRequest));
|
||||
return ResponseEntity.ok(Response.ok(TagConverter.INSTANCE.convertToResponse(tag)));
|
||||
}
|
||||
|
||||
@DeleteMapping
|
||||
public ResponseEntity<Response<Void>> deleteTag(@RequestParam(value = "ids") @Valid @Size(max = 10) List<String> ids) {
|
||||
try {
|
||||
tagApplicationService.deleteTag(ids.stream().filter(StringUtils::isNoneBlank).distinct().toList());
|
||||
return ResponseEntity.ok(Response.ok(null));
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ResponseEntity.badRequest().body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.deepdata.datamanagement.interfaces.scheduler;
|
||||
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.datamanagement.application.DatasetApplicationService;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetPagingQuery;
|
||||
import com.deepdata.datamanagement.interfaces.dto.DatasetResponse;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 数据集定时任务触发
|
||||
*
|
||||
* @since 2025/10/24
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetScheduler {
|
||||
private final DatasetApplicationService datasetApplicationService;
|
||||
|
||||
/**
|
||||
* 每天凌晨 00:00 扫描并删除超出保留期的数据集
|
||||
*/
|
||||
@Scheduled(cron = "0 0 0 * * ?")
|
||||
public void cleanupExpiredDatasets() {
|
||||
int pageNo = 1;
|
||||
int pageSize = 500;
|
||||
|
||||
while (true) {
|
||||
DatasetPagingQuery datasetPagingQuery = new DatasetPagingQuery();
|
||||
datasetPagingQuery.setPage(pageNo);
|
||||
datasetPagingQuery.setSize(pageSize);
|
||||
PagedResponse<DatasetResponse> datasets = datasetApplicationService.getDatasets(datasetPagingQuery);
|
||||
if (CollectionUtils.isEmpty(datasets.getContent())) {
|
||||
break;
|
||||
}
|
||||
|
||||
datasets.getContent().forEach(dataset -> {
|
||||
Integer retentionDays = dataset.getRetentionDays();
|
||||
LocalDateTime createdAt = dataset.getCreatedAt();
|
||||
if (retentionDays != null && retentionDays > 0 && createdAt != null) {
|
||||
LocalDateTime expireAt = createdAt.plusDays(retentionDays);
|
||||
if (expireAt.isBefore(LocalDateTime.now())) {
|
||||
try {
|
||||
log.info("Deleting dataset {}, expired at {} (retentionDays={})", dataset.getId(), expireAt, retentionDays);
|
||||
datasetApplicationService.deleteDataset(dataset.getId());
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to delete expired dataset {}: {}", dataset.getId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (datasets.getPage() >= datasets.getTotalPages()) {
|
||||
break;
|
||||
}
|
||||
pageNo++;
|
||||
}
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 文件名校验注解
|
||||
* 验证文件名不包含非法字符
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidFileNameValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidFileName {
|
||||
|
||||
String message() default "文件名包含非法字符";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 文件名校验器
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
public class ValidFileNameValidator implements ConstraintValidator<ValidFileName, String> {
|
||||
|
||||
/**
|
||||
* 文件名正则表达式
|
||||
* 不允许包含特殊字符: / \ : * ? " < > | \0
|
||||
* 允许字母、数字、中文、常见符号（- _ . space）
|
||||
*/
|
||||
private static final Pattern FILE_NAME_PATTERN = Pattern.compile(
|
||||
"^[^/\\\\:*?\"<>|\\x00]+$"
|
||||
);
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null || value.isEmpty()) {
|
||||
return true; // 空值由 @NotBlank 等其他注解处理
|
||||
}
|
||||
|
||||
boolean isValid = FILE_NAME_PATTERN.matcher(value).matches();
|
||||
|
||||
if (!isValid) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.FILE_NAME_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
}
|
||||
|
||||
return isValid;
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 文件路径校验注解
|
||||
* 验证文件路径不包含非法字符（允许 / 用于支持文件夹上传）
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/03/12
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidFilePathValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidFilePath {
|
||||
|
||||
String message() default "文件路径包含非法字符";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 文件路径校验器
|
||||
* 允许路径分隔符 / 用于支持文件夹上传
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/03/12
|
||||
*/
|
||||
public class ValidFilePathValidator implements ConstraintValidator<ValidFilePath, String> {
|
||||
|
||||
/**
|
||||
* 文件路径正则表达式
|
||||
* 不允许包含特殊字符: \ : * ? " < > | \0
|
||||
* 允许字母、数字、中文、常见符号（- _ . space /）
|
||||
* 注意：允许 / 是为了支持文件夹上传的相对路径
|
||||
*/
|
||||
private static final Pattern FILE_PATH_PATTERN = Pattern.compile(
|
||||
"^[^\\\\:*?\"<>|\\x00]+$"
|
||||
);
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null || value.isEmpty()) {
|
||||
return true; // 空值由 @NotBlank 等其他注解处理
|
||||
}
|
||||
|
||||
// 检查是否包含路径遍历序列 ..
|
||||
if (value.contains("..")) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
"文件路径不允许包含 '..'"
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean isValid = FILE_PATH_PATTERN.matcher(value).matches();
|
||||
|
||||
if (!isValid) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.FILE_NAME_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
}
|
||||
|
||||
return isValid;
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 十六进制颜色格式校验注解
|
||||
* 验证字符串是否符合十六进制颜色格式 (#RRGGBB 或 #RGB)
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidHexColorValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidHexColor {
|
||||
|
||||
String message() default "标签颜色格式不正确，应为十六进制颜色代码";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import com.deepdata.common.infrastructure.exception.ErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
import org.hibernate.validator.internal.constraintvalidators.bv.size.SizeValidatorForCharSequence;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 十六进制颜色格式校验器
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
public class ValidHexColorValidator implements ConstraintValidator<ValidHexColor, String> {
|
||||
|
||||
/**
|
||||
* 十六进制颜色正则表达式
|
||||
* 支持 #RGB 和 #RRGGBB 格式
|
||||
*/
|
||||
private static final Pattern HEX_COLOR_PATTERN = Pattern.compile("^#[0-9a-fA-F]{6}$");
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null) {
|
||||
return true; // null 值由 @NotBlank 等其他注解处理
|
||||
}
|
||||
|
||||
boolean isValid = HEX_COLOR_PATTERN.matcher(value).matches();
|
||||
|
||||
if (!isValid) {
|
||||
// 自定义错误消息和错误码
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.TAG_COLOR_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
}
|
||||
|
||||
return isValid;
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import jakarta.validation.Constraint;
|
||||
import jakarta.validation.Payload;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* 路径格式校验注解
|
||||
* 验证路径格式和长度限制
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
@Documented
|
||||
@Constraint(validatedBy = ValidPathValidator.class)
|
||||
@Target({ElementType.FIELD, ElementType.PARAMETER})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ValidPath {
|
||||
|
||||
String message() default "路径格式不正确";
|
||||
|
||||
Class<?>[] groups() default {};
|
||||
|
||||
Class<? extends Payload>[] payload() default {};
|
||||
|
||||
/**
|
||||
* 最大路径长度，默认500
|
||||
*/
|
||||
int maxLength() default 500;
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package com.deepdata.datamanagement.interfaces.validation;
|
||||
|
||||
import com.deepdata.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||
import jakarta.validation.ConstraintValidator;
|
||||
import jakarta.validation.ConstraintValidatorContext;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 路径格式校验器
|
||||
*
|
||||
* @author DeepData
|
||||
* @since 2026/02/11
|
||||
*/
|
||||
public class ValidPathValidator implements ConstraintValidator<ValidPath, String> {
|
||||
|
||||
/**
|
||||
* 路径正则表达式
|
||||
* 不允许以点开头（隐藏文件/目录）
|
||||
* 不允许包含特殊字符如 \0, <, >, :, ", |, ?, *
|
||||
*/
|
||||
private static final Pattern PATH_PATTERN = Pattern.compile(
|
||||
"^[^<>:\"|?*\\x00]+" // 不以点开头，不包含特殊字符
|
||||
);
|
||||
|
||||
private int maxLength = 500;
|
||||
|
||||
@Override
|
||||
public void initialize(ValidPath constraintAnnotation) {
|
||||
this.maxLength = constraintAnnotation.maxLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isValid(String value, ConstraintValidatorContext context) {
|
||||
if (value == null || value.isEmpty()) {
|
||||
return true; // 空值由其他注解处理
|
||||
}
|
||||
|
||||
// 检查长度
|
||||
if (value.length() > maxLength) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.PATH_TOO_LONG.getMessage()
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否以点开头
|
||||
if (value.startsWith(".")) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
DataManagementErrorCode.PREFIX_INVALID.getMessage()
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否包含路径遍历序列 ..
|
||||
if (value.contains("..")) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate(
|
||||
"路径不允许包含 '..'"
|
||||
).addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否包含非法字符
|
||||
if (!PATH_PATTERN.matcher(value).matches()) {
|
||||
context.disableDefaultConstraintViolation();
|
||||
context.buildConstraintViolationWithTemplate("路径包含非法字符").addConstraintViolation();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
deepdata:
|
||||
datamanagement:
|
||||
file-storage:
|
||||
upload-dir: ${FILE_UPLOAD_DIR:./uploads}
|
||||
max-file-size: 10485760 # 10MB
|
||||
max-request-size: 52428800 # 50MB
|
||||
allowed-source-dirs:
|
||||
- ${DATASET_BASE_PATH:/dataset}
|
||||
- ${FILE_UPLOAD_DIR:./uploads}
|
||||
cache:
|
||||
ttl: 3600
|
||||
max-size: 1000
|
||||
# MyBatis is configured centrally in main-application (mapper-locations & aliases)
|
||||
# to avoid list overriding issues when importing multiple module configs.
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper">
|
||||
<sql id="Base_Column_List">
|
||||
id, dataset_id, file_name, file_path, file_type, file_size, check_sum, tags, tags_updated_at, metadata, status,
|
||||
upload_time, last_access_time, created_at, updated_at
|
||||
</sql>
|
||||
|
||||
<select id="findById" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetId" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetIdAndStatus" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
AND status = #{status}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetIdAndFileType" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
AND file_type = #{fileType}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="countByDatasetId" parameterType="string" resultType="long">
|
||||
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
|
||||
</select>
|
||||
|
||||
<select id="countCompletedByDatasetId" parameterType="string" resultType="long">
|
||||
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId} AND status = 'COMPLETED'
|
||||
</select>
|
||||
|
||||
<select id="sumSizeByDatasetId" parameterType="string" resultType="long">
|
||||
SELECT COALESCE(SUM(file_size), 0) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
|
||||
</select>
|
||||
|
||||
<select id="findByDatasetIdAndFileName" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId} AND file_name = #{fileName}
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
<select id="findAllByDatasetId" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
<select id="findByCriteria" resultType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_dataset_files
|
||||
WHERE dataset_id = #{datasetId}
|
||||
<!-- Replace invalid XML '&&' with 'and' for MyBatis OGNL -->
|
||||
<if test="fileType != null and fileType != ''">
|
||||
AND file_type = #{fileType}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND status = #{status}
|
||||
</if>
|
||||
ORDER BY upload_time DESC
|
||||
</select>
|
||||
|
||||
|
||||
<update id="update" parameterType="com.deepdata.datamanagement.domain.model.dataset.DatasetFile">
|
||||
UPDATE t_dm_dataset_files
|
||||
SET file_name = #{fileName},
|
||||
file_path = #{filePath},
|
||||
file_type = #{fileType},
|
||||
file_size = #{fileSize},
|
||||
upload_time = #{uploadTime},
|
||||
last_access_time = #{lastAccessTime},
|
||||
status = #{status}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<delete id="deleteById" parameterType="string">
|
||||
DELETE FROM t_dm_dataset_files WHERE id = #{id}
|
||||
</delete>
|
||||
</mapper>
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.deepdata.datamanagement.infrastructure.persistence.mapper.DatasetMapper">
|
||||
|
||||
<sql id="Base_Column_List">
|
||||
id, name, description, dataset_type, category, path, format, schema_info, size_bytes, file_count, record_count,
|
||||
retention_days, tags, metadata, status, is_public, is_featured, version, created_at, updated_at, created_by, updated_by
|
||||
</sql>
|
||||
|
||||
<sql id="Alias_D_Column_List">
|
||||
d.id AS id,
|
||||
d.name AS name,
|
||||
d.description AS description,
|
||||
d.dataset_type AS dataset_type,
|
||||
d.category AS category,
|
||||
d.path AS path,
|
||||
d.format AS format,
|
||||
d.schema_info AS schema_info,
|
||||
d.size_bytes AS size_bytes,
|
||||
d.file_count AS file_count,
|
||||
d.record_count AS record_count,
|
||||
d.retention_days AS retention_days,
|
||||
d.tags AS tags,
|
||||
d.metadata AS metadata,
|
||||
d.status AS status,
|
||||
d.is_public AS is_public,
|
||||
d.is_featured AS is_featured,
|
||||
d.version AS version,
|
||||
d.created_at AS created_at,
|
||||
d.updated_at AS updated_at,
|
||||
d.created_by AS created_by,
|
||||
d.updated_by AS updated_by
|
||||
</sql>
|
||||
|
||||
<select id="findById" parameterType="string" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<select id="findByName" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE name = #{name}
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
<select id="findByStatus" parameterType="string"
|
||||
resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE status = #{status}
|
||||
ORDER BY updated_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByCreatedBy" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE created_by = #{createdBy}
|
||||
ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByTypeCode" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE dataset_type = #{typeCode}
|
||||
ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByTagNames" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT DISTINCT <include refid="Alias_D_Column_List"/>
|
||||
FROM t_dm_datasets d
|
||||
JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
|
||||
JOIN t_dm_tags t ON t.id = dt.tag_id
|
||||
WHERE t.name IN
|
||||
<foreach collection="tagNames" item="name" open="(" separator="," close=")">
|
||||
#{name}
|
||||
</foreach>
|
||||
ORDER BY d.created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByKeyword" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_datasets
|
||||
WHERE name LIKE CONCAT('%', #{keyword}, '%')
|
||||
OR description LIKE CONCAT('%', #{keyword}, '%')
|
||||
ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="findByCriteria" resultType="com.deepdata.datamanagement.domain.model.dataset.Dataset">
|
||||
SELECT DISTINCT <include refid="Alias_D_Column_List"/>
|
||||
FROM t_dm_datasets d
|
||||
LEFT JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
|
||||
LEFT JOIN t_dm_tags t ON t.id = dt.tag_id
|
||||
<where>
|
||||
<if test="typeCode != null and typeCode != ''">
|
||||
AND d.dataset_type = #{typeCode}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND d.status = #{status}
|
||||
</if>
|
||||
<if test="keyword != null and keyword != ''">
|
||||
AND (d.name LIKE CONCAT('%', #{keyword}, '%') OR d.description LIKE CONCAT('%', #{keyword}, '%'))
|
||||
</if>
|
||||
<if test="tagNames != null and tagNames.size > 0">
|
||||
AND t.name IN
|
||||
<foreach collection="tagNames" item="n" open="(" separator="," close=")">
|
||||
#{n}
|
||||
</foreach>
|
||||
</if>
|
||||
</where>
|
||||
ORDER BY d.created_at DESC
|
||||
</select>
|
||||
|
||||
<select id="countByCriteria" resultType="long">
|
||||
SELECT COUNT(DISTINCT d.id)
|
||||
FROM t_dm_datasets d
|
||||
LEFT JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
|
||||
LEFT JOIN t_dm_tags t ON t.id = dt.tag_id
|
||||
<where>
|
||||
<if test="typeCode != null and typeCode != ''">
|
||||
AND d.dataset_type = #{typeCode}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND d.status = #{status}
|
||||
</if>
|
||||
<if test="keyword != null and keyword != ''">
|
||||
AND (d.name LIKE CONCAT('%', #{keyword}, '%') OR d.description LIKE CONCAT('%', #{keyword}, '%'))
|
||||
</if>
|
||||
<if test="tagNames != null and tagNames.size > 0">
|
||||
AND t.name IN
|
||||
<foreach collection="tagNames" item="n" open="(" separator="," close=")">
|
||||
#{n}
|
||||
</foreach>
|
||||
</if>
|
||||
</where>
|
||||
</select>
|
||||
|
||||
<delete id="deleteById" parameterType="string">
|
||||
DELETE FROM t_dm_datasets WHERE id = #{id}
|
||||
</delete>
|
||||
|
||||
<select id="getAllDatasetStatistics" resultType="com.deepdata.datamanagement.interfaces.dto.AllDatasetStatisticsResponse">
|
||||
SELECT
|
||||
COUNT(*) AS total_datasets,
|
||||
SUM(size_bytes) AS total_size,
|
||||
SUM(file_count) AS total_files
|
||||
FROM t_dm_datasets;
|
||||
</select>
|
||||
</mapper>
|
||||
@@ -0,0 +1,111 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.deepdata.datamanagement.infrastructure.persistence.mapper.TagMapper">
|
||||
|
||||
<resultMap id="TagResultMap" type="com.deepdata.datamanagement.domain.model.dataset.Tag">
|
||||
<id column="id" property="id"/>
|
||||
<result column="name" property="name"/>
|
||||
<result column="description" property="description"/>
|
||||
<result column="category" property="category"/>
|
||||
<result column="color" property="color"/>
|
||||
<result column="usage_count" property="usageCount"/>
|
||||
<result column="created_at" property="createdAt"/>
|
||||
<result column="updated_at" property="updatedAt"/>
|
||||
</resultMap>
|
||||
|
||||
<sql id="Base_Column_List">
|
||||
id, name, description, category, color, usage_count, created_at, updated_at
|
||||
</sql>
|
||||
|
||||
<select id="findById" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<select id="findByName" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE name = #{name}
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
<select id="findByNameIn" parameterType="list" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE name IN
|
||||
<foreach collection="list" item="n" open="(" separator="," close=")">
|
||||
#{n}
|
||||
</foreach>
|
||||
</select>
|
||||
|
||||
<select id="findByKeyword" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE name LIKE CONCAT('%', #{keyword}, '%')
|
||||
ORDER BY usage_count DESC, name ASC
|
||||
</select>
|
||||
|
||||
<select id="findAllByOrderByUsageCountDesc" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
ORDER BY usage_count DESC, name ASC
|
||||
</select>
|
||||
|
||||
<insert id="insert" parameterType="com.deepdata.datamanagement.domain.model.dataset.Tag">
|
||||
INSERT INTO t_dm_tags (id, name, description, category, color, usage_count)
|
||||
VALUES (#{id}, #{name}, #{description}, #{category}, #{color}, #{usageCount})
|
||||
</insert>
|
||||
|
||||
<update id="update" parameterType="com.deepdata.datamanagement.domain.model.dataset.Tag">
|
||||
UPDATE t_dm_tags
|
||||
SET name = #{name},
|
||||
description = #{description},
|
||||
category = #{category},
|
||||
color = #{color},
|
||||
usage_count = #{usageCount}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<update id="updateUsageCount">
|
||||
UPDATE t_dm_tags
|
||||
SET usage_count = #{usageCount}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Dataset & Tag relations -->
|
||||
<insert id="insertDatasetTag">
|
||||
INSERT INTO t_dm_dataset_tags (dataset_id, tag_id)
|
||||
VALUES (#{datasetId}, #{tagId})
|
||||
</insert>
|
||||
|
||||
<delete id="deleteDatasetTagsByDatasetId">
|
||||
DELETE FROM t_dm_dataset_tags WHERE dataset_id = #{datasetId}
|
||||
</delete>
|
||||
|
||||
<select id="findByDatasetId" parameterType="string" resultMap="TagResultMap">
|
||||
SELECT t.id, t.name, t.description, t.category, t.color, t.usage_count, t.created_at, t.updated_at
|
||||
FROM t_dm_tags t
|
||||
JOIN t_dm_dataset_tags dt ON dt.tag_id = t.id
|
||||
WHERE dt.dataset_id = #{datasetId}
|
||||
ORDER BY t.usage_count DESC, t.name ASC
|
||||
</select>
|
||||
|
||||
<delete id="deleteTagsById">
|
||||
DELETE FROM t_dm_tags WHERE
|
||||
id IN
|
||||
<foreach collection="ids" item="id" open="(" separator="," close=")">
|
||||
#{id}
|
||||
</foreach>
|
||||
</delete>
|
||||
|
||||
<select id="findByIdIn" resultMap="TagResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dm_tags
|
||||
WHERE id IN
|
||||
<foreach collection="ids" item="id" open="(" separator="," close=")">
|
||||
#{id}
|
||||
</foreach>
|
||||
</select>
|
||||
</mapper>
|
||||
@@ -0,0 +1,112 @@
|
||||
# 主应用
|
||||
|
||||
## 概述
|
||||
|
||||
主应用是 DeepData 的核心 Spring Boot 服务，包含数据管理、数据清洗、算子市场、数据收集等主要功能模块。
|
||||
|
||||
## 架构
|
||||
|
||||
```
|
||||
backend/services/main-application/
|
||||
├── src/main/java/com/deepdata/main/
|
||||
│ ├── interfaces/
|
||||
│ │ ├── rest/ # Controllers
|
||||
│ │ ├── dto/ # Request/Response DTOs
|
||||
│ │ └── converter/ # MapStruct converters
|
||||
│ ├── application/ # Application services
|
||||
│ ├── domain/
|
||||
│ │ ├── model/ # Entities
|
||||
│ │ └── repository/ # Repository interfaces
|
||||
│ └── infrastructure/
|
||||
│ ├── persistence/ # Repository implementations
|
||||
│ ├── client/ # External API clients
|
||||
│ └── config/ # Service configuration
|
||||
└── src/main/resources/
|
||||
├── application.yml # 主配置
|
||||
├── config/application-datamanagement.yml # 数据管理配置
|
||||
└── config/application-datacollection.yml # 数据收集配置
|
||||
```
|
||||
|
||||
## 模块
|
||||
|
||||
### 1. 数据管理
|
||||
- 数据集 CRUD 操作
|
||||
- 文件上传/下载
|
||||
- 标签管理
|
||||
- 数据集版本控制
|
||||
|
||||
### 2. 数据收集
|
||||
- 数据源配置
|
||||
- 定时数据收集任务
|
||||
- 数据同步
|
||||
- 数据导入/导出
|
||||
|
||||
## 配置
|
||||
|
||||
### 端口
|
||||
- **默认**: 8080
|
||||
- **上下文路径**: `/api`
|
||||
|
||||
### 关键配置
|
||||
```yaml
|
||||
server:
|
||||
port: 8080
|
||||
servlet:
|
||||
context-path: /api
|
||||
|
||||
deepdata:
|
||||
data-management:
|
||||
base-path: /dataset
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 前置条件
|
||||
- JDK 21+
|
||||
- Maven 3.8+
|
||||
- PostgreSQL 12+
|
||||
- Redis 6+
|
||||
|
||||
### 构建
|
||||
```bash
|
||||
cd backend/services/main-application
|
||||
mvn clean install
|
||||
```
|
||||
|
||||
### 运行
|
||||
```bash
|
||||
cd backend/services/main-application
|
||||
mvn spring-boot:run
|
||||
```
|
||||
|
||||
## 开发
|
||||
|
||||
### 添加新模块
|
||||
1. 在 `domain/model/` 创建实体类
|
||||
2. 在 `domain/repository/` 创建 repository 接口
|
||||
3. 在 `infrastructure/persistence/` 实现 repository
|
||||
4. 在 `application/` 创建 application service
|
||||
5. 在 `interfaces/rest/` 创建 controller
|
||||
|
||||
## 测试
|
||||
|
||||
### 运行测试
|
||||
```bash
|
||||
cd backend/services/main-application
|
||||
mvn test
|
||||
```
|
||||
|
||||
### 运行特定测试
|
||||
```bash
|
||||
mvn test -Dtest=DatasetControllerTest
|
||||
```
|
||||
|
||||
## 文档
|
||||
|
||||
- **Spring Boot 文档**: https://docs.spring.io/spring-boot/
|
||||
- [AGENTS.md](../../shared/AGENTS.md)
|
||||
|
||||
## 相关链接
|
||||
|
||||
- [后端 README](../../README.md)
|
||||
- [API Gateway README](../../api-gateway/README.md)
|
||||
@@ -0,0 +1,112 @@
|
||||
# Main Application
|
||||
|
||||
## Overview
|
||||
|
||||
The Main Application is DeepData's core Spring Boot service, containing major functional modules including data management, data cleaning, operator marketplace, and data collection.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
backend/services/main-application/
|
||||
├── src/main/java/com/deepdata/main/
|
||||
│ ├── interfaces/
|
||||
│ │ ├── rest/ # Controllers
|
||||
│ │ ├── dto/ # Request/Response DTOs
|
||||
│ │ └── converter/ # MapStruct converters
|
||||
│ ├── application/ # Application services
|
||||
│ ├── domain/
|
||||
│ │ ├── model/ # Entities
|
||||
│ │ └── repository/ # Repository interfaces
|
||||
│ └── infrastructure/
|
||||
│ ├── persistence/ # Repository implementations
|
||||
│ ├── client/ # External API clients
|
||||
│ └── config/ # Service configuration
|
||||
└── src/main/resources/
|
||||
├── application.yml # Main configuration
|
||||
├── config/application-datamanagement.yml # Data management config
|
||||
└── config/application-datacollection.yml # Data collection config
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
### 1. Data Management
|
||||
- Dataset CRUD operations
|
||||
- File upload/download
|
||||
- Tag management
|
||||
- Dataset versioning
|
||||
|
||||
### 2. Data Collection
|
||||
- Data source configuration
|
||||
- Scheduled data collection tasks
|
||||
- Data synchronization
|
||||
- Data import/export
|
||||
|
||||
## Configuration
|
||||
|
||||
### Port
|
||||
- **Default**: 8080
|
||||
- **Context Path**: `/api`
|
||||
|
||||
### Key Configuration
|
||||
```yaml
|
||||
server:
|
||||
port: 8080
|
||||
servlet:
|
||||
context-path: /api
|
||||
|
||||
deepdata:
|
||||
data-management:
|
||||
base-path: /dataset
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
- JDK 21+
|
||||
- Maven 3.8+
|
||||
- PostgreSQL 12+
|
||||
- Redis 6+
|
||||
|
||||
### Build
|
||||
```bash
|
||||
cd backend/services/main-application
|
||||
mvn clean install
|
||||
```
|
||||
|
||||
### Run
|
||||
```bash
|
||||
cd backend/services/main-application
|
||||
mvn spring-boot:run
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
### Adding a New Module
|
||||
1. Create entity class in `domain/model/`
|
||||
2. Create repository interface in `domain/repository/`
|
||||
3. Implement repository in `infrastructure/persistence/`
|
||||
4. Create application service in `application/`
|
||||
5. Create controller in `interfaces/rest/`
|
||||
|
||||
## Testing
|
||||
|
||||
### Run Tests
|
||||
```bash
|
||||
cd backend/services/main-application
|
||||
mvn test
|
||||
```
|
||||
|
||||
### Run Specific Test
|
||||
```bash
|
||||
mvn test -Dtest=DatasetControllerTest
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
- **Spring Boot Docs**: https://docs.spring.io/spring-boot/
|
||||
- [AGENTS.md](../../shared/AGENTS.md)
|
||||
|
||||
## Related Links
|
||||
|
||||
- [Backend README](../../README.md)
|
||||
- [API Gateway README](../../api-gateway/README.md)
|
||||
@@ -0,0 +1,124 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>services</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>main-application</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>Data Mate Platform - Main Application</name>
|
||||
<description>主启动应用，集成所有服务模</description>
|
||||
|
||||
<dependencies>
|
||||
<!-- Spring Boot Starters -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-actuator</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jakarta.persistence</groupId>
|
||||
<artifactId>jakarta.persistence-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- 共享库依赖 -->
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>domain-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>security-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- 核心服务依赖 -->
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>data-management-service</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- RAG服务依赖 -->
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>rag-indexer-service</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Database -->
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- 测试依赖 -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-redis</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<configuration>
|
||||
<source>${maven.compiler.source}</source>
|
||||
<target>${maven.compiler.target}</target>
|
||||
<parameters>true</parameters>
|
||||
<compilerArgs>
|
||||
<arg>-parameters</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<version>${spring-boot.version}</version>
|
||||
<configuration>
|
||||
<finalName>deepdata</finalName>
|
||||
<mainClass>com.deepdata.main.DeepDataApplication</mainClass>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.deepdata.main;
|
||||
|
||||
import org.mybatis.spring.annotation.MapperScan;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.scheduling.annotation.EnableAsync;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
|
||||
/**
|
||||
* 数据引擎平台主应用
|
||||
* 聚合所有业务服务JAR包的微服务启动类
|
||||
*
|
||||
* @author Data Mate Team
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@SpringBootApplication
|
||||
@ComponentScan(basePackages = {"com.deepdata"})
|
||||
@MapperScan(basePackages = {"com.deepdata.**.mapper"})
|
||||
@EnableTransactionManagement
|
||||
@EnableAsync
|
||||
@EnableScheduling
|
||||
@EnableCaching
|
||||
@EnableFeignClients(basePackages = "com.deepdata.*")
|
||||
public class DeepDataApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(DeepDataApplication.class, args);
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.main.config;
|
||||
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
||||
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
|
||||
import org.springframework.security.web.SecurityFilterChain;
|
||||
|
||||
/**
|
||||
* 安全配置 - 暂时禁用所有认证
|
||||
* 开发阶段使用，生产环境需要启用认证
|
||||
*/
|
||||
@Configuration
|
||||
@EnableWebSecurity
|
||||
public class SecurityConfig {
|
||||
|
||||
@Bean
|
||||
public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
|
||||
http.csrf(csrf -> csrf.disable())
|
||||
.authorizeHttpRequests(authz -> authz
|
||||
.anyRequest().permitAll() // 允许所有请求无需认证
|
||||
);
|
||||
|
||||
return http.build();
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.deepdata.main.filter;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.config.DataScopeHandle;
|
||||
import com.deepdata.common.infrastructure.exception.CommonErrorCode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.servlet.*;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* 用户信息过滤器
|
||||
*
|
||||
* @since 2026/1/19
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class UserContextFilter implements Filter {
|
||||
private static final String USER_HEADER = "User";
|
||||
|
||||
@Value("${deepdata.jwt.enable:false}")
|
||||
private Boolean jwtEnable;
|
||||
|
||||
@Override
|
||||
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) {
|
||||
try {
|
||||
HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
|
||||
HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
|
||||
String user = httpRequest.getHeader(USER_HEADER);
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
if (jwtEnable && StringUtils.isBlank(user)) {
|
||||
httpResponse.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
|
||||
httpResponse.getWriter().write(objectMapper.writeValueAsString(Response.error(CommonErrorCode.UNAUTHORIZED)));
|
||||
return;
|
||||
}
|
||||
DataScopeHandle.setUserInfo(user);
|
||||
filterChain.doFilter(servletRequest, servletResponse);
|
||||
} catch (IOException | ServletException e) {
|
||||
log.error("Request failed!");
|
||||
throw new RuntimeException(e);
|
||||
} finally {
|
||||
DataScopeHandle.removeUserInfo();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
# 数据引擎平台 - 主应用配置
|
||||
spring:
|
||||
application:
|
||||
name: deepdata
|
||||
|
||||
# 暂时排除Spring Security自动配置（开发阶段使用）
|
||||
autoconfigure:
|
||||
exclude:
|
||||
- org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
|
||||
- org.springframework.boot.autoconfigure.security.servlet.UserDetailsServiceAutoConfiguration
|
||||
|
||||
# 数据源配置
|
||||
datasource:
|
||||
driver-class-name: org.postgresql.Driver
|
||||
url: jdbc:postgresql://deepdata-database:5432/deepdata?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
|
||||
username: ${DB_USERNAME:postgres}
|
||||
password: ${DB_PASSWORD}
|
||||
hikari:
|
||||
maximum-pool-size: 20
|
||||
minimum-idle: 5
|
||||
connection-timeout: 30000
|
||||
idle-timeout: 600000
|
||||
max-lifetime: 1800000
|
||||
|
||||
# Jackson配置
|
||||
jackson:
|
||||
time-zone: Asia/Shanghai
|
||||
date-format: yyyy-MM-dd HH:mm:ss
|
||||
serialization:
|
||||
write-dates-as-timestamps: false
|
||||
deserialization:
|
||||
fail-on-unknown-properties: false
|
||||
|
||||
# 文件上传配置
|
||||
servlet:
|
||||
multipart:
|
||||
max-file-size: 100MB
|
||||
max-request-size: 100MB
|
||||
|
||||
# 任务调度配置
|
||||
task:
|
||||
execution:
|
||||
pool:
|
||||
core-size: ${TASK_EXECUTION_CORE_SIZE:10}
|
||||
max-size: ${TASK_EXECUTION_MAX_SIZE:20}
|
||||
queue-capacity: ${TASK_EXECUTION_QUEUE_CAPACITY:100}
|
||||
keep-alive: ${TASK_EXECUTION_KEEP_ALIVE:60s}
|
||||
scheduling:
|
||||
pool:
|
||||
size: ${TASK_SCHEDULING_POOL_SIZE:5}
|
||||
config:
|
||||
import:
|
||||
- classpath:config/application-datacollection.yml
|
||||
- classpath:config/application-datamanagement.yml
|
||||
|
||||
# Redis配置
|
||||
data:
|
||||
redis:
|
||||
host: deepdata-redis
|
||||
port: 6379
|
||||
timeout: 2000
|
||||
password: ${REDIS_PASSWORD}
|
||||
lettuce:
|
||||
pool:
|
||||
max-active: 20
|
||||
max-idle: 10
|
||||
min-idle: 5
|
||||
max-wait: 1000ms
|
||||
|
||||
ai:
|
||||
mcp:
|
||||
server:
|
||||
enabled: true
|
||||
name: deepdata-mcp-server
|
||||
protocol: STATELESS
|
||||
capabilities:
|
||||
completion: false
|
||||
resource: false
|
||||
prompt: false
|
||||
tool: true
|
||||
mcp-endpoint: /mcp
|
||||
|
||||
|
||||
# MyBatis配置（需在顶层，不在 spring 下）
|
||||
mybatis-plus:
|
||||
configuration:
|
||||
map-underscore-to-camel-case: true
|
||||
default-fetch-size: 100
|
||||
default-statement-timeout: 30
|
||||
use-generated-keys: true
|
||||
cache-enabled: true
|
||||
lazy-loading-enabled: false
|
||||
multiple-result-sets-enabled: true
|
||||
use-column-label: true
|
||||
auto-mapping-behavior: partial
|
||||
auto-mapping-unknown-column-behavior: none
|
||||
default-executor-type: simple
|
||||
call-setters-on-nulls: false
|
||||
return-instance-for-empty-row: false
|
||||
log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl
|
||||
mapper-locations:
|
||||
- classpath*:mappers/**/*.xml
|
||||
|
||||
# 应用配置
|
||||
server:
|
||||
port: ${SERVER_PORT:8080}
|
||||
servlet:
|
||||
context-path: /api
|
||||
encoding:
|
||||
charset: UTF-8
|
||||
enabled: true
|
||||
force: true
|
||||
|
||||
# 日志配置
|
||||
logging:
|
||||
config: classpath:log4j2.xml
|
||||
|
||||
# Actuator配置
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
show-details: when-authorized
|
||||
health:
|
||||
elasticsearch:
|
||||
enabled: false # 禁用Elasticsearch健康检查
|
||||
|
||||
|
||||
# 平台配置
|
||||
deepdata:
|
||||
# JWT配置
|
||||
jwt:
|
||||
secret: ${JWT_SECRET}
|
||||
|
||||
# 通用配置
|
||||
|
||||
|
||||
# 文件存储配置
|
||||
storage:
|
||||
type: ${STORAGE_TYPE:local} # local, minio, s3
|
||||
local:
|
||||
base-path: ${STORAGE_LOCAL_PATH:./data/storage}
|
||||
|
||||
|
||||
# Ray执行器配置
|
||||
ray:
|
||||
enabled: ${RAY_ENABLED:false}
|
||||
address: ${RAY_ADDRESS:ray://localhost:10001}
|
||||
runtime-env:
|
||||
working-dir: ${RAY_WORKING_DIR:./runtime/python-executor}
|
||||
pip-packages:
|
||||
- "ray[default]==2.7.0"
|
||||
- "pandas"
|
||||
- "numpy"
|
||||
- "data-juicer"
|
||||
|
||||
# 模块配置
|
||||
|
||||
# 数据管理服务配置
|
||||
data-management:
|
||||
base-path: /dataset
|
||||
|
||||
# 数据归集服务配置（可由模块导入叠加）
|
||||
data-collection: {}
|
||||
|
||||
# 算子市场配置
|
||||
operator-market:
|
||||
repository-path: ${OPERATOR_REPO_PATH:./runtime/operators}
|
||||
registry-url: ${OPERATOR_REGISTRY_URL:}
|
||||
max-upload-size: ${OPERATOR_MAX_UPLOAD_SIZE:50MB}
|
||||
|
||||
# 数据处理配置
|
||||
data-processing:
|
||||
max-file-size: ${MAX_FILE_SIZE:1GB}
|
||||
temp-dir: ${TEMP_DIR:./data/temp}
|
||||
batch-size: ${BATCH_SIZE:1000}
|
||||
|
||||
# 标注配置
|
||||
annotation:
|
||||
auto-annotation:
|
||||
enabled: ${AUTO_ANNOTATION_ENABLED:true}
|
||||
model-endpoint: ${ANNOTATION_MODEL_ENDPOINT:}
|
||||
quality-control:
|
||||
enabled: ${QC_ENABLED:true}
|
||||
threshold: ${QC_THRESHOLD:0.8}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
deepdata:
|
||||
data-collection:
|
||||
# DataX配置
|
||||
datax:
|
||||
home-path: ${DATAX_HOME:/opt/datax}
|
||||
python-path: ${DATAX_PYTHON_PATH:python3}
|
||||
job-config-path: ${DATAX_JOB_PATH:./data/temp/datax/jobs}
|
||||
log-path: ${DATAX_LOG_PATH:./logs/datax}
|
||||
max-memory: ${DATAX_MAX_MEMORY:2048}
|
||||
channel-count: ${DATAX_CHANNEL_COUNT:5}
|
||||
|
||||
# 执行配置
|
||||
execution:
|
||||
max-concurrent-tasks: ${DATA_COLLECTION_MAX_CONCURRENT_TASKS:10}
|
||||
task-timeout-minutes: ${DATA_COLLECTION_TASK_TIMEOUT:120}
|
||||
retry-count: ${DATA_COLLECTION_RETRY_COUNT:3}
|
||||
retry-interval-seconds: ${DATA_COLLECTION_RETRY_INTERVAL:30}
|
||||
|
||||
# 监控配置
|
||||
monitoring:
|
||||
status-check-interval-seconds: ${DATA_COLLECTION_STATUS_CHECK_INTERVAL:30}
|
||||
log-retention-days: ${DATA_COLLECTION_LOG_RETENTION:30}
|
||||
enable-metrics: ${DATA_COLLECTION_ENABLE_METRICS:true}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
deepdata:
|
||||
datamanagement:
|
||||
file-storage:
|
||||
upload-dir: ${FILE_UPLOAD_DIR:./uploads}
|
||||
max-file-size: 10485760 # 10MB
|
||||
max-request-size: 52428800 # 50MB
|
||||
cache:
|
||||
ttl: 3600
|
||||
max-size: 1000
|
||||
# MyBatis is configured centrally in main-application (mapper-locations & aliases)
|
||||
# to avoid list overriding issues when importing multiple module configs.
|
||||
@@ -0,0 +1,42 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Configuration status="WARN" monitorInterval="30">
|
||||
<Properties>
|
||||
<Property name="LOG_PATH">/var/log/deepdata/backend</Property>
|
||||
<Property name="LOG_PATTERN">%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n</Property>
|
||||
<Property name="MAX_FILE_SIZE">${env:LOG_ROTATION_MAX_SIZE:-100MB}</Property>
|
||||
<Property name="MAX_HISTORY">${env:LOG_ROTATION_BACKUP_COUNT:-30}</Property>
|
||||
<Property name="LOG_LEVEL">INFO</Property>
|
||||
<Property name="SQL_LOG_LEVEL">WARN</Property>
|
||||
</Properties>
|
||||
|
||||
<Appenders>
|
||||
<!-- 控制台输出 -->
|
||||
<Console name="Console" target="SYSTEM_OUT">
|
||||
<PatternLayout pattern="${LOG_PATTERN}" />
|
||||
</Console>
|
||||
|
||||
<!-- 滚动文件输出 -->
|
||||
<RollingFile name="File" fileName="${LOG_PATH}/app.log" filePattern="${LOG_PATH}/app-%d{yyyy-MM-dd}-%i.log">
|
||||
<PatternLayout pattern="${LOG_PATTERN}" />
|
||||
<Policies>
|
||||
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
|
||||
<SizeBasedTriggeringPolicy size="${MAX_FILE_SIZE}" />
|
||||
</Policies>
|
||||
<DefaultRolloverStrategy max="${MAX_HISTORY}" />
|
||||
</RollingFile>
|
||||
</Appenders>
|
||||
|
||||
<Loggers>
|
||||
<!-- 自定义日志器 -->
|
||||
<Logger name="com.deepdata" level="${LOG_LEVEL}" />
|
||||
<Logger name="org.springframework.security" level="${LOG_LEVEL}" />
|
||||
<Logger name="org.hibernate.SQL" level="${SQL_LOG_LEVEL}" />
|
||||
<Logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="${SQL_LOG_LEVEL}" />
|
||||
|
||||
<!-- 根日志器 -->
|
||||
<Root level="INFO">
|
||||
<AppenderRef ref="Console" />
|
||||
<AppenderRef ref="File" />
|
||||
</Root>
|
||||
</Loggers>
|
||||
</Configuration>
|
||||
@@ -0,0 +1,108 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>deepdata</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>services</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
<name>Services</name>
|
||||
<description>Services</description>
|
||||
|
||||
<modules>
|
||||
<!-- 核心服务 -->
|
||||
<module>data-management-service</module>
|
||||
|
||||
<!-- RAG服务 -->
|
||||
<module>rag-indexer-service</module>
|
||||
|
||||
<!-- 主启动模块 -->
|
||||
<module>main-application</module>
|
||||
</modules>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-log4j2</artifactId>
|
||||
<version>${spring-boot.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.baomidou</groupId>
|
||||
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.baomidou</groupId>
|
||||
<artifactId>mybatis-plus-jsqlparser</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-mcp-server-webmvc</artifactId>
|
||||
<version>1.1.2</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<version>${spring-boot.version}</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<configuration>
|
||||
<source>${maven.compiler.source}</source>
|
||||
<target>${maven.compiler.target}</target>
|
||||
<parameters>true</parameters>
|
||||
<compilerArgs>
|
||||
<arg>-parameters</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,115 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>services</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>rag-indexer-service</artifactId>
|
||||
<name>RAG Indexer Service</name>
|
||||
<description>RAG文档索引服务</description>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-bom</artifactId>
|
||||
<version>1.8.0</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>domain-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>data-management-service</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.openapitools</groupId>
|
||||
<artifactId>jackson-databind-nullable</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>jakarta.validation</groupId>
|
||||
<artifactId>jakarta.validation-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-open-ai</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-document-parser-apache-pdfbox</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-document-parser-apache-tika</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-document-parser-apache-poi</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-document-parser-markdown</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-document-transformer-jsoup</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.langchain4j</groupId>
|
||||
<artifactId>langchain4j-milvus</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
package com.deepdata.rag.indexer.application;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.deepdata.common.domain.enums.EdgeType;
|
||||
import com.deepdata.common.domain.enums.NodeType;
|
||||
import com.deepdata.common.domain.model.LineageEdge;
|
||||
import com.deepdata.common.domain.model.LineageNode;
|
||||
import com.deepdata.common.domain.service.LineageService;
|
||||
import com.deepdata.common.infrastructure.exception.BusinessException;
|
||||
import com.deepdata.common.infrastructure.exception.KnowledgeBaseErrorCode;
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.common.interfaces.PagingQuery;
|
||||
import com.deepdata.common.setting.domain.entity.ModelConfig;
|
||||
import com.deepdata.common.setting.domain.repository.ModelConfigRepository;
|
||||
import com.deepdata.common.setting.infrastructure.client.ModelClient;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.Dataset;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
||||
import com.deepdata.rag.indexer.domain.model.FileStatus;
|
||||
import com.deepdata.rag.indexer.domain.model.KnowledgeBase;
|
||||
import com.deepdata.rag.indexer.domain.model.RagChunk;
|
||||
import com.deepdata.rag.indexer.domain.model.RagFile;
|
||||
import com.deepdata.rag.indexer.domain.repository.KnowledgeBaseRepository;
|
||||
import com.deepdata.rag.indexer.domain.repository.RagFileRepository;
|
||||
import com.deepdata.rag.indexer.infrastructure.event.DataInsertedEvent;
|
||||
import com.deepdata.rag.indexer.infrastructure.milvus.MilvusService;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.*;
|
||||
import dev.langchain4j.data.embedding.Embedding;
|
||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||
import io.milvus.v2.service.collection.request.DropCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.RenameCollectionReq;
|
||||
import io.milvus.v2.service.vector.request.DeleteReq;
|
||||
import io.milvus.v2.service.vector.request.QueryReq;
|
||||
import io.milvus.v2.service.vector.response.QueryResp;
|
||||
import io.milvus.v2.service.vector.response.SearchResp;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识库服务类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class KnowledgeBaseService {
|
||||
private final KnowledgeBaseRepository knowledgeBaseRepository;
|
||||
private final RagFileRepository ragFileRepository;
|
||||
private final ApplicationEventPublisher eventPublisher;
|
||||
private final ModelConfigRepository modelConfigRepository;
|
||||
private final DatasetRepository datasetRepository;
|
||||
private final DatasetFileRepository datasetFileRepository;
|
||||
private final MilvusService milvusService;
|
||||
private final LineageService lineageService;
|
||||
|
||||
/**
|
||||
* 创建知识库
|
||||
*
|
||||
* @param request 知识库创建请求
|
||||
* @return 知识库 ID
|
||||
*/
|
||||
public String create(KnowledgeBaseCreateReq request) {
|
||||
KnowledgeBase knowledgeBase = new KnowledgeBase();
|
||||
BeanUtils.copyProperties(request, knowledgeBase);
|
||||
knowledgeBaseRepository.save(knowledgeBase);
|
||||
return knowledgeBase.getId();
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新知识库
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
* @param request 知识库更新请求
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void update(String knowledgeBaseId, KnowledgeBaseUpdateReq request) {
|
||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||
if (StringUtils.hasText(request.getName()) && !knowledgeBase.getName().equals(request.getName())) {
|
||||
milvusService.getMilvusClient().renameCollection(RenameCollectionReq.builder()
|
||||
.collectionName(knowledgeBase.getName())
|
||||
.newCollectionName(request.getName())
|
||||
.build());
|
||||
knowledgeBase.setName(request.getName());
|
||||
}
|
||||
knowledgeBase.setDescription(request.getDescription());
|
||||
knowledgeBaseRepository.updateById(knowledgeBase);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 删除知识库
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void delete(String knowledgeBaseId) {
|
||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||
knowledgeBaseRepository.removeById(knowledgeBaseId);
|
||||
ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
|
||||
milvusService.getMilvusClient().dropCollection(DropCollectionReq.builder().collectionName(knowledgeBase.getName()).build());
|
||||
}
|
||||
|
||||
public KnowledgeBaseResp getById(String knowledgeBaseId) {
|
||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||
KnowledgeBaseResp resp = getKnowledgeBaseResp(knowledgeBase);
|
||||
resp.setEmbedding(modelConfigRepository.getById(knowledgeBase.getEmbeddingModel()));
|
||||
resp.setChat(modelConfigRepository.getById(knowledgeBase.getChatModel()));
|
||||
return resp;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private KnowledgeBaseResp getKnowledgeBaseResp(KnowledgeBase knowledgeBase) {
|
||||
KnowledgeBaseResp resp = new KnowledgeBaseResp();
|
||||
BeanUtils.copyProperties(knowledgeBase, resp);
|
||||
|
||||
// 获取该知识库的所有文件
|
||||
List<RagFile> files = ragFileRepository.findAllByKnowledgeBaseId(knowledgeBase.getId());
|
||||
resp.setFileCount((long) files.size());
|
||||
|
||||
// 计算分片总数
|
||||
long totalChunkCount = files.stream()
|
||||
.mapToLong(file -> file.getChunkCount() != null ? file.getChunkCount() : 0)
|
||||
.sum();
|
||||
resp.setChunkCount(totalChunkCount);
|
||||
return resp;
|
||||
}
|
||||
|
||||
public PagedResponse<KnowledgeBaseResp> list(KnowledgeBaseQueryReq request) {
|
||||
IPage<KnowledgeBase> page = new Page<>(request.getPage(), request.getSize());
|
||||
page = knowledgeBaseRepository.page(page, request);
|
||||
|
||||
// 将 KnowledgeBase 转换为 KnowledgeBaseResp，并计算 fileCount 和 chunkCount
|
||||
List<KnowledgeBaseResp> respList = page.getRecords().stream().map(this::getKnowledgeBaseResp).toList();
|
||||
return PagedResponse.of(page.getCurrent(), page.getSize(), page.getTotal(), page.getPages(), respList);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void addFiles(AddFilesReq request) {
|
||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(request.getKnowledgeBaseId()))
|
||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||
List<RagFile> ragFiles = request.getFiles().stream().map(fileInfo -> {
|
||||
RagFile ragFile = new RagFile();
|
||||
ragFile.setKnowledgeBaseId(knowledgeBase.getId());
|
||||
ragFile.setFileId(fileInfo.id());
|
||||
ragFile.setFileName(fileInfo.fileName());
|
||||
ragFile.setStatus(FileStatus.UNPROCESSED);
|
||||
return ragFile;
|
||||
}).toList();
|
||||
ragFileRepository.saveBatch(ragFiles, 100);
|
||||
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request));
|
||||
updateLineageGraph(knowledgeBase, request.getFiles());
|
||||
}
|
||||
|
||||
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {
|
||||
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
||||
request.setKnowledgeBaseId(knowledgeBaseId);
|
||||
page = ragFileRepository.page(page, request);
|
||||
return PagedResponse.of(page.getCurrent(), page.getSize(), page.getTotal(), page.getPages(), page.getRecords());
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void deleteFiles(String knowledgeBaseId, DeleteFilesReq request) {
|
||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||
ragFileRepository.removeByIds(request.getIds());
|
||||
milvusService.getMilvusClient().delete(DeleteReq.builder()
|
||||
.collectionName(knowledgeBase.getName())
|
||||
.filter("metadata[\"rag_file_id\"] in [" + org.apache.commons.lang3.StringUtils.join(request.getIds().stream().map(id -> "\"" + id + "\"").toArray(), ",") + "]")
|
||||
.build());
|
||||
}
|
||||
|
||||
public PagedResponse<RagChunk> getChunks(String knowledgeBaseId, String ragFileId, PagingQuery pagingQuery) {
|
||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||
QueryResp results = milvusService.getMilvusClient().query(QueryReq.builder()
|
||||
.collectionName(knowledgeBase.getName())
|
||||
.filter("metadata[\"rag_file_id\"] == \"" + ragFileId + "\"")
|
||||
.outputFields(Collections.singletonList("*"))
|
||||
.limit(Long.valueOf(pagingQuery.getSize()))
|
||||
.offset((long) (pagingQuery.getPage() - 1) * pagingQuery.getSize())
|
||||
.build());
|
||||
List<QueryResp.QueryResult> queryResults = results.getQueryResults();
|
||||
List<RagChunk> ragChunks = queryResults.stream()
|
||||
.map(QueryResp.QueryResult::getEntity)
|
||||
.map(item -> new RagChunk(
|
||||
item.get("id").toString(),
|
||||
item.get("text").toString(),
|
||||
item.get("metadata").toString()
|
||||
)).toList();
|
||||
|
||||
// 获取总数
|
||||
QueryResp countResults = milvusService.getMilvusClient().query(QueryReq.builder()
|
||||
.collectionName(knowledgeBase.getName())
|
||||
.filter("metadata[\"rag_file_id\"] == \"" + ragFileId + "\"")
|
||||
.outputFields(Collections.singletonList("count(*)"))
|
||||
.build());
|
||||
|
||||
long totalCount = Long.parseLong(countResults.getQueryResults().getFirst().getEntity().get("count(*)").toString());
|
||||
return PagedResponse.of(pagingQuery.getPage(), pagingQuery.getSize(), totalCount, (int) Math.ceil((double) totalCount / pagingQuery.getSize()), ragChunks);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检索知识库内容
|
||||
*
|
||||
* @param request 检索请求
|
||||
* @return 检索结果
|
||||
*/
|
||||
public List<SearchResp.SearchResult> retrieve(RetrieveReq request) {
|
||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(request.getKnowledgeBaseIds().getFirst()))
|
||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||
ModelConfig modelConfig = modelConfigRepository.getById(knowledgeBase.getEmbeddingModel());
|
||||
EmbeddingModel embeddingModel = ModelClient.invokeEmbeddingModel(modelConfig);
|
||||
Embedding embedding = embeddingModel.embed(request.getQuery()).content();
|
||||
SearchResp searchResp = milvusService.hybridSearch(knowledgeBase.getName(), request.getQuery(), embedding.vector(), request.getTopK());
|
||||
List<SearchResp.SearchResult> searchResults = searchResp.getSearchResults().getFirst();
|
||||
|
||||
searchResults.forEach(item -> {
|
||||
String metadata = item.getEntity().get("metadata").toString();
|
||||
item.getEntity().put("metadata", metadata);
|
||||
});
|
||||
return searchResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* 向知识库添加文件的时候，将相关数据集加入血缘图
|
||||
*
|
||||
* @param knowledgeBase 知识库
|
||||
* @param files 数据集中选择的文件
|
||||
*/
|
||||
private void updateLineageGraph(KnowledgeBase knowledgeBase, List<AddFilesReq.FileInfo> files) {
|
||||
LineageNode kbNode = lineageService.getNodeById(knowledgeBase.getId());
|
||||
if (kbNode == null) {
|
||||
kbNode = new LineageNode();
|
||||
kbNode.setId(knowledgeBase.getId());
|
||||
kbNode.setNodeType(NodeType.KNOWLEDGE_BASE);
|
||||
kbNode.setName(knowledgeBase.getName());
|
||||
kbNode.setDescription(knowledgeBase.getDescription());
|
||||
}
|
||||
|
||||
// 获取所有唯一的数据集ID
|
||||
Set<String> datasetIds = files.stream()
|
||||
.map(fileInfo -> {
|
||||
DatasetFile datasetFile = datasetFileRepository.getById(fileInfo.id());
|
||||
return datasetFile != null ? datasetFile.getDatasetId() : null;
|
||||
})
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
// 为每个数据集创建血缘关系
|
||||
for (String datasetId : datasetIds) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
if (dataset == null) continue;
|
||||
|
||||
// 创建源数据集节点
|
||||
LineageNode datasetNode = new LineageNode();
|
||||
datasetNode.setId(dataset.getId());
|
||||
datasetNode.setNodeType(NodeType.DATASET);
|
||||
datasetNode.setName(dataset.getName());
|
||||
datasetNode.setDescription(dataset.getDescription());
|
||||
|
||||
// 创建血缘边
|
||||
LineageEdge edge = new LineageEdge();
|
||||
edge.setProcessId(knowledgeBase.getId());
|
||||
edge.setName("");
|
||||
edge.setEdgeType(EdgeType.KNOWLEDGE_BASE);
|
||||
edge.setDescription("Add the files from dataset to the knowledge base.");
|
||||
edge.setFromNodeId(dataset.getId());
|
||||
edge.setToNodeId(knowledgeBase.getId());
|
||||
|
||||
// 生成血缘图
|
||||
lineageService.generateGraph(datasetNode, edge, kbNode);
|
||||
}
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.rag.indexer.domain.model;
|
||||
|
||||
/**
|
||||
* 文件状态枚举
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-29
|
||||
*/
|
||||
public enum FileStatus {
|
||||
/**
|
||||
* 未处理
|
||||
*/
|
||||
UNPROCESSED,
|
||||
/**
|
||||
* 处理中
|
||||
*/
|
||||
PROCESSING,
|
||||
/**
|
||||
* 已处理
|
||||
*/
|
||||
PROCESSED,
|
||||
/**
|
||||
* 处理失败
|
||||
*/
|
||||
PROCESS_FAILED
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.deepdata.rag.indexer.domain.model;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.deepdata.common.domain.model.base.BaseEntity;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.RagType;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 知识库实体类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@TableName("t_rag_knowledge_base")
|
||||
public class KnowledgeBase extends BaseEntity<String> {
|
||||
/**
|
||||
* 知识库名称
|
||||
*/
|
||||
private String name;
|
||||
|
||||
/**
|
||||
* 知识库描述
|
||||
*/
|
||||
private String description;
|
||||
|
||||
/**
|
||||
* RAG 类型
|
||||
*/
|
||||
private RagType type;
|
||||
|
||||
/**
|
||||
* 嵌入模型
|
||||
*/
|
||||
private String embeddingModel;
|
||||
|
||||
/**
|
||||
* 聊天模型
|
||||
*/
|
||||
private String chatModel;
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.deepdata.rag.indexer.domain.model;
|
||||
|
||||
/**
|
||||
* RAG 文档块实体类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-29
|
||||
*/
|
||||
|
||||
public record RagChunk(
|
||||
String id,
|
||||
String text,
|
||||
String metadata
|
||||
) {
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.deepdata.rag.indexer.domain.model;
|
||||
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
|
||||
import com.deepdata.common.domain.model.base.BaseEntity;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Rag 文件实体类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@TableName("t_rag_file")
|
||||
public class RagFile extends BaseEntity<String> {
|
||||
/**
|
||||
* 知识库ID
|
||||
*/
|
||||
private String knowledgeBaseId;
|
||||
/**
|
||||
* 文件名
|
||||
*/
|
||||
private String fileName;
|
||||
/**
|
||||
* 文件ID
|
||||
*/
|
||||
private String fileId;
|
||||
/**
|
||||
* 分块数量
|
||||
*/
|
||||
private Integer chunkCount;
|
||||
|
||||
/**
|
||||
* 元数据
|
||||
*/
|
||||
@TableField(typeHandler = JacksonTypeHandler.class)
|
||||
private Map<String, Object> metadata;
|
||||
|
||||
private FileStatus status;
|
||||
|
||||
private String errMsg;
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.deepdata.rag.indexer.domain.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.deepdata.rag.indexer.domain.model.KnowledgeBase;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.KnowledgeBaseQueryReq;
|
||||
|
||||
/**
|
||||
* 知识库仓储接口
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
public interface KnowledgeBaseRepository extends IRepository<KnowledgeBase> {
|
||||
/**
|
||||
* 分页查询知识库
|
||||
*
|
||||
* @param page 分页信息
|
||||
* @param request 查询请求
|
||||
* @return 知识库分页结果
|
||||
*/
|
||||
IPage<KnowledgeBase> page(IPage<KnowledgeBase> page, KnowledgeBaseQueryReq request);
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.rag.indexer.domain.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.deepdata.rag.indexer.domain.model.RagFile;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.RagFileReq;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 知识库文件仓储接口
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
public interface RagFileRepository extends IRepository<RagFile> {
|
||||
void removeByKnowledgeBaseId(String knowledgeBaseId);
|
||||
|
||||
List<RagFile> findNotSuccessByKnowledgeBaseId(String knowledgeBaseId);
|
||||
|
||||
List<RagFile> findAllByKnowledgeBaseId(String knowledgeBaseId);
|
||||
|
||||
IPage<RagFile> page(IPage<RagFile> page, RagFileReq request);
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.client;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
|
||||
/**
|
||||
* 知识图谱RAG客户端
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2026-01-15
|
||||
*/
|
||||
@FeignClient(name = "rag-service", url = "${collection.service.url:http://deepdata-backend-python:18000}")
|
||||
public interface GraphRagClient {
|
||||
/**
|
||||
* 启动知识图谱RAG任务
|
||||
* @param knowledgeBaseId 知识库ID
|
||||
* @return 任务详情
|
||||
*/
|
||||
@PostMapping("/api/rag/process/{id}")
|
||||
Response<?> startGraphRagTask(@PathVariable("id") String knowledgeBaseId, @RequestHeader("User") String authorization);
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.event;
|
||||
|
||||
import com.deepdata.rag.indexer.domain.model.KnowledgeBase;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.AddFilesReq;
|
||||
|
||||
/**
|
||||
* 数据插入事件
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-29
|
||||
*/
|
||||
public record DataInsertedEvent(KnowledgeBase knowledgeBase, AddFilesReq addFilesReq) {
|
||||
}
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.event;
|
||||
|
||||
import com.deepdata.common.setting.domain.entity.ModelConfig;
|
||||
import com.deepdata.common.setting.domain.repository.ModelConfigRepository;
|
||||
import com.deepdata.common.setting.infrastructure.client.ModelClient;
|
||||
import com.deepdata.datamanagement.domain.model.dataset.DatasetFile;
|
||||
import com.deepdata.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||
import com.deepdata.rag.indexer.domain.model.FileStatus;
|
||||
import com.deepdata.rag.indexer.domain.model.RagFile;
|
||||
import com.deepdata.rag.indexer.domain.repository.RagFileRepository;
|
||||
import com.deepdata.rag.indexer.infrastructure.client.GraphRagClient;
|
||||
import com.deepdata.rag.indexer.infrastructure.milvus.MilvusService;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.AddFilesReq;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.RagType;
|
||||
import com.google.common.collect.Lists;
|
||||
import dev.langchain4j.data.document.Document;
|
||||
import dev.langchain4j.data.document.DocumentParser;
|
||||
import dev.langchain4j.data.document.DocumentSplitter;
|
||||
import dev.langchain4j.data.document.loader.FileSystemDocumentLoader;
|
||||
import dev.langchain4j.data.document.parser.TextDocumentParser;
|
||||
import dev.langchain4j.data.document.parser.apache.pdfbox.ApachePdfBoxDocumentParser;
|
||||
import dev.langchain4j.data.document.parser.apache.poi.ApachePoiDocumentParser;
|
||||
import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser;
|
||||
import dev.langchain4j.data.document.parser.markdown.MarkdownDocumentParser;
|
||||
import dev.langchain4j.data.document.splitter.*;
|
||||
import dev.langchain4j.data.document.transformer.jsoup.HtmlToTextDocumentTransformer;
|
||||
import dev.langchain4j.data.embedding.Embedding;
|
||||
import dev.langchain4j.data.segment.TextSegment;
|
||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.event.TransactionPhase;
|
||||
import org.springframework.transaction.event.TransactionalEventListener;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
/**
|
||||
* RAG ETL服务
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-29
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RagEtlService {
|
||||
private static final Semaphore SEMAPHORE = new Semaphore(10);
|
||||
|
||||
private final MilvusService milvusService;
|
||||
|
||||
private final RagFileRepository ragFileRepository;
|
||||
|
||||
private final DatasetFileRepository datasetFileRepository;
|
||||
|
||||
private final ModelConfigRepository modelConfigRepository;
|
||||
|
||||
private final GraphRagClient graphRagClient;
|
||||
|
||||
private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
|
||||
|
||||
@Async
|
||||
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT)
|
||||
public void processAfterCommit(DataInsertedEvent event) {
|
||||
// 执行 RAG 处理流水线
|
||||
List<RagFile> ragFiles = ragFileRepository.findNotSuccessByKnowledgeBaseId(event.knowledgeBase().getId());
|
||||
if (RagType.GRAPH.equals(event.knowledgeBase().getType())){
|
||||
log.info("Knowledge base {} is of type GRAPH. Skipping RAG ETL processing.", event.knowledgeBase().getName());
|
||||
graphRagClient.startGraphRagTask(event.knowledgeBase().getId(), event.knowledgeBase().getCreatedBy());
|
||||
return;
|
||||
}
|
||||
|
||||
ragFiles.forEach(ragFile -> {
|
||||
try {
|
||||
SEMAPHORE.acquire();
|
||||
executor.submit(() -> {
|
||||
try {
|
||||
// 执行 RAG 处理流水线
|
||||
ragFile.setStatus(FileStatus.PROCESSING);
|
||||
ragFileRepository.updateById(ragFile);
|
||||
processRagFile(ragFile, event);
|
||||
// 更新文件状态为已处理
|
||||
ragFile.setStatus(FileStatus.PROCESSED);
|
||||
ragFileRepository.updateById(ragFile);
|
||||
} catch (Throwable e) {
|
||||
// 处理异常
|
||||
log.error("Error processing RAG file: {}", ragFile.getFileId(), e);
|
||||
ragFile.setStatus(FileStatus.PROCESS_FAILED);
|
||||
ragFile.setErrMsg(e.getMessage());
|
||||
ragFileRepository.updateById(ragFile);
|
||||
} finally {
|
||||
SEMAPHORE.release();
|
||||
}
|
||||
});
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private void processRagFile(RagFile ragFile, DataInsertedEvent event) {
|
||||
DatasetFile file = datasetFileRepository.getById(ragFile.getFileId());
|
||||
// 使用文档解析器解析文档
|
||||
DocumentParser parser = documentParser(file.getFileType());
|
||||
// 从文件系统读取文档
|
||||
Document document = FileSystemDocumentLoader.loadDocument(file.getFilePath(), parser);
|
||||
// 对html文档进行转换
|
||||
if (Arrays.asList("html", "htm").contains(file.getFileType().toLowerCase())) {
|
||||
document = new HtmlToTextDocumentTransformer().transform(document);
|
||||
}
|
||||
document.metadata().put("rag_file_id", ragFile.getId());
|
||||
document.metadata().put("original_file_id", ragFile.getFileId());
|
||||
// 使用文档分块器对文档进行分块
|
||||
DocumentSplitter splitter = documentSplitter(event.addFilesReq());
|
||||
List<TextSegment> split = splitter.split(document);
|
||||
|
||||
// 更新分块数量
|
||||
ragFile.setChunkCount(split.size());
|
||||
ragFileRepository.updateById(ragFile);
|
||||
|
||||
// 调用模型客户端获取嵌入模型
|
||||
ModelConfig model = modelConfigRepository.getById(event.knowledgeBase().getEmbeddingModel());
|
||||
EmbeddingModel embeddingModel = ModelClient.invokeEmbeddingModel(model);
|
||||
// 调用嵌入模型获取嵌入向量
|
||||
|
||||
if (!milvusService.hasCollection(event.knowledgeBase().getName())) {
|
||||
milvusService.createCollection(event.knowledgeBase().getName(), embeddingModel.dimension());
|
||||
}
|
||||
|
||||
Lists.partition(split, 20).forEach(partition -> {
|
||||
List<Embedding> embeddings = embeddingModel.embedAll(partition).content();
|
||||
milvusService.addAll(event.knowledgeBase().getName(),partition, embeddings);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据文件类型返回对应的文档解析器
|
||||
*x
|
||||
* @param fileType 文件类型
|
||||
* @return 文档解析器
|
||||
*/
|
||||
public DocumentParser documentParser(String fileType) {
|
||||
fileType = fileType.toLowerCase();
|
||||
return switch (fileType) {
|
||||
case "txt", "html", "htm" -> new TextDocumentParser();
|
||||
case "md" -> new MarkdownDocumentParser();
|
||||
case "pdf" -> new ApachePdfBoxDocumentParser();
|
||||
case "doc", "docx", "xls", "xlsx", "ppt", "pptx" -> new ApachePoiDocumentParser();
|
||||
default -> new ApacheTikaDocumentParser();
|
||||
};
|
||||
}
|
||||
|
||||
public DocumentSplitter documentSplitter(AddFilesReq req) {
|
||||
return switch (req.getProcessType()) {
|
||||
case PARAGRAPH_CHUNK -> new DocumentByParagraphSplitter(req.getChunkSize(), req.getOverlapSize());
|
||||
case SENTENCE_CHUNK -> new DocumentBySentenceSplitter(req.getChunkSize(), req.getOverlapSize());
|
||||
case LENGTH_CHUNK -> new DocumentByCharacterSplitter(req.getChunkSize(), req.getOverlapSize());
|
||||
case DEFAULT_CHUNK -> new DocumentByWordSplitter(req.getChunkSize(), req.getOverlapSize());
|
||||
case CUSTOM_SEPARATOR_CHUNK ->
|
||||
new DocumentByRegexSplitter(req.getDelimiter(), "", req.getChunkSize(), req.getOverlapSize());
|
||||
};
|
||||
}
|
||||
}
|
||||
+220
@@ -0,0 +1,220 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.milvus;
|
||||
|
||||
import com.google.gson.*;
|
||||
import dev.langchain4j.data.embedding.Embedding;
|
||||
import dev.langchain4j.data.segment.TextSegment;
|
||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||
import dev.langchain4j.store.embedding.EmbeddingStore;
|
||||
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
|
||||
import io.milvus.common.clientenum.FunctionType;
|
||||
import io.milvus.v2.client.ConnectConfig;
|
||||
import io.milvus.v2.client.MilvusClientV2;
|
||||
import io.milvus.v2.common.DataType;
|
||||
import io.milvus.v2.common.IndexParam;
|
||||
import io.milvus.v2.service.collection.request.AddFieldReq;
|
||||
import io.milvus.v2.service.collection.request.CreateCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.HasCollectionReq;
|
||||
import io.milvus.v2.service.vector.request.AnnSearchReq;
|
||||
import io.milvus.v2.service.vector.request.FunctionScore;
|
||||
import io.milvus.v2.service.vector.request.HybridSearchReq;
|
||||
import io.milvus.v2.service.vector.request.InsertReq;
|
||||
import io.milvus.v2.service.vector.request.data.BaseVector;
|
||||
import io.milvus.v2.service.vector.request.data.EmbeddedText;
|
||||
import io.milvus.v2.service.vector.request.data.FloatVec;
|
||||
import io.milvus.v2.service.vector.response.SearchResp;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import static dev.langchain4j.internal.Utils.randomUUID;
|
||||
|
||||
/**
|
||||
* Milvus 服务类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-11-17
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MilvusService {
|
||||
@Value("${deepdata.rag.milvus-uri:http://milvus-standalone:19530}")
|
||||
private String milvusUri;
|
||||
private static final Gson GSON;
|
||||
|
||||
static {
|
||||
GSON = (new GsonBuilder()).setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create();
|
||||
}
|
||||
|
||||
private volatile MilvusClientV2 milvusClient;
|
||||
|
||||
public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
|
||||
return MilvusEmbeddingStore.builder()
|
||||
.uri(milvusUri)
|
||||
.collectionName(knowledgeBaseName)
|
||||
.dimension(embeddingModel.dimension())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 单例模式获取 Milvus 客户端，不依赖 Spring 容器
|
||||
*
|
||||
* @return MilvusClient
|
||||
*/
|
||||
public MilvusClientV2 getMilvusClient() {
|
||||
if (milvusClient == null) {
|
||||
synchronized (this) {
|
||||
if (milvusClient == null) {
|
||||
try {
|
||||
ConnectConfig connectConfig = ConnectConfig.builder()
|
||||
.uri(milvusUri)
|
||||
.build();
|
||||
milvusClient = new MilvusClientV2(connectConfig);
|
||||
log.info("Milvus client connected successfully");
|
||||
} catch (Exception e) {
|
||||
log.error("Milvus client connection failed: {}", e.getMessage());
|
||||
throw new RuntimeException("Milvus client connection failed", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return milvusClient;
|
||||
}
|
||||
|
||||
|
||||
public boolean hasCollection(String collectionName) {
|
||||
HasCollectionReq request = HasCollectionReq.builder().collectionName(collectionName).build();
|
||||
return getMilvusClient().hasCollection(request);
|
||||
}
|
||||
|
||||
public void createCollection(String collectionName, int dimension) {
|
||||
CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder()
|
||||
.build();
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName("id")
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(36)
|
||||
.isPrimaryKey(true)
|
||||
.autoID(false)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName("text")
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(65535)
|
||||
.enableAnalyzer(true)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName("metadata")
|
||||
.dataType(DataType.JSON)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName("vector")
|
||||
.dataType(DataType.FloatVector)
|
||||
.dimension(dimension)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName("sparse")
|
||||
.dataType(DataType.SparseFloatVector)
|
||||
.build());
|
||||
schema.addFunction(CreateCollectionReq.Function.builder()
|
||||
.functionType(FunctionType.BM25)
|
||||
.name("text_bm25_emb")
|
||||
.inputFieldNames(Collections.singletonList("text"))
|
||||
.outputFieldNames(Collections.singletonList("sparse"))
|
||||
.build());
|
||||
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("inverted_index_algo", "DAAT_MAXSCORE");
|
||||
params.put("bm25_k1", 1.2);
|
||||
params.put("bm25_b", 0.75);
|
||||
|
||||
List<IndexParam> indexes = new ArrayList<>();
|
||||
indexes.add(IndexParam.builder()
|
||||
.fieldName("sparse")
|
||||
.indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
|
||||
.metricType(IndexParam.MetricType.BM25)
|
||||
.extraParams(params)
|
||||
.build());
|
||||
indexes.add(IndexParam.builder()
|
||||
.fieldName("vector")
|
||||
.indexType(IndexParam.IndexType.FLAT)
|
||||
.metricType(IndexParam.MetricType.COSINE)
|
||||
.extraParams(Map.of())
|
||||
.build());
|
||||
|
||||
CreateCollectionReq createCollectionReq = CreateCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.collectionSchema(schema)
|
||||
.indexParams(indexes)
|
||||
.build();
|
||||
this.getMilvusClient().createCollection(createCollectionReq);
|
||||
}
|
||||
|
||||
public void addAll(String collectionName, List<TextSegment> textSegments, List<Embedding> embeddings) {
|
||||
List<JsonObject> data = convertToJsonObjects(textSegments, embeddings);
|
||||
InsertReq insertReq = InsertReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.data(data)
|
||||
.build();
|
||||
this.getMilvusClient().insert(insertReq);
|
||||
}
|
||||
|
||||
public List<JsonObject> convertToJsonObjects(List<TextSegment> textSegments, List<Embedding> embeddings) {
|
||||
List<JsonObject> data = new ArrayList<>();
|
||||
for (int i = 0; i < textSegments.size(); i++) {
|
||||
JsonObject jsonObject = new JsonObject();
|
||||
jsonObject.addProperty("id", randomUUID());
|
||||
jsonObject.addProperty("text", textSegments.get(i).text());
|
||||
jsonObject.add("metadata", GSON.toJsonTree(textSegments.get(i).metadata().toMap()).getAsJsonObject());
|
||||
JsonArray vectorArray = new JsonArray();
|
||||
for (float f : embeddings.get(i).vector()) {
|
||||
vectorArray.add(f);
|
||||
}
|
||||
jsonObject.add("vector", vectorArray);
|
||||
data.add(jsonObject);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
public SearchResp hybridSearch(String collectionName, String query, float[] queryDense, int topK) {
|
||||
List<BaseVector> queryTexts = Collections.singletonList(new EmbeddedText(query));
|
||||
List<BaseVector> queryVectors = Collections.singletonList(new FloatVec(queryDense));
|
||||
|
||||
List<AnnSearchReq> searchRequests = new ArrayList<>();
|
||||
searchRequests.add(AnnSearchReq.builder()
|
||||
.vectorFieldName("vector")
|
||||
.vectors(queryVectors)
|
||||
.params("{\"nprobe\": 10}")
|
||||
.topK(topK)
|
||||
.build());
|
||||
searchRequests.add(AnnSearchReq.builder()
|
||||
.vectorFieldName("sparse")
|
||||
.vectors(queryTexts)
|
||||
.params("{\"drop_ratio_search\": 0.2}")
|
||||
.topK(topK)
|
||||
.build());
|
||||
|
||||
CreateCollectionReq.Function ranker = CreateCollectionReq.Function.builder()
|
||||
.name("weight")
|
||||
.functionType(FunctionType.RERANK)
|
||||
.param("reranker", "weighted")
|
||||
.param("weights", "[0.1, 0.9]")
|
||||
.param("norm_score", "true")
|
||||
.build();
|
||||
|
||||
FunctionScore functionScore = FunctionScore.builder()
|
||||
.functions(Collections.singletonList(ranker))
|
||||
.build();
|
||||
|
||||
|
||||
SearchResp searchResp = this.getMilvusClient().hybridSearch(HybridSearchReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.searchRequests(searchRequests)
|
||||
.functionScore(functionScore)
|
||||
.outFields(Arrays.asList("id", "text", "metadata"))
|
||||
.limit(topK)
|
||||
.build());
|
||||
return searchResp;
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.persistence.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.deepdata.rag.indexer.domain.model.KnowledgeBase;
|
||||
import com.deepdata.rag.indexer.domain.repository.KnowledgeBaseRepository;
|
||||
import com.deepdata.rag.indexer.infrastructure.persistence.mapper.KnowledgeBaseMapper;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.KnowledgeBaseQueryReq;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* 知识库仓储实现类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Repository
|
||||
public class KnowledgeBaseRepositoryImpl extends CrudRepository<KnowledgeBaseMapper, KnowledgeBase> implements KnowledgeBaseRepository {
|
||||
|
||||
@Override
|
||||
public IPage<KnowledgeBase> page(IPage<KnowledgeBase> page, KnowledgeBaseQueryReq request) {
|
||||
return this.page(page, new LambdaQueryWrapper<KnowledgeBase>()
|
||||
.like(StringUtils.hasText(request.getName()), KnowledgeBase::getName, request.getName())
|
||||
.like(StringUtils.hasText(request.getDescription()), KnowledgeBase::getDescription, request.getDescription())
|
||||
.like(StringUtils.hasText(request.getCreatedBy()), KnowledgeBase::getCreatedBy, request.getCreatedBy())
|
||||
.like(StringUtils.hasText(request.getUpdatedBy()), KnowledgeBase::getUpdatedBy, request.getUpdatedBy())
|
||||
.orderByDesc(KnowledgeBase::getCreatedAt));
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.persistence.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.deepdata.rag.indexer.domain.model.FileStatus;
|
||||
import com.deepdata.rag.indexer.domain.model.RagFile;
|
||||
import com.deepdata.rag.indexer.domain.repository.RagFileRepository;
|
||||
import com.deepdata.rag.indexer.infrastructure.persistence.mapper.RagFileMapper;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.RagFileReq;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 知识库文件仓储实现类
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Repository
|
||||
public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile> implements RagFileRepository {
|
||||
@Override
|
||||
public void removeByKnowledgeBaseId(String knowledgeBaseId) {
|
||||
lambdaUpdate().eq(RagFile::getKnowledgeBaseId, knowledgeBaseId).remove();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RagFile> findNotSuccessByKnowledgeBaseId(String knowledgeBaseId) {
|
||||
return lambdaQuery()
|
||||
.eq(RagFile::getKnowledgeBaseId, knowledgeBaseId)
|
||||
.in(RagFile::getStatus, FileStatus.UNPROCESSED, FileStatus.PROCESS_FAILED)
|
||||
.list();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RagFile> findAllByKnowledgeBaseId(String knowledgeBaseId) {
|
||||
return lambdaQuery()
|
||||
.eq(RagFile::getKnowledgeBaseId, knowledgeBaseId)
|
||||
.list();
|
||||
}
|
||||
|
||||
@Override
|
||||
public IPage<RagFile> page(IPage<RagFile> page, RagFileReq request) {
|
||||
return lambdaQuery()
|
||||
.eq(RagFile::getKnowledgeBaseId, request.getKnowledgeBaseId())
|
||||
.like(StringUtils.hasText(request.getFileName()), RagFile::getFileName, request.getFileName())
|
||||
.page(page);
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.persistence.mapper;
|
||||
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.deepdata.rag.indexer.domain.model.KnowledgeBase;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
/**
|
||||
* 知识库映射器接口
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Mapper
|
||||
public interface KnowledgeBaseMapper extends BaseMapper<KnowledgeBase> {
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.deepdata.rag.indexer.infrastructure.persistence.mapper;
|
||||
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.deepdata.common.infrastructure.config.IgnoreDataScopeAnnotation;
|
||||
import com.deepdata.rag.indexer.domain.model.RagFile;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
/**
|
||||
* RAG文件映射器接口
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Mapper
|
||||
@IgnoreDataScopeAnnotation
|
||||
public interface RagFileMapper extends BaseMapper<RagFile> {
|
||||
}
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
package com.deepdata.rag.indexer.interfaces;
|
||||
|
||||
import com.deepdata.common.interfaces.PagedResponse;
|
||||
import com.deepdata.common.interfaces.PagingQuery;
|
||||
import com.deepdata.rag.indexer.application.KnowledgeBaseService;
|
||||
import com.deepdata.rag.indexer.domain.model.RagChunk;
|
||||
import com.deepdata.rag.indexer.domain.model.RagFile;
|
||||
import com.deepdata.rag.indexer.interfaces.dto.*;
|
||||
import io.milvus.v2.service.vector.response.SearchResp;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/**
|
||||
* 知识库控制器
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-09-30
|
||||
*/
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/knowledge-base")
|
||||
public class KnowledgeBaseController {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
|
||||
/**
|
||||
* 创建知识库
|
||||
*
|
||||
* @param request 知识库创建请求
|
||||
* @return 知识库 ID
|
||||
*/
|
||||
@PostMapping("/create")
|
||||
public String create(@RequestBody @Valid KnowledgeBaseCreateReq request) {
|
||||
return knowledgeBaseService.create(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新知识库
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
* @param request 知识库更新请求
|
||||
*/
|
||||
@PutMapping("/{knowledgeBaseId}")
|
||||
public void update(@PathVariable("knowledgeBaseId") String knowledgeBaseId,
|
||||
@RequestBody @Valid KnowledgeBaseUpdateReq request) {
|
||||
knowledgeBaseService.update(knowledgeBaseId, request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除知识库
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
*/
|
||||
@DeleteMapping("/{knowledgeBaseId}")
|
||||
public void delete(@PathVariable("knowledgeBaseId") String knowledgeBaseId) {
|
||||
knowledgeBaseService.delete(knowledgeBaseId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取知识库
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
* @return 知识库
|
||||
*/
|
||||
@GetMapping("/{knowledgeBaseId}")
|
||||
public KnowledgeBaseResp get(@PathVariable("knowledgeBaseId") String knowledgeBaseId) {
|
||||
return knowledgeBaseService.getById(knowledgeBaseId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取知识库列表
|
||||
*
|
||||
* @return 知识库列表
|
||||
*/
|
||||
@PostMapping("/list")
|
||||
public PagedResponse<KnowledgeBaseResp> list(@RequestBody @Valid KnowledgeBaseQueryReq request) {
|
||||
return knowledgeBaseService.list(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加文件到知识库
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
* @param request 添加文件请求
|
||||
*/
|
||||
@PostMapping("/{knowledgeBaseId}/files")
|
||||
public void addFiles(@PathVariable("knowledgeBaseId") String knowledgeBaseId,
|
||||
@RequestBody @Valid AddFilesReq request) {
|
||||
request.setKnowledgeBaseId(knowledgeBaseId);
|
||||
knowledgeBaseService.addFiles(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取知识库文件列表
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
* @return 知识库文件列表
|
||||
*/
|
||||
@GetMapping("/{knowledgeBaseId}/files")
|
||||
public PagedResponse<RagFile> listFiles(@PathVariable("knowledgeBaseId") String knowledgeBaseId,
|
||||
RagFileReq request) {
|
||||
return knowledgeBaseService.listFiles(knowledgeBaseId, request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除知识库文件
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
* @param request 删除文件请求
|
||||
*/
|
||||
@DeleteMapping("/{knowledgeBaseId}/files")
|
||||
public void deleteFile(@PathVariable("knowledgeBaseId") String knowledgeBaseId,
|
||||
@RequestBody DeleteFilesReq request) {
|
||||
knowledgeBaseService.deleteFiles(knowledgeBaseId, request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 知识库文件详情
|
||||
*
|
||||
* @param knowledgeBaseId 知识库 ID
|
||||
* @param ragFileId 文件 ID
|
||||
* @return 文件详情
|
||||
*/
|
||||
@GetMapping("/{knowledgeBaseId}/files/{ragFileId}")
|
||||
public PagedResponse<RagChunk> getChunks(@PathVariable("knowledgeBaseId") String knowledgeBaseId,
|
||||
@PathVariable("ragFileId") String ragFileId,
|
||||
PagingQuery pagingQuery) {
|
||||
return knowledgeBaseService.getChunks(knowledgeBaseId, ragFileId, pagingQuery);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检索知识库内容
|
||||
*
|
||||
* @param request 检索请求
|
||||
* @return 检索结果
|
||||
*/
|
||||
@PostMapping("/retrieve")
|
||||
public List<SearchResp.SearchResult> retrieve(@RequestBody @Valid RetrieveReq request) {
|
||||
return knowledgeBaseService.retrieve(request);
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.deepdata.rag.indexer.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 添加文件请求
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-29
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class AddFilesReq {
|
||||
private String knowledgeBaseId;
|
||||
private ProcessType processType;
|
||||
private Integer chunkSize;
|
||||
private Integer overlapSize;
|
||||
private String delimiter;
|
||||
private List<FileInfo> files;
|
||||
|
||||
public record FileInfo(String id, String fileName) {
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.deepdata.rag.indexer.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 删除文件请求
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-29
|
||||
*/
|
||||
@Setter
|
||||
@Getter
|
||||
public class DeleteFilesReq {
|
||||
/**
|
||||
* Rag文件表主键ID列表
|
||||
*/
|
||||
private List<String> ids;
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.deepdata.rag.indexer.interfaces.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* 知识库创建请求
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-24
|
||||
*/
|
||||
@Setter
|
||||
@Getter
|
||||
public class KnowledgeBaseCreateReq {
|
||||
/**
|
||||
* 知识库名称
|
||||
*/
|
||||
@NotEmpty(message = "知识库名称不能为空")
|
||||
@Size(min = 1, max = 255, message = "知识库名称长度必须在 1 到 255 之间")
|
||||
@Pattern(regexp = "^[a-zA-Z][a-zA-Z0-9_]*$", message = "知识库名称只能包含字母、数字和下划线")
|
||||
private String name;
|
||||
/**
|
||||
* 知识库描述
|
||||
*/
|
||||
@Size(max = 512, message = "知识库描述长度必须在 0 到 512 之间")
|
||||
private String description;
|
||||
|
||||
private RagType type = RagType.DOCUMENT;
|
||||
|
||||
/**
|
||||
* 嵌入模型
|
||||
*/
|
||||
@NotEmpty(message = "嵌入模型不能为空")
|
||||
private String embeddingModel;
|
||||
|
||||
/**
|
||||
* 聊天模型
|
||||
*/
|
||||
private String chatModel;
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.deepdata.rag.indexer.interfaces.dto;
|
||||
|
||||
import com.deepdata.common.interfaces.PagingQuery;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
* @author dallas
|
||||
* @since 2025-10-29
|
||||
*/
|
||||
@Setter
|
||||
@Getter
|
||||
public class KnowledgeBaseQueryReq extends PagingQuery {
|
||||
private String name;
|
||||
private String description;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
private String createdBy;
|
||||
private String updatedBy;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user