deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,112 @@
+# 主应用
+
+## 概述
+
+主应用是 DeepData 的核心 Spring Boot 服务，包含数据管理、数据清洗、算子市场、数据收集等主要功能模块。
+
+## 架构
+
+```
+backend/services/main-application/
+├── src/main/java/com/deepdata/main/
+│   ├── interfaces/
+│   │   ├── rest/       # Controllers
+│   │   ├── dto/        # Request/Response DTOs
+│   │   └── converter/   # MapStruct converters
+│   ├── application/     # Application services
+│   ├── domain/
+│   │   ├── model/       # Entities
+│   │   └── repository/  # Repository interfaces
+│   └── infrastructure/
+│       ├── persistence/  # Repository implementations
+│       ├── client/       # External API clients
+│       └── config/       # Service configuration
+└── src/main/resources/
+    ├── application.yml                # 主配置
+    ├── config/application-datamanagement.yml  # 数据管理配置
+    └── config/application-datacollection.yml   # 数据收集配置
+```
+
+## 模块
+
+### 1. 数据管理
+- 数据集 CRUD 操作
+- 文件上传/下载
+- 标签管理
+- 数据集版本控制
+
+### 2. 数据收集
+- 数据源配置
+- 定时数据收集任务
+- 数据同步
+- 数据导入/导出
+
+## 配置
+
+### 端口
+- **默认**: 8080
+- **上下文路径**: `/api`
+
+### 关键配置
+```yaml
+server:
+  port: 8080
+  servlet:
+    context-path: /api
+
+deepdata:
+  data-management:
+    base-path: /dataset
+```
+
+## 快速开始
+
+### 前置条件
+- JDK 21+
+- Maven 3.8+
+- PostgreSQL 12+
+- Redis 6+
+
+### 构建
+```bash
+cd backend/services/main-application
+mvn clean install
+```
+
+### 运行
+```bash
+cd backend/services/main-application
+mvn spring-boot:run
+```
+
+## 开发
+
+### 添加新模块
+1. 在 `domain/model/` 创建实体类
+2. 在 `domain/repository/` 创建 repository 接口
+3. 在 `infrastructure/persistence/` 实现 repository
+4. 在 `application/` 创建 application service
+5. 在 `interfaces/rest/` 创建 controller
+
+## 测试
+
+### 运行测试
+```bash
+cd backend/services/main-application
+mvn test
+```
+
+### 运行特定测试
+```bash
+mvn test -Dtest=DatasetControllerTest
+```
+
+## 文档
+
+- **Spring Boot 文档**: https://docs.spring.io/spring-boot/
+- [AGENTS.md](../../shared/AGENTS.md)
+
+## 相关链接
+
+- [后端 README](../../README.md)
+- [API Gateway README](../../api-gateway/README.md)
@@ -0,0 +1,112 @@
+# Main Application
+
+## Overview
+
+The Main Application is DeepData's core Spring Boot service, containing major functional modules including data management, data cleaning, operator marketplace, and data collection.
+
+## Architecture
+
+```
+backend/services/main-application/
+├── src/main/java/com/deepdata/main/
+│   ├── interfaces/
+│   │   ├── rest/       # Controllers
+│   │   ├── dto/        # Request/Response DTOs
+│   │   └── converter/   # MapStruct converters
+│   ├── application/     # Application services
+│   ├── domain/
+│   │   ├── model/       # Entities
+│   │   └── repository/  # Repository interfaces
+│   └── infrastructure/
+│       ├── persistence/  # Repository implementations
+│       ├── client/       # External API clients
+│       └── config/       # Service configuration
+└── src/main/resources/
+    ├── application.yml                # Main configuration
+    ├── config/application-datamanagement.yml  # Data management config
+    └── config/application-datacollection.yml   # Data collection config
+```
+
+## Modules
+
+### 1. Data Management
+- Dataset CRUD operations
+- File upload/download
+- Tag management
+- Dataset versioning
+
+### 2. Data Collection
+- Data source configuration
+- Scheduled data collection tasks
+- Data synchronization
+- Data import/export
+
+## Configuration
+
+### Port
+- **Default**: 8080
+- **Context Path**: `/api`
+
+### Key Configuration
+```yaml
+server:
+  port: 8080
+  servlet:
+    context-path: /api
+
+deepdata:
+  data-management:
+    base-path: /dataset
+```
+
+## Quick Start
+
+### Prerequisites
+- JDK 21+
+- Maven 3.8+
+- PostgreSQL 12+
+- Redis 6+
+
+### Build
+```bash
+cd backend/services/main-application
+mvn clean install
+```
+
+### Run
+```bash
+cd backend/services/main-application
+mvn spring-boot:run
+```
+
+## Development
+
+### Adding a New Module
+1. Create entity class in `domain/model/`
+2. Create repository interface in `domain/repository/`
+3. Implement repository in `infrastructure/persistence/`
+4. Create application service in `application/`
+5. Create controller in `interfaces/rest/`
+
+## Testing
+
+### Run Tests
+```bash
+cd backend/services/main-application
+mvn test
+```
+
+### Run Specific Test
+```bash
+mvn test -Dtest=DatasetControllerTest
+```
+
+## Documentation
+
+- **Spring Boot Docs**: https://docs.spring.io/spring-boot/
+- [AGENTS.md](../../shared/AGENTS.md)
+
+## Related Links
+
+- [Backend README](../../README.md)
+- [API Gateway README](../../api-gateway/README.md)
@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+         http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.deepdata</groupId>
+        <artifactId>services</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>main-application</artifactId>
+    <packaging>jar</packaging>
+    <name>Data Mate Platform - Main Application</name>
+    <description>主启动应用，集成所有服务模</description>
+
+    <dependencies>
+        <!-- Spring Boot Starters -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+
+        <dependency>
+          <groupId>jakarta.persistence</groupId>
+          <artifactId>jakarta.persistence-api</artifactId>
+        </dependency>
+
+      <!-- 共享库依赖 -->
+        <dependency>
+            <groupId>com.deepdata</groupId>
+            <artifactId>domain-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.deepdata</groupId>
+            <artifactId>security-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+          <groupId>org.apache.commons</groupId>
+          <artifactId>commons-compress</artifactId>
+        </dependency>
+
+        <!-- 核心服务依赖 -->
+        <dependency>
+            <groupId>com.deepdata</groupId>
+            <artifactId>data-management-service</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <!-- RAG服务依赖 -->
+        <dependency>
+            <groupId>com.deepdata</groupId>
+            <artifactId>rag-indexer-service</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <!-- Database -->
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+        </dependency>
+
+        <!-- 测试依赖 -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-autoconfigure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-redis</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.11.0</version>
+                <configuration>
+                    <source>${maven.compiler.source}</source>
+                    <target>${maven.compiler.target}</target>
+                    <parameters>true</parameters>
+                    <compilerArgs>
+                        <arg>-parameters</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>${spring-boot.version}</version>
+                <configuration>
+                    <finalName>deepdata</finalName>
+                    <mainClass>com.deepdata.main.DeepDataApplication</mainClass>
+                </configuration>
+                <executions>
+                  <execution>
+                    <goals>
+                      <goal>repackage</goal>
+                    </goals>
+                  </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
+
@@ -0,0 +1,32 @@
+package com.deepdata.main;
+
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.cloud.openfeign.EnableFeignClients;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+/**
+ * 数据引擎平台主应用
+ * 聚合所有业务服务JAR包的微服务启动类
+ *
+ * @author Data Mate Team
+ * @version 1.0.0
+ */
+@SpringBootApplication
+@ComponentScan(basePackages = {"com.deepdata"})
+@MapperScan(basePackages = {"com.deepdata.**.mapper"})
+@EnableTransactionManagement
+@EnableAsync
+@EnableScheduling
+@EnableCaching
+@EnableFeignClients(basePackages = "com.deepdata.*")
+public class DeepDataApplication {
+    public static void main(String[] args) {
+        SpringApplication.run(DeepDataApplication.class, args);
+    }
+}
@@ -0,0 +1,26 @@
+package com.deepdata.main.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.web.SecurityFilterChain;
+
+/**
+ * 安全配置 - 暂时禁用所有认证
+ * 开发阶段使用，生产环境需要启用认证
+ */
+@Configuration
+@EnableWebSecurity
+public class SecurityConfig {
+
+    @Bean
+    public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
+        http.csrf(csrf -> csrf.disable())
+            .authorizeHttpRequests(authz -> authz
+                .anyRequest().permitAll()  // 允许所有请求无需认证
+            );
+
+        return http.build();
+    }
+}
@@ -0,0 +1,51 @@
+package com.deepdata.main.filter;
+
+import com.deepdata.common.infrastructure.common.Response;
+import com.deepdata.common.infrastructure.config.DataScopeHandle;
+import com.deepdata.common.infrastructure.exception.CommonErrorCode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.servlet.*;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+
+/**
+ * 用户信息过滤器
+ *
+ * @since 2026/1/19
+ */
+@Slf4j
+@Component
+public class UserContextFilter implements Filter {
+    private static final String USER_HEADER = "User";
+
+    @Value("${deepdata.jwt.enable:false}")
+    private Boolean jwtEnable;
+
+    @Override
+    public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) {
+        try {
+            HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
+            HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
+            String user = httpRequest.getHeader(USER_HEADER);
+            ObjectMapper objectMapper = new ObjectMapper();
+            if (jwtEnable && StringUtils.isBlank(user)) {
+                httpResponse.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+                httpResponse.getWriter().write(objectMapper.writeValueAsString(Response.error(CommonErrorCode.UNAUTHORIZED)));
+                return;
+            }
+            DataScopeHandle.setUserInfo(user);
+            filterChain.doFilter(servletRequest, servletResponse);
+        } catch (IOException | ServletException e) {
+            log.error("Request failed!");
+            throw new RuntimeException(e);
+        } finally {
+            DataScopeHandle.removeUserInfo();
+        }
+    }
+}
@@ -0,0 +1,188 @@
+# 数据引擎平台 - 主应用配置
+spring:
+  application:
+    name: deepdata
+
+  # 暂时排除Spring Security自动配置（开发阶段使用）
+  autoconfigure:
+    exclude:
+      - org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+      - org.springframework.boot.autoconfigure.security.servlet.UserDetailsServiceAutoConfiguration
+
+  # 数据源配置
+  datasource:
+    driver-class-name: org.postgresql.Driver
+    url: jdbc:postgresql://deepdata-database:5432/deepdata?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
+    username: ${DB_USERNAME:postgres}
+    password: ${DB_PASSWORD}
+    hikari:
+      maximum-pool-size: 20
+      minimum-idle: 5
+      connection-timeout: 30000
+      idle-timeout: 600000
+      max-lifetime: 1800000
+
+  # Jackson配置
+  jackson:
+    time-zone: Asia/Shanghai
+    date-format: yyyy-MM-dd HH:mm:ss
+    serialization:
+      write-dates-as-timestamps: false
+    deserialization:
+      fail-on-unknown-properties: false
+
+  # 文件上传配置
+  servlet:
+    multipart:
+      max-file-size: 100MB
+      max-request-size: 100MB
+
+  # 任务调度配置
+  task:
+    execution:
+      pool:
+        core-size: ${TASK_EXECUTION_CORE_SIZE:10}
+        max-size: ${TASK_EXECUTION_MAX_SIZE:20}
+        queue-capacity: ${TASK_EXECUTION_QUEUE_CAPACITY:100}
+        keep-alive: ${TASK_EXECUTION_KEEP_ALIVE:60s}
+    scheduling:
+      pool:
+        size: ${TASK_SCHEDULING_POOL_SIZE:5}
+  config:
+    import:
+      - classpath:config/application-datacollection.yml
+      - classpath:config/application-datamanagement.yml
+
+  # Redis配置
+  data:
+    redis:
+      host: deepdata-redis
+      port: 6379
+      timeout: 2000
+      password: ${REDIS_PASSWORD}
+      lettuce:
+        pool:
+          max-active: 20
+          max-idle: 10
+          min-idle: 5
+          max-wait: 1000ms
+
+  ai:
+    mcp:
+      server:
+        enabled: true
+        name: deepdata-mcp-server
+        protocol: STATELESS
+        capabilities:
+          completion: false
+          resource: false
+          prompt: false
+          tool: true
+        mcp-endpoint: /mcp
+
+
+# MyBatis配置（需在顶层，不在 spring 下）
+mybatis-plus:
+  configuration:
+    map-underscore-to-camel-case: true
+    default-fetch-size: 100
+    default-statement-timeout: 30
+    use-generated-keys: true
+    cache-enabled: true
+    lazy-loading-enabled: false
+    multiple-result-sets-enabled: true
+    use-column-label: true
+    auto-mapping-behavior: partial
+    auto-mapping-unknown-column-behavior: none
+    default-executor-type: simple
+    call-setters-on-nulls: false
+    return-instance-for-empty-row: false
+    log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl
+  mapper-locations:
+    - classpath*:mappers/**/*.xml
+
+# 应用配置
+server:
+  port: ${SERVER_PORT:8080}
+  servlet:
+    context-path: /api
+    encoding:
+      charset: UTF-8
+      enabled: true
+      force: true
+
+# 日志配置
+logging:
+  config: classpath:log4j2.xml
+
+# Actuator配置
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,metrics,prometheus
+  endpoint:
+    health:
+      show-details: when-authorized
+  health:
+    elasticsearch:
+      enabled: false  # 禁用Elasticsearch健康检查
+
+
+# 平台配置
+deepdata:
+  # JWT配置
+  jwt:
+    secret: ${JWT_SECRET}
+
+  # 通用配置
+
+
+  # 文件存储配置
+  storage:
+    type: ${STORAGE_TYPE:local} # local, minio, s3
+    local:
+      base-path: ${STORAGE_LOCAL_PATH:./data/storage}
+
+
+  # Ray执行器配置
+  ray:
+    enabled: ${RAY_ENABLED:false}
+    address: ${RAY_ADDRESS:ray://localhost:10001}
+    runtime-env:
+      working-dir: ${RAY_WORKING_DIR:./runtime/python-executor}
+      pip-packages:
+        - "ray[default]==2.7.0"
+        - "pandas"
+        - "numpy"
+        - "data-juicer"
+
+  # 模块配置
+
+  # 数据管理服务配置
+  data-management:
+    base-path: /dataset
+
+  # 数据归集服务配置（可由模块导入叠加）
+  data-collection: {}
+
+  # 算子市场配置
+  operator-market:
+    repository-path: ${OPERATOR_REPO_PATH:./runtime/operators}
+    registry-url: ${OPERATOR_REGISTRY_URL:}
+    max-upload-size: ${OPERATOR_MAX_UPLOAD_SIZE:50MB}
+
+  # 数据处理配置
+  data-processing:
+    max-file-size: ${MAX_FILE_SIZE:1GB}
+    temp-dir: ${TEMP_DIR:./data/temp}
+    batch-size: ${BATCH_SIZE:1000}
+
+  # 标注配置
+  annotation:
+    auto-annotation:
+      enabled: ${AUTO_ANNOTATION_ENABLED:true}
+      model-endpoint: ${ANNOTATION_MODEL_ENDPOINT:}
+    quality-control:
+      enabled: ${QC_ENABLED:true}
+      threshold: ${QC_THRESHOLD:0.8}
@@ -0,0 +1,23 @@
+deepdata:
+  data-collection:
+    # DataX配置
+    datax:
+      home-path: ${DATAX_HOME:/opt/datax}
+      python-path: ${DATAX_PYTHON_PATH:python3}
+      job-config-path: ${DATAX_JOB_PATH:./data/temp/datax/jobs}
+      log-path: ${DATAX_LOG_PATH:./logs/datax}
+      max-memory: ${DATAX_MAX_MEMORY:2048}
+      channel-count: ${DATAX_CHANNEL_COUNT:5}
+
+    # 执行配置
+    execution:
+      max-concurrent-tasks: ${DATA_COLLECTION_MAX_CONCURRENT_TASKS:10}
+      task-timeout-minutes: ${DATA_COLLECTION_TASK_TIMEOUT:120}
+      retry-count: ${DATA_COLLECTION_RETRY_COUNT:3}
+      retry-interval-seconds: ${DATA_COLLECTION_RETRY_INTERVAL:30}
+
+    # 监控配置
+    monitoring:
+      status-check-interval-seconds: ${DATA_COLLECTION_STATUS_CHECK_INTERVAL:30}
+      log-retention-days: ${DATA_COLLECTION_LOG_RETENTION:30}
+      enable-metrics: ${DATA_COLLECTION_ENABLE_METRICS:true}
@@ -0,0 +1,11 @@
+deepdata:
+  datamanagement:
+    file-storage:
+      upload-dir: ${FILE_UPLOAD_DIR:./uploads}
+      max-file-size: 10485760   # 10MB
+      max-request-size: 52428800 # 50MB
+    cache:
+      ttl: 3600
+      max-size: 1000
+# MyBatis is configured centrally in main-application (mapper-locations & aliases)
+# to avoid list overriding issues when importing multiple module configs.
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN" monitorInterval="30">
+    <Properties>
+        <Property name="LOG_PATH">/var/log/deepdata/backend</Property>
+        <Property name="LOG_PATTERN">%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n</Property>
+        <Property name="MAX_FILE_SIZE">${env:LOG_ROTATION_MAX_SIZE:-100MB}</Property>
+        <Property name="MAX_HISTORY">${env:LOG_ROTATION_BACKUP_COUNT:-30}</Property>
+        <Property name="LOG_LEVEL">INFO</Property>
+        <Property name="SQL_LOG_LEVEL">WARN</Property>
+    </Properties>
+
+    <Appenders>
+        <!-- 控制台输出 -->
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="${LOG_PATTERN}" />
+        </Console>
+
+        <!-- 滚动文件输出 -->
+        <RollingFile name="File" fileName="${LOG_PATH}/app.log" filePattern="${LOG_PATH}/app-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="${LOG_PATTERN}" />
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="${MAX_FILE_SIZE}" />
+            </Policies>
+            <DefaultRolloverStrategy max="${MAX_HISTORY}" />
+        </RollingFile>
+    </Appenders>
+
+    <Loggers>
+        <!-- 自定义日志器 -->
+        <Logger name="com.deepdata" level="${LOG_LEVEL}" />
+        <Logger name="org.springframework.security" level="${LOG_LEVEL}" />
+        <Logger name="org.hibernate.SQL" level="${SQL_LOG_LEVEL}" />
+        <Logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="${SQL_LOG_LEVEL}" />
+
+        <!-- 根日志器 -->
+        <Root level="INFO">
+            <AppenderRef ref="Console" />
+            <AppenderRef ref="File" />
+        </Root>
+    </Loggers>
+</Configuration>