deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,32 @@
+package com.deepdata.main;
+
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.cloud.openfeign.EnableFeignClients;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+/**
+ * 数据引擎平台主应用
+ * 聚合所有业务服务JAR包的微服务启动类
+ *
+ * @author Data Mate Team
+ * @version 1.0.0
+ */
+@SpringBootApplication
+@ComponentScan(basePackages = {"com.deepdata"})
+@MapperScan(basePackages = {"com.deepdata.**.mapper"})
+@EnableTransactionManagement
+@EnableAsync
+@EnableScheduling
+@EnableCaching
+@EnableFeignClients(basePackages = "com.deepdata.*")
+public class DeepDataApplication {
+    public static void main(String[] args) {
+        SpringApplication.run(DeepDataApplication.class, args);
+    }
+}
@@ -0,0 +1,26 @@
+package com.deepdata.main.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.web.SecurityFilterChain;
+
+/**
+ * 安全配置 - 暂时禁用所有认证
+ * 开发阶段使用，生产环境需要启用认证
+ */
+@Configuration
+@EnableWebSecurity
+public class SecurityConfig {
+
+    @Bean
+    public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
+        http.csrf(csrf -> csrf.disable())
+            .authorizeHttpRequests(authz -> authz
+                .anyRequest().permitAll()  // 允许所有请求无需认证
+            );
+
+        return http.build();
+    }
+}
@@ -0,0 +1,51 @@
+package com.deepdata.main.filter;
+
+import com.deepdata.common.infrastructure.common.Response;
+import com.deepdata.common.infrastructure.config.DataScopeHandle;
+import com.deepdata.common.infrastructure.exception.CommonErrorCode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.servlet.*;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+
+/**
+ * 用户信息过滤器
+ *
+ * @since 2026/1/19
+ */
+@Slf4j
+@Component
+public class UserContextFilter implements Filter {
+    private static final String USER_HEADER = "User";
+
+    @Value("${deepdata.jwt.enable:false}")
+    private Boolean jwtEnable;
+
+    @Override
+    public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) {
+        try {
+            HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
+            HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
+            String user = httpRequest.getHeader(USER_HEADER);
+            ObjectMapper objectMapper = new ObjectMapper();
+            if (jwtEnable && StringUtils.isBlank(user)) {
+                httpResponse.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+                httpResponse.getWriter().write(objectMapper.writeValueAsString(Response.error(CommonErrorCode.UNAUTHORIZED)));
+                return;
+            }
+            DataScopeHandle.setUserInfo(user);
+            filterChain.doFilter(servletRequest, servletResponse);
+        } catch (IOException | ServletException e) {
+            log.error("Request failed!");
+            throw new RuntimeException(e);
+        } finally {
+            DataScopeHandle.removeUserInfo();
+        }
+    }
+}
@@ -0,0 +1,188 @@
+# 数据引擎平台 - 主应用配置
+spring:
+  application:
+    name: deepdata
+
+  # 暂时排除Spring Security自动配置（开发阶段使用）
+  autoconfigure:
+    exclude:
+      - org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+      - org.springframework.boot.autoconfigure.security.servlet.UserDetailsServiceAutoConfiguration
+
+  # 数据源配置
+  datasource:
+    driver-class-name: org.postgresql.Driver
+    url: jdbc:postgresql://deepdata-database:5432/deepdata?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
+    username: ${DB_USERNAME:postgres}
+    password: ${DB_PASSWORD}
+    hikari:
+      maximum-pool-size: 20
+      minimum-idle: 5
+      connection-timeout: 30000
+      idle-timeout: 600000
+      max-lifetime: 1800000
+
+  # Jackson配置
+  jackson:
+    time-zone: Asia/Shanghai
+    date-format: yyyy-MM-dd HH:mm:ss
+    serialization:
+      write-dates-as-timestamps: false
+    deserialization:
+      fail-on-unknown-properties: false
+
+  # 文件上传配置
+  servlet:
+    multipart:
+      max-file-size: 100MB
+      max-request-size: 100MB
+
+  # 任务调度配置
+  task:
+    execution:
+      pool:
+        core-size: ${TASK_EXECUTION_CORE_SIZE:10}
+        max-size: ${TASK_EXECUTION_MAX_SIZE:20}
+        queue-capacity: ${TASK_EXECUTION_QUEUE_CAPACITY:100}
+        keep-alive: ${TASK_EXECUTION_KEEP_ALIVE:60s}
+    scheduling:
+      pool:
+        size: ${TASK_SCHEDULING_POOL_SIZE:5}
+  config:
+    import:
+      - classpath:config/application-datacollection.yml
+      - classpath:config/application-datamanagement.yml
+
+  # Redis配置
+  data:
+    redis:
+      host: deepdata-redis
+      port: 6379
+      timeout: 2000
+      password: ${REDIS_PASSWORD}
+      lettuce:
+        pool:
+          max-active: 20
+          max-idle: 10
+          min-idle: 5
+          max-wait: 1000ms
+
+  ai:
+    mcp:
+      server:
+        enabled: true
+        name: deepdata-mcp-server
+        protocol: STATELESS
+        capabilities:
+          completion: false
+          resource: false
+          prompt: false
+          tool: true
+        mcp-endpoint: /mcp
+
+
+# MyBatis配置（需在顶层，不在 spring 下）
+mybatis-plus:
+  configuration:
+    map-underscore-to-camel-case: true
+    default-fetch-size: 100
+    default-statement-timeout: 30
+    use-generated-keys: true
+    cache-enabled: true
+    lazy-loading-enabled: false
+    multiple-result-sets-enabled: true
+    use-column-label: true
+    auto-mapping-behavior: partial
+    auto-mapping-unknown-column-behavior: none
+    default-executor-type: simple
+    call-setters-on-nulls: false
+    return-instance-for-empty-row: false
+    log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl
+  mapper-locations:
+    - classpath*:mappers/**/*.xml
+
+# 应用配置
+server:
+  port: ${SERVER_PORT:8080}
+  servlet:
+    context-path: /api
+    encoding:
+      charset: UTF-8
+      enabled: true
+      force: true
+
+# 日志配置
+logging:
+  config: classpath:log4j2.xml
+
+# Actuator配置
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,metrics,prometheus
+  endpoint:
+    health:
+      show-details: when-authorized
+  health:
+    elasticsearch:
+      enabled: false  # 禁用Elasticsearch健康检查
+
+
+# 平台配置
+deepdata:
+  # JWT配置
+  jwt:
+    secret: ${JWT_SECRET}
+
+  # 通用配置
+
+
+  # 文件存储配置
+  storage:
+    type: ${STORAGE_TYPE:local} # local, minio, s3
+    local:
+      base-path: ${STORAGE_LOCAL_PATH:./data/storage}
+
+
+  # Ray执行器配置
+  ray:
+    enabled: ${RAY_ENABLED:false}
+    address: ${RAY_ADDRESS:ray://localhost:10001}
+    runtime-env:
+      working-dir: ${RAY_WORKING_DIR:./runtime/python-executor}
+      pip-packages:
+        - "ray[default]==2.7.0"
+        - "pandas"
+        - "numpy"
+        - "data-juicer"
+
+  # 模块配置
+
+  # 数据管理服务配置
+  data-management:
+    base-path: /dataset
+
+  # 数据归集服务配置（可由模块导入叠加）
+  data-collection: {}
+
+  # 算子市场配置
+  operator-market:
+    repository-path: ${OPERATOR_REPO_PATH:./runtime/operators}
+    registry-url: ${OPERATOR_REGISTRY_URL:}
+    max-upload-size: ${OPERATOR_MAX_UPLOAD_SIZE:50MB}
+
+  # 数据处理配置
+  data-processing:
+    max-file-size: ${MAX_FILE_SIZE:1GB}
+    temp-dir: ${TEMP_DIR:./data/temp}
+    batch-size: ${BATCH_SIZE:1000}
+
+  # 标注配置
+  annotation:
+    auto-annotation:
+      enabled: ${AUTO_ANNOTATION_ENABLED:true}
+      model-endpoint: ${ANNOTATION_MODEL_ENDPOINT:}
+    quality-control:
+      enabled: ${QC_ENABLED:true}
+      threshold: ${QC_THRESHOLD:0.8}
@@ -0,0 +1,23 @@
+deepdata:
+  data-collection:
+    # DataX配置
+    datax:
+      home-path: ${DATAX_HOME:/opt/datax}
+      python-path: ${DATAX_PYTHON_PATH:python3}
+      job-config-path: ${DATAX_JOB_PATH:./data/temp/datax/jobs}
+      log-path: ${DATAX_LOG_PATH:./logs/datax}
+      max-memory: ${DATAX_MAX_MEMORY:2048}
+      channel-count: ${DATAX_CHANNEL_COUNT:5}
+
+    # 执行配置
+    execution:
+      max-concurrent-tasks: ${DATA_COLLECTION_MAX_CONCURRENT_TASKS:10}
+      task-timeout-minutes: ${DATA_COLLECTION_TASK_TIMEOUT:120}
+      retry-count: ${DATA_COLLECTION_RETRY_COUNT:3}
+      retry-interval-seconds: ${DATA_COLLECTION_RETRY_INTERVAL:30}
+
+    # 监控配置
+    monitoring:
+      status-check-interval-seconds: ${DATA_COLLECTION_STATUS_CHECK_INTERVAL:30}
+      log-retention-days: ${DATA_COLLECTION_LOG_RETENTION:30}
+      enable-metrics: ${DATA_COLLECTION_ENABLE_METRICS:true}
@@ -0,0 +1,11 @@
+deepdata:
+  datamanagement:
+    file-storage:
+      upload-dir: ${FILE_UPLOAD_DIR:./uploads}
+      max-file-size: 10485760   # 10MB
+      max-request-size: 52428800 # 50MB
+    cache:
+      ttl: 3600
+      max-size: 1000
+# MyBatis is configured centrally in main-application (mapper-locations & aliases)
+# to avoid list overriding issues when importing multiple module configs.
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN" monitorInterval="30">
+    <Properties>
+        <Property name="LOG_PATH">/var/log/deepdata/backend</Property>
+        <Property name="LOG_PATTERN">%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n</Property>
+        <Property name="MAX_FILE_SIZE">${env:LOG_ROTATION_MAX_SIZE:-100MB}</Property>
+        <Property name="MAX_HISTORY">${env:LOG_ROTATION_BACKUP_COUNT:-30}</Property>
+        <Property name="LOG_LEVEL">INFO</Property>
+        <Property name="SQL_LOG_LEVEL">WARN</Property>
+    </Properties>
+
+    <Appenders>
+        <!-- 控制台输出 -->
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="${LOG_PATTERN}" />
+        </Console>
+
+        <!-- 滚动文件输出 -->
+        <RollingFile name="File" fileName="${LOG_PATH}/app.log" filePattern="${LOG_PATH}/app-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="${LOG_PATTERN}" />
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="${MAX_FILE_SIZE}" />
+            </Policies>
+            <DefaultRolloverStrategy max="${MAX_HISTORY}" />
+        </RollingFile>
+    </Appenders>
+
+    <Loggers>
+        <!-- 自定义日志器 -->
+        <Logger name="com.deepdata" level="${LOG_LEVEL}" />
+        <Logger name="org.springframework.security" level="${LOG_LEVEL}" />
+        <Logger name="org.hibernate.SQL" level="${SQL_LOG_LEVEL}" />
+        <Logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="${SQL_LOG_LEVEL}" />
+
+        <!-- 根日志器 -->
+        <Root level="INFO">
+            <AppenderRef ref="Console" />
+            <AppenderRef ref="File" />
+        </Root>
+    </Loggers>
+</Configuration>