deepdata: delivery stack (docker compose, helm, CI, build scripts)

2026-08-30 19:23:42 +08:00
parent bd3db30be9
commit 7bbc9d4ef0
852 changed files with 184030 additions and 0 deletions
@@ -0,0 +1,80 @@
+# syntax=docker/dockerfile:1
+FROM maven:3-eclipse-temurin-8 AS datax-builder
+
+# 仅安装必要的 git
+RUN apt-get update && \
+    apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/*
+
+# 🟢 修改 1：恢复 git clone 下载完整源码，本地先git clone DataX
+# RUN git clone https://github.com/alibaba/DataX.git
+COPY DataX/ /DataX/
+
+# 🟢 修改 2：注释掉 COPY，因为它覆盖了完整代码且不完整
+COPY runtime/datax/ DataX/
+
+# 🟢 优化 1：挂载 Maven 缓存 + 执行修复与编译
+# 注意：这里恢复了之前讨论的 MySQL 驱动修复（sed），因为源码完整了，路径也就对了
+RUN --mount=type=cache,target=/root/.m2 \
+    cd DataX && \
+    sed -i "s/com.mysql.jdbc.Driver/com.mysql.cj.jdbc.Driver/g" \
+    plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java && \
+    mvn -U clean package assembly:assembly -Dmaven.test.skip=true
+
+FROM python:3.12-slim
+
+# 🟢 优化 2：挂载 APT 缓存，加速 apt-get update
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt-get update && \
+    apt-get install -y --no-install-recommends vim openjdk-21-jre nfs-common glusterfs-client rsync dos2unix && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    POETRY_VERSION=2.2.1 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk
+ENV PATH="/root/.local/bin:$JAVA_HOME/bin:$PATH"
+
+WORKDIR /app
+
+# 安装 Poetry
+RUN pip install --upgrade --root-user-action=ignore pip \
+    && pip install --root-user-action=ignore pipx \
+    && pipx install "poetry==$POETRY_VERSION"
+
+# 复制依赖文件
+COPY runtime/deepdata-python/pyproject.toml runtime/deepdata-python/poetry.lock* /app/
+
+# 🟢 优化 3：挂载 Poetry 缓存，避免重复下载 Python 包
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
+    poetry install --no-root --only main
+
+# 🆕 新增：安装 Ray SDK (用于提交任务到 Ray 集群)
+# 注意：使用 [default] 确保包含 job submission client
+RUN pip install "ray[default]>=2.9.0"
+
+# 下载 NLTK 数据
+RUN python -c "import nltk; nltk.download(['punkt_tab','averaged_perceptron_tagger_eng'], download_dir='/usr/local/nltk_data')"
+ENV NLTK_DATA=/usr/local/nltk_data
+
+# 复制应用代码
+COPY runtime/deepdata-python /app
+COPY runtime/ops/examples/test_operator/test_operator.tar /app/test_operator.tar
+
+COPY --from=datax-builder /DataX/target/datax/datax /opt/datax
+RUN cp /opt/datax/plugin/reader/mysqlreader/libs/mysql* /opt/datax/plugin/reader/starrocksreader/libs/
+
+COPY runtime/deepdata-python/deploy/docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh \
+    && dos2unix /docker-entrypoint.sh || true \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+# Expose the application port
+EXPOSE 18000
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
@@ -0,0 +1,31 @@
+FROM maven:3-eclipse-temurin-21 AS builder
+COPY backend/ /opt/backend
+
+#RUN cd /opt/backend/ && mvn -U clean package -Dmaven.test.skip=true
+
+RUN --mount=type=cache,target=/root/.m2 \
+    cd /opt/backend/ && mvn -U clean package -Dmaven.test.skip=true
+
+
+FROM eclipse-temurin:21-jdk
+
+RUN apt-get update && \
+    apt-get install -y vim wget curl rsync python3 python3-pip python-is-python3 dos2unix && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /opt/backend/services/main-application/target/deepdata.jar /opt/backend/deepdata.jar
+
+COPY scripts/images/backend/start.sh /opt/backend/start.sh
+
+COPY runtime/ops/examples/test_operator/test_operator.tar /opt/backend/test_operator.tar
+
+RUN dos2unix /opt/backend/start.sh \
+    && chmod +x /opt/backend/start.sh \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+EXPOSE 8080
+
+ENTRYPOINT ["/opt/backend/start.sh"]
+
+CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/backend/deepdata.jar"]
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+umask 0022
+
+echo "Starting main application..."
+exec "$@"
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+set -euo pipefail
+
+run_once=false
+if [ "${1:-}" = "--once" ]; then
+    run_once=true
+    shift
+fi
+
+LOG_ROOT="${1:-${LOG_ROTATION_PATH:-/tmp/ray}}"
+LOG_ROTATION_MAX_SIZE="${LOG_ROTATION_MAX_SIZE:-100MB}"
+LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
+LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
+
+parse_size_bytes() {
+    local value
+    value="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
+    value="${value// /}"
+    case "$value" in
+        *GB) echo $((${value%GB} * 1024 * 1024 * 1024)) ;;
+        *G) echo $((${value%G} * 1024 * 1024 * 1024)) ;;
+        *MB) echo $((${value%MB} * 1024 * 1024)) ;;
+        *M) echo $((${value%M} * 1024 * 1024)) ;;
+        *KB) echo $((${value%KB} * 1024)) ;;
+        *K) echo $((${value%K} * 1024)) ;;
+        *B) echo "${value%B}" ;;
+        *) echo "$value" ;;
+    esac
+}
+
+file_size_bytes() {
+    wc -c < "$1" | tr -d ' '
+}
+
+backup_path() {
+    local file="$1"
+    local index="$2"
+    local ext="${file##*.}"
+    local stem="${file%.$ext}"
+    echo "${stem}.${index}.${ext}"
+}
+
+is_backup_file() {
+    local name
+    name="$(basename "$1")"
+    [[ "$name" =~ \.[0-9]+\.(log|out|err)$ ]]
+}
+
+rotate_file() {
+    local file="$1"
+    local max_bytes="$2"
+    local backup_count="$3"
+
+    [ -f "$file" ] || return 0
+    is_backup_file "$file" && return 0
+
+    local current_size
+    current_size="$(file_size_bytes "$file")"
+    [ "$current_size" -lt "$max_bytes" ] && return 0
+
+    rm -f "$(backup_path "$file" "$backup_count")"
+
+    local index
+    for ((index = backup_count - 1; index >= 1; index--)); do
+        local src dst
+        src="$(backup_path "$file" "$index")"
+        dst="$(backup_path "$file" "$((index + 1))")"
+        [ -f "$src" ] && mv "$src" "$dst"
+    done
+
+    cp -p "$file" "$(backup_path "$file" 1)"
+    : > "$file"
+}
+
+rotate_once() {
+    [ -d "$LOG_ROOT" ] || return 0
+
+    local max_bytes
+    max_bytes="$(parse_size_bytes "$LOG_ROTATION_MAX_SIZE")"
+
+    while IFS= read -r -d '' file; do
+        rotate_file "$file" "$max_bytes" "$LOG_ROTATION_BACKUP_COUNT"
+    done < <(
+        find "$LOG_ROOT" -type f \( -name '*.log' -o -name '*.out' -o -name '*.err' \) -print0 2>/dev/null || true
+    )
+}
+
+if [ "$run_once" = true ]; then
+    rotate_once
+    exit 0
+fi
+
+while true; do
+    rotate_once
+    sleep "$LOG_ROTATION_INTERVAL_SECONDS"
+done
@@ -0,0 +1,22 @@
+FROM postgres:15-trixie
+
+# 创建日志目录并设置权限
+USER root
+RUN mkdir -p /var/log/deepdata/database && \
+    chown -R postgres:postgres /var/log/deepdata && \
+    chmod -R 755 /var/log/deepdata
+
+ENV POSTGRES_INITDB_ARGS="--encoding=UTF8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8"
+# 创建配置目录
+RUN mkdir -p /etc/postgresql/
+# 复制 PostgreSQL 主配置文件
+COPY scripts/images/database/postgresql.cnf /etc/postgresql/postgresql.conf
+COPY scripts/images/database/prune-postgres-logs.sh /usr/local/bin/prune-postgres-logs.sh
+COPY scripts/images/database/start.sh /usr/local/bin/deepdata-database-start.sh
+
+COPY scripts/db/ /docker-entrypoint-initdb.d/
+
+RUN chmod +x /usr/local/bin/prune-postgres-logs.sh /usr/local/bin/deepdata-database-start.sh
+
+ENTRYPOINT ["/usr/local/bin/deepdata-database-start.sh"]
+CMD ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
@@ -0,0 +1,25 @@
+# 1. 字符集设置
+client_encoding = 'UTF8'
+# server_encoding = 'UTF8' # 只能启动时设置，无法通过修改配置文件设置
+# 2. 排序规则
+# lc_collate = 'en_US.UTF-8' # 只能启动时设置，无法通过修改配置文件设置
+# lc_ctype = 'en_US.UTF-8' # 只能启动时设置，无法通过修改配置文件设置
+# 3. 时区设置
+timezone = 'Asia/Shanghai'
+log_timezone = 'Asia/Shanghai'
+# 4. 日志设置
+log_destination = 'stderr'
+logging_collector = on
+log_directory = '/var/log/deepdata/database'
+log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'
+log_file_mode = 0644
+log_rotation_age = 1d
+log_rotation_size = 100MB
+log_truncate_on_rotation = off
+log_error_verbosity = default
+log_statement = 'ddl'
+log_duration = off
+log_lock_waits = on
+log_min_duration_statement = 1000
+# 5. 监听配置
+listen_addresses = '*'
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -euo pipefail
+
+LOG_DIR="${POSTGRES_LOG_DIR:-/var/log/deepdata/database}"
+LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
+LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
+
+prune_once() {
+    [ -d "$LOG_DIR" ] || return 0
+
+    mapfile -t files < <(
+        find "$LOG_DIR" -maxdepth 1 -type f -name 'postgresql-*.log' -printf '%T@ %p\n' 2>/dev/null \
+            | sort -rn \
+            | sed 's/^[^ ]* //'
+    )
+
+    local index
+    for ((index = LOG_ROTATION_BACKUP_COUNT; index < ${#files[@]}; index++)); do
+        rm -f "${files[$index]}"
+    done
+}
+
+if [ "${1:-}" = "--once" ]; then
+    prune_once
+    exit 0
+fi
+
+while true; do
+    prune_once
+    sleep "$LOG_ROTATION_INTERVAL_SECONDS"
+done
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+/usr/local/bin/prune-postgres-logs.sh &
+
+exec docker-entrypoint.sh "$@"
@@ -0,0 +1,29 @@
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm
+
+# Install uv.
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+
+# Install system dependencies including libpq
+RUN apt-get update && apt-get install -y \
+    libpq-dev git \
+    && rm -rf /var/lib/apt/lists/*
+    
+WORKDIR /app
+
+# Copy the application into the container.
+COPY deepflow /app/
+COPY runtime/deepflow/.env /app/.env
+COPY runtime/deepflow/conf.yaml /app/conf.yaml
+
+# Pre-cache the application dependencies.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-install-project
+
+# Install the application dependencies.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked
+
+EXPOSE 8000
+
+# Run the application.
+CMD ["uv", "run", "--no-sync", "python", "server.py", "--host", "0.0.0.0", "--port", "8000"]
@@ -0,0 +1,66 @@
+##### DEPENDENCIES
+
+FROM node:20-alpine AS deps
+RUN apk add --no-cache libc6-compat openssl
+WORKDIR /app
+
+# 原git拉取逻辑注释，改用本地COPY，本地上下文目录存在 deepflow 文件夹
+#     && mv /deepflow/web/* /app \
+#     && rm -rf /deepflow
+
+# 将本地 deepflow/web 下全部内容复制到镜像 /app
+COPY deepflow/web/ ./
+
+# Install dependencies based on the preferred package manager
+RUN if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+    elif [ -f package-lock.json ]; then npm ci; \
+    elif [ -f pnpm-lock.yaml ]; then npm install -g pnpm && pnpm i; \
+    else echo "Lockfile not found." && exit 1; \
+    fi
+
+##### BUILDER
+
+FROM node:20-alpine AS builder
+
+RUN apk add --no-cache git
+WORKDIR /app
+ARG NEXT_PUBLIC_API_URL="/deepflow-backend"
+
+# 原git拉取逻辑注释
+# RUN cd /deepflow \
+#     && mv /deepflow/web/* /app \
+#     && rm -rf /deepflow
+
+# 本地复制web目录源码
+COPY deepflow/web/ ./
+
+COPY --from=deps /app/node_modules ./node_modules
+
+ENV NEXT_TELEMETRY_DISABLED=1
+
+RUN if [ -f yarn.lock ]; then SKIP_ENV_VALIDATION=1 yarn build; \
+    elif [ -f package-lock.json ]; then SKIP_ENV_VALIDATION=1 npm run build; \
+    elif [ -f pnpm-lock.yaml ]; then npm install -g pnpm && SKIP_ENV_VALIDATION=1 pnpm run build; \
+    else echo "Lockfile not found." && exit 1; \
+    fi
+
+##### RUNNER
+
+FROM node:20-alpine AS runner
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+ENV NEXT_TELEMETRY_DISABLED=1
+
+COPY --from=builder /app/next.config.js ./
+COPY --from=builder /app/public ./public
+COPY --from=builder /app/package.json ./package.json
+
+COPY --from=builder /app/.next/standalone ./
+COPY --from=builder /app/.next/static ./.next/static
+
+EXPOSE 3000
+ENV PORT=3000
+
+CMD ["server.js"]
@@ -0,0 +1,41 @@
+FROM node:20-alpine AS builder
+
+WORKDIR /app
+
+COPY frontend ./
+
+RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi && \
+    npm run build
+
+FROM nginx:1.29 AS runner
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt update \
+    && apt install -y dos2unix python3 python3-dev python3-venv libaugeas-dev gcc vim cron \
+    && python3 -m venv /opt/certbot/ \
+    && /opt/certbot/bin/pip install --upgrade pip \
+    && /opt/certbot/bin/pip install certbot certbot-nginx \
+    && ln -s /opt/certbot/bin/certbot /usr/local/bin/certbot
+
+COPY --from=builder /app/dist /opt/frontend/statics
+COPY scripts/images/frontend/routes.inc /opt/frontend/routes.inc
+COPY scripts/images/frontend/http_backend.conf /opt/frontend/http_backend.conf
+COPY scripts/images/frontend/https_backend.conf /opt/frontend/https_backend.conf
+COPY scripts/images/frontend/nginx.conf /opt/frontend/nginx.conf
+COPY scripts/images/frontend/rotate-nginx-logs.sh /usr/local/bin/rotate-nginx-logs.sh
+
+COPY scripts/images/frontend/start.sh /opt/frontend/start.sh
+
+RUN dos2unix /opt/frontend/start.sh \
+    && dos2unix /usr/local/bin/rotate-nginx-logs.sh \
+    && chmod +x /opt/frontend/start.sh \
+    && chmod +x /usr/local/bin/rotate-nginx-logs.sh \
+    && mkdir -p /etc/nginx/cert \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && rm -f /etc/nginx/nginx.conf \
+    && rm -f /etc/nginx/conf.d/default.conf
+
+EXPOSE 3000
+
+CMD ["/opt/frontend/start.sh"]
@@ -0,0 +1,13 @@
+server {
+    listen       3000;
+    server_name  0.0.0.0;
+
+    access_log /var/log/deepdata/frontend/access.log main;
+    error_log /var/log/deepdata/frontend/error.log notice;
+
+    client_max_body_size 1024M;
+
+    add_header Set-Cookie "NEXT_LOCALE=zh";
+
+    include /etc/nginx/conf.d/routes.inc;
+}
@@ -0,0 +1,18 @@
+server {
+    listen       3000 ssl;
+    server_name  0.0.0.0;
+
+    access_log /var/log/deepdata/frontend/access.log main;
+    error_log /var/log/deepdata/frontend/error.log notice;
+
+    client_max_body_size 1024M;
+
+    ssl_certificate     /etc/nginx/cert/server.pem;
+    ssl_certificate_key /etc/nginx/cert/server.key;
+
+    error_page 497 https://$host$request_uri;
+
+    add_header Set-Cookie "NEXT_LOCALE=zh";
+
+    include /etc/nginx/conf.d/routes.inc;
+}
@@ -0,0 +1,32 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/deepdata/frontend/error.log notice;
+pid        /run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/deepdata/frontend/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -euo pipefail
+
+LOG_DIR="${NGINX_LOG_DIR:-/var/log/deepdata/frontend}"
+LOG_ROTATION_MAX_SIZE="${LOG_ROTATION_MAX_SIZE:-100MB}"
+LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
+LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
+
+parse_size_bytes() {
+    local value
+    value="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
+    value="${value// /}"
+    case "$value" in
+        *GB) echo $((${value%GB} * 1024 * 1024 * 1024)) ;;
+        *G) echo $((${value%G} * 1024 * 1024 * 1024)) ;;
+        *MB) echo $((${value%MB} * 1024 * 1024)) ;;
+        *M) echo $((${value%M} * 1024 * 1024)) ;;
+        *KB) echo $((${value%KB} * 1024)) ;;
+        *K) echo $((${value%K} * 1024)) ;;
+        *B) echo "${value%B}" ;;
+        *) echo "$value" ;;
+    esac
+}
+
+file_size_bytes() {
+    wc -c < "$1" | tr -d ' '
+}
+
+backup_path() {
+    local file="$1"
+    local index="$2"
+    local ext="${file##*.}"
+    local stem="${file%.$ext}"
+    echo "${stem}.${index}.${ext}"
+}
+
+rotate_file() {
+    local file="$1"
+    local max_bytes="$2"
+    local backup_count="$3"
+
+    [ -f "$file" ] || return 1
+
+    local current_size
+    current_size="$(file_size_bytes "$file")"
+    [ "$current_size" -lt "$max_bytes" ] && return 1
+
+    rm -f "$(backup_path "$file" "$backup_count")"
+
+    local index
+    for ((index = backup_count - 1; index >= 1; index--)); do
+        local src dst
+        src="$(backup_path "$file" "$index")"
+        dst="$(backup_path "$file" "$((index + 1))")"
+        [ -f "$src" ] && mv "$src" "$dst"
+    done
+
+    mv "$file" "$(backup_path "$file" 1)"
+    install -o nginx -g nginx -m 0644 /dev/null "$file"
+    return 0
+}
+
+rotate_once() {
+    mkdir -p "$LOG_DIR"
+    touch "$LOG_DIR/access.log" "$LOG_DIR/error.log"
+    chown -R nginx:nginx "$LOG_DIR" 2>/dev/null || true
+
+    local max_bytes
+    max_bytes="$(parse_size_bytes "$LOG_ROTATION_MAX_SIZE")"
+
+    local rotated=false
+    for file in "$LOG_DIR/access.log" "$LOG_DIR/error.log"; do
+        if rotate_file "$file" "$max_bytes" "$LOG_ROTATION_BACKUP_COUNT"; then
+            rotated=true
+        fi
+    done
+
+    if [ "$rotated" = true ]; then
+        nginx -s reopen || true
+    fi
+}
+
+if [ "${1:-}" = "--once" ]; then
+    rotate_once
+    exit 0
+fi
+
+while true; do
+    rotate_once
+    sleep "$LOG_ROTATION_INTERVAL_SECONDS"
+done
@@ -0,0 +1,40 @@
+location /api/ {
+    proxy_pass http://deepdata-gateway:8080/api/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+}
+
+location /chat {
+    proxy_pass http://deepdata-gateway:8080/chat;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+    proxy_intercept_errors on;
+    error_page 500 501 502 503 504 =200 @static_fallback;
+}
+
+location /_next {
+    proxy_pass http://deepdata-gateway:8080/_next;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+}
+
+location /deepflow-backend {
+    proxy_pass http://deepdata-gateway:8080/deepflow-backend;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+}
+
+location @static_fallback {
+    root   /opt/frontend/statics;
+    try_files $uri $uri/ /index.html;
+}
+
+location / {
+    root   /opt/frontend/statics;
+    try_files $uri $uri/ /index.html;
+}
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+function set_proxy_protocol() {
+    if [ "${REAL_IP_MODE}" != "proxy_protocol" ]; then
+        echo "REAL_IP_MODE is ${REAL_IP_MODE}, no need to update nginx configuration file."
+        return 0
+    fi
+    sed -i 's/listen       3000;/listen       3000 proxy_protocol;/' /opt/frontend/http_backend.conf
+    sed -i 's/listen       3000 ssl;/listen       3000 ssl proxy_protocol;/' /opt/frontend/https_backend.conf
+    sed -i '/access_log.*main/a\    set_real_ip_from 0.0.0.0/0;' /opt/frontend/nginx.conf
+    sed -i '/access_log.*main/a\    real_ip_header proxy_protocol;' /opt/frontend/nginx.conf
+    echo "Nginx configuration file updated."
+}
+
+mkdir -p /var/log/deepdata/frontend
+touch /var/log/deepdata/frontend/access.log /var/log/deepdata/frontend/error.log
+chown -R nginx:nginx /var/log/deepdata/frontend 2>/dev/null || true
+/usr/local/bin/rotate-nginx-logs.sh &
+
+if [ -f "/cert/server.pem" ]; then
+    cp /cert/server.pem /etc/nginx/cert/server.pem
+    chown nginx:nginx /etc/nginx/cert/server.pem
+fi
+
+if [ -f "/cert/server.key" ]; then
+    # Private key MUST be encrypted. Plaintext keys are not accepted.
+    if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then
+        if [ -z "$CERT_PASS" ]; then
+            echo "ERROR: CERT_PASS is required to decrypt the private key"
+            exit 1
+        fi
+        echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin
+        chmod 600 /etc/nginx/cert/server.key
+    else
+        echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment."
+        exit 1
+    fi
+    chown nginx:nginx /etc/nginx/cert/server.key
+fi
+
+set_proxy_protocol
+cp /opt/frontend/nginx.conf /etc/nginx/nginx.conf
+if [ -f "/etc/nginx/cert/server.pem" ]; then
+    cp /opt/frontend/https_backend.conf /etc/nginx/conf.d/default.conf
+    cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
+    echo "Switching to HTTPS config"
+else
+    cp /opt/frontend/http_backend.conf /etc/nginx/conf.d/default.conf
+    cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
+
+    if [ -n "$DOMAIN" ]; then
+      cron
+      certbot --nginx "-d ${DOMAIN//,/ -d }"
+      echo "Switching to HTTPS config, Domain: $DOMAIN"
+      echo "0 0 1 * * root /usr/local/bin/certbot renew --quiet" | tee /etc/cron.d/certbot-renew
+      chmod 0644 /etc/cron.d/certbot-renew
+    else
+      echo "Switching to HTTP config"
+    fi
+fi
+
+exec nginx -g "daemon off;"
@@ -0,0 +1,28 @@
+FROM maven:3-eclipse-temurin-21 AS builder
+
+COPY backend/ /opt/gateway
+
+RUN cd /opt/gateway/ && \
+    mvn -U clean package -Dmaven.test.skip=true
+
+
+FROM eclipse-temurin:21-jdk
+
+RUN apt-get update && \
+    apt-get install -y vim wget curl dos2unix && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /opt/gateway/api-gateway/target/gateway.jar /opt/gateway/gateway.jar
+
+COPY scripts/images/gateway/start.sh /opt/gateway/start.sh
+
+RUN dos2unix /opt/gateway/start.sh \
+    && chmod +x /opt/gateway/start.sh \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+EXPOSE 8080
+
+ENTRYPOINT ["/opt/gateway/start.sh"]
+
+CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/gateway/gateway.jar"]
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+echo "Starting main application..."
+exec "$@"
@@ -0,0 +1,188 @@
+ARG NODE_VERSION=22
+ARG PYTHON_VERSION=3.13
+ARG POETRY_VERSION=2.1.4
+ARG VERSION_OVERRIDE
+ARG BRANCH_OVERRIDE
+
+ARG GIT_REPO=https://github.com/ModelEngine-Group/Label-Studio.git
+ARG GIT_BRANCH=ls-release/1.21.0.wsi
+
+################################ Stage: downloader (Source Code)
+FROM alpine/git AS downloader
+ARG GIT_REPO
+ARG GIT_BRANCH
+WORKDIR /label-studio
+
+RUN git clone --branch ${GIT_BRANCH} ${GIT_REPO} .
+
+################################ Overview
+
+# This Dockerfile builds a Label Studio environment.
+# It consists of five main stages:
+# 1. "frontend-builder" - Compiles the frontend assets using Node.
+# 2. "frontend-version-generator" - Generates version files for frontend sources.
+# 3. "venv-builder" - Prepares the virtualenv environment.
+# 4. "py-version-generator" - Generates version files for python sources.
+# 5. "prod" - Creates the final production image with the Label Studio, Nginx, and other dependencies.
+
+################################ Stage: frontend-builder (build frontend assets)
+FROM --platform=${BUILDPLATFORM} node:${NODE_VERSION}-trixie AS frontend-builder
+ENV BUILD_NO_SERVER=true \
+    BUILD_NO_HASH=true \
+    BUILD_NO_CHUNKS=true \
+    BUILD_MODULE=true \
+    YARN_CACHE_FOLDER=/root/web/.yarn \
+    NX_CACHE_DIRECTORY=/root/web/.nx \
+    NODE_ENV=production
+
+WORKDIR /label-studio/web
+
+# Fix Docker Arm64 Build
+RUN yarn config set registry https://registry.npmjs.org/
+RUN yarn config set network-timeout 1200000 # HTTP timeout used when downloading packages, set to 20 minutes
+
+# [修改] 从 downloader 阶段复制 package.json 和 lock 文件
+COPY --from=downloader /label-studio/web/package.json .
+COPY --from=downloader /label-studio/web/yarn.lock .
+COPY --from=downloader /label-studio/web/tools tools
+
+RUN --mount=type=cache,target=/root/web/.yarn,id=yarn-cache,sharing=locked \
+    --mount=type=cache,target=/root/web/.nx,id=nx-cache,sharing=locked \
+    yarn install --prefer-offline --no-progress --pure-lockfile --frozen-lockfile --ignore-engines --non-interactive --production=false
+
+# [修改] 从 downloader 阶段复制前端源码
+COPY --from=downloader /label-studio/web/ .
+# [修改] 从 downloader 阶段复制 pyproject.toml
+COPY --from=downloader /label-studio/pyproject.toml ../pyproject.toml
+
+RUN --mount=type=cache,target=/root/web/.yarn,id=yarn-cache,sharing=locked \
+    --mount=type=cache,target=/root/web/.nx,id=nx-cache,sharing=locked \
+    yarn run build
+
+################################ Stage: frontend-version-generator
+FROM frontend-builder AS frontend-version-generator
+# [修改] 之前是 bind mount 本地 .git，现在需要将 downloader 的 .git 目录复制进来
+# 注意：WORKDIR 目前是 /label-studio/web，所以我们将 git 放在上级目录以匹配原逻辑
+COPY --from=downloader /label-studio/.git ../.git
+
+RUN --mount=type=cache,target=/root/web/.yarn,id=yarn-cache,sharing=locked \
+    --mount=type=cache,target=/root/web/.nx,id=nx-cache,sharing=locked \
+    yarn version:libs
+
+################################ Stage: venv-builder (prepare the virtualenv)
+FROM python:${PYTHON_VERSION}-slim-trixie AS venv-builder
+ARG POETRY_VERSION
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    PIP_CACHE_DIR="/.cache" \
+    POETRY_CACHE_DIR="/.poetry-cache" \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON=true \
+    PATH="/opt/poetry/bin:$PATH"
+
+ADD https://install.python-poetry.org /tmp/install-poetry.py
+RUN python /tmp/install-poetry.py
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+    build-essential git; \
+    apt-get autoremove -y
+
+WORKDIR /label-studio
+
+ENV VENV_PATH="/label-studio/.venv"
+ENV PATH="$VENV_PATH/bin:$PATH"
+
+## Starting from this line all packages will be installed in $VENV_PATH
+
+COPY --from=downloader /label-studio/pyproject.toml /label-studio/poetry.lock /label-studio/README.md ./
+
+# Set a default build argument for including dev dependencies
+ARG INCLUDE_DEV=false
+
+# Install dependencies
+RUN --mount=type=cache,target=/.poetry-cache,id=poetry-cache,sharing=locked \
+    poetry check --lock && \
+    if [ "$INCLUDE_DEV" = "true" ]; then \
+    poetry install --no-root --extras uwsgi --with test; \
+    else \
+    poetry install --no-root --without test --extras uwsgi; \
+    fi
+
+# Install LS
+COPY --from=downloader /label-studio/label_studio label_studio
+RUN --mount=type=cache,target=/.poetry-cache,id=poetry-cache,sharing=locked \
+    poetry install --only-root --extras uwsgi && \
+    python3 label_studio/manage.py collectstatic --no-input
+
+################################ Stage: py-version-generator
+FROM venv-builder AS py-version-generator
+ARG VERSION_OVERRIDE
+ARG BRANCH_OVERRIDE
+
+COPY --from=downloader /label-studio/.git ./.git
+
+# Create version_.py and ls-version_.py
+RUN VERSION_OVERRIDE=${VERSION_OVERRIDE} BRANCH_OVERRIDE=${BRANCH_OVERRIDE} poetry run python label_studio/core/version.py
+
+################################### Stage: prod
+FROM python:${PYTHON_VERSION}-slim-trixie AS production
+
+ENV LS_DIR=/label-studio \
+    HOME=/label-studio \
+    LABEL_STUDIO_BASE_DATA_DIR=/label-studio/data \
+    OPT_DIR=/opt/heartex/instance-data/etc \
+    PATH="/label-studio/.venv/bin:$PATH" \
+    DJANGO_SETTINGS_MODULE=core.settings.label_studio \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+WORKDIR $LS_DIR
+
+# install prerequisites for app
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    set -eux; \
+    apt-get update; \
+    apt-get upgrade -y; \
+    apt-get install --no-install-recommends -y libexpat1 libgl1 libglx-mesa0 libglib2.0-0t64 \
+        gnupg2 curl nginx \
+        libopenjp2-7 libnuma1 libspeex1 libvdpau1 libgomp1; \
+    apt-get autoremove -y
+
+RUN set -eux; \
+    mkdir -p $LS_DIR $LABEL_STUDIO_BASE_DATA_DIR $OPT_DIR && \
+    chown -R 1001:0 $LS_DIR $LABEL_STUDIO_BASE_DATA_DIR $OPT_DIR /var/log/nginx /etc/nginx
+
+COPY --chown=1001:0 --from=downloader /label-studio/deploy/default.conf /etc/nginx/nginx.conf
+
+# Copy essential files for installing Label Studio and its dependencies
+COPY --chown=1001:0 --from=downloader /label-studio/pyproject.toml .
+COPY --chown=1001:0 --from=downloader /label-studio/poetry.lock .
+COPY --chown=1001:0 --from=downloader /label-studio/README.md .
+COPY --chown=1001:0 --from=downloader /label-studio/LICENSE LICENSE
+COPY --chown=1001:0 --from=downloader /label-studio/licenses licenses
+COPY --chown=1001:0 --from=downloader /label-studio/deploy deploy
+
+# Copy files from build stages
+COPY --chown=1001:0 --from=venv-builder               $LS_DIR                                           $LS_DIR
+COPY --chown=1001:0 --from=py-version-generator       $LS_DIR/label_studio/core/version_.py             $LS_DIR/label_studio/core/version_.py
+COPY --chown=1001:0 --from=frontend-builder           $LS_DIR/web/dist                                  $LS_DIR/web/dist
+COPY --chown=1001:0 --from=frontend-version-generator $LS_DIR/web/dist/apps/labelstudio/version.json    $LS_DIR/web/dist/apps/labelstudio/version.json
+COPY --chown=1001:0 --from=frontend-version-generator $LS_DIR/web/dist/libs/editor/version.json         $LS_DIR/web/dist/libs/editor/version.json
+COPY --chown=1001:0 --from=frontend-version-generator $LS_DIR/web/dist/libs/datamanager/version.json    $LS_DIR/web/dist/libs/datamanager/version.json
+
+USER 1001
+
+EXPOSE 8080
+
+ENTRYPOINT ["./deploy/docker-entrypoint.sh"]
+CMD ["label-studio"]
@@ -0,0 +1,107 @@
+# ==============================================================================
+# Stage 1: DataX Builder
+# ==============================================================================
+FROM maven:3-eclipse-temurin-8 AS datax-builder
+
+# 安装 git
+RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+
+COPY DataX/ /DataX/
+COPY runtime/datax/ DataX/
+
+# 创建阿里云 settings.xml
+RUN mkdir -p /root/.m2 && \
+    cat > /root/.m2/settings.xml <<EOF
+<settings>
+  <mirrors>
+    <mirror>
+      <id>aliyunmaven</id>
+      <mirrorOf>central</mirrorOf>
+      <name>Aliyun Maven</name>
+      <url>https://maven.aliyun.com/repository/public</url>
+    </mirror>
+  </mirrors>
+</settings>
+EOF
+
+# 设置工作目录
+WORKDIR /DataX
+
+RUN find /DataX -type f -name "DataBaseType.java" -exec sed -i "s/com.mysql.jdbc.Driver/com.mysql.cj.jdbc.Driver/g" {} + || echo "⚠️ MySQL driver file not found, skipping." && \
+    sed -i 's/<maven-assembly-plugin.version>.*<\/maven-assembly-plugin.version>/<maven-assembly-plugin.version>2.6<\/maven-assembly-plugin.version>/' /DataX/pom.xml || echo "⚠️ Maven assembly version tag not found, skipping."
+
+# 编译 DataX (精确挂载 repository 目录，不覆盖 settings.xml)
+RUN --mount=type=cache,target=/root/.m2/repository \
+    mvn -s /root/.m2/settings.xml clean package assembly:assembly -Dmaven.test.skip=true
+
+# ==============================================================================
+# Stage 2: Ray Worker Runtime (保留原有的 AI 环境，增加 Java 支持)
+# ==============================================================================
+FROM ghcr.io/astral-sh/uv:python3.11-bookworm
+
+# 1. 安装系统依赖
+RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \
+    sed -i 's/security.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+       libgl1 libglib2.0-0 vim libmagic1 dos2unix swig \
+       poppler-utils tesseract-ocr openjdk-17-jre-headless \
+    && rm -rf /var/lib/apt/lists/*
+
+# 2. 下载 AI 模型 (保持不变)
+RUN mkdir -p /home/models \
+    && wget -q --show-progress https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-LCNet_x1_0_doc_ori_infer.tar \
+    && tar -xf PP-LCNet_x1_0_doc_ori_infer.tar -C /home/models \
+    && rm -f PP_*.tar
+
+# 3. 复制业务代码 (保持不变)
+COPY runtime/python-executor /opt/runtime
+COPY runtime/ops /opt/runtime/deepdata/ops
+COPY runtime/ops/user /opt/runtime/user
+COPY scripts/images/runtime-ray/start.sh /opt/runtime/start.sh
+COPY scripts/images/common/log-rotate-copytruncate.sh /usr/local/bin/log-rotate-copytruncate.sh
+
+# 4. 复制编译好的 DataX (从 builder 阶段复制)
+# ✅ 新增: 将编译好的 datax 复制到 /opt/datax
+COPY --from=datax-builder /DataX/target/datax/datax /opt/datax
+
+# ✅ 新增: 修复 MySQL 驱动路径 (参照 backend-python)
+RUN cp /opt/datax/plugin/reader/mysqlreader/libs/mysql* /opt/datax/plugin/reader/starrocksreader/libs/ || true
+
+# 5. 环境变量配置
+ENV PYTHONPATH=/opt/runtime/deepdata/
+ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
+ENV UV_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu
+ENV UV_INDEX_STRATEGY=unsafe-best-match
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+# ✅ 新增: 设置 JAVA_HOME，确保 datax.py 能正确调用 Java
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
+ENV PATH="$PATH:$JAVA_HOME/bin"
+
+WORKDIR /opt/runtime
+
+COPY scripts/images/runtime-ray/zh_core_web_sm-3.8.0-py3-none-any.whl /tmp/
+
+ENV UV_LINK_MODE=copy
+
+# 6. 安装 Python 依赖
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install -e . --system \
+    && uv pip install -r /opt/runtime/deepdata/ops/pyproject.toml --system \
+    && uv pip install "ray[default]>=2.9.0" --system \
+    && uv pip install /tmp/zh_core_web_sm-3.8.0-py3-none-any.whl --system \
+    && python -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')" \
+    && echo "/usr/local/lib/ops/site-packages" > /usr/local/lib/python3.11/site-packages/ops.pth
+
+   
+# 7. 初始化与权限
+RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && chmod +x /opt/runtime/start.sh \
+    && chmod +x /usr/local/bin/log-rotate-copytruncate.sh \
+    && dos2unix /opt/runtime/start.sh
+
+EXPOSE 8081
+
+ENTRYPOINT ["/opt/runtime/start.sh"]
@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+
+# 1. 初始化环境 (原有的逻辑：注入用户自定义算子)
+if [ -d "/opt/runtime/user" ]; then
+    cp -r /opt/runtime/user/* /opt/runtime/deepdata/ops/user 2>/dev/null || true
+fi
+
+# 2. 根据 RAY_MODE 环境变量决定启动模式
+if [ "${RAY_MODE}" = "head" ]; then
+    echo "🚀 Starting Ray Head..."
+    # ✅ 修复 Head 分支：
+    # Ray 2.x 已废弃 CLI 参数 --redis-address，改为自动读取环境变量 RAY_REDIS_ADDRESS。
+    # 只要确保 docker-compose.yml 中配置了 RAY_REDIS_ADDRESS 环境变量即可。
+    exec ray start --head \
+        --port=${RAY_HEAD_PORT:-6379} \
+        --redis-password=${RAY_REDIS_PASSWORD} \
+        --dashboard-host=0.0.0.0 \
+        --block
+
+elif [ "${RAY_MODE}" = "worker" ]; then
+    echo "🚀 Starting Ray Worker connecting to ${RAY_HEAD_ADDRESS}..."
+    # ✅ 修复 Worker 分支：
+    # Ray 2.x 移除了 --worker 参数。只要不传 --head，默认即为 Worker 模式。
+    exec ray start \
+        --address=${RAY_HEAD_ADDRESS} \
+        --redis-password=${RAY_REDIS_PASSWORD} \
+        --num-cpus=${NUM_CPUS:-4} \
+        --block
+
+else
+    echo "🔧 Starting Runtime (Legacy Mode / Job Execution)..."
+    # 默认行为：执行 docker-compose.yml command 传入的指令，或 Ray Job 的入口点
+    exec "$@"
+fi
@@ -0,0 +1,39 @@
+FROM ghcr.io/astral-sh/uv:python3.11-bookworm
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt update \
+    && apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix swig poppler-utils tesseract-ocr
+
+RUN mkdir -p /home/models \
+    && wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-LCNet_x1_0_doc_ori_infer.tar \
+    && tar -xf PP-LCNet_x1_0_doc_ori_infer.tar -C /home/models \
+    && rm -f PP_*.tar
+
+COPY runtime/python-executor /opt/runtime
+COPY runtime/ops /opt/runtime/deepdata/ops
+COPY runtime/ops/user /opt/runtime/user
+COPY scripts/images/runtime/start.sh /opt/runtime/start.sh
+COPY scripts/images/common/log-rotate-copytruncate.sh /usr/local/bin/log-rotate-copytruncate.sh
+
+ENV PYTHONPATH=/opt/runtime/deepdata/
+ENV UV_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
+ENV UV_INDEX_STRATEGY=unsafe-best-match
+
+WORKDIR /opt/runtime
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install -e . --system \
+    && uv pip install -r /opt/runtime/deepdata/ops/pyproject.toml --system \
+    && python -m spacy download zh_core_web_sm \
+    && python -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')" \
+    && echo "/usr/local/lib/ops/site-packages" > /usr/local/lib/python3.11/site-packages/ops.pth
+
+RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && chmod +x /opt/runtime/start.sh \
+    && chmod +x /usr/local/bin/log-rotate-copytruncate.sh \
+    && dos2unix /opt/runtime/start.sh
+
+EXPOSE 8081
+
+ENTRYPOINT ["/opt/runtime/start.sh"]
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+cp -r /opt/runtime/user/* /opt/runtime/deepdata/ops/user
+
+if [ "${LOG_ROTATION_ENABLED:-true}" = "true" ]; then
+  LOG_ROTATION_PATH="${LOG_ROTATION_PATH:-/tmp/ray}" /usr/local/bin/log-rotate-copytruncate.sh "${LOG_ROTATION_PATH:-/tmp/ray}" &
+fi
+
+echo "Starting main application..."
+exec "$@"