deepdata: delivery stack (docker compose, helm, CI, build scripts)

scripts/images/frontend/Dockerfile

@@ -0,0 +1,41 @@
+FROM node:20-alpine AS builder
+
+WORKDIR /app
+
+COPY frontend ./
+
+RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi && \
+    npm run build
+
+FROM nginx:1.29 AS runner
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt update \
+    && apt install -y dos2unix python3 python3-dev python3-venv libaugeas-dev gcc vim cron \
+    && python3 -m venv /opt/certbot/ \
+    && /opt/certbot/bin/pip install --upgrade pip \
+    && /opt/certbot/bin/pip install certbot certbot-nginx \
+    && ln -s /opt/certbot/bin/certbot /usr/local/bin/certbot
+
+COPY --from=builder /app/dist /opt/frontend/statics
+COPY scripts/images/frontend/routes.inc /opt/frontend/routes.inc
+COPY scripts/images/frontend/http_backend.conf /opt/frontend/http_backend.conf
+COPY scripts/images/frontend/https_backend.conf /opt/frontend/https_backend.conf
+COPY scripts/images/frontend/nginx.conf /opt/frontend/nginx.conf
+COPY scripts/images/frontend/rotate-nginx-logs.sh /usr/local/bin/rotate-nginx-logs.sh
+
+COPY scripts/images/frontend/start.sh /opt/frontend/start.sh
+
+RUN dos2unix /opt/frontend/start.sh \
+    && dos2unix /usr/local/bin/rotate-nginx-logs.sh \
+    && chmod +x /opt/frontend/start.sh \
+    && chmod +x /usr/local/bin/rotate-nginx-logs.sh \
+    && mkdir -p /etc/nginx/cert \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && rm -f /etc/nginx/nginx.conf \
+    && rm -f /etc/nginx/conf.d/default.conf
+
+EXPOSE 3000
+
+CMD ["/opt/frontend/start.sh"]

scripts/images/frontend/http_backend.conf

@@ -0,0 +1,13 @@
+server {
+    listen       3000;
+    server_name  0.0.0.0;
+
+    access_log /var/log/deepdata/frontend/access.log main;
+    error_log /var/log/deepdata/frontend/error.log notice;
+
+    client_max_body_size 1024M;
+
+    add_header Set-Cookie "NEXT_LOCALE=zh";
+
+    include /etc/nginx/conf.d/routes.inc;
+}

scripts/images/frontend/https_backend.conf

@@ -0,0 +1,18 @@
+server {
+    listen       3000 ssl;
+    server_name  0.0.0.0;
+
+    access_log /var/log/deepdata/frontend/access.log main;
+    error_log /var/log/deepdata/frontend/error.log notice;
+
+    client_max_body_size 1024M;
+
+    ssl_certificate     /etc/nginx/cert/server.pem;
+    ssl_certificate_key /etc/nginx/cert/server.key;
+
+    error_page 497 https://$host$request_uri;
+
+    add_header Set-Cookie "NEXT_LOCALE=zh";
+
+    include /etc/nginx/conf.d/routes.inc;
+}

scripts/images/frontend/nginx.conf

@@ -0,0 +1,32 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/deepdata/frontend/error.log notice;
+pid        /run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/deepdata/frontend/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}

scripts/images/frontend/rotate-nginx-logs.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -euo pipefail
+
+LOG_DIR="${NGINX_LOG_DIR:-/var/log/deepdata/frontend}"
+LOG_ROTATION_MAX_SIZE="${LOG_ROTATION_MAX_SIZE:-100MB}"
+LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
+LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
+
+parse_size_bytes() {
+    local value
+    value="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
+    value="${value// /}"
+    case "$value" in
+        *GB) echo $((${value%GB} * 1024 * 1024 * 1024)) ;;
+        *G) echo $((${value%G} * 1024 * 1024 * 1024)) ;;
+        *MB) echo $((${value%MB} * 1024 * 1024)) ;;
+        *M) echo $((${value%M} * 1024 * 1024)) ;;
+        *KB) echo $((${value%KB} * 1024)) ;;
+        *K) echo $((${value%K} * 1024)) ;;
+        *B) echo "${value%B}" ;;
+        *) echo "$value" ;;
+    esac
+}
+
+file_size_bytes() {
+    wc -c < "$1" | tr -d ' '
+}
+
+backup_path() {
+    local file="$1"
+    local index="$2"
+    local ext="${file##*.}"
+    local stem="${file%.$ext}"
+    echo "${stem}.${index}.${ext}"
+}
+
+rotate_file() {
+    local file="$1"
+    local max_bytes="$2"
+    local backup_count="$3"
+
+    [ -f "$file" ] || return 1
+
+    local current_size
+    current_size="$(file_size_bytes "$file")"
+    [ "$current_size" -lt "$max_bytes" ] && return 1
+
+    rm -f "$(backup_path "$file" "$backup_count")"
+
+    local index
+    for ((index = backup_count - 1; index >= 1; index--)); do
+        local src dst
+        src="$(backup_path "$file" "$index")"
+        dst="$(backup_path "$file" "$((index + 1))")"
+        [ -f "$src" ] && mv "$src" "$dst"
+    done
+
+    mv "$file" "$(backup_path "$file" 1)"
+    install -o nginx -g nginx -m 0644 /dev/null "$file"
+    return 0
+}
+
+rotate_once() {
+    mkdir -p "$LOG_DIR"
+    touch "$LOG_DIR/access.log" "$LOG_DIR/error.log"
+    chown -R nginx:nginx "$LOG_DIR" 2>/dev/null || true
+
+    local max_bytes
+    max_bytes="$(parse_size_bytes "$LOG_ROTATION_MAX_SIZE")"
+
+    local rotated=false
+    for file in "$LOG_DIR/access.log" "$LOG_DIR/error.log"; do
+        if rotate_file "$file" "$max_bytes" "$LOG_ROTATION_BACKUP_COUNT"; then
+            rotated=true
+        fi
+    done
+
+    if [ "$rotated" = true ]; then
+        nginx -s reopen || true
+    fi
+}
+
+if [ "${1:-}" = "--once" ]; then
+    rotate_once
+    exit 0
+fi
+
+while true; do
+    rotate_once
+    sleep "$LOG_ROTATION_INTERVAL_SECONDS"
+done

scripts/images/frontend/routes.inc

@@ -0,0 +1,40 @@
+location /api/ {
+    proxy_pass http://deepdata-gateway:8080/api/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+}
+
+location /chat {
+    proxy_pass http://deepdata-gateway:8080/chat;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+    proxy_intercept_errors on;
+    error_page 500 501 502 503 504 =200 @static_fallback;
+}
+
+location /_next {
+    proxy_pass http://deepdata-gateway:8080/_next;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+}
+
+location /deepflow-backend {
+    proxy_pass http://deepdata-gateway:8080/deepflow-backend;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+}
+
+location @static_fallback {
+    root   /opt/frontend/statics;
+    try_files $uri $uri/ /index.html;
+}
+
+location / {
+    root   /opt/frontend/statics;
+    try_files $uri $uri/ /index.html;
+}

scripts/images/frontend/start.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+function set_proxy_protocol() {
+    if [ "${REAL_IP_MODE}" != "proxy_protocol" ]; then
+        echo "REAL_IP_MODE is ${REAL_IP_MODE}, no need to update nginx configuration file."
+        return 0
+    fi
+    sed -i 's/listen       3000;/listen       3000 proxy_protocol;/' /opt/frontend/http_backend.conf
+    sed -i 's/listen       3000 ssl;/listen       3000 ssl proxy_protocol;/' /opt/frontend/https_backend.conf
+    sed -i '/access_log.*main/a\    set_real_ip_from 0.0.0.0/0;' /opt/frontend/nginx.conf
+    sed -i '/access_log.*main/a\    real_ip_header proxy_protocol;' /opt/frontend/nginx.conf
+    echo "Nginx configuration file updated."
+}
+
+mkdir -p /var/log/deepdata/frontend
+touch /var/log/deepdata/frontend/access.log /var/log/deepdata/frontend/error.log
+chown -R nginx:nginx /var/log/deepdata/frontend 2>/dev/null || true
+/usr/local/bin/rotate-nginx-logs.sh &
+
+if [ -f "/cert/server.pem" ]; then
+    cp /cert/server.pem /etc/nginx/cert/server.pem
+    chown nginx:nginx /etc/nginx/cert/server.pem
+fi
+
+if [ -f "/cert/server.key" ]; then
+    # Private key MUST be encrypted. Plaintext keys are not accepted.
+    if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then
+        if [ -z "$CERT_PASS" ]; then
+            echo "ERROR: CERT_PASS is required to decrypt the private key"
+            exit 1
+        fi
+        echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin
+        chmod 600 /etc/nginx/cert/server.key
+    else
+        echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment."
+        exit 1
+    fi
+    chown nginx:nginx /etc/nginx/cert/server.key
+fi
+
+set_proxy_protocol
+cp /opt/frontend/nginx.conf /etc/nginx/nginx.conf
+if [ -f "/etc/nginx/cert/server.pem" ]; then
+    cp /opt/frontend/https_backend.conf /etc/nginx/conf.d/default.conf
+    cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
+    echo "Switching to HTTPS config"
+else
+    cp /opt/frontend/http_backend.conf /etc/nginx/conf.d/default.conf
+    cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
+
+    if [ -n "$DOMAIN" ]; then
+      cron
+      certbot --nginx "-d ${DOMAIN//,/ -d }"
+      echo "Switching to HTTPS config, Domain: $DOMAIN"
+      echo "0 0 1 * * root /usr/local/bin/certbot renew --quiet" | tee /etc/cron.d/certbot-renew
+      chmod 0644 /etc/cron.d/certbot-renew
+    else
+      echo "Switching to HTTP config"
+    fi
+fi
+
+exec nginx -g "daemon off;"