deepdata: delivery stack (docker compose, helm, CI, build scripts)

.github/workflows/codeql.yml

@@ -0,0 +1,86 @@
+# 对于大多数项目，此工作流文件无需更改；只需提交到您的仓库即可。
+#
+# 您可以根据需要修改此文件，以覆盖分析的语言集，或提供自定义查询或构建逻辑。
+#
+# ******** 注意 ********
+# 我们已尝试检测您的仓库中的语言。请检查下面定义的 `language` 矩阵，确保包含了所有受支持的 CodeQL 语言。
+name: "CodeQL Advanced"
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '23 5 * * 2'
+  workflow_dispatch:
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner 的规格会影响 CodeQL 分析时间。详情请参阅：
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (仅限 GitHub.com)
+    # 建议使用更高规格的 Runner 或更大资源的机器以提升分析速度。
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # 所有工作流都需要
+      security-events: write
+      # 获取内部或私有 CodeQL 包时需要
+      packages: read
+      # 仅私有仓库工作流需要
+      actions: read
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: actions
+          build-mode: none
+        - language: javascript-typescript
+          build-mode: none
+        - language: python
+          build-mode: none
+        - language: java-kotlin
+          build-mode: none
+        # CodeQL 支持以下 'language' 关键字：'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift'
+        # 使用 `c-cpp` 可分析 C、C++ 或两者的代码
+        # 使用 'java-kotlin' 可分析 Java、Kotlin 或两者的代码
+        # 使用 'javascript-typescript' 可分析 JavaScript、TypeScript 或两者的代码
+        # 了解如何更改分析语言或自定义分析模式，请参阅：
+        # https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning
+        # 如果分析编译型语言，可修改 'build-mode' 以自定义分析方式，详见：
+        # https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    # 在运行 `github/codeql-action/init` 之前可添加任何设置步骤。
+    # 包括安装编译器或运行环境（如 `actions/setup-node` 等）。通常仅手动构建时需要。
+    # - name: Setup runtime (示例)
+    #   uses: actions/setup-example@v1
+    # 初始化 CodeQL 工具以进行扫描。
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # 如需指定自定义查询，可在此处或配置文件中设置。
+        # 默认情况下，此处列出的查询会覆盖配置文件中指定的查询。
+        # 在列表前加 "+" 可同时使用此处和配置文件中的查询。
+        # 更多 CodeQL 查询包详情请参阅：
+        # https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+    # 如果某种语言的 analyze 步骤失败，提示“无法自动构建您的代码”，请在上方矩阵中将该语言的 build-mode 设置为 "manual"，并在此步骤中添加构建命令。
+    # ℹ️ 可使用操作系统 shell 运行命令行程序。
+    # 📚 详见 https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo '如果您为一种或多种语言使用了 "manual" 构建模式，请在此处替换为您的构建命令，例如：'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"
+

.github/workflows/docker-image-backend-python.yml

@@ -0,0 +1,30 @@
+name: Python Backend Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/images/backend-python/**'
+      - 'runtime/deepdata-python/**'
+      - '.github/workflows/docker-image-backend-python.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/images/backend-python/**'
+      - 'runtime/deepdata-python/**'
+      - '.github/workflows/docker-image-backend-python.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Backend Python Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: backend-python
+      build_dir: .

.github/workflows/docker-image-backend.yml

@@ -0,0 +1,32 @@
+name: Backend Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'backend/**'
+      - '!backend/api-gateway/**'
+      - 'scripts/images/backend/**'
+      - '.github/workflows/docker-image-backend.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'backend/**'
+      - '!backend/api-gateway/**'
+      - 'scripts/images/backend/**'
+      - '.github/workflows/docker-image-backend.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Backend Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: backend
+      build_dir: .

.github/workflows/docker-image-database.yml

@@ -0,0 +1,30 @@
+name: Database Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/db/**'
+      - 'scripts/images/database/**'
+      - '.github/workflows/docker-image-database.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/db/**'
+      - 'scripts/images/database/**'
+      - '.github/workflows/docker-image-database.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Database Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: database
+      build_dir: .

.github/workflows/docker-image-deepflow.yml

@@ -0,0 +1,151 @@
+name: Deer Flow Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'runtime/deepflow/**'
+      - 'scripts/images/deepflow-backend/**'
+      - 'scripts/images/deepflow-frontend/**'
+      - '.github/workflows/docker-image-deepflow.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'runtime/deepflow/**'
+      - 'scripts/images/deepflow-backend/**'
+      - 'scripts/images/deepflow-frontend/**'
+      - '.github/workflows/docker-image-deepflow.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build-and-push-amd:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Docker Image Tag
+        id: set-tag
+        run: |
+          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+            TAG=${GITHUB_REF#refs/tags/v}
+            echo "TAGS=amd64-$TAG" >> $GITHUB_OUTPUT
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            echo "TAGS=amd64" >> $GITHUB_OUTPUT
+          else
+            echo "TAGS=amd64-temp" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build Docker Image
+        run: |
+          docker build -t deepflow-backend:amd64 . -f scripts/images/deepflow-backend/Dockerfile
+          docker build -t deepflow-frontend:amd64 . -f scripts/images/deepflow-frontend/Dockerfile
+
+      - name: Tag & Push Docker Image
+        if: github.event_name != 'pull_request'
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          docker tag deepflow-backend:amd64 ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
+          docker tag deepflow-frontend:amd64 ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
+          docker push ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
+          docker push ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
+          
+
+  build-and-push-arm:
+    runs-on: ubuntu-24.04-arm
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Docker Image Tag
+        id: set-tag
+        run: |
+          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+            TAG=${GITHUB_REF#refs/tags/v}
+            echo "TAGS=arm64-$TAG" >> $GITHUB_OUTPUT
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            echo "TAGS=arm64" >> $GITHUB_OUTPUT
+          else
+            echo "TAGS=arm64-temp" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build Docker Image
+        run: |
+          docker build -t deepflow-backend:arm64 . -f scripts/images/deepflow-backend/Dockerfile
+          docker build -t deepflow-frontend:arm64 . -f scripts/images/deepflow-frontend/Dockerfile
+
+      - name: Tag & Push Docker Image
+        if: github.event_name != 'pull_request'
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          docker tag deepflow-backend:arm64 ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
+          docker tag deepflow-frontend:arm64 ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
+          docker push ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
+          docker push ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
+
+  manifest:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    needs: [ build-and-push-amd, build-and-push-arm ]
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Docker Image Tag
+        id: set-tag
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
+          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+            TAG=${GITHUB_REF#refs/tags/v}
+            echo "TAGS=$TAG" >> $GITHUB_OUTPUT
+            echo "ARM_TAGS=arm64-$TAG" >> $GITHUB_OUTPUT
+            echo "AMD_TAGS=amd64-$TAG" >> $GITHUB_OUTPUT
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            echo "TAGS=latest" >> $GITHUB_OUTPUT
+            echo "ARM_TAGS=arm64" >> $GITHUB_OUTPUT
+            echo "AMD_TAGS=amd64" >> $GITHUB_OUTPUT
+          else
+            echo "TAGS=temp" >> $GITHUB_OUTPUT
+            echo "ARM_TAGS=arm64-temp" >> $GITHUB_OUTPUT
+            echo "AMD_TAGS=amd64-temp" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Manifest Docker Image
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          docker manifest create ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }} \
+            ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.AMD_TAGS }} \
+            ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.ARM_TAGS }}
+          docker manifest push ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
+          docker manifest create ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }} \
+            ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.AMD_TAGS }} \
+            ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.ARM_TAGS }}
+          docker manifest push ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}

.github/workflows/docker-image-frontend.yml

@@ -0,0 +1,30 @@
+name: Frontend Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'frontend/**'
+      - 'scripts/images/frontend/**'
+      - '.github/workflows/docker-image-frontend.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'frontend/**'
+      - 'scripts/images/frontend/**'
+      - '.github/workflows/docker-image-frontend.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Frontend Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: frontend
+      build_dir: .

.github/workflows/docker-image-gateway.yml

@@ -0,0 +1,30 @@
+name: Gateway Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'backend/api-gateway/**'
+      - 'scripts/images/gateway/**'
+      - '.github/workflows/docker-image-gateway.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'backend/api-gateway/**'
+      - 'scripts/images/gateway/**'
+      - '.github/workflows/docker-image-gateway.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Gateway Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: gateway
+      build_dir: .

.github/workflows/docker-image-label-studio.yml

@@ -0,0 +1,28 @@
+name: Label Studio Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/images/label-studio/**'
+      - '.github/workflows/docker-image-label-studio.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/images/label-studio/**'
+      - '.github/workflows/docker-image-label-studio.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Label Studio Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: label-studio
+      build_dir: .

.github/workflows/docker-image-runtime.yml

@@ -0,0 +1,32 @@
+name: Runtime Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'runtime/ops/**'
+      - 'runtime/python-executor/**'
+      - 'scripts/images/runtime/**'
+      - '.github/workflows/docker-image-runtime.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'runtime/ops/**'
+      - 'runtime/python-executor/**'
+      - 'scripts/images/runtime/**'
+      - '.github/workflows/docker-image-runtime.yml'
+      - '.github/workflows/docker-images-reusable.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Runtime Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: runtime
+      build_dir: .

.github/workflows/docker-image-save.yml

@@ -0,0 +1,27 @@
+name: docker-image-save.yml
+on:
+  workflow_call:
+    inputs:
+      service_name:
+        required: true
+        type: string
+
+jobs:
+  pull-and-save:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pull Docker Image
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          docker pull ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}:latest
+          docker tag ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}:latest deepdata-${{ inputs.service_name }}:latest
+
+      - name: Save Docker Image
+        run: |
+          docker save -o deepdata-${{ inputs.service_name }}.tar deepdata-${{ inputs.service_name }}:latest
+
+      - name: Upload Docker Image
+        uses: actions/upload-artifact@v4
+        with:
+          name: deepdata-${{ inputs.service_name }}
+          path: deepdata-${{ inputs.service_name }}.tar

.github/workflows/docker-images-release.yml

@@ -0,0 +1,55 @@
+name: Release Docker Images
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+
+jobs:
+  frontend:
+    name: Frontend Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    with:
+      service_name: frontend
+      build_dir: .
+
+  gateway:
+    name: Gateway Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    with:
+      service_name: gateway
+      build_dir: .
+
+  backend:
+    name: Backend Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    with:
+      service_name: backend
+      build_dir: .
+
+  backend-python:
+    name: Backend Python Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    with:
+      service_name: backend-python
+      build_dir: .
+
+  database:
+    name: Database Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    with:
+      service_name: database
+      build_dir: .
+
+  runtime:
+    name: Runtime Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    with:
+      service_name: runtime
+      build_dir: .
+
+  deepflow:
+    name: Deer Flow Images
+    uses: ./.github/workflows/docker-image-deepflow.yml
+

.github/workflows/docker-images-reusable.yml

@@ -0,0 +1,158 @@
+name: Docker Image Build & Push
+
+on:
+  workflow_call:
+    inputs:
+      service_name:
+        required: true
+        type: string
+      build_dir:
+        required: true
+        type: string
+
+jobs:
+  build-and-push-amd:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Docker Image Tag
+        id: set-tag
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
+          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+            TAG=${GITHUB_REF#refs/tags/v}
+            echo "TAGS=$BASE_IMAGE:amd64-$TAG" >> $GITHUB_OUTPUT
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            echo "TAGS=$BASE_IMAGE:amd64" >> $GITHUB_OUTPUT
+          else
+            # 使用分支名作为 tag
+            if [[ $GITHUB_REF == refs/heads/* ]]; then
+              BRANCH_NAME=${GITHUB_REF#refs/heads/}
+              # 清理分支名，将特殊字符替换为连字符
+              BRANCH_TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9._-]/-/g')
+              echo "TAGS=$BASE_IMAGE:amd64-$BRANCH_TAG" >> $GITHUB_OUTPUT
+            else
+              echo "TAGS=$BASE_IMAGE:amd64-temp" >> $GITHUB_OUTPUT
+            fi
+          fi
+
+      - name: Build Docker Image
+        run: |
+          docker build -t deepdata-${{ inputs.service_name }}:amd64 . -f scripts/images/${{ inputs.service_name }}/Dockerfile --platform amd64
+
+      - name: Tag & Push Docker Image
+        if: github.event_name != 'pull_request'
+        run: |
+          docker tag deepdata-${{ inputs.service_name }}:amd64 ${{ steps.set-tag.outputs.TAGS }}
+          docker push ${{ steps.set-tag.outputs.TAGS }}
+
+  build-and-push-arm:
+    runs-on: ubuntu-24.04-arm
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Docker Image Tag
+        id: set-tag
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
+          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+            TAG=${GITHUB_REF#refs/tags/v}
+            echo "TAGS=$BASE_IMAGE:arm64-$TAG" >> $GITHUB_OUTPUT
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            echo "TAGS=$BASE_IMAGE:arm64" >> $GITHUB_OUTPUT
+          else
+            # 使用分支名作为 tag
+            if [[ $GITHUB_REF == refs/heads/* ]]; then
+              BRANCH_NAME=${GITHUB_REF#refs/heads/}
+              # 清理分支名，将特殊字符替换为连字符
+              BRANCH_TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9._-]/-/g')
+              echo "TAGS=$BASE_IMAGE:arm64-$BRANCH_TAG" >> $GITHUB_OUTPUT
+            else
+              echo "TAGS=$BASE_IMAGE:arm64-temp" >> $GITHUB_OUTPUT
+            fi
+          fi
+
+      - name: Build Docker Image
+        run: |
+          docker build -t deepdata-${{ inputs.service_name }}:arm64 . -f scripts/images/${{ inputs.service_name }}/Dockerfile --platform arm64
+
+      - name: Tag & Push Docker Image
+        if: github.event_name != 'pull_request'
+        run: |
+          docker tag deepdata-${{ inputs.service_name }}:arm64 ${{ steps.set-tag.outputs.TAGS }}
+          docker push ${{ steps.set-tag.outputs.TAGS }}
+
+  manifest:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    needs: [ build-and-push-amd, build-and-push-arm ]
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Docker Image Tag
+        id: set-tag
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
+          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+            TAG=${GITHUB_REF#refs/tags/v}
+            echo "TAGS=$BASE_IMAGE:$TAG" >> $GITHUB_OUTPUT
+            echo "ARM_TAGS=$BASE_IMAGE:arm64-$TAG" >> $GITHUB_OUTPUT
+            echo "AMD_TAGS=$BASE_IMAGE:amd64-$TAG" >> $GITHUB_OUTPUT
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            echo "TAGS=$BASE_IMAGE:latest" >> $GITHUB_OUTPUT
+            echo "ARM_TAGS=$BASE_IMAGE:arm64" >> $GITHUB_OUTPUT
+            echo "AMD_TAGS=$BASE_IMAGE:amd64" >> $GITHUB_OUTPUT
+          else
+            # 使用分支名作为 tag
+            if [[ $GITHUB_REF == refs/heads/* ]]; then
+              BRANCH_NAME=${GITHUB_REF#refs/heads/}
+              # 清理分支名，将特殊字符替换为连字符
+              BRANCH_TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9._-]/-/g')
+              echo "TAGS=$BASE_IMAGE:$BRANCH_TAG" >> $GITHUB_OUTPUT
+              echo "ARM_TAGS=$BASE_IMAGE:arm64-$BRANCH_TAG" >> $GITHUB_OUTPUT
+              echo "AMD_TAGS=$BASE_IMAGE:amd64-$BRANCH_TAG" >> $GITHUB_OUTPUT
+            else
+              echo "TAGS=$BASE_IMAGE:temp" >> $GITHUB_OUTPUT
+              echo "ARM_TAGS=$BASE_IMAGE:arm64-temp" >> $GITHUB_OUTPUT
+              echo "AMD_TAGS=$BASE_IMAGE:amd64-temp" >> $GITHUB_OUTPUT
+            fi
+          fi
+
+      - name: Manifest Docker Image
+        run: |
+          docker buildx imagetools create -t ${{ steps.set-tag.outputs.TAGS }} \
+            ${{ steps.set-tag.outputs.AMD_TAGS }} \
+            ${{ steps.set-tag.outputs.ARM_TAGS }}

.github/workflows/docker-push-to-huaweicloud-reusable.yml

@@ -0,0 +1,48 @@
+name: docker-push-to-huaweicloud-reusable.yml
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: 'image version (e.g. latest or v0.0.1-beta)'
+        required: true
+        default: 'latest'
+        type: string
+      service_name:
+        description: 'image version (e.g. latest or v0.0.1-beta)'
+        required: true
+        default: 'latest'
+        type: string
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD:
+        description: 'login pwd'
+        required: true
+
+jobs:
+  retag-and-push:
+    name: Retag and Push to Huawei Cloud SWR
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pull source image and retag (amd)
+        shell: bash
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository }}" | tr "[:upper:]" "[:lower:]")
+          SOURCE_IMAGE=ghcr.io/${LOWERCASE_REPO}/deepdata-${{ inputs.service_name}}:${{ inputs.version}}
+          TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/deepdata/${{ inputs.service_name}}:${{ inputs.version}}
+          docker pull "${SOURCE_IMAGE}"
+          docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"-amd
+      - name: Pull source image and retag (arm)
+        shell: bash
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository }}" | tr "[:upper:]" "[:lower:]")
+          SOURCE_IMAGE=ghcr.io/${LOWERCASE_REPO}/deepdata-${{ inputs.service_name}}:${{ inputs.version}}
+          TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/deepdata/${{ inputs.service_name}}:${{ inputs.version}}
+          docker pull "${SOURCE_IMAGE}" --platform=linux/arm64
+          docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"-arm
+      - name: Push image to Huawei SWR using
+        shell: bash
+        run: |
+          TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/deepdata/${{ inputs.service_name}}:${{ inputs.version}}
+          echo "${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}" | docker login swr.cn-east-3.myhuaweicloud.com --username=cn-east-3@HPUA3ZGOTQFAFMM59DB9 --password-stdin
+          docker push "${TARGET_IMAGE}"-amd
+          docker push "${TARGET_IMAGE}"-arm
+          docker buildx imagetools create -t "${TARGET_IMAGE}" "${TARGET_IMAGE}"-amd "${TARGET_IMAGE}"-arm

.github/workflows/docker-push-to-huaweicloud.yml

@@ -0,0 +1,82 @@
+name: docker-push-to-huaweicloud.yml
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'image version (e.g. latest or v0.0.1-beta)'
+        required: true
+        default: 'latest'
+        type: string
+
+jobs:
+  frontend:
+    name: Push Frontend Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepdata-frontend
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
+
+  gateway:
+    name: Push Gateway Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepdata-gateway
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
+
+  backend:
+    name: Push Backend Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepdata-backend
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
+
+  backend-python:
+    name: Push Backend-Python Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepdata-backend-python
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
+
+  runtime:
+    name: Push Runtime Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepdata-runtime
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
+
+  database:
+    name: Push Database Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepdata-database
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
+
+  deepflow-backend:
+    name: Push deepflow-backend Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepflow-backend
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
+
+  deepflow-frontend:
+    name: Push deepflow-frontend Image
+    uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
+    with:
+      service_name: deepflow-frontend
+      version: ${{ inputs.version}}
+    secrets:
+      HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}

Makefile

@@ -0,0 +1,581 @@
+MAKEFLAGS += --no-print-directory
+
+WITH_MINERU ?= false  # 默认不构建mineru
+VERSION ?= latest
+NAMESPACE ?= deepdata
+
+# Registry configuration: use --dev for local images, otherwise use GitHub registry
+ifdef dev
+    REGISTRY :=
+else
+    REGISTRY ?= 
+endif
+
+ifdef COMSPEC
+    # Windows
+    MAKE := "C:/Program Files (x86)/GnuWin32/bin/make"
+else
+    # Linux/Mac
+    MAKE := make
+endif
+
+# ========== Help ==========
+
+.PHONY: help
+help:
+	@echo "DeepData Makefile - Available Commands"
+	@echo ""
+	@echo "Usage: make <target> [options]"
+	@echo ""
+	@echo "Options:"
+	@echo "  dev=true             Use local images instead of registry (empty REGISTRY)"
+	@echo "  VERSION=<version>    Set image version (default: latest)"
+	@echo "  NAMESPACE=<name>     Set Kubernetes namespace (default: deepdata)"
+	@echo "  INSTALLER=<type>     Set installer type: docker or k8s"
+	@echo "  PLATFORM=<platform>  Set platform for downloads (default: linux/amd64)"
+	@echo "                       Options: linux/amd64, linux/arm64"
+	@echo "  SAVE=true            Save images to dist/ during download (default: false)"
+	@echo ""
+	@echo "Build Commands:"
+	@echo "  make build                     Build all core images"
+	@echo "  make <service>-docker-build    Build specific service image"
+	@echo "    Valid services: backend, database, frontend, runtime,"
+	@echo "                    backend-python, deepflow, mineru"
+	@echo ""
+	@echo "Install Commands:"
+	@echo "  make install                        Install deepdata + milvus (prompts for method)"
+	@echo "  make install INSTALLER=docker       Install using Docker Compose"
+	@echo "  make install INSTALLER=k8s          Install using Kubernetes/Helm"
+	@echo "  make install INSTALLER=k8s          (requires Sealed Secrets Controller)"
+	@echo "  make install-<component>            Install specific component (prompts)"
+	@echo "  make <component>-docker-install     Install component via Docker"
+	@echo "  make <component>-k8s-install        Install component via Kubernetes"
+	@echo "    Valid components: deepdata, milvus, deepflow, mineru"
+	@echo "    Valid services: backend, frontend, runtime, label-studio"
+	@echo ""
+	@echo "Uninstall Commands:"
+	@echo "  make uninstall                      Uninstall deepdata + milvus (prompts)"
+	@echo "  make uninstall INSTALLER=docker     Uninstall using Docker Compose"
+	@echo "  make uninstall INSTALLER=k8s        Uninstall using Kubernetes/Helm"
+	@echo "  make uninstall-<component>          Uninstall specific component (prompts)"
+	@echo "  make <component>-docker-uninstall   Uninstall component via Docker"
+	@echo "  make <component>-k8s-uninstall      Uninstall component via Kubernetes"
+	@echo "  Note: Docker uninstall will prompt whether to delete volumes"
+	@echo ""
+	@echo "Upgrade Commands:"
+	@echo "  make deepdata-docker-upgrade   Upgrade deepdata deployment"
+	@echo ""
+	@echo "Download Commands:"
+	@echo "  make download                       Pull all images (no save by default)"
+	@echo "  make download VERSION=<version>     Pull all images with specific version"
+	@echo "  make download REGISTRY=<registry>   Pull images from specific registry"
+	@echo "  make load-images                    Load all downloaded images from dist/"
+	@echo "  make download-sealed-secrets        Download Sealed Secrets image (for offline)"
+	@echo ""
+	@echo "Utility Commands:"
+	@echo "  make create-namespace          Create Kubernetes namespace"
+	@echo "  make node-setup               Configure dedicated nodes interactively"
+	@echo "  make node-cleanup             Remove node labels and taints"
+	@echo "  make help                      Show this help message"
+	@echo ""
+	@echo "Examples:"
+	@echo "  make build dev=true            Build all images for local development"
+	@echo "  make install INSTALLER=docker  Install via Docker Compose"
+	@echo "  make install dev=true          Install using local images"
+	@echo "  make deepdata-docker-upgrade   Upgrade running deepdata services"
+	@echo ""
+
+.DEFAULT_GOAL := help
+
+# ========== Functions ==========
+
+# Prompt user to choose installer if not specified
+define prompt-installer
+	@echo "Choose a deployment method:"
+	@echo "1. Docker/Docker-Compose"
+	@echo "2. Kubernetes/Helm"
+	@echo -n "Enter choice: "
+	@read choice; \
+	case $$choice in \
+		1) INSTALLER=docker ;; \
+		2) INSTALLER=k8s ;; \
+		*) echo "Invalid choice" && exit 1 ;; \
+	esac; \
+	$(MAKE) $(1)
+endef
+
+# Prompt user to choose installer and volume deletion for uninstall
+define prompt-uninstaller
+	@echo "Choose a deployment method:"
+	@echo "1. Docker/Docker-Compose"
+	@echo "2. Kubernetes/Helm"
+	@echo -n "Enter choice: "
+	@read installer_choice; \
+	case $$installer_choice in \
+		1) INSTALLER=docker ;; \
+		2) INSTALLER=k8s ;; \
+		*) echo "Invalid choice" && exit 1 ;; \
+	esac; \
+	if [ "$$INSTALLER" = "docker" ]; then \
+		echo "Delete volumes? (This will remove all data)"; \
+		echo "1. Yes - Delete volumes"; \
+		echo "2. No - Keep volumes"; \
+		echo -n "Enter choice (default: 2): "; \
+		read DELETE_VOLUMES_CHOICE; \
+		$(MAKE) $(1) DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
+	else \
+		$(MAKE) $(1); \
+	fi
+endef
+
+# Generic docker build function
+# Usage: $(call docker-build,service-name,image-name)
+define docker-build
+	docker build -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile
+endef
+
+# Generic docker compose service action
+# Usage: $(call docker-compose-service,service-name,action,compose-dir)
+define docker-compose-service
+	cd $(3) && docker compose $(2) $(1)
+endef
+
+# Prompt user to choose whether to delete volumes
+define prompt-volume-deletion
+	@echo "Delete volumes? (This will remove all data)"
+	@echo "1. Yes - Delete volumes"
+	@echo "2. No - Keep volumes"
+	@echo -n "Enter choice (default: 2): "
+	@read choice; \
+	case $$choice in \
+		1) echo "-v" ;; \
+		*) echo "" ;; \
+	esac
+endef
+
+# ========== Build Targets ==========
+
+# Valid build targets
+VALID_BUILD_TARGETS := frontend backend gateway database runtime runtime-ray backend-python deepflow label-studio mineru mineru-910B mineru-910C mineru-310P
+
+# Generic docker build target with service name as parameter
+# Automatically prefixes image names with "deepdata-" unless it's deepflow
+.PHONY: %-docker-build
+%-docker-build:
+	@if ! echo " $(VALID_BUILD_TARGETS) " | grep -q " $* "; then \
+		echo "Error: Unknown build target '$*'"; \
+		echo "Valid build targets are:"; \
+		for target in $(VALID_BUILD_TARGETS); do \
+			echo "  - $$target"; \
+		done; \
+		exit 1; \
+	fi
+	@if [ "$*" = "deepflow" ]; then \
+		$(call docker-build,deepflow-backend,deepflow-backend); \
+        # $(call docker-build,deepflow-frontend,deepflow-frontend); \
+		docker build -t deepflow-frontend:$(VERSION) . \
+            -f scripts/images/deepflow-frontend/Dockerfile \
+            --build-arg NEXT_PUBLIC_API_URL=/deepflow-backend; \
+	elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ]; then \
+		wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .; \
+	elif [ "$*" = "mineru-910C" ]; then \
+		wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .; \
+	elif [ "$*" = "mineru-310P" ]; then \
+		wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .; \
+	else \
+		$(call docker-build,$*,deepdata-$*); \
+	fi
+
+.PHONY: build-%
+build-%: %-docker-build
+	@:
+
+.PHONY: build
+build: database-docker-build gateway-docker-build backend-docker-build frontend-docker-build runtime-docker-build backend-python-docker-build
+
+# ========== Utility Targets ==========
+
+.PHONY: create-namespace
+create-namespace:
+	kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
+
+# ========== Node Setup/Cleanup Targets ==========
+
+.PHONY: node-setup
+node-setup:
+	@echo "Configure dedicated nodes for DeepData deployment?"
+	@echo "This will apply labels and optional taints to selected nodes."
+	@echo "1. Yes - Configure nodes interactively"
+	@echo "2. No - Use default scheduling"
+	@echo -n "Enter choice (default: 2): "
+	@read NODE_SETUP_CHOICE; \
+	if [ "$$NODE_SETUP_CHOICE" = "1" ]; then \
+		chmod +x scripts/k8s/node-setup.sh; \
+		./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
+	fi
+
+.PHONY: node-cleanup
+node-cleanup:
+	@chmod +x scripts/k8s/node-cleanup.sh
+	@./scripts/k8s/node-cleanup.sh --namespace $(NAMESPACE)
+
+# ========== Generic Install/Uninstall Targets (Redirect to prompt-installer) ==========
+
+.PHONY: install-%
+install-%:
+ifeq ($(origin INSTALLER), undefined)
+	$(call prompt-installer,$*-$$INSTALLER-install)
+else
+	$(MAKE) $*-$(INSTALLER)-install
+endif
+
+.PHONY: install
+install:
+ifeq ($(origin INSTALLER), undefined)
+	$(call prompt-installer,deepdata-$$INSTALLER-install milvus-$$INSTALLER-install)
+else
+	$(MAKE) deepdata-$(INSTALLER)-install
+	$(MAKE) milvus-$(INSTALLER)-install
+	@rm -f /tmp/deepdata-helm-args.sh
+endif
+
+.PHONY: uninstall-%
+uninstall-%:
+ifeq ($(origin INSTALLER), undefined)
+	$(call prompt-uninstaller,$*-$$INSTALLER-uninstall)
+else
+	$(MAKE) $*-$(INSTALLER)-uninstall
+endif
+
+.PHONY: uninstall
+uninstall:
+ifeq ($(origin INSTALLER), undefined)
+	$(call prompt-uninstaller,label-studio-$$INSTALLER-uninstall milvus-$$INSTALLER-uninstall deepflow-$$INSTALLER-uninstall deepdata-$$INSTALLER-uninstall)
+else
+	@if [ "$(INSTALLER)" = "docker" ]; then \
+		echo "Delete volumes? (This will remove all data)"; \
+    	echo "1. Yes - Delete volumes"; \
+    	echo "2. No - Keep volumes"; \
+    	echo -n "Enter choice (default: 2): "; \
+    	read DELETE_VOLUMES_CHOICE; \
+    	export DELETE_VOLUMES_CHOICE; \
+	fi; \
+	$(MAKE) label-studio-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
+	$(MAKE) milvus-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
+	$(MAKE) deepflow-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
+	$(MAKE) deepdata-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE
+endif
+
+# ========== Docker Install/Uninstall Targets ==========
+
+# Valid service targets for docker install/uninstall
+VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
+
+# Generic docker service install target
+.PHONY: %-docker-install
+%-docker-install:
+	@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
+		echo "Error: Unknown service target '$*'"; \
+		echo "Valid service targets are:"; \
+		for target in $(VALID_SERVICE_TARGETS); do \
+			echo "  - $$target"; \
+		done; \
+		exit 1; \
+	fi
+	@if [ ! -f deployment/docker/deepdata/.env ]; then \
+		echo "ERROR: deployment/docker/deepdata/.env not found."; \
+		echo "Create it from the template:"; \
+		echo "  cp deployment/docker/deepdata/.env.example deployment/docker/deepdata/.env"; \
+		echo "Then edit it with your actual passwords."; \
+		exit 1; \
+	fi
+	@if [ "$*" = "label-studio" ]; then \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile label-studio up -d; \
+	elif [ "$*" = "deepdata" ]; then \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d; \
+	elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
+	elif [ "$*" = "mineru-310P" ]; then \
+		REGISTRY=$(REGISTRY) EXTRA_ARGS="--enforce-eager --dtype float16" docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
+	elif [ "$*" = "data-juicer" ]; then \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile data-juicer up -d deepdata-data-juicer; \
+	elif [ "$*" = "redis" ]; then \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile redis up -d deepdata-redis; \
+	elif [ "$*" = "milvus" ]; then \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d; \
+	elif [ "$*" = "deepflow" ]; then \
+		cp runtime/deepflow/.env deployment/docker/deepdata/.env; \
+		cp runtime/deepflow/conf.yaml deployment/docker/deepdata/conf.yaml; \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile deepflow up -d; \
+	else \
+		REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d deepdata-$*; \
+	fi
+
+# Generic docker service uninstall target
+.PHONY: %-docker-uninstall
+%-docker-uninstall:
+	@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
+		echo "Error: Unknown service target '$*'"; \
+		echo "Valid service targets are:"; \
+		for target in $(VALID_SERVICE_TARGETS); do \
+			echo "  - $$target"; \
+		done; \
+		exit 1; \
+	fi
+	@if [ "$*" = "label-studio" ]; then \
+		docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s label-studio; \
+	elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ] || [ "$*" = "mineru-310P" ]; then \
+		docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-mineru; \
+	elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
+		docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-data-juicer; \
+	elif [ "$*" = "redis" ]; then \
+		docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-redis; \
+	elif [ "$*" = "deepdata" ]; then \
+		if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
+			docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down -v; \
+		else \
+			docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down; \
+		fi; \
+	elif [ "$*" = "deepflow" ]; then \
+		docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepflow-backend deepflow-frontend; \
+	elif [ "$*" = "milvus" ]; then \
+		docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s milvus etcd minio; \
+	else \
+		$(call docker-compose-service,$*,down,deployment/docker/deepdata); \
+	fi
+
+# ========== Kubernetes Install/Uninstall Targets ==========
+
+# Valid k8s targets
+VALID_K8S_TARGETS := deepdata deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
+
+# Generic k8s install target
+.PHONY: %-k8s-install
+%-k8s-install: create-namespace
+	@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
+		echo "Error: Unknown k8s target '$*'"; \
+		echo "Valid k8s targets are:"; \
+		for target in $(VALID_K8S_TARGETS); do \
+			echo "  - $$target"; \
+		done; \
+		exit 1; \
+	fi
+	@if [ "$*" = "label-studio" ]; then \
+		if [ -f /tmp/deepdata-helm-args.sh ]; then . /tmp/deepdata-helm-args.sh; fi; \
+		helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install $${HELM_LABEL_STUDIO_TOLERATIONS:-}; \
+    elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
+		kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
+	elif [ "$*" = "mineru-310P" ]; then \
+		kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
+	elif [ "$*" = "deepdata" ]; then \
+		echo ""; \
+		chmod +x scripts/k8s/node-setup.sh; \
+		./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
+		if [ -f /tmp/deepdata-helm-args.sh ]; then \
+			. /tmp/deepdata-helm-args.sh; \
+		fi; \
+		chmod +x scripts/k8s/collect-secrets.sh; \
+		eval $$(NAMESPACE=$(NAMESPACE) bash scripts/k8s/collect-secrets.sh); \
+		if [ "$$SECRETS_CREATE" = "SKIP" ]; then \
+			echo "[SKIP] Secrets collection failed — skipping deepdata Helm install"; \
+			rm -f /tmp/deepdata-helm-args.sh; \
+			exit 0; \
+		fi; \
+		if [ -n "$$HELM_VALUES_FILE" ] && [ -f "$$HELM_VALUES_FILE" ]; then \
+			HELM_EXTRA_ARGS="-f $$HELM_VALUES_FILE"; \
+		else \
+			HELM_EXTRA_ARGS=""; \
+		fi; \
+		if [ -n "$$HELM_NODE_SELECTOR_ARGS" ] || [ -n "$$HELM_TOLERATIONS_ARGS" ]; then \
+			helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS $$HELM_NODE_SELECTOR_ARGS $$HELM_TOLERATIONS_ARGS; \
+		else \
+			helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS; \
+		fi; \
+		rm -f /tmp/deepdata-secret-values-*.yaml; \
+	elif [ "$*" = "deepflow" ]; then \
+		cp runtime/deepflow/.env deployment/helm/deepflow/charts/public/.env; \
+		cp runtime/deepflow/conf.yaml deployment/helm/deepflow/charts/public/conf.yaml; \
+		helm upgrade deepflow deployment/helm/deepflow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \
+	elif [ "$*" = "milvus" ]; then \
+		chmod +x scripts/k8s/collect-secrets.sh; \
+		bash scripts/k8s/collect-secrets.sh --component milvus -n $(NAMESPACE); \
+		MILVUS_MINIO_ACCESS_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accesskey}' | base64 -d); \
+		MILVUS_MINIO_SECRET_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretkey}' | base64 -d); \
+		if [ -f /tmp/deepdata-helm-args.sh ]; then \
+			. /tmp/deepdata-helm-args.sh; \
+		fi; \
+		helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
+			--set minio.accessKey="$$MILVUS_MINIO_ACCESS_KEY" \
+			--set minio.secretKey="$$MILVUS_MINIO_SECRET_KEY" \
+			--set log.persistence.persistentVolumeClaim.accessModes=ReadWriteOnce \
+			$$HELM_MILVUS_TOLERATIONS; \
+	elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
+		kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
+	fi
+
+# Generic k8s uninstall target
+.PHONY: %-k8s-uninstall
+%-k8s-uninstall:
+	@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
+		echo "Error: Unknown k8s target '$*'"; \
+		echo "Valid k8s targets are:"; \
+		for target in $(VALID_K8S_TARGETS); do \
+			echo "  - $$target"; \
+		done; \
+		exit 1; \
+	fi
+	@if [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
+		kubectl delete -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
+	elif [ "$*" = "mineru-310P" ]; then \
+		kubectl delete -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
+	elif [ "$*" = "deepdata" ]; then \
+		helm uninstall deepdata -n $(NAMESPACE) --ignore-not-found; \
+		$(MAKE) node-cleanup; \
+	elif [ "$*" = "deepflow" ]; then \
+		helm uninstall deepflow -n $(NAMESPACE) --ignore-not-found; \
+	elif [ "$*" = "milvus" ]; then \
+		helm uninstall milvus -n $(NAMESPACE) --ignore-not-found; \
+	elif [ "$*" = "label-studio" ]; then \
+		helm uninstall label-studio -n $(NAMESPACE) --ignore-not-found; \
+	elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
+		kubectl delete -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
+	fi
+
+# ========== Download Targets ==========
+
+# List of all images to download
+DOWNLOAD_IMAGES := \
+	deepdata-backend \
+	deepdata-frontend \
+	deepdata-database \
+	deepdata-runtime \
+	deepdata-backend-python \
+	deepdata-gateway \
+
+# Detect architecture for smart default
+CURRENT_ARCH := $(shell uname -m 2>/dev/null)
+ifeq ($(CURRENT_ARCH),aarch64)
+	DEFAULT_OPTION := 2
+else ifeq ($(CURRENT_ARCH),arm64)
+	DEFAULT_OPTION := 2
+else
+	DEFAULT_OPTION := 1
+endif
+
+
+#Usage: make download VERSION=1.0.0 REGISTRY=<registry-prefix>/ SAVE=true
+.PHONY: download
+download:
+	@echo "Select target platform for images:"
+	@echo "  1) x86_64 (Intel/AMD, linux/amd64)"
+	@echo "  2) ARM64  (Apple Silicon, Raspberry Pi 4+, linux/arm64)"
+	@{ \
+		read -p "Enter choice [1/2] (default: $(DEFAULT_OPTION)): " plat_choice; \
+		case "$${plat_choice:-$(DEFAULT_OPTION)}" in \
+			1) PLATFORM="linux/amd64" ;; \
+			2) PLATFORM="linux/arm64" ;; \
+			*) echo "Invalid choice. Using default option $(DEFAULT_OPTION)."; \
+			   PLATFORM=$$([ "$(DEFAULT_OPTION)" = "1" ] && echo "linux/amd64" || echo "linux/arm64");; \
+		esac; \
+		\
+		read -p "Save images to dist/ for offline use? (y/n): " save_resp; \
+		case "$$save_resp" in \
+			[yY]|[yY][eE][sS]) SAVE="true" ;; \
+			*) SAVE="false" ;; \
+		esac; \
+		\
+		if [ -z "$(REGISTRY)" ] && [ "$$SAVE" != "true" ]; then \
+			echo "Error: REGISTRY unset and SAVE=false. Nothing to do."; \
+			exit 1; \
+		fi; \
+		\
+		if [ "$$SAVE" = "true" ]; then mkdir -p dist; fi; \
+		failed=0; \
+		\
+		for image in $(DOWNLOAD_IMAGES); do \
+			if [ -z "$(REGISTRY)" ]; then \
+				src_image="$$image:$(VERSION)"; \
+				echo "Using local image: $$src_image"; \
+			else \
+				src_image="$(REGISTRY)$$image:$(VERSION)"; \
+				echo "Pulling $$src_image ($$PLATFORM)..."; \
+				if ! docker pull --platform "$$PLATFORM" "$$src_image"; then \
+					echo "✗ Failed to pull $$src_image"; \
+					failed=$$(($$failed + 1)); \
+					continue; \
+				fi; \
+			fi; \
+			\
+			if [ "$$SAVE" = "true" ]; then \
+				output="dist/$$image-$(VERSION).tar"; \
+				echo "Saving to $$output..."; \
+				if docker save -o "$$output" "$$src_image"; then \
+					echo "✓ Saved $$image"; \
+				else \
+					echo "✗ Failed to save $$src_image"; \
+					failed=$$(($$failed + 1)); \
+				fi; \
+			else \
+				echo "✓ Ready: $$src_image"; \
+			fi; \
+			echo ""; \
+		done; \
+		\
+		if [ $$failed -eq 0 ]; then \
+			if [ "$$SAVE" = "true" ]; then \
+				echo "✅ All images saved to dist/"; \
+				echo "Load on target: docker load -i <image>.tar"; \
+			else \
+				echo "✅ All images ready in local Docker daemon"; \
+			fi; \
+		else \
+			echo "❌ $$failed image(s) failed"; \
+			exit 1; \
+		fi; \
+	}
+
+DEEPFLOW_IMAGES := \
+	deepflow-backend \
+	deepflow-frontend
+
+.PHONY: download-deepflow
+download-deepflow:
+	$(MAKE) download DOWNLOAD_IMAGES="$(DEEPFLOW_IMAGES)"
+
+# Download Sealed Secrets controller image for offline/air-gapped environments
+SEALED_SECRETS_IMAGE := bitnami/sealed-secrets-controller:latest
+.PHONY: download-sealed-secrets
+download-sealed-secrets:
+	@echo "Pulling Sealed Secrets controller image..."
+	@mkdir -p dist
+	docker pull $(SEALED_SECRETS_IMAGE)
+	docker save $(SEALED_SECRETS_IMAGE) -o dist/sealed-secrets-controller.tar
+	@echo "✅ Saved to dist/sealed-secrets-controller.tar"
+	@echo "Transfer to offline environment and load with: docker load -i dist/sealed-secrets-controller.tar"
+
+# Load all downloaded images from dist/ directory
+.PHONY: load-images
+load-images:
+	@if [ ! -d "dist" ]; then \
+		echo "Error: dist/ directory not found"; \
+		echo "Please run 'make download' first to download images"; \
+		exit 1; \
+	fi
+	@echo "Loading images from dist/..."
+	@count=0; \
+	for tarfile in dist/*.tar.gz; do \
+		if [ -f "$$tarfile" ]; then \
+			echo "Loading $$tarfile..."; \
+			docker load -i "$$tarfile"; \
+			count=$$((count + 1)); \
+			echo "✓ Loaded $$tarfile"; \
+			echo ""; \
+		fi; \
+	done; \
+	if [ $$count -eq 0 ]; then \
+		echo "No image files found in dist/"; \
+		echo "Please run 'make download' first"; \
+		exit 1; \
+	else \
+		echo "Successfully loaded $$count image(s)"; \
+	fi

deployment/docker/deepdata/.env.example

@@ -0,0 +1,30 @@
+# DeepData Environment Variables Template
+# Copy this file to .env and fill in the values
+#   cp .env.example .env
+# IMPORTANT: Never commit .env to git! It is already in .gitignore.
+#
+# For K8s/Helm deployment: secrets are managed via Sealed Secrets.
+# For Docker deployment: use this .env file (gitignored, local only).
+
+# Database
+DB_PASSWORD=your-secure-password-here
+
+# JWT Authentication
+JWT_SECRET=your-secure-jwt-secret-here
+DEEPDATA_JWT_ENABLE=false
+
+# MinIO (for Milvus storage)
+MINIO_ACCESS_KEY=your-minio-access-key
+MINIO_SECRET_KEY=your-minio-secret-key
+
+# Label Studio
+LABEL_STUDIO_USERNAME=admin@demo.com
+LABEL_STUDIO_PASSWORD=your-labelstudio-password
+LABEL_STUDIO_USER_TOKEN=your-labelstudio-token
+LABEL_STUDIO_HOST=
+
+# Optional: SSL Certificate Password (for encrypted private keys)
+CERT_PASS=
+
+# Optional: Domain for HTTPS
+DOMAIN=

deployment/docker/deepdata/docker-compose.yml

@@ -0,0 +1,419 @@
+services:
+  # ==============================
+  # Core DeepData Services (always enabled)
+  # ==============================
+
+  deepdata-backend:
+    container_name: deepdata-backend
+    image: ${REGISTRY:-}deepdata-backend:${VERSION:-latest}
+    restart: on-failure
+    privileged: true
+    environment:
+      - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
+      - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file}
+      - deepdata.jwt.enable=${DEEPDATA_JWT_ENABLE:-false}
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
+    volumes:
+      - dataset_volume:/dataset
+      - flow_volume:/flow
+      - log_volume:/var/log/deepdata
+      - operator-upload-volume:/operators/upload
+      - operator-runtime-volume:/operators/extract
+    networks: [ deepdata ]
+    depends_on:
+      - deepdata-database
+
+  deepdata-backend-python:
+    container_name: deepdata-backend-python
+    image: ${REGISTRY:-}deepdata-backend-python:${VERSION:-latest}
+    restart: on-failure
+    privileged: true
+    ports:
+      - "18000:18000"
+    environment:
+      - log_level=DEBUG
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
+      - LOG_PVC_MONITOR_ENABLED=${LOG_PVC_MONITOR_ENABLED:-true}
+      - LOG_PVC_MONITOR_PATH=${LOG_PVC_MONITOR_PATH:-/var/log/deepdata}
+      - LOG_PVC_MONITOR_THRESHOLD=${LOG_PVC_MONITOR_THRESHOLD:-0.9}
+      - LOG_PVC_MONITOR_INTERVAL_SECONDS=${LOG_PVC_MONITOR_INTERVAL_SECONDS:-300}
+      - LOG_PVC_MONITOR_DELETE_BATCH_SIZE=${LOG_PVC_MONITOR_DELETE_BATCH_SIZE:-10}
+      - LOG_PVC_MONITOR_FILE_SUFFIXES=${LOG_PVC_MONITOR_FILE_SUFFIXES:-log,out,err}
+      - PGSQL_HOST=deepdata-database
+      - PGSQL_PORT=5432
+      - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
+      - deepdata_jwt_enable=${DEEPDATA_JWT_ENABLE:-false}
+      - milvus_uri=${MILVUS_URI:-http://milvus:19530}
+      - LABEL_STUDIO_USERNAME=${LABEL_STUDIO_USERNAME:-admin@demo.com}
+      - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-}
+      - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-}
+    volumes:
+      - dataset_volume:/dataset
+      - flow_volume:/flow
+      - log_volume:/var/log/deepdata
+      - graph_data_volume:/data/rag_storage
+      - operator-upload-volume:/operators/upload
+      - operator-runtime-volume:/operators/extract
+    networks: [ deepdata ]
+    depends_on:
+      - deepdata-database
+
+  deepdata-gateway:
+    container_name: deepdata-gateway
+    image: ${REGISTRY:-}deepdata-gateway:${VERSION:-latest}
+    restart: on-failure
+    privileged: true
+    ports:
+      - '8080:8080'
+    environment:
+      - JWT_SECRET=${JWT_SECRET:-}
+      - deepdata.jwt.enable=${DEEPDATA_JWT_ENABLE:-false}
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
+    volumes:
+      - log_volume:/var/log/deepdata
+    networks: [ deepdata ]
+
+  deepdata-frontend:
+    container_name: deepdata-frontend
+    image: ${REGISTRY:-}deepdata-frontend:${VERSION:-latest}
+    restart: on-failure
+    ports:
+      - "30000:3000"
+    environment:
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
+    volumes:
+      - frontend_log_volume:/var/log/deepdata/frontend
+    networks: [ deepdata ]
+    depends_on:
+      - deepdata-backend
+      - deepdata-backend-python
+
+  deepdata-database:
+    container_name: deepdata-database
+    image: ${REGISTRY:-}deepdata-database:${VERSION:-latest}
+    restart: on-failure
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
+    volumes:
+      - postgresql_volume:/var/lib/postgresql/data
+      - database_log_volume:/var/log/deepdata/database
+    ports:
+      - "5432:5432"
+    networks: [ deepdata ]
+
+  deepdata-runtime:
+    container_name: deepdata-runtime
+    image: ${REGISTRY:-}deepdata-runtime:${VERSION:-latest}
+    restart: on-failure
+    environment:
+      RAY_DEDUP_LOGS: "0"
+      RAY_TQDM_PATCH_PRINT: "0"
+      LOG_ROTATION_MAX_SIZE: ${LOG_ROTATION_MAX_SIZE:-100MB}
+      LOG_ROTATION_BACKUP_COUNT: ${LOG_ROTATION_BACKUP_COUNT:-30}
+      PG_HOST: "deepdata-database"
+      PG_PORT: "5432"
+      PG_USER: "postgres"
+      PG_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
+      PG_DATABASE: "deepdata"
+    command:
+      - python
+      - /opt/runtime/deepdata/operator_runtime.py
+      - --port
+      - "8081"
+    volumes:
+      - ray_log_volume:/tmp/ray
+      - log_volume:/var/log/deepdata
+      - dataset_volume:/dataset
+      - flow_volume:/flow
+      - operator-runtime-volume:/opt/runtime/deepdata/ops/user
+      - operator-packages-volume:/usr/local/lib/ops/site-packages
+    networks: [ deepdata ]
+
+  # =============================
+  # Optional: Mineru NPU Engine (profile: mineru)
+  # ==============================
+  deepdata-mineru:
+    container_name: deepdata-mineru
+    image: deepdata-mineru
+    restart: on-failure
+    environment:
+      MINERU_MODEL_SOURCE: local
+      MINERU_DEVICE_MODE: npu
+      VLLM_WORKER_MULTIPROC_METHOD: spawn
+    privileged: true
+    entrypoint: mineru-openai-server
+    command:
+      --engine vllm
+      --host 0.0.0.0
+      --port 8000
+      ${EXTRA_ARGS:-}
+    volumes:
+      - /var/log/npu/:/usr/slog
+      - /usr/local/dcmi:/usr/local/dcmi
+      - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
+      - /usr/local/Ascend/driver:/usr/local/Ascend/driver
+    networks: [ deepdata ]
+    profiles: [ mineru ]
+    devices:
+      - /dev/davinci0
+      - /dev/davinci_manager
+      - /dev/devmm_svm
+      - /dev/hisi_hdc
+
+  # ==============================
+  # Optional: Redis (profile: redis)
+  # ==============================
+  deepdata-redis:
+    container_name: deepdata-redis
+    image: redis:8.2.3
+    restart: on-failure
+    ports:
+      - "6379:6379"
+    networks: [ deepdata ]
+    profiles: [ redis ]
+
+  # ==============================
+  # Optional: Data Juicer (profile: data-juicer)
+  # ==============================
+  deepdata-data-juicer:
+    container_name: deepdata-data-juicer
+    image: datajuicer/data-juicer:v1.4.4
+    restart: on-failure
+    command:
+      - uvicorn
+      - service:app
+      - --host
+      - "0.0.0.0"
+    volumes:
+      - dataset_volume:/dataset
+      - flow_volume:/flow
+    networks: [ deepdata ]
+    profiles: [ data-juicer ]
+
+
+  # ==============================
+  # Optional: Deer Flow (profile: deepflow)
+  # ==============================
+  deepflow-backend:
+    image: ${REGISTRY:-}deepflow-backend
+    container_name: deepflow-backend
+    env_file:
+      - .env
+    volumes:
+      - ./conf.yaml:/app/conf.yaml:ro
+      - deepflow-log-volume:/var/log/deepflow
+    restart: on-failure
+    networks:
+      - deepdata
+    profiles:
+      - deepflow
+
+  deepflow-frontend:
+    image: ${REGISTRY:-}deepflow-frontend
+    container_name: deepflow-frontend
+    env_file:
+      - .env
+    depends_on:
+      - deepflow-backend
+    restart: on-failure
+    networks:
+      - deepdata
+    profiles:
+      - deepflow
+
+  # ==============================
+  # Optional: Label Studio (profile: label-studio)
+  # ==============================
+  label-studio-pgbouncer:
+    container_name: label-studio-pgbouncer
+    image: edoburu/pgbouncer:latest
+    restart: on-failure
+    environment:
+      - DB_HOST=deepdata-database
+      - DB_PORT=5432
+      - DB_NAME=labelstudio
+      - DB_USER=postgres
+      - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
+      - AUTH_TYPE=scram-sha-256
+      - POOL_MODE=transaction
+      - MAX_CLIENT_CONN=100
+      - DEFAULT_POOL_SIZE=20
+      - MAX_DB_CONNECTIONS=20
+    networks:
+      - deepdata
+    profiles: [ label-studio ]
+
+  label-studio:
+    container_name: label-studio
+    stdin_open: true
+    tty: true
+    image: heartexlabs/label-studio:latest
+    privileged: true
+    restart: on-failure
+    user: root
+    expose:
+      - "8000"
+    ports:
+      - "30001:8000"
+    depends_on:
+      - label-studio-pgbouncer
+    environment:
+      - DJANGO_DB=default
+      - POSTGRE_NAME=labelstudio
+      - POSTGRE_USER=postgres
+      - POSTGRE_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
+      - POSTGRE_PORT=5432
+      - POSTGRE_HOST=label-studio-pgbouncer
+      - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-}
+      - LOCAL_FILES_SERVING_ENABLED=true
+      - LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local
+      - USE_USERNAME_FOR_LOGIN=true
+      - LABEL_STUDIO_USERNAME=admin@demo.com
+      - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-}
+      - LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN=true
+      - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-}
+      - LOG_LEVEL=DEBUG
+    volumes:
+      - label-studio-data:/label-studio/data:rw
+      - dataset_volume:/label-studio/local:rw
+    networks:
+      - deepdata
+    command: label-studio-uwsgi
+    profiles: [ label-studio ]
+
+  # ==============================
+  # Optional: Milvus (profile: milvus)
+  # ==============================
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.18
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - etcd-volume:/etcd
+    command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    restart: always
+    networks:
+      - deepdata
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    profiles:
+      - milvus
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2024-12-18T13-15-44Z
+    environment:
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-}
+    ports:
+      - "9001:9001"
+      - "9000:9000"
+    volumes:
+      - minio-volume:/data
+    command: minio server /data --console-address ":9001"
+    networks:
+      - deepdata
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    profiles:
+      - milvus
+
+  milvus:
+    container_name: milvus-standalone
+    restart: on-failure
+    image: milvusdb/milvus:v2.6.5
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+    - seccomp:unconfined
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+      MQ_TYPE: woodpecker
+    volumes:
+      - milvus-volume:/var/lib/milvus
+    networks:
+      - deepdata
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+      - "minio"
+    profiles:
+      - milvus
+
+# ==============================
+# Volumes
+# ==============================
+volumes:
+  dataset_volume:
+    name: deepdata-dataset-volume
+  flow_volume:
+    name: deepdata-flow-volume
+  log_volume:
+    name: deepdata-log-volume
+  postgresql_volume:
+    name: deepdata-postgresql-volume
+  ray_log_volume:
+    name: deepdata-ray-log-volume
+  frontend_log_volume:
+    name: deepdata-frontend-log-volume
+  database_log_volume:
+    name: deepdata-database-log-volume
+  operator-upload-volume:
+    name: deepdata-operator-upload-volume
+  operator-runtime-volume:
+    name: deepdata-operator-runtime-volume
+  operator-packages-volume:
+    name: deepdata-operator-packages-volume
+  graph_data_volume:
+    name: deepdata-graph-data-volume
+
+  # Deer Flow
+  deepflow-log-volume:
+    name: deepflow-log-volume
+
+  # Label Studio
+  label-studio-data:
+    name: label-studio-data-volume
+  label-studio-db:
+    name: label-studio-db-volume
+
+  # Milvus
+  etcd-volume:
+    name: milvus-etcd-volume
+  minio-volume:
+    name: milvus-minio-volume
+  milvus-volume:
+    name: milvus-milvus-volume
+
+
+# ==============================
+# Networks
+# ==============================
+networks:
+  deepdata:
+    driver: bridge
+    name: deepdata-network

deployment/helm/deepdata/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/Chart.yaml

@@ -0,0 +1,48 @@
+apiVersion: v2
+name: deepdata
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"
+
+dependencies:
+  - name: public
+    version: 0.0.1
+  - name: backend
+    version: 0.0.1
+  - name: gateway
+    version: 0.0.1
+  - name: frontend
+    version: 0.0.1
+  - name: database
+    version: 0.0.1
+  - name: backend-python
+    version: 0.0.1
+  - name: kuberay-operator
+    version: 1.4.2
+    condition: ray-cluster.enabled
+  - name: ray-cluster
+    version: 1.4.2
+    condition: ray-cluster.enabled
+  - name: runtime
+    version: 0.0.1
+    condition: runtime.enabled
+

deployment/helm/deepdata/charts/backend-python/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/charts/backend-python/Chart.yaml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: backend-python
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"
+
+dependencies:
+  - name: database
+    repository: file://../database
+    version: 0.0.1

deployment/helm/deepdata/charts/backend-python/templates/_helpers.tpl

@@ -0,0 +1,75 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "backend-python.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "backend-python.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "backend-python.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "backend-python.labels" -}}
+helm.sh/chart: {{ include "backend-python.chart" . }}
+{{ include "backend-python.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "backend-python.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "backend-python.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "backend-python.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "backend-python.fullname" .) .Values.serviceAccount.name -}}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of image
+*/}}
+{{- define "backend-python.image" -}}
+{{- $name := default .Values.image.repository .Values.global.image.backendPython.name }}
+{{- $tag := default .Values.image.tag .Values.global.image.backendPython.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/backend-python/templates/deployment.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "backend-python.fullname" . }}
+  labels:
+    {{- include "backend-python.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "backend-python.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "backend-python.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "backend-python.serviceAccountName" . }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ include "backend-python.image" . }}"
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployment/helm/deepdata/charts/backend-python/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "backend-python.fullname" . }}
+  labels:
+    {{- include "backend-python.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.port }}
+      protocol: TCP
+      name: {{ .Chart.Name }}
+  selector:
+    {{- include "backend.selectorLabels" . | nindent 4 }}

deployment/helm/deepdata/charts/backend-python/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "backend-python.serviceAccountName" . }}
+  labels:
+    {{- include "backend-python.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

deployment/helm/deepdata/charts/backend-python/values.yaml

@@ -0,0 +1,108 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: "deepdata-backend-python"
+  # This sets the pull policy for images.
+  pullPolicy: "IfNotPresent"
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: "deepdata-backend-python"
+fullnameOverride: "deepdata-backend-python"
+
+env: []
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: ClusterIP
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  port: 18000
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+# livenessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+# readinessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

deployment/helm/deepdata/charts/backend/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/charts/backend/Chart.yaml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: backend
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"
+
+dependencies:
+  - name: database
+    repository: file://../database
+    version: 0.0.1

deployment/helm/deepdata/charts/backend/templates/_helpers.tpl

@@ -0,0 +1,76 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "backend.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "backend.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "backend.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "backend.labels" -}}
+helm.sh/chart: {{ include "backend.chart" . }}
+app: {{ .Release.Name }}
+{{ include "backend.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "backend.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "backend.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "backend.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "backend.fullname" .) .Values.serviceAccount.name -}}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of image
+*/}}
+{{- define "backend.image" -}}
+{{- $name := default .Values.image.repository .Values.global.image.backend.name }}
+{{- $tag := default .Values.image.tag .Values.global.image.backend.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/backend/templates/deployment.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "backend.fullname" . }}
+  labels:
+    {{- include "backend.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "backend.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "backend.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "backend.serviceAccountName" . }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ include "backend.image" . }}"
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployment/helm/deepdata/charts/backend/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "backend.fullname" . }}
+  labels:
+    {{- include "backend.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.port }}
+      protocol: TCP
+      name: {{ .Chart.Name }}
+  selector:
+    {{- include "backend.selectorLabels" . | nindent 4 }}

deployment/helm/deepdata/charts/backend/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "backend.serviceAccountName" . }}
+  labels:
+    {{- include "backend.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

deployment/helm/deepdata/charts/backend/values.yaml

@@ -0,0 +1,108 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: "deepdata-backend"
+  # This sets the pull policy for images.
+  pullPolicy: "IfNotPresent"
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: "deepdata-backend"
+fullnameOverride: "deepdata-backend"
+
+env: []
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: ClusterIP
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  port: 8080
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+# livenessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+# readinessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

deployment/helm/deepdata/charts/database/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/charts/database/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: database
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"

deployment/helm/deepdata/charts/database/templates/_helpers.tpl

@@ -0,0 +1,75 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "database.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "database.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "database.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "database.labels" -}}
+helm.sh/chart: {{ include "database.chart" . }}
+{{ include "database.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "database.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "database.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "database.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "database.fullname" .) .Values.serviceAccount.name -}}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of image
+*/}}
+{{- define "database.image" -}}
+{{- $name := default .Values.image.repository .Values.global.image.database.name }}
+{{- $tag := default .Values.image.tag .Values.global.image.database.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/database/templates/deployment.yaml

@@ -0,0 +1,91 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "database.fullname" . }}
+  labels:
+    {{- include "database.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "database.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "database.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "database.serviceAccountName" . }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.initContainers }}
+      initContainers:
+        {{- $root := $ }}
+        {{- range . }}
+        {{- $defaultImage := dict "image" (include "database.image" $root) }}
+        {{- $containerConfig := merge (deepCopy .) $defaultImage }}
+        - {{ toYaml $containerConfig | nindent 10 | trim }}
+        {{- end }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ include "database.image" . }}"
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployment/helm/deepdata/charts/database/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "database.fullname" . }}
+  labels:
+    {{- include "database.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.port }}
+      protocol: TCP
+      name: {{ .Chart.Name }}
+  selector:
+    {{- include "database.selectorLabels" . | nindent 4 }}

deployment/helm/deepdata/charts/database/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "database.serviceAccountName" . }}
+  labels:
+    {{- include "database.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

deployment/helm/deepdata/charts/database/values.yaml

@@ -0,0 +1,110 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: "deepdata-database"
+  # This sets the pull policy for images.
+  pullPolicy: "IfNotPresent"
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: "deepdata-database"
+fullnameOverride: "deepdata-database"
+
+env: []
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: ClusterIP
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  port: 5432
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+# livenessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+# readinessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+initContainers: []

deployment/helm/deepdata/charts/frontend/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/charts/frontend/Chart.yaml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: frontend
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"
+
+dependencies:
+  - name: backend
+    repository: file://../backend
+    version: 0.0.1

deployment/helm/deepdata/charts/frontend/templates/_helpers.tpl

@@ -0,0 +1,76 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "frontend.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "frontend.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "frontend.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "frontend.labels" -}}
+helm.sh/chart: {{ include "frontend.chart" . }}
+{{ include "frontend.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "frontend.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "frontend.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app: {{ include "frontend.name" . }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "frontend.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "frontend.fullname" .) .Values.serviceAccount.name -}}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of image
+*/}}
+{{- define "frontend.image" -}}
+{{- $name := default .Values.image.repository .Values.global.image.frontend.name }}
+{{- $tag := default .Values.image.tag .Values.global.image.frontend.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/frontend/templates/deployment.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "frontend.fullname" . }}
+  labels:
+    {{- include "frontend.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "frontend.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "frontend.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "frontend.serviceAccountName" . }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ include "frontend.image" . }}"
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployment/helm/deepdata/charts/frontend/templates/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "frontend.fullname" . }}
+  labels:
+    {{- include "frontend.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.port }}
+      protocol: TCP
+      name: {{ .Chart.Name }}
+      {{- if eq .Values.service.type "NodePort" }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
+  selector:
+    {{- include "frontend.selectorLabels" . | nindent 4 }}

deployment/helm/deepdata/charts/frontend/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "frontend.serviceAccountName" . }}
+  labels:
+    {{- include "frontend.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

deployment/helm/deepdata/charts/frontend/values.yaml

@@ -0,0 +1,109 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: "deepdata-frontend"
+  # This sets the pull policy for images.
+  pullPolicy: "IfNotPresent"
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: "deepdata-frontend"
+fullnameOverride: "deepdata-frontend"
+
+env: []
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: NodePort
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  port: 3000
+  nodePort: 30000
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+# livenessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+# readinessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

deployment/helm/deepdata/charts/gateway/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/charts/gateway/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: gateway
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"

deployment/helm/deepdata/charts/gateway/templates/_helpers.tpl

@@ -0,0 +1,76 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "gateway.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "gateway.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "gateway.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "gateway.labels" -}}
+helm.sh/chart: {{ include "gateway.chart" . }}
+app: {{ .Release.Name }}
+{{ include "gateway.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "gateway.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "gateway.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "gateway.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "gateway.fullname" .) .Values.serviceAccount.name -}}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of image
+*/}}
+{{- define "gateway.image" -}}
+{{- $name := default .Values.image.repository .Values.global.image.gateway.name }}
+{{- $tag := default .Values.image.tag .Values.global.image.gateway.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/gateway/templates/deployment.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "gateway.fullname" . }}
+  labels:
+    {{- include "gateway.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "gateway.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "gateway.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "gateway.serviceAccountName" . }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ include "gateway.image" . }}"
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployment/helm/deepdata/charts/gateway/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "gateway.fullname" . }}
+  labels:
+    {{- include "gateway.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.port }}
+      protocol: TCP
+      name: {{ .Chart.Name }}
+  selector:
+    {{- include "gateway.selectorLabels" . | nindent 4 }}

deployment/helm/deepdata/charts/gateway/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "gateway.serviceAccountName" . }}
+  labels:
+    {{- include "gateway.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

deployment/helm/deepdata/charts/gateway/values.yaml

@@ -0,0 +1,108 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: "deepdata-gateway"
+  # This sets the pull policy for images.
+  pullPolicy: "IfNotPresent"
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: "deepdata-gateway"
+fullnameOverride: "deepdata-gateway"
+
+env: []
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: ClusterIP
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  port: 8080
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+# livenessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+# readinessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

deployment/helm/deepdata/charts/kuberay-operator/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+
+name: kuberay-operator
+
+description: A Helm chart for deploying the Kuberay operator on Kubernetes.
+
+version: 1.4.2
+
+type: application
+
+keywords:
+- ray
+- ray operator
+- distributed computing
+- data processing
+- machine learning
+- deep learning
+- hyperparameter tuning
+- reinforcement learning
+- model serving
+
+home: https://github.com/ray-project/kuberay
+
+icon: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png

deployment/helm/deepdata/charts/kuberay-operator/templates/_helpers.tpl

@@ -0,0 +1,335 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "kuberay-operator.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Allow the component label to be overridden, otherwise provide a default value.
+*/}}
+{{- define "kuberay-operator.component" -}}
+{{- default .Chart.Name .Values.componentOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "kuberay-operator.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "kuberay-operator.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "kuberay-operator.labels" -}}
+app.kubernetes.io/name: {{ include "kuberay-operator.name" . }}
+helm.sh/chart: {{ include "kuberay-operator.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{- /* Create the name of the deployment to use. */ -}}
+{{- define "kuberay-operator.deployment.name" -}}
+{{- include "kuberay-operator.fullname" . }}
+{{- end -}}
+
+{{/*
+FeatureGates
+*/}}
+{{- define "kuberay.featureGates" -}}
+{{- $features := "" }}
+{{- range .Values.featureGates }}
+  {{- $str := printf "%s=%t," .name .enabled }}
+  {{- $features = print $features $str }}
+{{- end }}
+{{- with .Values.featureGates }}
+--feature-gates={{ $features | trimSuffix "," }}
+{{- end }}
+{{- end }}
+
+{{- /* Create the name of the service to use. */ -}}
+{{- define "kuberay-operator.service.name" -}}
+{{- include "kuberay-operator.fullname" . }}
+{{- end -}}
+
+{{- /* Create the name of the service account to use. */ -}}
+{{- define "kuberay-operator.serviceAccount.name" -}}
+{{- if .Values.serviceAccount.create -}}
+{{- default (include "kuberay-operator.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{- /* Create the name of the cluster role to use. */ -}}
+{{- define "kuberay-operator.clusterRole.name" -}}
+{{- include "kuberay-operator.fullname" . -}}
+{{- end -}}
+
+{{- /* Create the name of the cluster role binding to use. */ -}}
+{{- define "kuberay-operator.clusterRoleBinding.name" -}}
+{{- include "kuberay-operator.fullname" . -}}
+{{- end -}}
+
+{{- /* Create the name of the role to use. */ -}}
+{{- define "kuberay-operator.role.name" -}}
+{{- include "kuberay-operator.fullname" . -}}
+{{- end -}}
+
+{{- /* Create the name of the role binding to use. */ -}}
+{{- define "kuberay-operator.roleBinding.name" -}}
+{{- include "kuberay-operator.fullname" . -}}
+{{- end -}}
+
+{{- /* Create the name of the leader election role to use. */ -}}
+{{- define "kuberay-operator.leaderElectionRole.name" -}}
+{{- include "kuberay-operator.fullname" . -}}-leader-election
+{{- end -}}
+
+{{- /* Create the name of the leader election role binding to use. */ -}}
+{{- define "kuberay-operator.leaderElectionRoleBinding.name" -}}
+{{- include "kuberay-operator.fullname" . -}}-leader-election
+{{- end -}}
+
+{{/*
+Create a template to ensure consistency for Role and ClusterRole.
+*/}}
+{{- define "role.consistentRules" -}}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  - pods/status
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/proxy
+  - services/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - create
+  - get
+  - patch
+  - update
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - update
+- apiGroups:
+  - extensions
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ray.io
+  resources:
+  - rayclusters
+  - rayjobs
+  - rayservices
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ray.io
+  resources:
+  - rayclusters/finalizers
+  - rayjobs/finalizers
+  - rayservices/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - ray.io
+  resources:
+  - rayclusters/status
+  - rayjobs/status
+  - rayservices/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - route.openshift.io
+  resources:
+  - routes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+{{- if or .batchSchedulerEnabled (eq .batchSchedulerName "volcano") }}
+- apiGroups:
+  - scheduling.volcano.sh
+  resources:
+  - podgroups
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+{{- end -}}
+{{- if or .batchSchedulerEnabled (eq .batchSchedulerName "scheduler-plugins") }}
+- apiGroups:
+  - scheduling.x-k8s.io
+  resources:
+  - podgroups
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+{{- end -}}
+{{- end -}}
+
+{{/*
+Name of image
+*/}}
+{{- define "kuberay.image" -}}
+{{- $name := .Values.image.repository }}
+{{- $tag := .Values.image.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/deployment.yaml

@@ -0,0 +1,150 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "kuberay-operator.deployment.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "kuberay-operator.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "kuberay-operator.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: {{ include "kuberay-operator.component" . }}
+        {{- with .Values.labels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- with .Values.annotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      serviceAccountName: {{ include "kuberay-operator.serviceAccount.name" . }}
+      {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }}
+      volumes:
+        - name: kuberay-logs
+          {{- if .Values.logging.sizeLimit }}
+          emptyDir:
+            sizeLimit: {{ .Values.logging.sizeLimit }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+      {{- end }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ include "kuberay.image" . }}"
+          {{- with .Values.image.pullPolicy }}
+          imagePullPolicy: {{ . }}
+          {{- end }}
+          {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }}
+          volumeMounts:
+            - name: kuberay-logs
+              mountPath: "{{ .Values.logging.baseDir }}"
+          {{- end }}
+          command:
+            - {{ .Values.operatorCommand }}
+          args:
+            {{- $argList := list -}}
+            {{- $argList = append $argList (include "kuberay.featureGates" . | trim) -}}
+            {{- if .Values.batchScheduler -}}
+            {{- if .Values.batchScheduler.enabled -}}
+            {{- $argList = append $argList "--enable-batch-scheduler" -}}
+            {{- end -}}
+            {{- if .Values.batchScheduler.name -}}
+            {{- $argList = append $argList (printf "--batch-scheduler=%s" .Values.batchScheduler.name) -}}
+            {{- end -}}
+            {{- end -}}
+            {{- $watchNamespace := "" -}}
+            {{- if and .Values.singleNamespaceInstall (not .Values.watchNamespace) -}}
+            {{- $watchNamespace = .Release.Namespace -}}
+            {{- else if .Values.watchNamespace -}}
+            {{- $watchNamespace = join "," .Values.watchNamespace -}}
+            {{- end -}}
+            {{- if $watchNamespace -}}
+            {{- $argList = append $argList "--watch-namespace" -}}
+            {{- $argList = append $argList $watchNamespace -}}
+            {{- end -}}
+            {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) -}}
+            {{- $argList = append $argList "--log-file-path" -}}
+            {{- $argList = append $argList (printf "%s/%s" .Values.logging.baseDir .Values.logging.fileName) -}}
+            {{- end -}}
+            {{- if .Values.logging.stdoutEncoder -}}
+            {{- $argList = append $argList "--log-stdout-encoder" -}}
+            {{- $argList = append $argList .Values.logging.stdoutEncoder -}}
+            {{- end -}}
+            {{- if .Values.logging.fileEncoder -}}
+            {{- $argList = append $argList "--log-file-encoder" -}}
+            {{- $argList = append $argList .Values.logging.fileEncoder -}}
+            {{- end -}}
+            {{- if hasKey .Values "useKubernetesProxy" -}}
+            {{- $argList = append $argList (printf "--use-kubernetes-proxy=%t" .Values.useKubernetesProxy) -}}
+            {{- end -}}
+            {{- if hasKey .Values "leaderElectionEnabled" -}}
+            {{- $argList = append $argList (printf "--enable-leader-election=%t" .Values.leaderElectionEnabled) -}}
+            {{- end -}}
+            {{- if and (hasKey .Values "metrics") (hasKey .Values.metrics "enabled") }}
+            {{- $argList = append $argList (printf "--enable-metrics=%t" .Values.metrics.enabled) -}}
+            {{- end -}}
+            {{- (printf "\n") -}}
+            {{- $argList | toYaml | indent 12 }}
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          livenessProbe:
+            httpGet:
+              path: /metrics
+              port: http
+            initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+            failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
+          readinessProbe:
+            httpGet:
+              path: /metrics
+              port: http
+            initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+            failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/leader_election_role.yaml

@@ -0,0 +1,37 @@
+{{- if .Values.rbacEnable }}
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "kuberay-operator.leaderElectionRole.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - update
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/leader_election_role_binding.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.rbacEnable -}}
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "kuberay-operator.leaderElectionRoleBinding.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "kuberay-operator.leaderElectionRole.name" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "kuberay-operator.serviceAccount.name" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/multiple_namespaces_role.yaml

@@ -0,0 +1,13 @@
+{{- if and .Values.rbacEnable .Values.singleNamespaceInstall .Values.crNamespacedRbacEnable }}
+{{- $watchNamespaces := default (list .Release.Namespace) .Values.watchNamespace }}
+{{- range $namespace := $watchNamespaces }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "kuberay-operator.fullname" $ }}
+  namespace: {{ $namespace }}
+  labels: {{ include "kuberay-operator.labels" $ | nindent 4 }}
+{{ include "role.consistentRules" (dict "batchSchedulerEnabled" $.Values.batchScheduler.enabled) }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/multiple_namespaces_rolebinding.yaml

@@ -0,0 +1,20 @@
+{{- if and .Values.rbacEnable .Values.singleNamespaceInstall .Values.crNamespacedRbacEnable }}
+{{- $watchNamespaces := default (list .Release.Namespace) .Values.watchNamespace }}
+{{- range $namespace := $watchNamespaces }}
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "kuberay-operator.fullname" $ }}
+  namespace: {{ $namespace }}
+  labels: {{ include "kuberay-operator.labels" $ | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "kuberay-operator.fullname" $ }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "kuberay-operator.serviceAccount.name" $ }}
+  namespace: {{ $.Release.Namespace }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/ray_rayjob_editor_role.yaml

@@ -0,0 +1,28 @@
+{{- /* ClusterRole for end users to view and edit RayJob. */ -}}
+{{- if and .Values.rbacEnable (not .Values.singleNamespaceInstall) }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rayjob-editor-role
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ray.io
+  resources:
+  - rayjobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ray.io
+  resources:
+  - rayjobs/status
+  verbs:
+  - get
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/ray_rayjob_viewer_role.yaml

@@ -0,0 +1,24 @@
+{{- /* ClusterRole for end users to view RayJob. */ -}}
+{{- if and .Values.rbacEnable (not .Values.singleNamespaceInstall) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rayjob-viewer-role
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ray.io
+  resources:
+  - rayjobs
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ray.io
+  resources:
+  - rayjobs/status
+  verbs:
+  - get
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/ray_rayservice_editor_role.yaml

@@ -0,0 +1,28 @@
+{{- /* ClusterRole for end users to view and edit RayService. */ -}}
+{{- if and .Values.rbacEnable (not .Values.singleNamespaceInstall) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rayservice-editor-role
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ray.io
+  resources:
+  - rayservices
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ray.io
+  resources:
+  - rayservices/status
+  verbs:
+  - get
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/ray_rayservice_viewer_role.yaml

@@ -0,0 +1,24 @@
+{{- /* ClusterRole for end users to view RayService. */ -}}
+{{- if and .Values.rbacEnable (not .Values.singleNamespaceInstall) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rayservice-viewer-role
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ray.io
+  resources:
+  - rayservices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ray.io
+  resources:
+  - rayservices/status
+  verbs:
+  - get
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/role.yaml

@@ -0,0 +1,9 @@
+{{- if and .Values.rbacEnable (not .Values.singleNamespaceInstall) }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "kuberay-operator.clusterRole.name" . }}
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+{{ include "role.consistentRules" (dict "batchSchedulerEnabled" .Values.batchScheduler.enabled "batchSchedulerName" .Values.batchScheduler.name) }}
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/rolebinding.yaml

@@ -0,0 +1,16 @@
+{{- if and .Values.rbacEnable (not .Values.singleNamespaceInstall) }}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "kuberay-operator.clusterRoleBinding.name" . }}
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "kuberay-operator.serviceAccount.name" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "kuberay-operator.clusterRole.name" . }}
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/templates/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "kuberay-operator.service.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: {{ include "kuberay-operator.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}

deployment/helm/deepdata/charts/kuberay-operator/templates/serviceaccount.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "kuberay-operator.serviceAccount.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kuberay-operator.labels" . | nindent 4 }}
+{{- end -}}

deployment/helm/deepdata/charts/kuberay-operator/templates/servicemonitor.yaml

@@ -0,0 +1,23 @@
+{{- if .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "kuberay-operator.fullname" . }}
+  namespace: {{ .Values.metrics.serviceMonitor.namespace | default .Release.Namespace }}
+  labels:
+    {{- with .Values.metrics.serviceMonitor.selector }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+    - path: /metrics
+      targetPort: http
+      interval: {{ .Values.metrics.serviceMonitor.interval }}
+      honorLabels: {{ .Values.metrics.serviceMonitor.honorLabels }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "kuberay-operator.name" . }}
+{{- end }}

deployment/helm/deepdata/charts/kuberay-operator/values.yaml

@@ -0,0 +1,227 @@
+# Default values for kuberay-operator.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# -- String to partially override release name.
+nameOverride: deepdata-kuberay-operator
+
+# -- String to fully override release name.
+fullnameOverride: deepdata-kuberay-operator
+
+# -- String to override component name.
+componentOverride: deepdata-kuberay-operator
+
+image:
+  # -- Image repository.
+  repository: quay.io/kuberay/operator
+
+  # -- Image tag.
+  tag: v1.4.2
+
+  # -- Image pull policy.
+  pullPolicy: IfNotPresent
+
+# -- Extra labels.
+labels: {}
+
+# -- Extra annotations.
+annotations: {}
+
+serviceAccount:
+  # -- Specifies whether a service account should be created.
+  create: true
+  # -- The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template.
+  name: deepdata-kuberay-operator
+
+logging:
+  # -- Log encoder to use for stdout (one of `json` or `console`).
+  stdoutEncoder: json
+  # -- Log encoder to use for file logging (one of `json` or `console`).
+  fileEncoder: json
+  # -- Directory for kuberay-operator log file.
+  baseDir: ""
+  # -- File name for kuberay-operator log file.
+  fileName: ""
+  # -- EmptyDir volume size limit for kuberay-operator log file.
+  sizeLimit: ""
+
+# Enable customized Kubernetes scheduler integration. If enabled, Ray workloads will be scheduled
+# by the customized scheduler.
+#  * "enabled" is the legacy option and will be deprecated soon.
+#  * "name" is the standard option, expecting a scheduler name, supported values are
+#    "default", "volcano", "yunikorn", and "scheduler-plugins".
+#
+# Note: "enabled" and "name" should not be set at the same time. If both are set, an error will be thrown.
+#
+# Examples:
+#  1. Use volcano (deprecated)
+#       batchScheduler:
+#         enabled: true
+#
+#  2. Use volcano
+#       batchScheduler:
+#         name: volcano
+#
+#  3. Use yunikorn
+#       batchScheduler:
+#         name: yunikorn
+#
+#  4. Use PodGroup
+#       batchScheduler:
+#         name: scheduler-plugins
+#
+batchScheduler:
+  # Deprecated. This option will be removed in the future.
+  # Note, for backwards compatibility. When it sets to true, it enables volcano scheduler integration.
+  enabled: false
+  # Set the customized scheduler name, supported values are "volcano" or "yunikorn", do not set
+  # "batchScheduler.enabled=true" at the same time as it will override this option.
+  name: ""
+
+featureGates:
+- name: RayClusterStatusConditions
+  enabled: true
+- name: RayJobDeletionPolicy
+  enabled: false
+
+# Configurations for KubeRay operator metrics.
+metrics:
+  # -- Whether KubeRay operator should emit control plane metrics.
+  enabled: false
+  serviceMonitor:
+    # -- Enable a prometheus ServiceMonitor
+    enabled: false
+    # -- Prometheus ServiceMonitor interval
+    interval: 30s
+    # -- When true, honorLabels preserves the metric’s labels when they collide with the target’s labels.
+    honorLabels: true
+    # -- Prometheus ServiceMonitor selector
+    selector: {}
+      #  release: prometheus
+    # -- Prometheus ServiceMonitor namespace
+    namespace: ""  # "monitoring"
+
+# -- Path to the operator binary
+operatorCommand: /manager
+
+# if userKubernetesProxy is set to true, the KubeRay operator will be configured with the --use-kubernetes-proxy flag.
+# Using this option to configure kuberay-operator to comunitcate to Ray head pods by proxying through the Kubernetes API Server.
+# useKubernetesProxy: true
+
+# -- If leaderElectionEnabled is set to true, the KubeRay operator will use leader election for high availability.
+leaderElectionEnabled: true
+
+# -- If rbacEnable is set to false, no RBAC resources will be created, including the Role for leader election, the Role for Pods and Services, and so on.
+rbacEnable: true
+
+# -- When crNamespacedRbacEnable is set to true, the KubeRay operator will create a Role for RayCluster preparation (e.g., Pods, Services)
+# and a corresponding RoleBinding for each namespace listed in the "watchNamespace" parameter. Please note that even if crNamespacedRbacEnable
+# is set to false, the Role and RoleBinding for leader election will still be created.
+#
+# Note:
+# (1) This variable is only effective when rbacEnable and singleNamespaceInstall are both set to true.
+# (2) In most cases, it should be set to true, unless you are using a Kubernetes cluster managed by GitOps tools such as ArgoCD.
+crNamespacedRbacEnable: true
+
+# -- When singleNamespaceInstall is true:
+# - Install namespaced RBAC resources such as Role and RoleBinding instead of cluster-scoped ones like ClusterRole and ClusterRoleBinding so that
+#   the chart can be installed by users with permissions restricted to a single namespace.
+#   (Please note that this excludes the CRDs, which can only be installed at the cluster scope.)
+# - If "watchNamespace" is not set, the KubeRay operator will, by default, only listen
+#   to resource events within its own namespace.
+singleNamespaceInstall: true
+
+# The KubeRay operator will watch the custom resources in the namespaces listed in the "watchNamespace" parameter.
+# watchNamespace:
+#   - n1
+#   - n2
+
+# -- Environment variables.
+
+env:
+# If not set or set to true, kuberay auto injects an init container waiting for ray GCS.
+# If false, you will need to inject your own init container to ensure ray GCS is up before the ray workers start.
+# Warning: we highly recommend setting to true and let kuberay handle for you.
+# - name: ENABLE_INIT_CONTAINER_INJECTION
+#   value: "true"
+# If set to true, kuberay creates a normal ClusterIP service for a Ray Head instead of a Headless service. Default to false.
+- name: ENABLE_RAY_HEAD_CLUSTER_IP_SERVICE
+  value: "true"
+# If not set or set to "", kuberay will pick up the default k8s cluster domain `cluster.local`
+# Otherwise, kuberay will use your custom domain
+# - name: CLUSTER_DOMAIN
+#   value: ""
+# If not set or set to false, when running on OpenShift with Ingress creation enabled, kuberay will create OpenShift route
+# Otherwise, regardless of the type of cluster with Ingress creation enabled, kuberay will create Ingress
+# - name: USE_INGRESS_ON_OPENSHIFT
+#   value: "true"
+# Unconditionally requeue after the number of seconds specified in the
+# environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV. If the
+# environment variable is not set, requeue after the default value (300).
+# - name: RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV
+#   value: 300
+# If not set or set to "true", KubeRay will clean up the Redis storage namespace when a GCS FT-enabled RayCluster is deleted.
+# - name: ENABLE_GCS_FT_REDIS_CLEANUP
+#   value: "true"
+# For LLM serving, some users might not have sufficient GPU resources to run two RayClusters simultaneously.
+# Therefore, KubeRay offers ENABLE_ZERO_DOWNTIME as a feature flag for zero-downtime upgrades.
+# - name: ENABLE_ZERO_DOWNTIME
+#   value: "true"
+# This environment variable for the KubeRay operator is used to determine whether to enable
+# the injection of readiness and liveness probes into Ray head and worker containers.
+# Enabling this feature contributes to the robustness of Ray clusters.
+# - name: ENABLE_PROBES_INJECTION
+#   value: "true"
+# If set to true, the RayJob CR itself will be deleted if shutdownAfterJobFinishes is set to true. Note that all resources created by the RayJob CR will be deleted, including the K8s Job. Otherwise, only the RayCluster CR will be deleted. Default is false.
+# - name: DELETE_RAYJOB_CR_AFTER_JOB_FINISHES
+#   value: "false"
+
+# -- Resource requests and limits for containers.
+resources:
+  limits:
+    cpu: 100m
+    # Anecdotally, managing 500 Ray pods requires roughly 500MB memory.
+    # Monitor memory usage and adjust as needed.
+    memory: 512Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 512Mi
+
+# @Ignore -- Pod liveness probe configuration.
+livenessProbe:
+  initialDelaySeconds: 10
+  periodSeconds: 5
+  failureThreshold: 5
+
+# @Ignore -- Pod readiness probe configuration.
+readinessProbe:
+  initialDelaySeconds: 10
+  periodSeconds: 5
+  failureThreshold: 5
+
+# -- Set up `securityContext` to improve Pod security.
+podSecurityContext: {}
+
+# @ignore -- Set up `securityContext` to improve container security.
+securityContext:
+  allowPrivilegeEscalation: false
+  readOnlyRootFilesystem: true
+  capabilities:
+    drop:
+    - ALL
+  runAsNonRoot: true
+  seccompProfile:
+    type: RuntimeDefault
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+service:
+  # -- Service type.
+  type: ClusterIP
+  # -- Service port.
+  port: 8080

deployment/helm/deepdata/charts/public/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/charts/public/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: public
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"

deployment/helm/deepdata/charts/public/templates/_helpers.tpl

@@ -0,0 +1,51 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "public.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "public.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "public.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "public.labels" -}}
+helm.sh/chart: {{ include "public.chart" . }}
+{{ include "public.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "public.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "public.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

deployment/helm/deepdata/charts/public/templates/database-pvc.yaml

@@ -0,0 +1,72 @@
+{{- if and .Values.persistentVolumeClaim.storageClass (eq .Values.persistentVolumeClaim.storageClass "local-storage") }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Values.global.namespace | default "deepdata" }}-database-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistentVolumeClaim.size.database | default "10Gi" }}
+  volumeMode: Filesystem
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  local: # local类型
+    path: {{ .Values.persistentVolumeClaim.storagePath | default "/opt/deepdata/data" }}/database
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: deepdata-database-pvc
+    namespace: {{ .Values.global.namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistentVolumeClaim.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-database-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  volumeName: {{ .Values.global.namespace | default "deepdata" }}-database-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.database | default "10Gi" }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+
+{{- else }}
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-database-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.database | default "10Gi" }}
+{{- end }}

deployment/helm/deepdata/charts/public/templates/dataset-pvc.yaml

@@ -0,0 +1,72 @@
+{{- if and .Values.persistentVolumeClaim.storageClass (eq .Values.persistentVolumeClaim.storageClass "local-storage") }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Values.global.namespace | default "deepdata" }}-dataset-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistentVolumeClaim.size.dataset | default "10Gi" }}
+  volumeMode: Filesystem
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  local: # local类型
+    path: {{ .Values.persistentVolumeClaim.storagePath | default "/opt/deepdata/data" }}/dataset
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: deepdata-dataset-pvc
+    namespace: {{ .Values.global.namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+          - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistentVolumeClaim.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-dataset-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  volumeName: {{ .Values.global.namespace | default "deepdata" }}-dataset-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.dataset | default "10Gi" }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+
+{{- else }}
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-dataset-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.dataset | default "10Gi" }}
+{{- end }}

deployment/helm/deepdata/charts/public/templates/flow-pvc.yaml

@@ -0,0 +1,72 @@
+{{- if and .Values.persistentVolumeClaim.storageClass (eq .Values.persistentVolumeClaim.storageClass "local-storage") }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Values.global.namespace | default "deepdata" }}-flow-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistentVolumeClaim.size.flow | default "10Gi" }}
+  volumeMode: Filesystem
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  local: # local类型
+    path: {{ .Values.persistentVolumeClaim.storagePath | default "/opt/deepdata/data" }}/flow
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: deepdata-flow-pvc
+    namespace: {{ .Values.global.namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistentVolumeClaim.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-flow-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  volumeName: {{ .Values.global.namespace | default "deepdata" }}-flow-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.flow | default "10Gi" }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  
+{{- else }}
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-flow-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.flow | default "10Gi" }}
+{{- end }}

deployment/helm/deepdata/charts/public/templates/log-pvc.yaml

@@ -0,0 +1,72 @@
+{{- if and .Values.persistentVolumeClaim.storageClass (eq .Values.persistentVolumeClaim.storageClass "local-storage") }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Values.global.namespace | default "deepdata" }}-log-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistentVolumeClaim.size.log | default "10Gi" }}
+  volumeMode: Filesystem
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  local: # local类型
+    path: {{ .Values.persistentVolumeClaim.storagePath | default "/opt/deepdata/data" }}/log
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: deepdata-log-pvc
+    namespace: {{ .Values.global.namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistentVolumeClaim.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-log-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  volumeName: {{ .Values.global.namespace | default "deepdata" }}-log-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.log | default "10Gi" }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  
+{{- else }}
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-log-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.log | default "10Gi" }}
+{{- end }}

deployment/helm/deepdata/charts/public/templates/operator-pvc.yaml

@@ -0,0 +1,72 @@
+{{- if and .Values.persistentVolumeClaim.storageClass (eq .Values.persistentVolumeClaim.storageClass "local-storage") }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Values.global.namespace | default "deepdata" }}-operator-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistentVolumeClaim.size.operator | default "10Gi" }}
+  volumeMode: Filesystem
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  local: # local类型
+    path: {{ .Values.persistentVolumeClaim.storagePath | default "/opt/deepdata/data" }}/operator
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: deepdata-operator-pvc
+    namespace: {{ .Values.global.namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistentVolumeClaim.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-operator-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  volumeName: {{ .Values.global.namespace | default "deepdata" }}-operator-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.operator | default "10Gi" }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  
+{{- else }}
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: deepdata-operator-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.operator | default "10Gi" }}
+{{- end }}

deployment/helm/deepdata/charts/public/templates/secret.yaml

@@ -0,0 +1,10 @@
+{{- if not (eq (.Values.secrets.create | toString) "false") }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: deepdata-conf
+data:
+  {{- range $key, $val := .Values.secrets.data }}
+  {{ $key }}: {{ $val | toString | b64enc | quote }}
+  {{- end }}
+{{- end }}

deployment/helm/deepdata/charts/public/values.yaml

@@ -0,0 +1,28 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This is to override the chart name.
+nameOverride: "deepdata-public"
+fullnameOverride: "deepdata-public"
+
+# This is for setting Kubernetes Annotations to a Pvc.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+annotations: {}
+
+persistentVolumeClaim:
+  ## If defined, storageClassName: <storageClass>
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ##   set, choosing the default provisioner.
+  ##
+  storageClass: ""
+  accessModes: ReadWriteMany
+  size:
+    dataset: 10Gi
+    flow: 1Gi
+    log: 1Gi
+    database: 1Gi
+    operator: 1Gi
+
+secrets:
+  create: true  # Set to false when using Sealed Secrets

deployment/helm/deepdata/charts/ray-cluster/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+description: A Helm chart for Kubernetes
+name: ray-cluster
+version: 1.4.2
+icon: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png

deployment/helm/deepdata/charts/ray-cluster/templates/_helpers.tpl

@@ -0,0 +1,68 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "ray-cluster.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "ray-cluster.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "ray-cluster.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "ray-cluster.labels" -}}
+helm.sh/chart: {{ include "ray-cluster.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "ray-cluster.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "ray-cluster.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Name of image
+*/}}
+{{- define "ray-cluster.image" -}}
+{{- $name := default .Values.image.repository .Values.global.image.runtime.name }}
+{{- $tag := default .Values.image.tag .Values.global.image.runtime.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/ray-cluster/templates/raycluster-cluster.yaml

@@ -0,0 +1,415 @@
+apiVersion: ray.io/v1
+kind: RayCluster
+metadata:
+  name: {{ include "ray-cluster.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "ray-cluster.labels" . | nindent 4 }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.head.rayVersion }}
+  rayVersion: {{ . }}
+  {{- end }}
+  {{- with .Values.head.enableInTreeAutoscaling }}
+  enableInTreeAutoscaling: {{ . }}
+  {{- end }}
+  {{- with .Values.head.autoscalerOptions }}
+  autoscalerOptions:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  headGroupSpec:
+    {{- with .Values.head.headService }}
+    headService:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.service.type }}
+    serviceType: {{ . }}
+    {{- end }}
+    {{- if or .Values.head.rayStartParams .Values.head.initArgs }}
+    rayStartParams:
+      {{- range $key, $val := .Values.head.rayStartParams }}
+        {{ $key }}: {{ $val | quote }}
+      {{- end }}
+      {{- /* initArgs is a deprecated alias for rayStartParams. */}}
+      {{- range $key, $val := .Values.head.initArgs }}
+        {{ $key }}: {{ $val | quote }}
+      {{- end }}
+    {{- else }}
+    rayStartParams: {}
+    {{- end }}
+    template:
+      metadata:
+        labels:
+          {{- include "ray-cluster.labels" . | nindent 10 }}
+        {{- with .Values.head.labels }}
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.head.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      spec:
+        {{- with .Values.head.initContainers }}
+        initContainers:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        containers:
+        - name: ray-head
+          image: {{ include "ray-cluster.image" . }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          {{- with .Values.head.command }}
+          command:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.head.args }}
+          args:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with concat .Values.common.containerEnv .Values.head.containerEnv }}
+          env:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.head.envFrom }}
+          envFrom:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{ with .Values.head.volumeMounts }}
+          volumeMounts:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.head.ports }}
+          ports:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.head.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.head.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.head.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- $default := include "ray-cluster.image" . -}}
+        {{- with .Values.head.sidecarContainers }}
+        {{- range $index, $container := . }}
+        {{- $image := "" -}}
+        {{- if $container.image -}}
+        {{- if $.Values.global.image.repository -}}
+        {{- $image = printf "%s/%s" (trimSuffix "/" $.Values.global.image.repository) $container.image -}}
+        {{- else -}}
+        {{- $image = $container.image -}}
+        {{- end -}}
+        {{- else -}}
+        {{- $image = $default -}}
+        {{- end -}}
+        {{- $mergedObj := mergeOverwrite $container (dict "image" $image) }}
+        - {{- toYaml $mergedObj | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- with .Values.imagePullSecrets }}
+        imagePullSecrets:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.head.volumes }}
+        volumes:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.head.dnsConfig }}
+        dnsConfig:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.head.nodeSelector }}
+        nodeSelector:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.head.affinity }}
+        affinity:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.head.tolerations }}
+        tolerations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.head.priorityClassName }}
+        priorityClassName: {{ . }}
+        {{- end }}
+        {{- with .Values.head.priority }}
+        priority: {{ . }}
+        {{- end }}
+        {{- with .Values.head.topologySpreadConstraints }}
+        topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.head.restartPolicy }}
+        restartPolicy: {{ . }}
+        {{- end }}
+        {{- with .Values.head.serviceAccountName }}
+        serviceAccountName: {{ . }}
+        {{- end }}
+        {{- with .Values.head.podSecurityContext }}
+        securityContext:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+  workerGroupSpecs:
+  {{- if not .Values.worker.disabled }}
+  - groupName: {{ .Values.worker.groupName }}
+    replicas: {{ .Values.worker.replicas }}
+    minReplicas: {{ .Values.worker.minReplicas | default 0 }}
+    maxReplicas: {{ .Values.worker.maxReplicas | default 2147483647 }}
+    numOfHosts: {{ .Values.worker.numOfHosts | default 1 }}
+    {{- if or .Values.worker.rayStartParams .Values.worker.initArgs }}
+    rayStartParams:
+      {{- range $key, $val := .Values.worker.rayStartParams }}
+      {{ $key }}: {{ $val | quote }}
+      {{- end }}
+      {{- /* initArgs is a deprecated alias for rayStartParams. */}}
+      {{- range $key, $val := .Values.worker.initArgs }}
+      {{ $key }}: {{ $val | quote }}
+      {{- end }}
+    {{- else }}
+    rayStartParams: {}
+    {{- end }}
+    template:
+      metadata:
+        labels:
+          {{- include "ray-cluster.labels" . | nindent 10 }}
+        {{- with .Values.worker.labels }}
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.worker.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      spec:
+        {{- with .Values.worker.initContainers }}
+        initContainers:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        containers:
+        - name: ray-worker
+          image: {{ include "ray-cluster.image" . }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          {{- with .Values.worker.command }}
+          command:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.worker.args }}
+          args:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with concat .Values.common.containerEnv .Values.worker.containerEnv }}
+          env:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.worker.envFrom }}
+          envFrom:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.worker.volumeMounts }}
+          volumeMounts:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.worker.ports }}
+          ports:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.worker.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.worker.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.worker.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- with .Values.worker.sidecarContainers }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.imagePullSecrets }}
+        imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.worker.volumes }}
+        volumes:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.worker.dnsConfig }}
+        dnsConfig:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.worker.nodeSelector }}
+        nodeSelector:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.worker.affinity }}
+        affinity:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.worker.tolerations }}
+        tolerations:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.worker.priorityClassName }}
+        priorityClassName: {{ . }}
+        {{- end }}
+        {{- with .Values.worker.priority }}
+        priority: {{ . }}
+        {{- end }}
+        {{- with .Values.worker.topologySpreadConstraints }}
+        topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.worker.restartPolicy }}
+        restartPolicy: {{ . }}
+        {{- end }}
+        {{- with .Values.worker.serviceAccountName }}
+        serviceAccountName: {{ . }}
+        {{- end }}
+        {{- with .Values.worker.podSecurityContext }}
+        securityContext:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+  {{- end }}
+  {{- range $groupName, $values := .Values.additionalWorkerGroups }}
+  {{- if not $values.disabled }}
+  - groupName: {{ $groupName }}
+    replicas: {{ $values.replicas }}
+    minReplicas: {{ $values.minReplicas | default 0 }}
+    maxReplicas: {{ $values.maxReplicas | default 2147483647 }}
+    numOfHosts: {{ $values.numOfHosts | default 1 }}
+    {{- if or $values.rayStartParams $values.initArgs }}
+    rayStartParams:
+      {{- range $key, $val := $values.rayStartParams }}
+        {{ $key }}: {{ $val | quote }}
+      {{- end }}
+      {{- /* initArgs is a deprecated alias for rayStartParams. */}}
+      {{- range $key, $val := $values.initArgs }}
+        {{ $key }}: {{ $val | quote }}
+      {{- end }}
+    {{- else }}
+    rayStartParams: {}
+    {{- end }}
+    template:
+      metadata:
+        labels:
+          {{- include "ray-cluster.labels" $ | nindent 10 }}
+        {{- with $values.labels }}
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with $values.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      spec:
+        {{- with $values.initContainers }}
+        initContainers:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        containers:
+        - name: ray-worker
+          {{- if $values.image }}
+          image: {{ $values.image.repository }}:{{ $values.image.tag }}
+          {{- if $values.image.pullPolicy }}
+          imagePullPolicy: {{ $values.image.pullPolicy }}
+          {{- else }}
+          imagePullPolicy: {{ default $.Values.image.pullPolicy $.Values.global.image.pullPolicy }}
+          {{- end }}
+          {{- else }}
+          image: {{ include "ray-cluster.image" $ }}
+          imagePullPolicy: {{ default $.Values.image.pullPolicy $.Values.global.image.pullPolicy }}
+          {{- end }}
+          {{- with $values.command }}
+          command:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with $values.args }}
+          args:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with concat $.Values.common.containerEnv ($values.containerEnv | default list) }}
+          env:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with $values.envFrom }}
+          envFrom:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with $values.volumeMounts }}
+          volumeMounts:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with $values.ports }}
+          ports:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with $values.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with $values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with $values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- with $values.sidecarContainers }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with $.Values.imagePullSecrets }}
+        imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with $values.volumes }}
+        volumes:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with $values.dnsConfig }}
+        dnsConfig:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with $values.nodeSelector }}
+        nodeSelector:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with $values.affinity }}
+        affinity:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with $values.tolerations }}
+        tolerations:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with $values.priorityClassName }}
+        priorityClassName: {{ . }}
+        {{- end }}
+        {{- with $values.priority }}
+        priority: {{ . }}
+        {{- end }}
+        {{- with $values.topologySpreadConstraints }}
+        topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with $values.restartPolicy }}
+        restartPolicy: {{ . }}
+        {{- end }}
+        {{- with $values.serviceAccountName }}
+        serviceAccountName: {{ . }}
+        {{- end }}
+        {{- with $values.podSecurityContext }}
+        securityContext:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+  {{- end }}
+  {{- end }}

deployment/helm/deepdata/charts/ray-cluster/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: deepdata-runtime
+  labels:
+    ray.io/node-type: head
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8081
+      targetPort: 8081
+      protocol: TCP
+  selector:
+    ray.io/node-type: head
+

deployment/helm/deepdata/charts/ray-cluster/templates/worker-networkpolicy.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.workerNetworkPolicy.enabled .Values.workerNetworkPolicy.blockedCIDRs }}
+apiVersion: crd.projectcalico.org/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ include "ray-cluster.fullname" . }}-worker-egress
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "ray-cluster.labels" . | nindent 4 }}
+spec:
+  selector: ray.io/node-type == "worker"
+  types:
+    - Egress
+  egress:
+    - action: Deny
+      destination:
+        nets:
+{{- range .Values.workerNetworkPolicy.blockedCIDRs }}
+          - {{ . }}
+{{- end }}
+    - action: Allow
+{{- end }}

deployment/helm/deepdata/charts/ray-cluster/values.yaml

@@ -0,0 +1,291 @@
+# Default values for ray-cluster.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# The KubeRay community welcomes PRs to expose additional configuration
+# in this Helm chart.
+
+image:
+  repository: deepdata-runtime
+  tag: latest
+  pullPolicy: IfNotPresent
+
+nameOverride: "kuberay"
+fullnameOverride: "deepdata-raycluster"
+
+imagePullSecrets: []
+# - name: an-existing-secret
+
+workerNetworkPolicy:
+  enabled: false
+  # CIDRs to block from Ray worker pod egress. Use /32 for a single IPv4 address.
+  blockedCIDRs: []
+  # - 192.0.2.10/32
+
+# common defined values shared between the head and worker
+common:
+  # containerEnv specifies environment variables for the Ray head and worker containers.
+  # Follows standard K8s container env schema.
+  containerEnv: []
+  #  - name: BLAH
+  #    value: VAL
+head:
+  # rayVersion determines the autoscaler's image version.
+  # It should match the Ray version in the image of the containers.
+  # rayVersion: 2.46.0
+  # If enableInTreeAutoscaling is true, the autoscaler sidecar will be added to the Ray head pod.
+  # Ray autoscaler integration is supported only for Ray versions >= 1.11.0
+  # Ray autoscaler integration is Beta with KubeRay >= 0.3.0 and Ray >= 2.0.0.
+  # enableInTreeAutoscaling: true
+  # autoscalerOptions is an OPTIONAL field specifying configuration overrides for the Ray autoscaler.
+  # The example configuration shown below represents the DEFAULT values.
+  # autoscalerOptions:
+  # upscalingMode: Default
+  # idleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources.
+  # idleTimeoutSeconds: 60
+  # imagePullPolicy optionally overrides the autoscaler container's default image pull policy (IfNotPresent).
+  # imagePullPolicy: IfNotPresent
+  # Optionally specify the autoscaler container's securityContext.
+  # securityContext: {}
+  # env: []
+  # envFrom: []
+  # resources specifies optional resource request and limit overrides for the autoscaler container.
+  # For large Ray clusters, we recommend monitoring container resource usage to determine if overriding the defaults is required.
+  # resources:
+  #   limits:
+  #     cpu: "500m"
+  #     memory: "512Mi"
+  #   requests:
+  #     cpu: "500m"
+  #     memory: "512Mi"
+  initContainers: []
+  labels: {}
+  # Note: From KubeRay v0.6.0, users need to create the ServiceAccount by themselves if they specify the `serviceAccountName`
+  # in the headGroupSpec. See https://github.com/ray-project/kuberay/pull/1128 for more details.
+  serviceAccountName: ""
+  restartPolicy: ""
+  rayStartParams: {}
+  # containerEnv specifies environment variables for the Ray container,
+  # Follows standard K8s container env schema.
+  containerEnv: []
+  # - name: EXAMPLE_ENV
+  #   value: "1"
+  envFrom: []
+    # - secretRef:
+  #     name: my-env-secret
+  # ports optionally allows specifying ports for the Ray container.
+  # ports: []
+  # resource requests and limits for the Ray head container.
+  # Modify as needed for your application.
+  # Note that the resources in this example are much too small for production;
+  # we don't recommend allocating less than 8G memory for a Ray pod in production.
+  # Ray pods should be sized to take up entire K8s nodes when possible.
+  # Always set CPU and memory limits for Ray pods.
+  # It is usually best to set requests equal to limits.
+  # See https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#resources
+  # for further guidance.
+  resources: {}
+  annotations: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  # Pod security context.
+  podSecurityContext: {}
+  # Ray container security context.
+  securityContext: {}
+  # Optional: The following volumes/volumeMounts configurations are optional but recommended because
+  # Ray writes logs to /tmp/ray/session_latests/logs instead of stdout/stderr.
+  volumes: []
+  volumeMounts: []
+  # sidecarContainers specifies additional containers to attach to the Ray pod.
+  # Follows standard K8s container spec.
+  sidecarContainers: []
+  # See docs/guidance/pod-command.md for more details about how to specify
+  # container command for head Pod.
+  command: []
+  args: []
+  # Optional, for the user to provide any additional fields to the service.
+  # See https://pkg.go.dev/k8s.io/Kubernetes/pkg/api/v1#Service
+  headService: {}
+    # metadata:
+    #   annotations:
+  #     prometheus.io/scrape: "true"
+
+  # Custom pod DNS configuration
+  # See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config
+  # dnsConfig:
+  #   nameservers:
+  #     - 8.8.8.8
+  #   searches:
+  #     - example.local
+  #   options:
+  #     - name: ndots
+  #       value: "2"
+  #     - name: edns0
+  topologySpreadConstraints: []
+
+
+worker:
+  # If you want to disable the default workergroup
+  # uncomment the line below
+  # disabled: true
+  groupName: workergroup
+  replicas: 1
+  minReplicas: 1
+  maxReplicas: 1
+  labels: {}
+  serviceAccountName: ""
+  restartPolicy: ""
+  rayStartParams: {}
+  initContainers: []
+  # containerEnv specifies environment variables for the Ray container,
+  # Follows standard K8s container env schema.
+  containerEnv: []
+  # - name: EXAMPLE_ENV
+  #   value: "1"
+  envFrom: []
+    # - secretRef:
+  #     name: my-env-secret
+  # ports optionally allows specifying ports for the Ray container.
+  # ports: []
+  # resource requests and limits for the Ray head container.
+  # Modify as needed for your application.
+  # Note that the resources in this example are much too small for production;
+  # we don't recommend allocating less than 8G memory for a Ray pod in production.
+  # Ray pods should be sized to take up entire K8s nodes when possible.
+  # Always set CPU and memory limits for Ray pods.
+  # It is usually best to set requests equal to limits.
+  # See https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#resources
+  # for further guidance.
+  resources: {}
+  annotations: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  # Pod security context.
+  podSecurityContext: {}
+  # Ray container security context.
+  securityContext: {}
+  # Optional: The following volumes/volumeMounts configurations are optional but recommended because
+  # Ray writes logs to /tmp/ray/session_latests/logs instead of stdout/stderr.
+  volumes: []
+  volumeMounts: []
+  # sidecarContainers specifies additional containers to attach to the Ray pod.
+  # Follows standard K8s container spec.
+  sidecarContainers: []
+  # See docs/guidance/pod-command.md for more details about how to specify
+  # container command for worker Pod.
+  command: []
+  args: []
+  topologySpreadConstraints: []
+
+
+  # Custom pod DNS configuration
+  # See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config
+  # dnsConfig:
+  #   nameservers:
+  #     - 8.8.8.8
+  #   searches:
+  #     - example.local
+  #   options:
+  #     - name: ndots
+  #       value: "2"
+  #     - name: edns0
+
+# The map's key is used as the groupName.
+# For example, key:small-group in the map below
+# will be used as the groupName
+additionalWorkerGroups:
+  smallGroup:
+    # Disabled by default
+    disabled: true
+    replicas: 0
+    minReplicas: 0
+    maxReplicas: 3
+    labels: {}
+    serviceAccountName: ""
+    restartPolicy: ""
+    rayStartParams: {}
+    # containerEnv specifies environment variables for the Ray container,
+    # Follows standard K8s container env schema.
+    containerEnv: []
+      # - name: EXAMPLE_ENV
+    #   value: "1"
+    envFrom: []
+      # - secretRef:
+    #     name: my-env-secret
+    # ports optionally allows specifying ports for the Ray container.
+    # ports: []
+    # resource requests and limits for the Ray head container.
+    # Modify as needed for your application.
+    # Note that the resources in this example are much too small for production;
+    # we don't recommend allocating less than 8G memory for a Ray pod in production.
+    # Ray pods should be sized to take up entire K8s nodes when possible.
+    # Always set CPU and memory limits for Ray pods.
+    # It is usually best to set requests equal to limits.
+    # See https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#resources
+    # for further guidance.
+    resources:
+      limits:
+        cpu: 1
+        memory: "1G"
+      requests:
+        cpu: 1
+        memory: "1G"
+    annotations: {}
+    nodeSelector: {}
+    tolerations: []
+    affinity: {}
+    # Pod security context.
+    podSecurityContext: {}
+    # Ray container security context.
+    securityContext: {}
+    # Optional: The following volumes/volumeMounts configurations are optional but recommended because
+    # Ray writes logs to /tmp/ray/session_latests/logs instead of stdout/stderr.
+    volumes:
+      - name: log-volume
+        hostPath:
+          path: /opt/deepdata/data/log
+          type: DirectoryOrCreate
+      - name: dataset-volume
+        hostPath:
+          path: /opt/deepdata/data/dataset
+          type: DirectoryOrCreate
+      - name: flow-volume
+        hostPath:
+          path: /opt/deepdata/data/flow
+          type: DirectoryOrCreate
+    volumeMounts:
+      - mountPath: /tmp/ray
+        name: log-volume
+        subPath: ray
+      - mountPath: /dataset
+        name: dataset-volume
+      - mountPath: /flow
+        name: flow-volume
+    sidecarContainers: []
+    # See docs/guidance/pod-command.md for more details about how to specify
+    # container command for worker Pod.
+    command: []
+    args: []
+
+    # Topology Spread Constraints for worker pods
+    # See: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
+    topologySpreadConstraints: []
+
+    # Custom pod DNS configuration
+    # See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config
+    # dnsConfig:
+    #   nameservers:
+    #     - 8.8.8.8
+    #   searches:
+    #     - example.local
+    #   options:
+    #     - name: ndots
+    #       value: "2"
+    #     - name: edns0
+
+# Configuration for Head's Kubernetes Service
+service:
+  # This is optional, and the default is ClusterIP.
+  type: NodePort

deployment/helm/deepdata/charts/runtime/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepdata/charts/runtime/Chart.yaml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: runtime
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"
+
+dependencies:
+  - name: database
+    repository: file://../database
+    version: 0.0.1

deployment/helm/deepdata/charts/runtime/templates/_helpers.tpl

@@ -0,0 +1,76 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "runtime.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "runtime.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "runtime.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "runtime.labels" -}}
+helm.sh/chart: {{ include "runtime.chart" . }}
+app: {{ .Release.Name }}
+{{ include "runtime.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "runtime.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "runtime.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "runtime.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "runtime.fullname" .) .Values.serviceAccount.name -}}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of image
+*/}}
+{{- define "runtime.image" -}}
+{{- $name := default .Values.image.repository .Values.global.image.runtime.name }}
+{{- $tag := default .Values.image.tag .Values.global.image.runtime.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
+{{- $name }}:{{ $tag }}
+{{- end }}
+{{- end }}

deployment/helm/deepdata/charts/runtime/templates/deployment.yaml

@@ -0,0 +1,86 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "runtime.fullname" . }}
+  labels:
+    {{- include "runtime.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "runtime.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "runtime.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "runtime.serviceAccountName" . }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          {{- with .Values.args }}
+          args:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: "{{ include "runtime.image" . }}"
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployment/helm/deepdata/charts/runtime/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "runtime.fullname" . }}
+  labels:
+    {{- include "runtime.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.port }}
+      protocol: TCP
+      name: {{ .Chart.Name }}
+  selector:
+    {{- include "runtime.selectorLabels" . | nindent 4 }}

deployment/helm/deepdata/charts/runtime/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "runtime.serviceAccountName" . }}
+  labels:
+    {{- include "runtime.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

deployment/helm/deepdata/charts/runtime/values.yaml

@@ -0,0 +1,108 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: "deepdata-runtime"
+  # This sets the pull policy for images.
+  pullPolicy: "IfNotPresent"
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: "deepdata-runtime"
+fullnameOverride: "deepdata-runtime"
+
+env: []
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: ClusterIP
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  port: 8081
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+# livenessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+# readinessProbe:
+#   httpGet:
+#     path: /
+#     port: http
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

deployment/helm/deepdata/values.yaml

@@ -0,0 +1,617 @@
+# Default values for deepdata.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+global:
+  namespace: deepdata
+  # Default nodeSelector for all pods in deepdata namespace
+  # Override in subchart values.yaml if needed for specific services
+  nodeSelector: {}
+    # Example: To schedule pods on nodes with specific labels
+    # node-role.kubernetes.io/deepdata: "true"
+  # Default tolerations for all pods (used when nodes have taints)
+  tolerations: []
+    # Example: To tolerate taints on dedicated nodes
+    # - key: "node-role.kubernetes.io/deepdata"
+    #   operator: "Equal"
+    #   value: "true"
+    #   effect: "NoSchedule"
+  image:
+    repository: ""
+    pullPolicy: "IfNotPresent"
+    backend:
+      name: "deepdata-backend"
+      tag: "latest"
+    backendPython:
+      name: "deepdata-backend-python"
+      tag: "latest"
+    gateway:
+      name: "deepdata-gateway"
+      tag: "latest"
+    frontend:
+      name: "deepdata-frontend"
+      tag: "latest"
+    runtime:
+      name: "deepdata-runtime"
+      tag: "latest"
+    database:
+      name: "deepdata-database"
+      tag: "latest"
+
+public:
+  persistentVolumeClaim:
+    storageClass: ""
+    storagePath: ""
+    storageNode: ""
+    size:
+      dataset: 10Gi
+      flow: 1Gi
+      log: 1Gi
+      database: 1Gi
+      operator: 1Gi
+  secrets:
+    # Set to false when using Sealed Secrets (managed by install script)
+    create: false
+    data:
+      DB_PASSWORD: ""  # Set via install script or --set
+      CERT_PASS: ""    # Set via install script for encrypted SSL keys
+      DOMAIN: ""
+      HOME_PAGE_URL: ""
+      JWT_SECRET: ""               # Auto-generated by install script
+      LABEL_STUDIO_PASSWORD: ""    # Set via install script
+      LABEL_STUDIO_USER_TOKEN: ""  # Auto-generated by install script
+
+datasetVolume: &datasetVolume
+  name: dataset-volume
+  persistentVolumeClaim:
+    claimName: deepdata-dataset-pvc
+
+flowVolume: &flowVolume
+  name: flow-volume
+  persistentVolumeClaim:
+    claimName: deepdata-flow-pvc
+
+logVolume: &logVolume
+  name: log-volume
+  persistentVolumeClaim:
+    claimName: deepdata-log-pvc
+
+dataVolume: &dataVolume
+  name: data-volume
+  persistentVolumeClaim:
+    claimName: deepdata-database-pvc
+
+operatorVolume: &operatorVolume
+  name: operator-volume
+  persistentVolumeClaim:
+    claimName: deepdata-operator-pvc
+
+database:
+  nodeSelector: {}
+  tolerations: []
+  initContainers:
+    - name: init-chown
+      imagePullPolicy: IfNotPresent
+      command:
+        - sh
+        - -c
+        - "chown -R 999:999 /var/log/deepdata/database && chmod 750 /var/log/deepdata/database"
+      securityContext:
+        runAsUser: 0
+      volumeMounts:
+        - name: log-volume
+          mountPath: /var/log/deepdata/database
+          subPath: database
+  env:
+    - name: POSTGRES_USER
+      value: postgres
+    - name: POSTGRES_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: DB_PASSWORD
+    - name: HOME_PAGE_URL
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: HOME_PAGE_URL
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: "30"
+  volumes:
+    - *dataVolume
+    - *logVolume
+  volumeMounts:
+    - name: data-volume
+      mountPath: /var/lib/postgresql/data
+      subPath: postgres
+    - name: log-volume
+      mountPath: /var/log/deepdata/database
+      subPath: database
+
+backend:
+  nodeSelector: {}
+  tolerations: []
+  securityContext:
+    capabilities:
+      add:
+        - SYS_ADMIN
+  env:
+    - name: DB_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: DB_PASSWORD
+    - name: JWT_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: JWT_SECRET
+    - name: deepdata.rag.milvus-uri
+      value: "http://deepdata-milvus:19530"
+    - name: deepdata.jwt.enable
+      value: &DEEPDATA_JWT_ENABLE 'false'
+    - name: LOG_ROTATION_MAX_SIZE
+      value: &LOG_ROTATION_MAX_SIZE "100MB"
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: &LOG_ROTATION_BACKUP_COUNT "30"
+  volumes:
+    - *datasetVolume
+    - *flowVolume
+    - *logVolume
+    - *operatorVolume
+  volumeMounts:
+    - name: dataset-volume
+      mountPath: /dataset
+    - name: flow-volume
+      mountPath: /flow
+    - name: log-volume
+      mountPath: /var/log/deepdata
+    - name: operator-volume
+      mountPath: /operators
+
+backend-python:
+  nodeSelector: {}
+  tolerations: []
+  env:
+    - name: PGSQL_HOST
+      value: "deepdata-database"
+    - name: PGSQL_PORT
+      value: "5432"
+    - name: pgsql_password
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: DB_PASSWORD
+    - name: deepdata_jwt_enable
+      value: *DEEPDATA_JWT_ENABLE
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
+    - name: LOG_PVC_MONITOR_ENABLED
+      value: "true"
+    - name: LOG_PVC_MONITOR_PATH
+      value: "/var/log/deepdata"
+    - name: LOG_PVC_MONITOR_THRESHOLD
+      value: "0.9"
+    - name: LOG_PVC_MONITOR_INTERVAL_SECONDS
+      value: "300"
+    - name: LOG_PVC_MONITOR_DELETE_BATCH_SIZE
+      value: "10"
+    - name: LOG_PVC_MONITOR_FILE_SUFFIXES
+      value: "log,out,err"
+    - name: milvus_uri
+      value: "http://deepdata-milvus:19530"
+    - name: LABEL_STUDIO_USER_TOKEN
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: LABEL_STUDIO_USER_TOKEN
+    - name: LABEL_STUDIO_USERNAME
+      value: "admin@demo.com"
+    - name: LABEL_STUDIO_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: LABEL_STUDIO_PASSWORD
+  volumes:
+    - *datasetVolume
+    - *flowVolume
+    - *logVolume
+    - *operatorVolume
+  volumeMounts:
+    - name: dataset-volume
+      mountPath: /dataset
+    - name: flow-volume
+      mountPath: /flow
+    - name: log-volume
+      mountPath: /var/log/deepdata
+    - name: operator-volume
+      mountPath: /operators
+
+gateway:
+  nodeSelector: {}
+  tolerations: []
+  env:
+    - name: JWT_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: JWT_SECRET
+    - name: deepdata.jwt.enable
+      value: *DEEPDATA_JWT_ENABLE
+    - name: OMS_AUTH_ENABLED
+      value: "false"
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
+  volumes:
+    - *logVolume
+  volumeMounts:
+    - name: log-volume
+      mountPath: /var/log/deepdata/gateway
+      subPath: gateway
+
+frontend:
+  nodeSelector: {}
+  tolerations: []
+  service:
+    type: NodePort
+    nodePort: 30000
+  env:
+    - name: CERT_PASS
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: CERT_PASS
+    - name: DOMAIN
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: DOMAIN
+    - name: REAL_IP_MODE
+      value: "off"
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
+  volumes:
+    - *logVolume
+    - name: cert-volume
+      configMap:
+        name: cube-security-nginx-cert
+        optional: true
+  volumeMounts:
+    - name: log-volume
+      mountPath: /var/log/deepdata/frontend
+      subPath: frontend
+    - name: cert-volume
+      mountPath: /cert
+
+runtime:
+  nodeSelector: {}
+  tolerations: []
+  enabled: false
+  args: &runtimeArgs
+    - python
+    - /opt/runtime/deepdata/operator_runtime.py
+    - --port
+    - "8081"
+  env: &runtimeEnv
+    - name: PG_HOST
+      value: "deepdata-database"
+    - name: PG_PORT
+      value: "5432"
+    - name: PG_USER
+      value: "postgres"
+    - name: PG_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: deepdata-conf
+          key: DB_PASSWORD
+    - name: PG_DATABASE
+      value: "deepdata"
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
+  volumes:
+    - *datasetVolume
+    - *flowVolume
+    - *logVolume
+    - *operatorVolume
+  volumeMounts: &runtimeVolumeMounts
+    - mountPath: /tmp/ray
+      name: log-volume
+      subPath: ray/head
+    - mountPath: /var/log/deepdata
+      name: log-volume
+    - mountPath: /dataset
+      name: dataset-volume
+    - mountPath: /flow
+      name: flow-volume
+    - mountPath: /opt/runtime/deepdata/ops/user
+      name: operator-volume
+      subPath: extract
+    - mountPath: /usr/local/lib/ops/site-packages
+      name: operator-volume
+      subPath: site-packages
+
+ray-cluster:
+  enabled: true
+  workerNetworkPolicy:
+    enabled: true
+    # Use CIDR notation, for example 192.0.2.10/32 for a single IPv4 address.
+    blockedCIDRs: []
+  head:
+    nodeSelector: {}
+    tolerations: []
+    enableInTreeAutoscaling: true
+    autoscalerOptions:
+      upscalingMode: Default
+      idleTimeoutSeconds: 60
+      imagePullPolicy: IfNotPresent
+      resources:
+        limits:
+          cpu: "500m"
+          memory: "512Mi"
+        requests:
+          cpu: "500m"
+          memory: "512Mi"
+    rayStartParams:
+      num-cpus: "0"
+    containerEnv:
+      - name: RAY_DEDUP_LOGS
+        value: "0"
+      - name: RAY_TQDM_PATCH_PRINT
+        value: "0"
+      - name: LOG_ROTATION_MAX_SIZE
+        value: *LOG_ROTATION_MAX_SIZE
+      - name: LOG_ROTATION_BACKUP_COUNT
+        value: *LOG_ROTATION_BACKUP_COUNT
+      - name: PG_HOST
+        value: "deepdata-database"
+      - name: PG_PORT
+        value: "5432"
+      - name: PG_USER
+        value: "postgres"
+      - name: PG_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: deepdata-conf
+            key: DB_PASSWORD
+      - name: PG_DATABASE
+        value: "deepdata"
+      - name: RAY_enable_autoscaler_v2
+        value: "1"
+    resources:
+      limits:
+        cpu: "4"
+        memory: "16G"
+      requests:
+        cpu: "1"
+        memory: "2G"
+    volumes:
+      - *datasetVolume
+      - *flowVolume
+      - *logVolume
+      - *operatorVolume
+    volumeMounts:
+      - mountPath: /tmp/ray
+        name: log-volume
+        subPath: ray/head
+      - mountPath: /dataset
+        name: dataset-volume
+      - mountPath: /flow
+        name: flow-volume
+      - mountPath: /opt/runtime/deepdata/ops/user
+        name: operator-volume
+        subPath: extract
+      - mountPath: /usr/local/lib/ops/site-packages
+        name: operator-volume
+        subPath: site-packages
+    sidecarContainers:
+      - name: runtime
+        imagePullPolicy: IfNotPresent
+        args: *runtimeArgs
+        env: *runtimeEnv
+        ports:
+          - containerPort: 8081
+        volumeMounts: *runtimeVolumeMounts
+  worker:
+    nodeSelector: {}
+    tolerations: []
+    containerEnv:
+      - name: RAY_DEDUP_LOGS
+        value: "0"
+      - name: RAY_TQDM_PATCH_PRINT
+        value: "0"
+      - name: LOG_ROTATION_MAX_SIZE
+        value: *LOG_ROTATION_MAX_SIZE
+      - name: LOG_ROTATION_BACKUP_COUNT
+        value: *LOG_ROTATION_BACKUP_COUNT
+      - name: PG_HOST
+        value: "deepdata-database"
+      - name: PG_PORT
+        value: "5432"
+      - name: PG_USER
+        value: "postgres"
+      - name: PG_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: deepdata-conf
+            key: DB_PASSWORD
+      - name: PG_DATABASE
+        value: "deepdata"
+      - name: POD_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.name
+    resources:
+      limits:
+        cpu: "8"
+        memory: "64G"
+      requests:
+        cpu: "1"
+        memory: "2G"
+    lifecycle: &rayLogRotationLifecycle
+      postStart:
+        exec:
+          command:
+            - /bin/bash
+            - -c
+            - "nohup /usr/local/bin/log-rotate-copytruncate.sh /tmp/ray >/tmp/log-rotation.out 2>/tmp/log-rotation.err &"
+    volumes:
+      - *datasetVolume
+      - *flowVolume
+      - *logVolume
+      - *operatorVolume
+    volumeMounts:
+      - mountPath: /tmp/ray
+        name: log-volume
+        subPathExpr: ray/$(POD_NAME)
+      - mountPath: /dataset
+        name: dataset-volume
+      - mountPath: /flow
+        name: flow-volume
+      - mountPath: /opt/runtime/deepdata/ops/user
+        name: operator-volume
+        subPath: extract
+      - mountPath: /usr/local/lib/ops/site-packages
+        name: operator-volume
+        subPath: site-packages
+  additionalWorkerGroups:
+    npuGroup:
+      nodeSelector: {}
+      tolerations: []
+      disabled: false
+      replicas: 0
+      minReplicas: 0
+      maxReplicas: 8
+      rayStartParams:
+        resources: '"{\"npu\": 1}"'
+      containerEnv:
+        - name: RAY_DEDUP_LOGS
+          value: "0"
+        - name: RAY_TQDM_PATCH_PRINT
+          value: "0"
+        - name: LOG_ROTATION_MAX_SIZE
+          value: *LOG_ROTATION_MAX_SIZE
+        - name: LOG_ROTATION_BACKUP_COUNT
+          value: *LOG_ROTATION_BACKUP_COUNT
+        - name: PG_HOST
+          value: "deepdata-database"
+        - name: PG_PORT
+          value: "5432"
+        - name: PG_USER
+          value: "postgres"
+        - name: PG_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: deepdata-conf
+              key: DB_PASSWORD
+        - name: PG_DATABASE
+          value: "deepdata"
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+      resources:
+        limits:
+          cpu: "8"
+          memory: "64G"
+          huawei.com/Ascend910: 1
+        requests:
+          cpu: "1"
+          memory: "2G"
+          huawei.com/Ascend910: 1
+      lifecycle: *rayLogRotationLifecycle
+      volumes:
+        - *datasetVolume
+        - *flowVolume
+        - *logVolume
+        - *operatorVolume
+        - name: ascend
+          hostPath:
+            path: /usr/local/Ascend
+      volumeMounts:
+        - mountPath: /tmp/ray
+          name: log-volume
+          subPathExpr: ray/$(POD_NAME)
+        - mountPath: /dataset
+          name: dataset-volume
+        - mountPath: /flow
+          name: flow-volume
+        - mountPath: /opt/runtime/deepdata/ops/user
+          name: operator-volume
+          subPath: extract
+        - mountPath: /usr/local/lib/ops/site-packages
+          name: operator-volume
+          subPath: site-packages
+        - mountPath: /usr/local/Ascend
+          name: ascend
+    gpuGroup:
+      nodeSelector: {}
+      tolerations: []
+      disabled: false
+      replicas: 0
+      minReplicas: 0
+      maxReplicas: 8
+      rayStartParams:
+        resources: '"{\"gpu\": 1}"'
+      containerEnv:
+        - name: RAY_DEDUP_LOGS
+          value: "0"
+        - name: RAY_TQDM_PATCH_PRINT
+          value: "0"
+        - name: LOG_ROTATION_MAX_SIZE
+          value: *LOG_ROTATION_MAX_SIZE
+        - name: LOG_ROTATION_BACKUP_COUNT
+          value: *LOG_ROTATION_BACKUP_COUNT
+        - name: PG_HOST
+          value: "deepdata-database"
+        - name: PG_PORT
+          value: "5432"
+        - name: PG_USER
+          value: "postgres"
+        - name: PG_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: deepdata-conf
+              key: DB_PASSWORD
+        - name: PG_DATABASE
+          value: "deepdata"
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+      resources:
+        limits:
+          cpu: "8"
+          memory: "64G"
+          nvidia.com/gpu: 1
+        requests:
+          cpu: "1"
+          memory: "2G"
+          nvidia.com/gpu: 1
+      lifecycle: *rayLogRotationLifecycle
+      volumes:
+        - *datasetVolume
+        - *flowVolume
+        - *logVolume
+        - *operatorVolume
+      volumeMounts:
+        - mountPath: /tmp/ray
+          name: log-volume
+          subPathExpr: ray/$(POD_NAME)
+        - mountPath: /dataset
+          name: dataset-volume
+        - mountPath: /flow
+          name: flow-volume
+        - mountPath: /opt/runtime/deepdata/ops/user
+          name: operator-volume
+          subPath: extract
+        - mountPath: /usr/local/lib/ops/site-packages
+          name: operator-volume
+          subPath: site-packages

deployment/helm/deepflow/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deepflow/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: deepflow
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"