deepdata/deployment/helm/deepdata/values.yaml

# Default values for deepdata.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

global:
  namespace: deepdata
  # Default nodeSelector for all pods in deepdata namespace
  # Override in subchart values.yaml if needed for specific services
  nodeSelector: {}
    # Example: To schedule pods on nodes with specific labels
    # node-role.kubernetes.io/deepdata: "true"
  # Default tolerations for all pods (used when nodes have taints)
  tolerations: []
    # Example: To tolerate taints on dedicated nodes
    # - key: "node-role.kubernetes.io/deepdata"
    #   operator: "Equal"
    #   value: "true"
    #   effect: "NoSchedule"
  image:
    repository: ""
    pullPolicy: "IfNotPresent"
    backend:
      name: "deepdata-backend"
      tag: "latest"
    backendPython:
      name: "deepdata-backend-python"
      tag: "latest"
    gateway:
      name: "deepdata-gateway"
      tag: "latest"
    frontend:
      name: "deepdata-frontend"
      tag: "latest"
    runtime:
      name: "deepdata-runtime"
      tag: "latest"
    database:
      name: "deepdata-database"
      tag: "latest"

public:
  persistentVolumeClaim:
    storageClass: ""
    storagePath: ""
    storageNode: ""
    size:
      dataset: 10Gi
      flow: 1Gi
      log: 1Gi
      database: 1Gi
      operator: 1Gi
  secrets:
    # Set to false when using Sealed Secrets (managed by install script)
    create: false
    data:
      DB_PASSWORD: ""  # Set via install script or --set
      CERT_PASS: ""    # Set via install script for encrypted SSL keys
      DOMAIN: ""
      HOME_PAGE_URL: ""
      JWT_SECRET: ""               # Auto-generated by install script
      LABEL_STUDIO_PASSWORD: ""    # Set via install script
      LABEL_STUDIO_USER_TOKEN: ""  # Auto-generated by install script

datasetVolume: &datasetVolume
  name: dataset-volume
  persistentVolumeClaim:
    claimName: deepdata-dataset-pvc

flowVolume: &flowVolume
  name: flow-volume
  persistentVolumeClaim:
    claimName: deepdata-flow-pvc

logVolume: &logVolume
  name: log-volume
  persistentVolumeClaim:
    claimName: deepdata-log-pvc

dataVolume: &dataVolume
  name: data-volume
  persistentVolumeClaim:
    claimName: deepdata-database-pvc

operatorVolume: &operatorVolume
  name: operator-volume
  persistentVolumeClaim:
    claimName: deepdata-operator-pvc

database:
  nodeSelector: {}
  tolerations: []
  initContainers:
    - name: init-chown
      imagePullPolicy: IfNotPresent
      command:
        - sh
        - -c
        - "chown -R 999:999 /var/log/deepdata/database && chmod 750 /var/log/deepdata/database"
      securityContext:
        runAsUser: 0
      volumeMounts:
        - name: log-volume
          mountPath: /var/log/deepdata/database
          subPath: database
  env:
    - name: POSTGRES_USER
      value: postgres
    - name: POSTGRES_PASSWORD
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: DB_PASSWORD
    - name: HOME_PAGE_URL
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: HOME_PAGE_URL
    - name: LOG_ROTATION_BACKUP_COUNT
      value: "30"
  volumes:
    - *dataVolume
    - *logVolume
  volumeMounts:
    - name: data-volume
      mountPath: /var/lib/postgresql/data
      subPath: postgres
    - name: log-volume
      mountPath: /var/log/deepdata/database
      subPath: database

backend:
  nodeSelector: {}
  tolerations: []
  securityContext:
    capabilities:
      add:
        - SYS_ADMIN
  env:
    - name: DB_PASSWORD
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: DB_PASSWORD
    - name: JWT_SECRET
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: JWT_SECRET
    - name: deepdata.rag.milvus-uri
      value: "http://deepdata-milvus:19530"
    - name: deepdata.jwt.enable
      value: &DEEPDATA_JWT_ENABLE 'false'
    - name: LOG_ROTATION_MAX_SIZE
      value: &LOG_ROTATION_MAX_SIZE "100MB"
    - name: LOG_ROTATION_BACKUP_COUNT
      value: &LOG_ROTATION_BACKUP_COUNT "30"
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts:
    - name: dataset-volume
      mountPath: /dataset
    - name: flow-volume
      mountPath: /flow
    - name: log-volume
      mountPath: /var/log/deepdata
    - name: operator-volume
      mountPath: /operators

backend-python:
  nodeSelector: {}
  tolerations: []
  env:
    - name: PGSQL_HOST
      value: "deepdata-database"
    - name: PGSQL_PORT
      value: "5432"
    - name: pgsql_password
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: DB_PASSWORD
    - name: deepdata_jwt_enable
      value: *DEEPDATA_JWT_ENABLE
    - name: LOG_ROTATION_MAX_SIZE
      value: *LOG_ROTATION_MAX_SIZE
    - name: LOG_ROTATION_BACKUP_COUNT
      value: *LOG_ROTATION_BACKUP_COUNT
    - name: LOG_PVC_MONITOR_ENABLED
      value: "true"
    - name: LOG_PVC_MONITOR_PATH
      value: "/var/log/deepdata"
    - name: LOG_PVC_MONITOR_THRESHOLD
      value: "0.9"
    - name: LOG_PVC_MONITOR_INTERVAL_SECONDS
      value: "300"
    - name: LOG_PVC_MONITOR_DELETE_BATCH_SIZE
      value: "10"
    - name: LOG_PVC_MONITOR_FILE_SUFFIXES
      value: "log,out,err"
    - name: milvus_uri
      value: "http://deepdata-milvus:19530"
    - name: LABEL_STUDIO_USER_TOKEN
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: LABEL_STUDIO_USER_TOKEN
    - name: LABEL_STUDIO_USERNAME
      value: "admin@demo.com"
    - name: LABEL_STUDIO_PASSWORD
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: LABEL_STUDIO_PASSWORD
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts:
    - name: dataset-volume
      mountPath: /dataset
    - name: flow-volume
      mountPath: /flow
    - name: log-volume
      mountPath: /var/log/deepdata
    - name: operator-volume
      mountPath: /operators

gateway:
  nodeSelector: {}
  tolerations: []
  env:
    - name: JWT_SECRET
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: JWT_SECRET
    - name: deepdata.jwt.enable
      value: *DEEPDATA_JWT_ENABLE
    - name: OMS_AUTH_ENABLED
      value: "false"
    - name: LOG_ROTATION_MAX_SIZE
      value: *LOG_ROTATION_MAX_SIZE
    - name: LOG_ROTATION_BACKUP_COUNT
      value: *LOG_ROTATION_BACKUP_COUNT
  volumes:
    - *logVolume
  volumeMounts:
    - name: log-volume
      mountPath: /var/log/deepdata/gateway
      subPath: gateway

frontend:
  nodeSelector: {}
  tolerations: []
  service:
    type: NodePort
    nodePort: 30000
  env:
    - name: CERT_PASS
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: CERT_PASS
    - name: DOMAIN
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: DOMAIN
    - name: REAL_IP_MODE
      value: "off"
    - name: LOG_ROTATION_MAX_SIZE
      value: *LOG_ROTATION_MAX_SIZE
    - name: LOG_ROTATION_BACKUP_COUNT
      value: *LOG_ROTATION_BACKUP_COUNT
  volumes:
    - *logVolume
    - name: cert-volume
      configMap:
        name: cube-security-nginx-cert
        optional: true
  volumeMounts:
    - name: log-volume
      mountPath: /var/log/deepdata/frontend
      subPath: frontend
    - name: cert-volume
      mountPath: /cert

runtime:
  nodeSelector: {}
  tolerations: []
  enabled: false
  args: &runtimeArgs
    - python
    - /opt/runtime/deepdata/operator_runtime.py
    - --port
    - "8081"
  env: &runtimeEnv
    - name: PG_HOST
      value: "deepdata-database"
    - name: PG_PORT
      value: "5432"
    - name: PG_USER
      value: "postgres"
    - name: PG_PASSWORD
      valueFrom:
        secretKeyRef:
          name: deepdata-conf
          key: DB_PASSWORD
    - name: PG_DATABASE
      value: "deepdata"
    - name: LOG_ROTATION_MAX_SIZE
      value: *LOG_ROTATION_MAX_SIZE
    - name: LOG_ROTATION_BACKUP_COUNT
      value: *LOG_ROTATION_BACKUP_COUNT
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts: &runtimeVolumeMounts
    - mountPath: /tmp/ray
      name: log-volume
      subPath: ray/head
    - mountPath: /var/log/deepdata
      name: log-volume
    - mountPath: /dataset
      name: dataset-volume
    - mountPath: /flow
      name: flow-volume
    - mountPath: /opt/runtime/deepdata/ops/user
      name: operator-volume
      subPath: extract
    - mountPath: /usr/local/lib/ops/site-packages
      name: operator-volume
      subPath: site-packages

ray-cluster:
  enabled: true
  workerNetworkPolicy:
    enabled: true
    # Use CIDR notation, for example 192.0.2.10/32 for a single IPv4 address.
    blockedCIDRs: []
  head:
    nodeSelector: {}
    tolerations: []
    enableInTreeAutoscaling: true
    autoscalerOptions:
      upscalingMode: Default
      idleTimeoutSeconds: 60
      imagePullPolicy: IfNotPresent
      resources:
        limits:
          cpu: "500m"
          memory: "512Mi"
        requests:
          cpu: "500m"
          memory: "512Mi"
    rayStartParams:
      num-cpus: "0"
    containerEnv:
      - name: RAY_DEDUP_LOGS
        value: "0"
      - name: RAY_TQDM_PATCH_PRINT
        value: "0"
      - name: LOG_ROTATION_MAX_SIZE
        value: *LOG_ROTATION_MAX_SIZE
      - name: LOG_ROTATION_BACKUP_COUNT
        value: *LOG_ROTATION_BACKUP_COUNT
      - name: PG_HOST
        value: "deepdata-database"
      - name: PG_PORT
        value: "5432"
      - name: PG_USER
        value: "postgres"
      - name: PG_PASSWORD
        valueFrom:
          secretKeyRef:
            name: deepdata-conf
            key: DB_PASSWORD
      - name: PG_DATABASE
        value: "deepdata"
      - name: RAY_enable_autoscaler_v2
        value: "1"
    resources:
      limits:
        cpu: "4"
        memory: "16G"
      requests:
        cpu: "1"
        memory: "2G"
    volumes:
      - *datasetVolume
      - *flowVolume
      - *logVolume
      - *operatorVolume
    volumeMounts:
      - mountPath: /tmp/ray
        name: log-volume
        subPath: ray/head
      - mountPath: /dataset
        name: dataset-volume
      - mountPath: /flow
        name: flow-volume
      - mountPath: /opt/runtime/deepdata/ops/user
        name: operator-volume
        subPath: extract
      - mountPath: /usr/local/lib/ops/site-packages
        name: operator-volume
        subPath: site-packages
    sidecarContainers:
      - name: runtime
        imagePullPolicy: IfNotPresent
        args: *runtimeArgs
        env: *runtimeEnv
        ports:
          - containerPort: 8081
        volumeMounts: *runtimeVolumeMounts
  worker:
    nodeSelector: {}
    tolerations: []
    containerEnv:
      - name: RAY_DEDUP_LOGS
        value: "0"
      - name: RAY_TQDM_PATCH_PRINT
        value: "0"
      - name: LOG_ROTATION_MAX_SIZE
        value: *LOG_ROTATION_MAX_SIZE
      - name: LOG_ROTATION_BACKUP_COUNT
        value: *LOG_ROTATION_BACKUP_COUNT
      - name: PG_HOST
        value: "deepdata-database"
      - name: PG_PORT
        value: "5432"
      - name: PG_USER
        value: "postgres"
      - name: PG_PASSWORD
        valueFrom:
          secretKeyRef:
            name: deepdata-conf
            key: DB_PASSWORD
      - name: PG_DATABASE
        value: "deepdata"
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
    resources:
      limits:
        cpu: "8"
        memory: "64G"
      requests:
        cpu: "1"
        memory: "2G"
    lifecycle: &rayLogRotationLifecycle
      postStart:
        exec:
          command:
            - /bin/bash
            - -c
            - "nohup /usr/local/bin/log-rotate-copytruncate.sh /tmp/ray >/tmp/log-rotation.out 2>/tmp/log-rotation.err &"
    volumes:
      - *datasetVolume
      - *flowVolume
      - *logVolume
      - *operatorVolume
    volumeMounts:
      - mountPath: /tmp/ray
        name: log-volume
        subPathExpr: ray/$(POD_NAME)
      - mountPath: /dataset
        name: dataset-volume
      - mountPath: /flow
        name: flow-volume
      - mountPath: /opt/runtime/deepdata/ops/user
        name: operator-volume
        subPath: extract
      - mountPath: /usr/local/lib/ops/site-packages
        name: operator-volume
        subPath: site-packages
  additionalWorkerGroups:
    npuGroup:
      nodeSelector: {}
      tolerations: []
      disabled: false
      replicas: 0
      minReplicas: 0
      maxReplicas: 8
      rayStartParams:
        resources: '"{\"npu\": 1}"'
      containerEnv:
        - name: RAY_DEDUP_LOGS
          value: "0"
        - name: RAY_TQDM_PATCH_PRINT
          value: "0"
        - name: LOG_ROTATION_MAX_SIZE
          value: *LOG_ROTATION_MAX_SIZE
        - name: LOG_ROTATION_BACKUP_COUNT
          value: *LOG_ROTATION_BACKUP_COUNT
        - name: PG_HOST
          value: "deepdata-database"
        - name: PG_PORT
          value: "5432"
        - name: PG_USER
          value: "postgres"
        - name: PG_PASSWORD
          valueFrom:
            secretKeyRef:
              name: deepdata-conf
              key: DB_PASSWORD
        - name: PG_DATABASE
          value: "deepdata"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
      resources:
        limits:
          cpu: "8"
          memory: "64G"
          huawei.com/Ascend910: 1
        requests:
          cpu: "1"
          memory: "2G"
          huawei.com/Ascend910: 1
      lifecycle: *rayLogRotationLifecycle
      volumes:
        - *datasetVolume
        - *flowVolume
        - *logVolume
        - *operatorVolume
        - name: ascend
          hostPath:
            path: /usr/local/Ascend
      volumeMounts:
        - mountPath: /tmp/ray
          name: log-volume
          subPathExpr: ray/$(POD_NAME)
        - mountPath: /dataset
          name: dataset-volume
        - mountPath: /flow
          name: flow-volume
        - mountPath: /opt/runtime/deepdata/ops/user
          name: operator-volume
          subPath: extract
        - mountPath: /usr/local/lib/ops/site-packages
          name: operator-volume
          subPath: site-packages
        - mountPath: /usr/local/Ascend
          name: ascend
    gpuGroup:
      nodeSelector: {}
      tolerations: []
      disabled: false
      replicas: 0
      minReplicas: 0
      maxReplicas: 8
      rayStartParams:
        resources: '"{\"gpu\": 1}"'
      containerEnv:
        - name: RAY_DEDUP_LOGS
          value: "0"
        - name: RAY_TQDM_PATCH_PRINT
          value: "0"
        - name: LOG_ROTATION_MAX_SIZE
          value: *LOG_ROTATION_MAX_SIZE
        - name: LOG_ROTATION_BACKUP_COUNT
          value: *LOG_ROTATION_BACKUP_COUNT
        - name: PG_HOST
          value: "deepdata-database"
        - name: PG_PORT
          value: "5432"
        - name: PG_USER
          value: "postgres"
        - name: PG_PASSWORD
          valueFrom:
            secretKeyRef:
              name: deepdata-conf
              key: DB_PASSWORD
        - name: PG_DATABASE
          value: "deepdata"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
      resources:
        limits:
          cpu: "8"
          memory: "64G"
          nvidia.com/gpu: 1
        requests:
          cpu: "1"
          memory: "2G"
          nvidia.com/gpu: 1
      lifecycle: *rayLogRotationLifecycle
      volumes:
        - *datasetVolume
        - *flowVolume
        - *logVolume
        - *operatorVolume
      volumeMounts:
        - mountPath: /tmp/ray
          name: log-volume
          subPathExpr: ray/$(POD_NAME)
        - mountPath: /dataset
          name: dataset-volume
        - mountPath: /flow
          name: flow-volume
        - mountPath: /opt/runtime/deepdata/ops/user
          name: operator-volume
          subPath: extract
        - mountPath: /usr/local/lib/ops/site-packages
          name: operator-volume
          subPath: site-packages