deepdata: delivery stack (docker compose, helm, CI, build scripts)

deployment/helm/label-studio/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v2
+name: label-studio
+version: 0.1.0
+description: A Helm chart for deploying Label Studio with PostgreSQL on Kubernetes
+
+# Application metadata
+appVersion: "latest"
+
+# This chart is designed to mirror the behavior of deployment/docker/label-studio/docker-compose.yml
+

deployment/helm/label-studio/templates/_helpers.tpl

@@ -0,0 +1,25 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "label-studio.name" -}}
+{{- default .Values.nameOverride .Chart.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "label-studio.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Values.nameOverride .Chart.Name -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "label-studio.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

deployment/helm/label-studio/templates/deployment.yaml

@@ -0,0 +1,118 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "label-studio.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "label-studio.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "label-studio.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: label-studio
+          {{- if .Values.global.imageRegistry }}
+          image: "{{ .Values.global.imageRegistry | trimSuffix "/" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          {{- else }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          {{- end }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args: ["label-studio-uwsgi"]
+          env:
+            - name: DJANGO_DB
+              value: {{ .Values.env.DJANGO_DB | quote }}
+            - name: POSTGRE_NAME
+              value: {{ .Values.env.POSTGRE_NAME | quote }}
+            - name: POSTGRE_USER
+              value: {{ .Values.env.POSTGRE_USER | quote }}
+            - name: POSTGRE_PASSWORD
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: POSTGRE_PASSWORD
+            {{- else }}
+              value: {{ .Values.env.POSTGRE_PASSWORD | quote }}
+            {{- end }}
+            - name: POSTGRE_PORT
+              value: "5432"
+            - name: POSTGRE_HOST
+              value: {{ if .Values.pgbouncer.enabled }}{{ printf "%s-pgbouncer" (include "label-studio.fullname" .) | quote }}{{ else }}{{ .Values.env.POSTGRE_HOST | quote }}{{ end }}
+            - name: LABEL_STUDIO_HOST
+              value: {{ .Values.env.LABEL_STUDIO_HOST | quote }}
+            - name: LOCAL_FILES_SERVING_ENABLED
+              value: {{ .Values.env.LOCAL_FILES_SERVING_ENABLED | quote }}
+            - name: LOCAL_FILES_DOCUMENT_ROOT
+              value: {{ .Values.env.LOCAL_FILES_DOCUMENT_ROOT | quote }}
+            - name: USE_USERNAME_FOR_LOGIN
+              value: {{ .Values.env.USE_USERNAME_FOR_LOGIN | quote }}
+            - name: LABEL_STUDIO_USERNAME
+              value: {{ .Values.env.LABEL_STUDIO_USERNAME | quote }}
+            - name: LABEL_STUDIO_PASSWORD
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: LABEL_STUDIO_PASSWORD
+            {{- else }}
+              value: {{ .Values.env.LABEL_STUDIO_PASSWORD | quote }}
+            {{- end }}
+            - name: LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN
+              value: {{ .Values.env.LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN | quote }}
+            - name: LABEL_STUDIO_USER_TOKEN
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: LABEL_STUDIO_USER_TOKEN
+            {{- else }}
+              value: {{ .Values.env.LABEL_STUDIO_USER_TOKEN | quote }}
+            {{- end }}
+            - name: LOG_LEVEL
+              value: {{ .Values.env.LOG_LEVEL | quote }}
+          ports:
+            - name: http
+              containerPort: 8000
+              protocol: TCP
+          volumeMounts:
+            - name: data
+              mountPath: /label-studio/data
+            - name: dataset
+              mountPath: /label-studio/local
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{ else }}{{ include "label-studio.fullname" . }}-data{{ end }}
+        - name: dataset
+          {{- if and .Values.datasetVolume.enabled .Values.datasetVolume.claimName }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.datasetVolume.claimName }}
+          {{- else }}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{ else }}{{ include "label-studio.fullname" . }}-dataset{{ end }}
+          {{- end }}
+

deployment/helm/label-studio/templates/ingress.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "label-studio.fullname" . }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.className }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
+            backend:
+              service:
+                name: {{ include "label-studio.fullname" $ }}
+                port:
+                  number: {{ $.Values.service.port }}
+          {{- end }}
+    {{- end }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- toYaml .Values.ingress.tls | nindent 4 }}
+  {{- end }}
+{{- end }}
+

deployment/helm/label-studio/templates/pgbouncer-deployment.yaml

@@ -0,0 +1,87 @@
+{{- if .Values.pgbouncer.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "label-studio.fullname" . }}-pgbouncer
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" .}}-pgbouncer
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.pgbouncer.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: pgbouncer
+          {{- if .Values.global.imageRegistry }}
+          image: "{{ .Values.global.imageRegistry | trimSuffix "/" }}/{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}"
+          {{- else }}
+          image: "{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}"
+          {{- end }}
+          imagePullPolicy: {{ .Values.pgbouncer.image.pullPolicy }}
+          ports:
+            - name: pgbouncer
+              containerPort: 5432
+              protocol: TCP
+          env:
+            - name: DB_HOST
+              value: {{ .Values.env.POSTGRE_HOST | quote }}
+            - name: DB_PORT
+              value: {{ .Values.env.POSTGRE_PORT | quote }}
+            - name: DB_NAME
+              value: {{ .Values.env.POSTGRE_NAME | quote }}
+            - name: DB_USER
+              value: {{ .Values.env.POSTGRE_USER | quote }}
+            - name: DB_PASSWORD
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: POSTGRE_PASSWORD
+            {{- else }}
+              value: {{ .Values.env.POSTGRE_PASSWORD | quote }}
+            {{- end }}
+            - name: POOL_MODE
+              value: {{ .Values.pgbouncer.poolMode | quote }}
+            - name: MAX_CLIENT_CONN
+              value: {{ .Values.pgbouncer.maxClientConn | quote }}
+            - name: DEFAULT_POOL_SIZE
+              value: {{ .Values.pgbouncer.defaultPoolSize | quote }}
+            - name: MAX_DB_CONNECTIONS
+              value: {{ .Values.pgbouncer.maxDbConnections | quote }}
+            - name: AUTH_TYPE
+              value: {{ .Values.pgbouncer.authType | quote }}
+          livenessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          readinessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            {{- toYaml .Values.pgbouncer.resources | nindent 12 }}
+      {{- with .Values.pgbouncer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.pgbouncer.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.pgbouncer.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

deployment/helm/label-studio/templates/pgbouncer-service.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.pgbouncer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "label-studio.fullname" . }}-pgbouncer
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: 5432
+      targetPort: pgbouncer
+      protocol: TCP
+      name: pgbouncer
+  selector:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

deployment/helm/label-studio/templates/pvc.yaml

@@ -0,0 +1,123 @@
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+{{- if and .Values.persistence.storageClass (eq .Values.persistence.storageClass "local-storage") }}
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "label-studio.fullname" . }}-data-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistence.size }}
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistence.storageClass }}
+  local:
+    path: {{ .Values.persistence.storagePath | default "/opt/k8s/deepdata" }}/data
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: {{ include "label-studio.fullname" . }}-data
+    namespace: {{ .Release.Namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistence.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "label-studio.fullname" . }}-dataset-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistence.size }}
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistence.storageClass }}
+  local:
+    path: {{ .Values.persistence.storagePath | default "/opt/k8s/deepdata" }}/dataset
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: {{ include "label-studio.fullname" . }}-dataset
+    namespace: {{ .Release.Namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistence.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeName: {{ include "label-studio.fullname" . }}-data-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-dataset
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeName: {{ include "label-studio.fullname" . }}-dataset-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+
+{{- else }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  {{- if .Values.persistence.storageClass }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+  {{- end }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-dataset
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  {{- if .Values.persistence.storageClass }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+  {{- end }}
+{{- end }}
+
+{{- end }}
+

deployment/helm/label-studio/templates/service.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "label-studio.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+      {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
+  selector:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+

deployment/helm/label-studio/values.yaml

@@ -0,0 +1,99 @@
+# Default values for label-studio Helm chart.
+# This mirrors the configuration from deployment/docker/label-studio/docker-compose.yml
+fullnameOverride: label-studio
+
+replicaCount: 1
+
+global:
+  imageRegistry: ""
+
+image:
+  repository: heartexlabs/label-studio
+  tag: "latest"
+  pullPolicy: IfNotPresent
+
+service:
+  type: NodePort
+  port: 8000
+  nodePort: 30001
+
+# Corresponds to docker-compose port mapping 30001:8000
+ingress:
+  enabled: false
+  className: ""
+  annotations: {}
+  hosts:
+    - host: ""
+      paths:
+        - path: /
+          pathType: Prefix
+  tls: []
+
+resources: {}
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+env:
+  DJANGO_DB: "default"
+  POSTGRE_NAME: "labelstudio"
+  POSTGRE_USER: "postgres"
+  POSTGRE_PASSWORD: ""  # Set via secrets.yaml or --set
+  POSTGRE_PORT: 5432
+  POSTGRE_HOST: "deepdata-database"
+  LABEL_STUDIO_HOST: "" # can be overridden
+  LOCAL_FILES_SERVING_ENABLED: "true"
+  LOCAL_FILES_DOCUMENT_ROOT: "/label-studio/local"
+  USE_USERNAME_FOR_LOGIN: "true"
+  LABEL_STUDIO_USERNAME: "admin@demo.com"
+  LABEL_STUDIO_PASSWORD: ""  # Set via secrets.yaml or --set
+  LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN: "true"
+  LABEL_STUDIO_USER_TOKEN: ""  # Set via secrets.yaml or --set
+  LOG_LEVEL: "DEBUG"
+
+# Use an existing K8s Secret instead of creating one from values.
+# Set to "label-studio-env" to use the SealedSecret-managed secret.
+existingSecret: "label-studio-env"
+
+persistence:
+  enabled: true
+  existingClaim: "" # if you already have PVC that maps to external deepdata-dataset-volume
+  size: 10Gi
+  storageClass: ""
+  storageNode: ""
+  storagePath: ""
+
+# Name of the Kubernetes PVC to simulate external Docker volume `deepdata-dataset-volume`
+# If not set and persistence.enabled=true, a PVC will be created automatically.
+datasetVolume:
+  enabled: true
+  claimName: deepdata-dataset-pvc # if empty, uses same PVC as persistence or creates a dedicated one
+
+# PgBouncer configuration for connection pooling
+pgbouncer:
+  enabled: true
+  replicaCount: 1
+
+  image:
+    repository: edoburu/pgbouncer
+    tag: "v1.25.1-p0"
+    pullPolicy: IfNotPresent
+
+  # Connection pool settings
+  poolMode: transaction  # transaction, session, or statement
+  maxClientConn: 100     # Maximum number of client connections
+  defaultPoolSize: 20    # Default pool size per database (max connections to PostgreSQL)
+  maxDbConnections: 20   # Maximum database connections (hard limit)
+  authType: scram-sha-256
+
+  resources: {}
+
+  nodeSelector: {}
+
+  tolerations: []
+
+  affinity: {}
+