deepdata: delivery stack (docker compose, helm, CI, build scripts)

deployment/helm/label-studio/templates/_helpers.tpl

@@ -0,0 +1,25 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "label-studio.name" -}}
+{{- default .Values.nameOverride .Chart.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "label-studio.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Values.nameOverride .Chart.Name -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "label-studio.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

deployment/helm/label-studio/templates/deployment.yaml

@@ -0,0 +1,118 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "label-studio.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "label-studio.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "label-studio.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: label-studio
+          {{- if .Values.global.imageRegistry }}
+          image: "{{ .Values.global.imageRegistry | trimSuffix "/" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          {{- else }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          {{- end }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args: ["label-studio-uwsgi"]
+          env:
+            - name: DJANGO_DB
+              value: {{ .Values.env.DJANGO_DB | quote }}
+            - name: POSTGRE_NAME
+              value: {{ .Values.env.POSTGRE_NAME | quote }}
+            - name: POSTGRE_USER
+              value: {{ .Values.env.POSTGRE_USER | quote }}
+            - name: POSTGRE_PASSWORD
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: POSTGRE_PASSWORD
+            {{- else }}
+              value: {{ .Values.env.POSTGRE_PASSWORD | quote }}
+            {{- end }}
+            - name: POSTGRE_PORT
+              value: "5432"
+            - name: POSTGRE_HOST
+              value: {{ if .Values.pgbouncer.enabled }}{{ printf "%s-pgbouncer" (include "label-studio.fullname" .) | quote }}{{ else }}{{ .Values.env.POSTGRE_HOST | quote }}{{ end }}
+            - name: LABEL_STUDIO_HOST
+              value: {{ .Values.env.LABEL_STUDIO_HOST | quote }}
+            - name: LOCAL_FILES_SERVING_ENABLED
+              value: {{ .Values.env.LOCAL_FILES_SERVING_ENABLED | quote }}
+            - name: LOCAL_FILES_DOCUMENT_ROOT
+              value: {{ .Values.env.LOCAL_FILES_DOCUMENT_ROOT | quote }}
+            - name: USE_USERNAME_FOR_LOGIN
+              value: {{ .Values.env.USE_USERNAME_FOR_LOGIN | quote }}
+            - name: LABEL_STUDIO_USERNAME
+              value: {{ .Values.env.LABEL_STUDIO_USERNAME | quote }}
+            - name: LABEL_STUDIO_PASSWORD
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: LABEL_STUDIO_PASSWORD
+            {{- else }}
+              value: {{ .Values.env.LABEL_STUDIO_PASSWORD | quote }}
+            {{- end }}
+            - name: LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN
+              value: {{ .Values.env.LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN | quote }}
+            - name: LABEL_STUDIO_USER_TOKEN
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: LABEL_STUDIO_USER_TOKEN
+            {{- else }}
+              value: {{ .Values.env.LABEL_STUDIO_USER_TOKEN | quote }}
+            {{- end }}
+            - name: LOG_LEVEL
+              value: {{ .Values.env.LOG_LEVEL | quote }}
+          ports:
+            - name: http
+              containerPort: 8000
+              protocol: TCP
+          volumeMounts:
+            - name: data
+              mountPath: /label-studio/data
+            - name: dataset
+              mountPath: /label-studio/local
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{ else }}{{ include "label-studio.fullname" . }}-data{{ end }}
+        - name: dataset
+          {{- if and .Values.datasetVolume.enabled .Values.datasetVolume.claimName }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.datasetVolume.claimName }}
+          {{- else }}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{ else }}{{ include "label-studio.fullname" . }}-dataset{{ end }}
+          {{- end }}
+

deployment/helm/label-studio/templates/ingress.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "label-studio.fullname" . }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.className }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
+            backend:
+              service:
+                name: {{ include "label-studio.fullname" $ }}
+                port:
+                  number: {{ $.Values.service.port }}
+          {{- end }}
+    {{- end }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- toYaml .Values.ingress.tls | nindent 4 }}
+  {{- end }}
+{{- end }}
+

deployment/helm/label-studio/templates/pgbouncer-deployment.yaml

@@ -0,0 +1,87 @@
+{{- if .Values.pgbouncer.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "label-studio.fullname" . }}-pgbouncer
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" .}}-pgbouncer
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.pgbouncer.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: pgbouncer
+          {{- if .Values.global.imageRegistry }}
+          image: "{{ .Values.global.imageRegistry | trimSuffix "/" }}/{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}"
+          {{- else }}
+          image: "{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}"
+          {{- end }}
+          imagePullPolicy: {{ .Values.pgbouncer.image.pullPolicy }}
+          ports:
+            - name: pgbouncer
+              containerPort: 5432
+              protocol: TCP
+          env:
+            - name: DB_HOST
+              value: {{ .Values.env.POSTGRE_HOST | quote }}
+            - name: DB_PORT
+              value: {{ .Values.env.POSTGRE_PORT | quote }}
+            - name: DB_NAME
+              value: {{ .Values.env.POSTGRE_NAME | quote }}
+            - name: DB_USER
+              value: {{ .Values.env.POSTGRE_USER | quote }}
+            - name: DB_PASSWORD
+            {{- if .Values.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.existingSecret | quote }}
+                  key: POSTGRE_PASSWORD
+            {{- else }}
+              value: {{ .Values.env.POSTGRE_PASSWORD | quote }}
+            {{- end }}
+            - name: POOL_MODE
+              value: {{ .Values.pgbouncer.poolMode | quote }}
+            - name: MAX_CLIENT_CONN
+              value: {{ .Values.pgbouncer.maxClientConn | quote }}
+            - name: DEFAULT_POOL_SIZE
+              value: {{ .Values.pgbouncer.defaultPoolSize | quote }}
+            - name: MAX_DB_CONNECTIONS
+              value: {{ .Values.pgbouncer.maxDbConnections | quote }}
+            - name: AUTH_TYPE
+              value: {{ .Values.pgbouncer.authType | quote }}
+          livenessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          readinessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            {{- toYaml .Values.pgbouncer.resources | nindent 12 }}
+      {{- with .Values.pgbouncer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.pgbouncer.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.pgbouncer.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

deployment/helm/label-studio/templates/pgbouncer-service.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.pgbouncer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "label-studio.fullname" . }}-pgbouncer
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: 5432
+      targetPort: pgbouncer
+      protocol: TCP
+      name: pgbouncer
+  selector:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}-pgbouncer
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

deployment/helm/label-studio/templates/pvc.yaml

@@ -0,0 +1,123 @@
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+{{- if and .Values.persistence.storageClass (eq .Values.persistence.storageClass "local-storage") }}
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "label-studio.fullname" . }}-data-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistence.size }}
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistence.storageClass }}
+  local:
+    path: {{ .Values.persistence.storagePath | default "/opt/k8s/deepdata" }}/data
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: {{ include "label-studio.fullname" . }}-data
+    namespace: {{ .Release.Namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistence.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "label-studio.fullname" . }}-dataset-pv
+spec:
+  capacity:
+    storage: {{ .Values.persistence.size }}
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.persistence.storageClass }}
+  local:
+    path: {{ .Values.persistence.storagePath | default "/opt/k8s/deepdata" }}/dataset
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: {{ include "label-studio.fullname" . }}-dataset
+    namespace: {{ .Release.Namespace | default "deepdata" }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - {{ .Values.persistence.storageNode }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeName: {{ include "label-studio.fullname" . }}-data-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-dataset
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeName: {{ include "label-studio.fullname" . }}-dataset-pv
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+
+{{- else }}
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  {{- if .Values.persistence.storageClass }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+  {{- end }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "label-studio.fullname" . }}-dataset
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  {{- if .Values.persistence.storageClass }}
+  storageClassName: {{ .Values.persistence.storageClass }}
+  {{- end }}
+{{- end }}
+
+{{- end }}
+

deployment/helm/label-studio/templates/service.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "label-studio.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}
+    helm.sh/chart: {{ include "label-studio.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+      {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
+  selector:
+    app.kubernetes.io/name: {{ include "label-studio.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+