Files
figmar 7bbc9d4ef0
CodeQL Advanced / Analyze (actions) (push) Canceled after 0s
CodeQL Advanced / Analyze (java-kotlin) (push) Canceled after 0s
CodeQL Advanced / Analyze (javascript-typescript) (push) Canceled after 0s
CodeQL Advanced / Analyze (python) (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-arm (push) Canceled after 0s
Deer Flow Docker Image CI / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Python Backend Docker Image CI / Build and Push Backend Python Docker Image (push) Canceled after 0s
Backend Docker Image CI / Build and Push Backend Docker Image (push) Canceled after 0s
Database Docker Image CI / Build and Push Database Docker Image (push) Canceled after 0s
Frontend Docker Image CI / Build and Push Frontend Docker Image (push) Canceled after 0s
Gateway Docker Image CI / Build and Push Gateway Docker Image (push) Canceled after 0s
Label Studio Docker Image CI / Build and Push Label Studio Docker Image (push) Canceled after 0s
Runtime Docker Image CI / Build and Push Runtime Docker Image (push) Canceled after 0s
deepdata: delivery stack (docker compose, helm, CI, build scripts)
2026-08-30 19:23:42 +08:00

582 lines
23 KiB
Makefile

MAKEFLAGS += --no-print-directory
WITH_MINERU ?= false # 默认不构建mineru
VERSION ?= latest
NAMESPACE ?= deepdata
# Registry configuration: use --dev for local images, otherwise use GitHub registry
ifdef dev
REGISTRY :=
else
REGISTRY ?=
endif
ifdef COMSPEC
# Windows
MAKE := "C:/Program Files (x86)/GnuWin32/bin/make"
else
# Linux/Mac
MAKE := make
endif
# ========== Help ==========
.PHONY: help
help:
@echo "DeepData Makefile - Available Commands"
@echo ""
@echo "Usage: make <target> [options]"
@echo ""
@echo "Options:"
@echo " dev=true Use local images instead of registry (empty REGISTRY)"
@echo " VERSION=<version> Set image version (default: latest)"
@echo " NAMESPACE=<name> Set Kubernetes namespace (default: deepdata)"
@echo " INSTALLER=<type> Set installer type: docker or k8s"
@echo " PLATFORM=<platform> Set platform for downloads (default: linux/amd64)"
@echo " Options: linux/amd64, linux/arm64"
@echo " SAVE=true Save images to dist/ during download (default: false)"
@echo ""
@echo "Build Commands:"
@echo " make build Build all core images"
@echo " make <service>-docker-build Build specific service image"
@echo " Valid services: backend, database, frontend, runtime,"
@echo " backend-python, deepflow, mineru"
@echo ""
@echo "Install Commands:"
@echo " make install Install deepdata + milvus (prompts for method)"
@echo " make install INSTALLER=docker Install using Docker Compose"
@echo " make install INSTALLER=k8s Install using Kubernetes/Helm"
@echo " make install INSTALLER=k8s (requires Sealed Secrets Controller)"
@echo " make install-<component> Install specific component (prompts)"
@echo " make <component>-docker-install Install component via Docker"
@echo " make <component>-k8s-install Install component via Kubernetes"
@echo " Valid components: deepdata, milvus, deepflow, mineru"
@echo " Valid services: backend, frontend, runtime, label-studio"
@echo ""
@echo "Uninstall Commands:"
@echo " make uninstall Uninstall deepdata + milvus (prompts)"
@echo " make uninstall INSTALLER=docker Uninstall using Docker Compose"
@echo " make uninstall INSTALLER=k8s Uninstall using Kubernetes/Helm"
@echo " make uninstall-<component> Uninstall specific component (prompts)"
@echo " make <component>-docker-uninstall Uninstall component via Docker"
@echo " make <component>-k8s-uninstall Uninstall component via Kubernetes"
@echo " Note: Docker uninstall will prompt whether to delete volumes"
@echo ""
@echo "Upgrade Commands:"
@echo " make deepdata-docker-upgrade Upgrade deepdata deployment"
@echo ""
@echo "Download Commands:"
@echo " make download Pull all images (no save by default)"
@echo " make download VERSION=<version> Pull all images with specific version"
@echo " make download REGISTRY=<registry> Pull images from specific registry"
@echo " make load-images Load all downloaded images from dist/"
@echo " make download-sealed-secrets Download Sealed Secrets image (for offline)"
@echo ""
@echo "Utility Commands:"
@echo " make create-namespace Create Kubernetes namespace"
@echo " make node-setup Configure dedicated nodes interactively"
@echo " make node-cleanup Remove node labels and taints"
@echo " make help Show this help message"
@echo ""
@echo "Examples:"
@echo " make build dev=true Build all images for local development"
@echo " make install INSTALLER=docker Install via Docker Compose"
@echo " make install dev=true Install using local images"
@echo " make deepdata-docker-upgrade Upgrade running deepdata services"
@echo ""
.DEFAULT_GOAL := help
# ========== Functions ==========
# Prompt user to choose installer if not specified
define prompt-installer
@echo "Choose a deployment method:"
@echo "1. Docker/Docker-Compose"
@echo "2. Kubernetes/Helm"
@echo -n "Enter choice: "
@read choice; \
case $$choice in \
1) INSTALLER=docker ;; \
2) INSTALLER=k8s ;; \
*) echo "Invalid choice" && exit 1 ;; \
esac; \
$(MAKE) $(1)
endef
# Prompt user to choose installer and volume deletion for uninstall
define prompt-uninstaller
@echo "Choose a deployment method:"
@echo "1. Docker/Docker-Compose"
@echo "2. Kubernetes/Helm"
@echo -n "Enter choice: "
@read installer_choice; \
case $$installer_choice in \
1) INSTALLER=docker ;; \
2) INSTALLER=k8s ;; \
*) echo "Invalid choice" && exit 1 ;; \
esac; \
if [ "$$INSTALLER" = "docker" ]; then \
echo "Delete volumes? (This will remove all data)"; \
echo "1. Yes - Delete volumes"; \
echo "2. No - Keep volumes"; \
echo -n "Enter choice (default: 2): "; \
read DELETE_VOLUMES_CHOICE; \
$(MAKE) $(1) DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
else \
$(MAKE) $(1); \
fi
endef
# Generic docker build function
# Usage: $(call docker-build,service-name,image-name)
define docker-build
docker build -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile
endef
# Generic docker compose service action
# Usage: $(call docker-compose-service,service-name,action,compose-dir)
define docker-compose-service
cd $(3) && docker compose $(2) $(1)
endef
# Prompt user to choose whether to delete volumes
define prompt-volume-deletion
@echo "Delete volumes? (This will remove all data)"
@echo "1. Yes - Delete volumes"
@echo "2. No - Keep volumes"
@echo -n "Enter choice (default: 2): "
@read choice; \
case $$choice in \
1) echo "-v" ;; \
*) echo "" ;; \
esac
endef
# ========== Build Targets ==========
# Valid build targets
VALID_BUILD_TARGETS := frontend backend gateway database runtime runtime-ray backend-python deepflow label-studio mineru mineru-910B mineru-910C mineru-310P
# Generic docker build target with service name as parameter
# Automatically prefixes image names with "deepdata-" unless it's deepflow
.PHONY: %-docker-build
%-docker-build:
@if ! echo " $(VALID_BUILD_TARGETS) " | grep -q " $* "; then \
echo "Error: Unknown build target '$*'"; \
echo "Valid build targets are:"; \
for target in $(VALID_BUILD_TARGETS); do \
echo " - $$target"; \
done; \
exit 1; \
fi
@if [ "$*" = "deepflow" ]; then \
$(call docker-build,deepflow-backend,deepflow-backend); \
# $(call docker-build,deepflow-frontend,deepflow-frontend); \
docker build -t deepflow-frontend:$(VERSION) . \
-f scripts/images/deepflow-frontend/Dockerfile \
--build-arg NEXT_PUBLIC_API_URL=/deepflow-backend; \
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ]; then \
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .; \
elif [ "$*" = "mineru-910C" ]; then \
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .; \
elif [ "$*" = "mineru-310P" ]; then \
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .; \
else \
$(call docker-build,$*,deepdata-$*); \
fi
.PHONY: build-%
build-%: %-docker-build
@:
.PHONY: build
build: database-docker-build gateway-docker-build backend-docker-build frontend-docker-build runtime-docker-build backend-python-docker-build
# ========== Utility Targets ==========
.PHONY: create-namespace
create-namespace:
kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
# ========== Node Setup/Cleanup Targets ==========
.PHONY: node-setup
node-setup:
@echo "Configure dedicated nodes for DeepData deployment?"
@echo "This will apply labels and optional taints to selected nodes."
@echo "1. Yes - Configure nodes interactively"
@echo "2. No - Use default scheduling"
@echo -n "Enter choice (default: 2): "
@read NODE_SETUP_CHOICE; \
if [ "$$NODE_SETUP_CHOICE" = "1" ]; then \
chmod +x scripts/k8s/node-setup.sh; \
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
fi
.PHONY: node-cleanup
node-cleanup:
@chmod +x scripts/k8s/node-cleanup.sh
@./scripts/k8s/node-cleanup.sh --namespace $(NAMESPACE)
# ========== Generic Install/Uninstall Targets (Redirect to prompt-installer) ==========
.PHONY: install-%
install-%:
ifeq ($(origin INSTALLER), undefined)
$(call prompt-installer,$*-$$INSTALLER-install)
else
$(MAKE) $*-$(INSTALLER)-install
endif
.PHONY: install
install:
ifeq ($(origin INSTALLER), undefined)
$(call prompt-installer,deepdata-$$INSTALLER-install milvus-$$INSTALLER-install)
else
$(MAKE) deepdata-$(INSTALLER)-install
$(MAKE) milvus-$(INSTALLER)-install
@rm -f /tmp/deepdata-helm-args.sh
endif
.PHONY: uninstall-%
uninstall-%:
ifeq ($(origin INSTALLER), undefined)
$(call prompt-uninstaller,$*-$$INSTALLER-uninstall)
else
$(MAKE) $*-$(INSTALLER)-uninstall
endif
.PHONY: uninstall
uninstall:
ifeq ($(origin INSTALLER), undefined)
$(call prompt-uninstaller,label-studio-$$INSTALLER-uninstall milvus-$$INSTALLER-uninstall deepflow-$$INSTALLER-uninstall deepdata-$$INSTALLER-uninstall)
else
@if [ "$(INSTALLER)" = "docker" ]; then \
echo "Delete volumes? (This will remove all data)"; \
echo "1. Yes - Delete volumes"; \
echo "2. No - Keep volumes"; \
echo -n "Enter choice (default: 2): "; \
read DELETE_VOLUMES_CHOICE; \
export DELETE_VOLUMES_CHOICE; \
fi; \
$(MAKE) label-studio-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
$(MAKE) milvus-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
$(MAKE) deepflow-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
$(MAKE) deepdata-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE
endif
# ========== Docker Install/Uninstall Targets ==========
# Valid service targets for docker install/uninstall
VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
# Generic docker service install target
.PHONY: %-docker-install
%-docker-install:
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
echo "Error: Unknown service target '$*'"; \
echo "Valid service targets are:"; \
for target in $(VALID_SERVICE_TARGETS); do \
echo " - $$target"; \
done; \
exit 1; \
fi
@if [ ! -f deployment/docker/deepdata/.env ]; then \
echo "ERROR: deployment/docker/deepdata/.env not found."; \
echo "Create it from the template:"; \
echo " cp deployment/docker/deepdata/.env.example deployment/docker/deepdata/.env"; \
echo "Then edit it with your actual passwords."; \
exit 1; \
fi
@if [ "$*" = "label-studio" ]; then \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile label-studio up -d; \
elif [ "$*" = "deepdata" ]; then \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d; \
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
elif [ "$*" = "mineru-310P" ]; then \
REGISTRY=$(REGISTRY) EXTRA_ARGS="--enforce-eager --dtype float16" docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
elif [ "$*" = "data-juicer" ]; then \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile data-juicer up -d deepdata-data-juicer; \
elif [ "$*" = "redis" ]; then \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile redis up -d deepdata-redis; \
elif [ "$*" = "milvus" ]; then \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d; \
elif [ "$*" = "deepflow" ]; then \
cp runtime/deepflow/.env deployment/docker/deepdata/.env; \
cp runtime/deepflow/conf.yaml deployment/docker/deepdata/conf.yaml; \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile deepflow up -d; \
else \
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d deepdata-$*; \
fi
# Generic docker service uninstall target
.PHONY: %-docker-uninstall
%-docker-uninstall:
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
echo "Error: Unknown service target '$*'"; \
echo "Valid service targets are:"; \
for target in $(VALID_SERVICE_TARGETS); do \
echo " - $$target"; \
done; \
exit 1; \
fi
@if [ "$*" = "label-studio" ]; then \
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s label-studio; \
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ] || [ "$*" = "mineru-310P" ]; then \
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-mineru; \
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-data-juicer; \
elif [ "$*" = "redis" ]; then \
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-redis; \
elif [ "$*" = "deepdata" ]; then \
if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down -v; \
else \
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down; \
fi; \
elif [ "$*" = "deepflow" ]; then \
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepflow-backend deepflow-frontend; \
elif [ "$*" = "milvus" ]; then \
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s milvus etcd minio; \
else \
$(call docker-compose-service,$*,down,deployment/docker/deepdata); \
fi
# ========== Kubernetes Install/Uninstall Targets ==========
# Valid k8s targets
VALID_K8S_TARGETS := deepdata deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
# Generic k8s install target
.PHONY: %-k8s-install
%-k8s-install: create-namespace
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
echo "Error: Unknown k8s target '$*'"; \
echo "Valid k8s targets are:"; \
for target in $(VALID_K8S_TARGETS); do \
echo " - $$target"; \
done; \
exit 1; \
fi
@if [ "$*" = "label-studio" ]; then \
if [ -f /tmp/deepdata-helm-args.sh ]; then . /tmp/deepdata-helm-args.sh; fi; \
helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install $${HELM_LABEL_STUDIO_TOLERATIONS:-}; \
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
elif [ "$*" = "mineru-310P" ]; then \
kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
elif [ "$*" = "deepdata" ]; then \
echo ""; \
chmod +x scripts/k8s/node-setup.sh; \
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
if [ -f /tmp/deepdata-helm-args.sh ]; then \
. /tmp/deepdata-helm-args.sh; \
fi; \
chmod +x scripts/k8s/collect-secrets.sh; \
eval $$(NAMESPACE=$(NAMESPACE) bash scripts/k8s/collect-secrets.sh); \
if [ "$$SECRETS_CREATE" = "SKIP" ]; then \
echo "[SKIP] Secrets collection failed — skipping deepdata Helm install"; \
rm -f /tmp/deepdata-helm-args.sh; \
exit 0; \
fi; \
if [ -n "$$HELM_VALUES_FILE" ] && [ -f "$$HELM_VALUES_FILE" ]; then \
HELM_EXTRA_ARGS="-f $$HELM_VALUES_FILE"; \
else \
HELM_EXTRA_ARGS=""; \
fi; \
if [ -n "$$HELM_NODE_SELECTOR_ARGS" ] || [ -n "$$HELM_TOLERATIONS_ARGS" ]; then \
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS $$HELM_NODE_SELECTOR_ARGS $$HELM_TOLERATIONS_ARGS; \
else \
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS; \
fi; \
rm -f /tmp/deepdata-secret-values-*.yaml; \
elif [ "$*" = "deepflow" ]; then \
cp runtime/deepflow/.env deployment/helm/deepflow/charts/public/.env; \
cp runtime/deepflow/conf.yaml deployment/helm/deepflow/charts/public/conf.yaml; \
helm upgrade deepflow deployment/helm/deepflow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \
elif [ "$*" = "milvus" ]; then \
chmod +x scripts/k8s/collect-secrets.sh; \
bash scripts/k8s/collect-secrets.sh --component milvus -n $(NAMESPACE); \
MILVUS_MINIO_ACCESS_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accesskey}' | base64 -d); \
MILVUS_MINIO_SECRET_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretkey}' | base64 -d); \
if [ -f /tmp/deepdata-helm-args.sh ]; then \
. /tmp/deepdata-helm-args.sh; \
fi; \
helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
--set minio.accessKey="$$MILVUS_MINIO_ACCESS_KEY" \
--set minio.secretKey="$$MILVUS_MINIO_SECRET_KEY" \
--set log.persistence.persistentVolumeClaim.accessModes=ReadWriteOnce \
$$HELM_MILVUS_TOLERATIONS; \
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
fi
# Generic k8s uninstall target
.PHONY: %-k8s-uninstall
%-k8s-uninstall:
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
echo "Error: Unknown k8s target '$*'"; \
echo "Valid k8s targets are:"; \
for target in $(VALID_K8S_TARGETS); do \
echo " - $$target"; \
done; \
exit 1; \
fi
@if [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
kubectl delete -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
elif [ "$*" = "mineru-310P" ]; then \
kubectl delete -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
elif [ "$*" = "deepdata" ]; then \
helm uninstall deepdata -n $(NAMESPACE) --ignore-not-found; \
$(MAKE) node-cleanup; \
elif [ "$*" = "deepflow" ]; then \
helm uninstall deepflow -n $(NAMESPACE) --ignore-not-found; \
elif [ "$*" = "milvus" ]; then \
helm uninstall milvus -n $(NAMESPACE) --ignore-not-found; \
elif [ "$*" = "label-studio" ]; then \
helm uninstall label-studio -n $(NAMESPACE) --ignore-not-found; \
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
kubectl delete -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
fi
# ========== Download Targets ==========
# List of all images to download
DOWNLOAD_IMAGES := \
deepdata-backend \
deepdata-frontend \
deepdata-database \
deepdata-runtime \
deepdata-backend-python \
deepdata-gateway \
# Detect architecture for smart default
CURRENT_ARCH := $(shell uname -m 2>/dev/null)
ifeq ($(CURRENT_ARCH),aarch64)
DEFAULT_OPTION := 2
else ifeq ($(CURRENT_ARCH),arm64)
DEFAULT_OPTION := 2
else
DEFAULT_OPTION := 1
endif
#Usage: make download VERSION=1.0.0 REGISTRY=<registry-prefix>/ SAVE=true
.PHONY: download
download:
@echo "Select target platform for images:"
@echo " 1) x86_64 (Intel/AMD, linux/amd64)"
@echo " 2) ARM64 (Apple Silicon, Raspberry Pi 4+, linux/arm64)"
@{ \
read -p "Enter choice [1/2] (default: $(DEFAULT_OPTION)): " plat_choice; \
case "$${plat_choice:-$(DEFAULT_OPTION)}" in \
1) PLATFORM="linux/amd64" ;; \
2) PLATFORM="linux/arm64" ;; \
*) echo "Invalid choice. Using default option $(DEFAULT_OPTION)."; \
PLATFORM=$$([ "$(DEFAULT_OPTION)" = "1" ] && echo "linux/amd64" || echo "linux/arm64");; \
esac; \
\
read -p "Save images to dist/ for offline use? (y/n): " save_resp; \
case "$$save_resp" in \
[yY]|[yY][eE][sS]) SAVE="true" ;; \
*) SAVE="false" ;; \
esac; \
\
if [ -z "$(REGISTRY)" ] && [ "$$SAVE" != "true" ]; then \
echo "Error: REGISTRY unset and SAVE=false. Nothing to do."; \
exit 1; \
fi; \
\
if [ "$$SAVE" = "true" ]; then mkdir -p dist; fi; \
failed=0; \
\
for image in $(DOWNLOAD_IMAGES); do \
if [ -z "$(REGISTRY)" ]; then \
src_image="$$image:$(VERSION)"; \
echo "Using local image: $$src_image"; \
else \
src_image="$(REGISTRY)$$image:$(VERSION)"; \
echo "Pulling $$src_image ($$PLATFORM)..."; \
if ! docker pull --platform "$$PLATFORM" "$$src_image"; then \
echo "✗ Failed to pull $$src_image"; \
failed=$$(($$failed + 1)); \
continue; \
fi; \
fi; \
\
if [ "$$SAVE" = "true" ]; then \
output="dist/$$image-$(VERSION).tar"; \
echo "Saving to $$output..."; \
if docker save -o "$$output" "$$src_image"; then \
echo "✓ Saved $$image"; \
else \
echo "✗ Failed to save $$src_image"; \
failed=$$(($$failed + 1)); \
fi; \
else \
echo "✓ Ready: $$src_image"; \
fi; \
echo ""; \
done; \
\
if [ $$failed -eq 0 ]; then \
if [ "$$SAVE" = "true" ]; then \
echo "✅ All images saved to dist/"; \
echo "Load on target: docker load -i <image>.tar"; \
else \
echo "✅ All images ready in local Docker daemon"; \
fi; \
else \
echo "❌ $$failed image(s) failed"; \
exit 1; \
fi; \
}
DEEPFLOW_IMAGES := \
deepflow-backend \
deepflow-frontend
.PHONY: download-deepflow
download-deepflow:
$(MAKE) download DOWNLOAD_IMAGES="$(DEEPFLOW_IMAGES)"
# Download Sealed Secrets controller image for offline/air-gapped environments
SEALED_SECRETS_IMAGE := bitnami/sealed-secrets-controller:latest
.PHONY: download-sealed-secrets
download-sealed-secrets:
@echo "Pulling Sealed Secrets controller image..."
@mkdir -p dist
docker pull $(SEALED_SECRETS_IMAGE)
docker save $(SEALED_SECRETS_IMAGE) -o dist/sealed-secrets-controller.tar
@echo "✅ Saved to dist/sealed-secrets-controller.tar"
@echo "Transfer to offline environment and load with: docker load -i dist/sealed-secrets-controller.tar"
# Load all downloaded images from dist/ directory
.PHONY: load-images
load-images:
@if [ ! -d "dist" ]; then \
echo "Error: dist/ directory not found"; \
echo "Please run 'make download' first to download images"; \
exit 1; \
fi
@echo "Loading images from dist/..."
@count=0; \
for tarfile in dist/*.tar.gz; do \
if [ -f "$$tarfile" ]; then \
echo "Loading $$tarfile..."; \
docker load -i "$$tarfile"; \
count=$$((count + 1)); \
echo "✓ Loaded $$tarfile"; \
echo ""; \
fi; \
done; \
if [ $$count -eq 0 ]; then \
echo "No image files found in dist/"; \
echo "Please run 'make download' first"; \
exit 1; \
else \
echo "Successfully loaded $$count image(s)"; \
fi