CodeQL Advanced / Analyze (actions) (push) Canceled after 0s
CodeQL Advanced / Analyze (java-kotlin) (push) Canceled after 0s
CodeQL Advanced / Analyze (javascript-typescript) (push) Canceled after 0s
CodeQL Advanced / Analyze (python) (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-arm (push) Canceled after 0s
Deer Flow Docker Image CI / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Python Backend Docker Image CI / Build and Push Backend Python Docker Image (push) Canceled after 0s
Backend Docker Image CI / Build and Push Backend Docker Image (push) Canceled after 0s
Database Docker Image CI / Build and Push Database Docker Image (push) Canceled after 0s
Frontend Docker Image CI / Build and Push Frontend Docker Image (push) Canceled after 0s
Gateway Docker Image CI / Build and Push Gateway Docker Image (push) Canceled after 0s
Label Studio Docker Image CI / Build and Push Label Studio Docker Image (push) Canceled after 0s
Runtime Docker Image CI / Build and Push Runtime Docker Image (push) Canceled after 0s
582 lines
23 KiB
Makefile
582 lines
23 KiB
Makefile
MAKEFLAGS += --no-print-directory
|
|
|
|
WITH_MINERU ?= false # 默认不构建mineru
|
|
VERSION ?= latest
|
|
NAMESPACE ?= deepdata
|
|
|
|
# Registry configuration: use --dev for local images, otherwise use GitHub registry
|
|
ifdef dev
|
|
REGISTRY :=
|
|
else
|
|
REGISTRY ?=
|
|
endif
|
|
|
|
ifdef COMSPEC
|
|
# Windows
|
|
MAKE := "C:/Program Files (x86)/GnuWin32/bin/make"
|
|
else
|
|
# Linux/Mac
|
|
MAKE := make
|
|
endif
|
|
|
|
# ========== Help ==========
|
|
|
|
.PHONY: help
|
|
help:
|
|
@echo "DeepData Makefile - Available Commands"
|
|
@echo ""
|
|
@echo "Usage: make <target> [options]"
|
|
@echo ""
|
|
@echo "Options:"
|
|
@echo " dev=true Use local images instead of registry (empty REGISTRY)"
|
|
@echo " VERSION=<version> Set image version (default: latest)"
|
|
@echo " NAMESPACE=<name> Set Kubernetes namespace (default: deepdata)"
|
|
@echo " INSTALLER=<type> Set installer type: docker or k8s"
|
|
@echo " PLATFORM=<platform> Set platform for downloads (default: linux/amd64)"
|
|
@echo " Options: linux/amd64, linux/arm64"
|
|
@echo " SAVE=true Save images to dist/ during download (default: false)"
|
|
@echo ""
|
|
@echo "Build Commands:"
|
|
@echo " make build Build all core images"
|
|
@echo " make <service>-docker-build Build specific service image"
|
|
@echo " Valid services: backend, database, frontend, runtime,"
|
|
@echo " backend-python, deepflow, mineru"
|
|
@echo ""
|
|
@echo "Install Commands:"
|
|
@echo " make install Install deepdata + milvus (prompts for method)"
|
|
@echo " make install INSTALLER=docker Install using Docker Compose"
|
|
@echo " make install INSTALLER=k8s Install using Kubernetes/Helm"
|
|
@echo " make install INSTALLER=k8s (requires Sealed Secrets Controller)"
|
|
@echo " make install-<component> Install specific component (prompts)"
|
|
@echo " make <component>-docker-install Install component via Docker"
|
|
@echo " make <component>-k8s-install Install component via Kubernetes"
|
|
@echo " Valid components: deepdata, milvus, deepflow, mineru"
|
|
@echo " Valid services: backend, frontend, runtime, label-studio"
|
|
@echo ""
|
|
@echo "Uninstall Commands:"
|
|
@echo " make uninstall Uninstall deepdata + milvus (prompts)"
|
|
@echo " make uninstall INSTALLER=docker Uninstall using Docker Compose"
|
|
@echo " make uninstall INSTALLER=k8s Uninstall using Kubernetes/Helm"
|
|
@echo " make uninstall-<component> Uninstall specific component (prompts)"
|
|
@echo " make <component>-docker-uninstall Uninstall component via Docker"
|
|
@echo " make <component>-k8s-uninstall Uninstall component via Kubernetes"
|
|
@echo " Note: Docker uninstall will prompt whether to delete volumes"
|
|
@echo ""
|
|
@echo "Upgrade Commands:"
|
|
@echo " make deepdata-docker-upgrade Upgrade deepdata deployment"
|
|
@echo ""
|
|
@echo "Download Commands:"
|
|
@echo " make download Pull all images (no save by default)"
|
|
@echo " make download VERSION=<version> Pull all images with specific version"
|
|
@echo " make download REGISTRY=<registry> Pull images from specific registry"
|
|
@echo " make load-images Load all downloaded images from dist/"
|
|
@echo " make download-sealed-secrets Download Sealed Secrets image (for offline)"
|
|
@echo ""
|
|
@echo "Utility Commands:"
|
|
@echo " make create-namespace Create Kubernetes namespace"
|
|
@echo " make node-setup Configure dedicated nodes interactively"
|
|
@echo " make node-cleanup Remove node labels and taints"
|
|
@echo " make help Show this help message"
|
|
@echo ""
|
|
@echo "Examples:"
|
|
@echo " make build dev=true Build all images for local development"
|
|
@echo " make install INSTALLER=docker Install via Docker Compose"
|
|
@echo " make install dev=true Install using local images"
|
|
@echo " make deepdata-docker-upgrade Upgrade running deepdata services"
|
|
@echo ""
|
|
|
|
.DEFAULT_GOAL := help
|
|
|
|
# ========== Functions ==========
|
|
|
|
# Prompt user to choose installer if not specified
|
|
define prompt-installer
|
|
@echo "Choose a deployment method:"
|
|
@echo "1. Docker/Docker-Compose"
|
|
@echo "2. Kubernetes/Helm"
|
|
@echo -n "Enter choice: "
|
|
@read choice; \
|
|
case $$choice in \
|
|
1) INSTALLER=docker ;; \
|
|
2) INSTALLER=k8s ;; \
|
|
*) echo "Invalid choice" && exit 1 ;; \
|
|
esac; \
|
|
$(MAKE) $(1)
|
|
endef
|
|
|
|
# Prompt user to choose installer and volume deletion for uninstall
|
|
define prompt-uninstaller
|
|
@echo "Choose a deployment method:"
|
|
@echo "1. Docker/Docker-Compose"
|
|
@echo "2. Kubernetes/Helm"
|
|
@echo -n "Enter choice: "
|
|
@read installer_choice; \
|
|
case $$installer_choice in \
|
|
1) INSTALLER=docker ;; \
|
|
2) INSTALLER=k8s ;; \
|
|
*) echo "Invalid choice" && exit 1 ;; \
|
|
esac; \
|
|
if [ "$$INSTALLER" = "docker" ]; then \
|
|
echo "Delete volumes? (This will remove all data)"; \
|
|
echo "1. Yes - Delete volumes"; \
|
|
echo "2. No - Keep volumes"; \
|
|
echo -n "Enter choice (default: 2): "; \
|
|
read DELETE_VOLUMES_CHOICE; \
|
|
$(MAKE) $(1) DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
|
else \
|
|
$(MAKE) $(1); \
|
|
fi
|
|
endef
|
|
|
|
# Generic docker build function
|
|
# Usage: $(call docker-build,service-name,image-name)
|
|
define docker-build
|
|
docker build -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile
|
|
endef
|
|
|
|
# Generic docker compose service action
|
|
# Usage: $(call docker-compose-service,service-name,action,compose-dir)
|
|
define docker-compose-service
|
|
cd $(3) && docker compose $(2) $(1)
|
|
endef
|
|
|
|
# Prompt user to choose whether to delete volumes
|
|
define prompt-volume-deletion
|
|
@echo "Delete volumes? (This will remove all data)"
|
|
@echo "1. Yes - Delete volumes"
|
|
@echo "2. No - Keep volumes"
|
|
@echo -n "Enter choice (default: 2): "
|
|
@read choice; \
|
|
case $$choice in \
|
|
1) echo "-v" ;; \
|
|
*) echo "" ;; \
|
|
esac
|
|
endef
|
|
|
|
# ========== Build Targets ==========
|
|
|
|
# Valid build targets
|
|
VALID_BUILD_TARGETS := frontend backend gateway database runtime runtime-ray backend-python deepflow label-studio mineru mineru-910B mineru-910C mineru-310P
|
|
|
|
# Generic docker build target with service name as parameter
|
|
# Automatically prefixes image names with "deepdata-" unless it's deepflow
|
|
.PHONY: %-docker-build
|
|
%-docker-build:
|
|
@if ! echo " $(VALID_BUILD_TARGETS) " | grep -q " $* "; then \
|
|
echo "Error: Unknown build target '$*'"; \
|
|
echo "Valid build targets are:"; \
|
|
for target in $(VALID_BUILD_TARGETS); do \
|
|
echo " - $$target"; \
|
|
done; \
|
|
exit 1; \
|
|
fi
|
|
@if [ "$*" = "deepflow" ]; then \
|
|
$(call docker-build,deepflow-backend,deepflow-backend); \
|
|
# $(call docker-build,deepflow-frontend,deepflow-frontend); \
|
|
docker build -t deepflow-frontend:$(VERSION) . \
|
|
-f scripts/images/deepflow-frontend/Dockerfile \
|
|
--build-arg NEXT_PUBLIC_API_URL=/deepflow-backend; \
|
|
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ]; then \
|
|
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .; \
|
|
elif [ "$*" = "mineru-910C" ]; then \
|
|
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .; \
|
|
elif [ "$*" = "mineru-310P" ]; then \
|
|
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .; \
|
|
else \
|
|
$(call docker-build,$*,deepdata-$*); \
|
|
fi
|
|
|
|
.PHONY: build-%
|
|
build-%: %-docker-build
|
|
@:
|
|
|
|
.PHONY: build
|
|
build: database-docker-build gateway-docker-build backend-docker-build frontend-docker-build runtime-docker-build backend-python-docker-build
|
|
|
|
# ========== Utility Targets ==========
|
|
|
|
.PHONY: create-namespace
|
|
create-namespace:
|
|
kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
|
|
|
|
# ========== Node Setup/Cleanup Targets ==========
|
|
|
|
.PHONY: node-setup
|
|
node-setup:
|
|
@echo "Configure dedicated nodes for DeepData deployment?"
|
|
@echo "This will apply labels and optional taints to selected nodes."
|
|
@echo "1. Yes - Configure nodes interactively"
|
|
@echo "2. No - Use default scheduling"
|
|
@echo -n "Enter choice (default: 2): "
|
|
@read NODE_SETUP_CHOICE; \
|
|
if [ "$$NODE_SETUP_CHOICE" = "1" ]; then \
|
|
chmod +x scripts/k8s/node-setup.sh; \
|
|
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
|
|
fi
|
|
|
|
.PHONY: node-cleanup
|
|
node-cleanup:
|
|
@chmod +x scripts/k8s/node-cleanup.sh
|
|
@./scripts/k8s/node-cleanup.sh --namespace $(NAMESPACE)
|
|
|
|
# ========== Generic Install/Uninstall Targets (Redirect to prompt-installer) ==========
|
|
|
|
.PHONY: install-%
|
|
install-%:
|
|
ifeq ($(origin INSTALLER), undefined)
|
|
$(call prompt-installer,$*-$$INSTALLER-install)
|
|
else
|
|
$(MAKE) $*-$(INSTALLER)-install
|
|
endif
|
|
|
|
.PHONY: install
|
|
install:
|
|
ifeq ($(origin INSTALLER), undefined)
|
|
$(call prompt-installer,deepdata-$$INSTALLER-install milvus-$$INSTALLER-install)
|
|
else
|
|
$(MAKE) deepdata-$(INSTALLER)-install
|
|
$(MAKE) milvus-$(INSTALLER)-install
|
|
@rm -f /tmp/deepdata-helm-args.sh
|
|
endif
|
|
|
|
.PHONY: uninstall-%
|
|
uninstall-%:
|
|
ifeq ($(origin INSTALLER), undefined)
|
|
$(call prompt-uninstaller,$*-$$INSTALLER-uninstall)
|
|
else
|
|
$(MAKE) $*-$(INSTALLER)-uninstall
|
|
endif
|
|
|
|
.PHONY: uninstall
|
|
uninstall:
|
|
ifeq ($(origin INSTALLER), undefined)
|
|
$(call prompt-uninstaller,label-studio-$$INSTALLER-uninstall milvus-$$INSTALLER-uninstall deepflow-$$INSTALLER-uninstall deepdata-$$INSTALLER-uninstall)
|
|
else
|
|
@if [ "$(INSTALLER)" = "docker" ]; then \
|
|
echo "Delete volumes? (This will remove all data)"; \
|
|
echo "1. Yes - Delete volumes"; \
|
|
echo "2. No - Keep volumes"; \
|
|
echo -n "Enter choice (default: 2): "; \
|
|
read DELETE_VOLUMES_CHOICE; \
|
|
export DELETE_VOLUMES_CHOICE; \
|
|
fi; \
|
|
$(MAKE) label-studio-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
|
$(MAKE) milvus-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
|
$(MAKE) deepflow-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
|
$(MAKE) deepdata-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE
|
|
endif
|
|
|
|
# ========== Docker Install/Uninstall Targets ==========
|
|
|
|
# Valid service targets for docker install/uninstall
|
|
VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
|
|
|
|
# Generic docker service install target
|
|
.PHONY: %-docker-install
|
|
%-docker-install:
|
|
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
|
echo "Error: Unknown service target '$*'"; \
|
|
echo "Valid service targets are:"; \
|
|
for target in $(VALID_SERVICE_TARGETS); do \
|
|
echo " - $$target"; \
|
|
done; \
|
|
exit 1; \
|
|
fi
|
|
@if [ ! -f deployment/docker/deepdata/.env ]; then \
|
|
echo "ERROR: deployment/docker/deepdata/.env not found."; \
|
|
echo "Create it from the template:"; \
|
|
echo " cp deployment/docker/deepdata/.env.example deployment/docker/deepdata/.env"; \
|
|
echo "Then edit it with your actual passwords."; \
|
|
exit 1; \
|
|
fi
|
|
@if [ "$*" = "label-studio" ]; then \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile label-studio up -d; \
|
|
elif [ "$*" = "deepdata" ]; then \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d; \
|
|
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
|
|
elif [ "$*" = "mineru-310P" ]; then \
|
|
REGISTRY=$(REGISTRY) EXTRA_ARGS="--enforce-eager --dtype float16" docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
|
|
elif [ "$*" = "data-juicer" ]; then \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile data-juicer up -d deepdata-data-juicer; \
|
|
elif [ "$*" = "redis" ]; then \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile redis up -d deepdata-redis; \
|
|
elif [ "$*" = "milvus" ]; then \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d; \
|
|
elif [ "$*" = "deepflow" ]; then \
|
|
cp runtime/deepflow/.env deployment/docker/deepdata/.env; \
|
|
cp runtime/deepflow/conf.yaml deployment/docker/deepdata/conf.yaml; \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile deepflow up -d; \
|
|
else \
|
|
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d deepdata-$*; \
|
|
fi
|
|
|
|
# Generic docker service uninstall target
|
|
.PHONY: %-docker-uninstall
|
|
%-docker-uninstall:
|
|
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
|
echo "Error: Unknown service target '$*'"; \
|
|
echo "Valid service targets are:"; \
|
|
for target in $(VALID_SERVICE_TARGETS); do \
|
|
echo " - $$target"; \
|
|
done; \
|
|
exit 1; \
|
|
fi
|
|
@if [ "$*" = "label-studio" ]; then \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s label-studio; \
|
|
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ] || [ "$*" = "mineru-310P" ]; then \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-mineru; \
|
|
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-data-juicer; \
|
|
elif [ "$*" = "redis" ]; then \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-redis; \
|
|
elif [ "$*" = "deepdata" ]; then \
|
|
if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down -v; \
|
|
else \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down; \
|
|
fi; \
|
|
elif [ "$*" = "deepflow" ]; then \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepflow-backend deepflow-frontend; \
|
|
elif [ "$*" = "milvus" ]; then \
|
|
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s milvus etcd minio; \
|
|
else \
|
|
$(call docker-compose-service,$*,down,deployment/docker/deepdata); \
|
|
fi
|
|
|
|
# ========== Kubernetes Install/Uninstall Targets ==========
|
|
|
|
# Valid k8s targets
|
|
VALID_K8S_TARGETS := deepdata deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
|
|
|
|
# Generic k8s install target
|
|
.PHONY: %-k8s-install
|
|
%-k8s-install: create-namespace
|
|
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
|
|
echo "Error: Unknown k8s target '$*'"; \
|
|
echo "Valid k8s targets are:"; \
|
|
for target in $(VALID_K8S_TARGETS); do \
|
|
echo " - $$target"; \
|
|
done; \
|
|
exit 1; \
|
|
fi
|
|
@if [ "$*" = "label-studio" ]; then \
|
|
if [ -f /tmp/deepdata-helm-args.sh ]; then . /tmp/deepdata-helm-args.sh; fi; \
|
|
helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install $${HELM_LABEL_STUDIO_TOLERATIONS:-}; \
|
|
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
|
kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
|
|
elif [ "$*" = "mineru-310P" ]; then \
|
|
kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
|
|
elif [ "$*" = "deepdata" ]; then \
|
|
echo ""; \
|
|
chmod +x scripts/k8s/node-setup.sh; \
|
|
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
|
|
if [ -f /tmp/deepdata-helm-args.sh ]; then \
|
|
. /tmp/deepdata-helm-args.sh; \
|
|
fi; \
|
|
chmod +x scripts/k8s/collect-secrets.sh; \
|
|
eval $$(NAMESPACE=$(NAMESPACE) bash scripts/k8s/collect-secrets.sh); \
|
|
if [ "$$SECRETS_CREATE" = "SKIP" ]; then \
|
|
echo "[SKIP] Secrets collection failed — skipping deepdata Helm install"; \
|
|
rm -f /tmp/deepdata-helm-args.sh; \
|
|
exit 0; \
|
|
fi; \
|
|
if [ -n "$$HELM_VALUES_FILE" ] && [ -f "$$HELM_VALUES_FILE" ]; then \
|
|
HELM_EXTRA_ARGS="-f $$HELM_VALUES_FILE"; \
|
|
else \
|
|
HELM_EXTRA_ARGS=""; \
|
|
fi; \
|
|
if [ -n "$$HELM_NODE_SELECTOR_ARGS" ] || [ -n "$$HELM_TOLERATIONS_ARGS" ]; then \
|
|
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS $$HELM_NODE_SELECTOR_ARGS $$HELM_TOLERATIONS_ARGS; \
|
|
else \
|
|
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS; \
|
|
fi; \
|
|
rm -f /tmp/deepdata-secret-values-*.yaml; \
|
|
elif [ "$*" = "deepflow" ]; then \
|
|
cp runtime/deepflow/.env deployment/helm/deepflow/charts/public/.env; \
|
|
cp runtime/deepflow/conf.yaml deployment/helm/deepflow/charts/public/conf.yaml; \
|
|
helm upgrade deepflow deployment/helm/deepflow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \
|
|
elif [ "$*" = "milvus" ]; then \
|
|
chmod +x scripts/k8s/collect-secrets.sh; \
|
|
bash scripts/k8s/collect-secrets.sh --component milvus -n $(NAMESPACE); \
|
|
MILVUS_MINIO_ACCESS_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accesskey}' | base64 -d); \
|
|
MILVUS_MINIO_SECRET_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretkey}' | base64 -d); \
|
|
if [ -f /tmp/deepdata-helm-args.sh ]; then \
|
|
. /tmp/deepdata-helm-args.sh; \
|
|
fi; \
|
|
helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
|
|
--set minio.accessKey="$$MILVUS_MINIO_ACCESS_KEY" \
|
|
--set minio.secretKey="$$MILVUS_MINIO_SECRET_KEY" \
|
|
--set log.persistence.persistentVolumeClaim.accessModes=ReadWriteOnce \
|
|
$$HELM_MILVUS_TOLERATIONS; \
|
|
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
|
kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
|
|
fi
|
|
|
|
# Generic k8s uninstall target
|
|
.PHONY: %-k8s-uninstall
|
|
%-k8s-uninstall:
|
|
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
|
|
echo "Error: Unknown k8s target '$*'"; \
|
|
echo "Valid k8s targets are:"; \
|
|
for target in $(VALID_K8S_TARGETS); do \
|
|
echo " - $$target"; \
|
|
done; \
|
|
exit 1; \
|
|
fi
|
|
@if [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
|
kubectl delete -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
|
|
elif [ "$*" = "mineru-310P" ]; then \
|
|
kubectl delete -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
|
|
elif [ "$*" = "deepdata" ]; then \
|
|
helm uninstall deepdata -n $(NAMESPACE) --ignore-not-found; \
|
|
$(MAKE) node-cleanup; \
|
|
elif [ "$*" = "deepflow" ]; then \
|
|
helm uninstall deepflow -n $(NAMESPACE) --ignore-not-found; \
|
|
elif [ "$*" = "milvus" ]; then \
|
|
helm uninstall milvus -n $(NAMESPACE) --ignore-not-found; \
|
|
elif [ "$*" = "label-studio" ]; then \
|
|
helm uninstall label-studio -n $(NAMESPACE) --ignore-not-found; \
|
|
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
|
kubectl delete -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
|
|
fi
|
|
|
|
# ========== Download Targets ==========
|
|
|
|
# List of all images to download
|
|
DOWNLOAD_IMAGES := \
|
|
deepdata-backend \
|
|
deepdata-frontend \
|
|
deepdata-database \
|
|
deepdata-runtime \
|
|
deepdata-backend-python \
|
|
deepdata-gateway \
|
|
|
|
# Detect architecture for smart default
|
|
CURRENT_ARCH := $(shell uname -m 2>/dev/null)
|
|
ifeq ($(CURRENT_ARCH),aarch64)
|
|
DEFAULT_OPTION := 2
|
|
else ifeq ($(CURRENT_ARCH),arm64)
|
|
DEFAULT_OPTION := 2
|
|
else
|
|
DEFAULT_OPTION := 1
|
|
endif
|
|
|
|
|
|
#Usage: make download VERSION=1.0.0 REGISTRY=<registry-prefix>/ SAVE=true
|
|
.PHONY: download
|
|
download:
|
|
@echo "Select target platform for images:"
|
|
@echo " 1) x86_64 (Intel/AMD, linux/amd64)"
|
|
@echo " 2) ARM64 (Apple Silicon, Raspberry Pi 4+, linux/arm64)"
|
|
@{ \
|
|
read -p "Enter choice [1/2] (default: $(DEFAULT_OPTION)): " plat_choice; \
|
|
case "$${plat_choice:-$(DEFAULT_OPTION)}" in \
|
|
1) PLATFORM="linux/amd64" ;; \
|
|
2) PLATFORM="linux/arm64" ;; \
|
|
*) echo "Invalid choice. Using default option $(DEFAULT_OPTION)."; \
|
|
PLATFORM=$$([ "$(DEFAULT_OPTION)" = "1" ] && echo "linux/amd64" || echo "linux/arm64");; \
|
|
esac; \
|
|
\
|
|
read -p "Save images to dist/ for offline use? (y/n): " save_resp; \
|
|
case "$$save_resp" in \
|
|
[yY]|[yY][eE][sS]) SAVE="true" ;; \
|
|
*) SAVE="false" ;; \
|
|
esac; \
|
|
\
|
|
if [ -z "$(REGISTRY)" ] && [ "$$SAVE" != "true" ]; then \
|
|
echo "Error: REGISTRY unset and SAVE=false. Nothing to do."; \
|
|
exit 1; \
|
|
fi; \
|
|
\
|
|
if [ "$$SAVE" = "true" ]; then mkdir -p dist; fi; \
|
|
failed=0; \
|
|
\
|
|
for image in $(DOWNLOAD_IMAGES); do \
|
|
if [ -z "$(REGISTRY)" ]; then \
|
|
src_image="$$image:$(VERSION)"; \
|
|
echo "Using local image: $$src_image"; \
|
|
else \
|
|
src_image="$(REGISTRY)$$image:$(VERSION)"; \
|
|
echo "Pulling $$src_image ($$PLATFORM)..."; \
|
|
if ! docker pull --platform "$$PLATFORM" "$$src_image"; then \
|
|
echo "✗ Failed to pull $$src_image"; \
|
|
failed=$$(($$failed + 1)); \
|
|
continue; \
|
|
fi; \
|
|
fi; \
|
|
\
|
|
if [ "$$SAVE" = "true" ]; then \
|
|
output="dist/$$image-$(VERSION).tar"; \
|
|
echo "Saving to $$output..."; \
|
|
if docker save -o "$$output" "$$src_image"; then \
|
|
echo "✓ Saved $$image"; \
|
|
else \
|
|
echo "✗ Failed to save $$src_image"; \
|
|
failed=$$(($$failed + 1)); \
|
|
fi; \
|
|
else \
|
|
echo "✓ Ready: $$src_image"; \
|
|
fi; \
|
|
echo ""; \
|
|
done; \
|
|
\
|
|
if [ $$failed -eq 0 ]; then \
|
|
if [ "$$SAVE" = "true" ]; then \
|
|
echo "✅ All images saved to dist/"; \
|
|
echo "Load on target: docker load -i <image>.tar"; \
|
|
else \
|
|
echo "✅ All images ready in local Docker daemon"; \
|
|
fi; \
|
|
else \
|
|
echo "❌ $$failed image(s) failed"; \
|
|
exit 1; \
|
|
fi; \
|
|
}
|
|
|
|
DEEPFLOW_IMAGES := \
|
|
deepflow-backend \
|
|
deepflow-frontend
|
|
|
|
.PHONY: download-deepflow
|
|
download-deepflow:
|
|
$(MAKE) download DOWNLOAD_IMAGES="$(DEEPFLOW_IMAGES)"
|
|
|
|
# Download Sealed Secrets controller image for offline/air-gapped environments
|
|
SEALED_SECRETS_IMAGE := bitnami/sealed-secrets-controller:latest
|
|
.PHONY: download-sealed-secrets
|
|
download-sealed-secrets:
|
|
@echo "Pulling Sealed Secrets controller image..."
|
|
@mkdir -p dist
|
|
docker pull $(SEALED_SECRETS_IMAGE)
|
|
docker save $(SEALED_SECRETS_IMAGE) -o dist/sealed-secrets-controller.tar
|
|
@echo "✅ Saved to dist/sealed-secrets-controller.tar"
|
|
@echo "Transfer to offline environment and load with: docker load -i dist/sealed-secrets-controller.tar"
|
|
|
|
# Load all downloaded images from dist/ directory
|
|
.PHONY: load-images
|
|
load-images:
|
|
@if [ ! -d "dist" ]; then \
|
|
echo "Error: dist/ directory not found"; \
|
|
echo "Please run 'make download' first to download images"; \
|
|
exit 1; \
|
|
fi
|
|
@echo "Loading images from dist/..."
|
|
@count=0; \
|
|
for tarfile in dist/*.tar.gz; do \
|
|
if [ -f "$$tarfile" ]; then \
|
|
echo "Loading $$tarfile..."; \
|
|
docker load -i "$$tarfile"; \
|
|
count=$$((count + 1)); \
|
|
echo "✓ Loaded $$tarfile"; \
|
|
echo ""; \
|
|
fi; \
|
|
done; \
|
|
if [ $$count -eq 0 ]; then \
|
|
echo "No image files found in dist/"; \
|
|
echo "Please run 'make download' first"; \
|
|
exit 1; \
|
|
else \
|
|
echo "Successfully loaded $$count image(s)"; \
|
|
fi
|