feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
This commit is contained in:
沃巴蒂
@@ -1,581 +1,221 @@
|
||||
MAKEFLAGS += --no-print-directory
|
||||
|
||||
WITH_MINERU ?= false # 默认不构建mineru
|
||||
MAKEFLAGS += --no-print-directory
|
||||
WITH_MINERU ?= false
|
||||
VERSION ?= latest
|
||||
BUILD_PLATFORM ?= linux/amd64
|
||||
NAMESPACE ?= deepdata
|
||||
|
||||
# Registry configuration: use --dev for local images, otherwise use GitHub registry
|
||||
# Registry
|
||||
ifdef dev
|
||||
REGISTRY :=
|
||||
else
|
||||
REGISTRY ?=
|
||||
REGISTRY ?= ghcr.io/modelengine-group/
|
||||
endif
|
||||
|
||||
ifdef COMSPEC
|
||||
# Windows
|
||||
MAKE := "C:/Program Files (x86)/GnuWin32/bin/make"
|
||||
MAKE := "C:/Program Files/GnuWin32/bin/make.exe"
|
||||
else
|
||||
# Linux/Mac
|
||||
MAKE := make
|
||||
endif
|
||||
|
||||
# ========== Help ==========
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "DeepData Makefile - Available Commands"
|
||||
@echo "DeepData Make Commands"
|
||||
@echo ""
|
||||
@echo "Usage: make <target> [options]"
|
||||
@echo "Build targets (using docker buildx):"
|
||||
@echo " make frontend-docker-build"
|
||||
@echo " make backend-docker-build"
|
||||
@echo " make database-docker-build"
|
||||
@echo " make gateway-docker-build"
|
||||
@echo " make runtime-docker-build"
|
||||
@echo " make runtime-ray-docker-build"
|
||||
@echo " make backend-python-docker-build"
|
||||
@echo " make deep-flow-docker-build"
|
||||
@echo " make mineru-docker-build / mineru-910B-docker-build / mineru-910C-docker-build / mineru-310P-docker-build"
|
||||
@echo " make label-studio-docker-build"
|
||||
@echo ""
|
||||
@echo "Options:"
|
||||
@echo " dev=true Use local images instead of registry (empty REGISTRY)"
|
||||
@echo " VERSION=<version> Set image version (default: latest)"
|
||||
@echo " NAMESPACE=<name> Set Kubernetes namespace (default: deepdata)"
|
||||
@echo " INSTALLER=<type> Set installer type: docker or k8s"
|
||||
@echo " PLATFORM=<platform> Set platform for downloads (default: linux/amd64)"
|
||||
@echo " Options: linux/amd64, linux/arm64"
|
||||
@echo " SAVE=true Save images to dist/ during download (default: false)"
|
||||
@echo "Build multi-arch example:"
|
||||
@echo " make frontend-docker-build BUILD_PLATFORM=linux/arm64 VERSION=latest dev=true"
|
||||
@echo ""
|
||||
@echo "Build Commands:"
|
||||
@echo " make build Build all core images"
|
||||
@echo " make <service>-docker-build Build specific service image"
|
||||
@echo " Valid services: backend, database, frontend, runtime,"
|
||||
@echo " backend-python, deepflow, mineru"
|
||||
@echo "Init buildx builder (run once): docker buildx create --name multi-builder --use --bootstrap"
|
||||
@echo ""
|
||||
@echo "Install Commands:"
|
||||
@echo " make install Install deepdata + milvus (prompts for method)"
|
||||
@echo " make install INSTALLER=docker Install using Docker Compose"
|
||||
@echo " make install INSTALLER=k8s Install using Kubernetes/Helm"
|
||||
@echo " make install INSTALLER=k8s (requires Sealed Secrets Controller)"
|
||||
@echo " make install-<component> Install specific component (prompts)"
|
||||
@echo " make <component>-docker-install Install component via Docker"
|
||||
@echo " make <component>-k8s-install Install component via Kubernetes"
|
||||
@echo " Valid components: deepdata, milvus, deepflow, mineru"
|
||||
@echo " Valid services: backend, frontend, runtime, label-studio"
|
||||
@echo ""
|
||||
@echo "Uninstall Commands:"
|
||||
@echo " make uninstall Uninstall deepdata + milvus (prompts)"
|
||||
@echo " make uninstall INSTALLER=docker Uninstall using Docker Compose"
|
||||
@echo " make uninstall INSTALLER=k8s Uninstall using Kubernetes/Helm"
|
||||
@echo " make uninstall-<component> Uninstall specific component (prompts)"
|
||||
@echo " make <component>-docker-uninstall Uninstall component via Docker"
|
||||
@echo " make <component>-k8s-uninstall Uninstall component via Kubernetes"
|
||||
@echo " Note: Docker uninstall will prompt whether to delete volumes"
|
||||
@echo ""
|
||||
@echo "Upgrade Commands:"
|
||||
@echo " make deepdata-docker-upgrade Upgrade deepdata deployment"
|
||||
@echo ""
|
||||
@echo "Download Commands:"
|
||||
@echo " make download Pull all images (no save by default)"
|
||||
@echo " make download VERSION=<version> Pull all images with specific version"
|
||||
@echo " make download REGISTRY=<registry> Pull images from specific registry"
|
||||
@echo " make load-images Load all downloaded images from dist/"
|
||||
@echo " make download-sealed-secrets Download Sealed Secrets image (for offline)"
|
||||
@echo ""
|
||||
@echo "Utility Commands:"
|
||||
@echo " make create-namespace Create Kubernetes namespace"
|
||||
@echo " make node-setup Configure dedicated nodes interactively"
|
||||
@echo " make node-cleanup Remove node labels and taints"
|
||||
@echo " make help Show this help message"
|
||||
@echo ""
|
||||
@echo "Examples:"
|
||||
@echo " make build dev=true Build all images for local development"
|
||||
@echo " make install INSTALLER=docker Install via Docker Compose"
|
||||
@echo " make install dev=true Install using local images"
|
||||
@echo " make deepdata-docker-upgrade Upgrade running deepdata services"
|
||||
@echo ""
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
# ========== Functions ==========
|
||||
|
||||
# Prompt user to choose installer if not specified
|
||||
define prompt-installer
|
||||
@echo "Choose a deployment method:"
|
||||
@echo "1. Docker/Docker-Compose"
|
||||
@echo "2. Kubernetes/Helm"
|
||||
@echo -n "Enter choice: "
|
||||
@read choice; \
|
||||
case $$choice in \
|
||||
1) INSTALLER=docker ;; \
|
||||
2) INSTALLER=k8s ;; \
|
||||
*) echo "Invalid choice" && exit 1 ;; \
|
||||
esac; \
|
||||
$(MAKE) $(1)
|
||||
endef
|
||||
|
||||
# Prompt user to choose installer and volume deletion for uninstall
|
||||
define prompt-uninstaller
|
||||
@echo "Choose a deployment method:"
|
||||
@echo "1. Docker/Docker-Compose"
|
||||
@echo "2. Kubernetes/Helm"
|
||||
@echo -n "Enter choice: "
|
||||
@read installer_choice; \
|
||||
case $$installer_choice in \
|
||||
1) INSTALLER=docker ;; \
|
||||
2) INSTALLER=k8s ;; \
|
||||
*) echo "Invalid choice" && exit 1 ;; \
|
||||
esac; \
|
||||
if [ "$$INSTALLER" = "docker" ]; then \
|
||||
echo "Delete volumes? (This will remove all data)"; \
|
||||
echo "1. Yes - Delete volumes"; \
|
||||
echo "2. No - Keep volumes"; \
|
||||
echo -n "Enter choice (default: 2): "; \
|
||||
read DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) $(1) DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
else \
|
||||
$(MAKE) $(1); \
|
||||
fi
|
||||
endef
|
||||
|
||||
# Generic docker build function
|
||||
# Usage: $(call docker-build,service-name,image-name)
|
||||
# ========== Function definitions ==========
|
||||
define docker-build
|
||||
docker build -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile
|
||||
@if ! docker buildx version >/dev/null 2>&1; then echo "ERROR: docker buildx not installed!"; exit 1; fi
|
||||
@echo "docker buildx build --platform $(BUILD_PLATFORM) -t $(2):$(VERSION) -f scripts/images/$(1)/Dockerfile . --load"
|
||||
docker buildx build --platform $(BUILD_PLATFORM) -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile --load
|
||||
endef
|
||||
|
||||
# Generic docker compose service action
|
||||
# Usage: $(call docker-compose-service,service-name,action,compose-dir)
|
||||
define docker-compose-service
|
||||
cd $(3) && docker compose $(2) $(1)
|
||||
endef
|
||||
|
||||
# Prompt user to choose whether to delete volumes
|
||||
define prompt-volume-deletion
|
||||
@echo "Delete volumes? (This will remove all data)"
|
||||
@echo "1. Yes - Delete volumes"
|
||||
@echo "2. No - Keep volumes"
|
||||
@echo -n "Enter choice (default: 2): "
|
||||
@read choice; \
|
||||
case $$choice in \
|
||||
1) echo "-v" ;; \
|
||||
*) echo "" ;; \
|
||||
esac
|
||||
define prompt-yesno
|
||||
@read -p "$(1) [y/N] " ans; case $${ans:-N} in [yY]) exit 0 ;; *) exit 1 ;; esac
|
||||
endef
|
||||
|
||||
# ========== Build Targets ==========
|
||||
# ========== All build targets, static PHONY (no pattern rule) ==========
|
||||
.PHONY: frontend-docker-build backend-docker-build database-docker-build gateway-docker-build
|
||||
.PHONY: runtime-docker-build runtime-ray-docker-build backend-python-docker-build
|
||||
.PHONY: deep-flow-backend-docker-build deep-flow-frontend-docker-build deep-flow-docker-build
|
||||
.PHONY: mineru-docker-build mineru-910B-docker-build mineru-910C-docker-build mineru-310P-docker-build
|
||||
.PHONY: label-studio-docker-build
|
||||
|
||||
# Valid build targets
|
||||
VALID_BUILD_TARGETS := frontend backend gateway database runtime runtime-ray backend-python deepflow label-studio mineru mineru-910B mineru-910C mineru-310P
|
||||
frontend-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-frontend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,frontend,deepdata-frontend)
|
||||
@echo "✅ deepdata-frontend build done"
|
||||
|
||||
# Generic docker build target with service name as parameter
|
||||
# Automatically prefixes image names with "deepdata-" unless it's deepflow
|
||||
.PHONY: %-docker-build
|
||||
%-docker-build:
|
||||
@if ! echo " $(VALID_BUILD_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown build target '$*'"; \
|
||||
echo "Valid build targets are:"; \
|
||||
for target in $(VALID_BUILD_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "deepflow" ]; then \
|
||||
$(call docker-build,deepflow-backend,deepflow-backend); \
|
||||
# $(call docker-build,deepflow-frontend,deepflow-frontend); \
|
||||
docker build -t deepflow-frontend:$(VERSION) . \
|
||||
-f scripts/images/deepflow-frontend/Dockerfile \
|
||||
--build-arg NEXT_PUBLIC_API_URL=/deepflow-backend; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ]; then \
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .; \
|
||||
elif [ "$*" = "mineru-910C" ]; then \
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .; \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .; \
|
||||
else \
|
||||
$(call docker-build,$*,deepdata-$*); \
|
||||
fi
|
||||
backend-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-backend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,backend,deepdata-backend)
|
||||
@echo "✅ deepdata-backend build done"
|
||||
|
||||
.PHONY: build-%
|
||||
build-%: %-docker-build
|
||||
@:
|
||||
database-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-database:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,database,deepdata-database)
|
||||
@echo "✅ deepdata-database build done"
|
||||
|
||||
.PHONY: build
|
||||
build: database-docker-build gateway-docker-build backend-docker-build frontend-docker-build runtime-docker-build backend-python-docker-build
|
||||
gateway-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-gateway:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,gateway,deepdata-gateway)
|
||||
@echo "✅ deepdata-gateway build done"
|
||||
|
||||
runtime-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-runtime:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,runtime,deepdata-runtime)
|
||||
@echo "✅ deepdata-runtime build done"
|
||||
|
||||
runtime-ray-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-runtime-ray:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,runtime-ray,deepdata-runtime-ray)
|
||||
@echo "✅ deepdata-runtime-ray build done"
|
||||
|
||||
backend-python-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-backend-python:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,backend-python,deepdata-backend-python)
|
||||
@echo "✅ deepdata-backend-python build done"
|
||||
|
||||
deep-flow-backend-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deep-flow-backend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,deep-flow-backend,deep-flow-backend)
|
||||
@echo "✅ deep-flow-backend build done"
|
||||
|
||||
deep-flow-frontend-docker-build:
|
||||
@if ! docker buildx version >/dev/null 2>&1; then echo "ERROR: docker buildx not installed!"; exit 1; fi
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deep-flow-frontend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
docker buildx build --platform $(BUILD_PLATFORM) -t deep-flow-frontend:$(VERSION) . \
|
||||
-f scripts/images/deep-flow-frontend/Dockerfile \
|
||||
--build-arg NEXT_PUBLIC_API_URL=/deep-flow-backend \
|
||||
--load
|
||||
@echo "✅ deep-flow-frontend build done"
|
||||
|
||||
deep-flow-docker-build: deep-flow-backend-docker-build deep-flow-frontend-docker-build
|
||||
|
||||
mineru-docker-build mineru-910B-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-mineru:$(VERSION)"
|
||||
@echo "=================================================="
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .
|
||||
@echo "✅ mineru build done"
|
||||
|
||||
mineru-910C-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-mineru 910C:$(VERSION)"
|
||||
@echo "=================================================="
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .
|
||||
@echo "✅ mineru-910C build done"
|
||||
|
||||
mineru-310P-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-mineru 310P:$(VERSION)"
|
||||
@echo "=================================================="
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .
|
||||
@echo "✅ mineru-310P build done"
|
||||
|
||||
label-studio-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-label-studio:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,label-studio,deepdata-label-studio)
|
||||
@echo "✅ deepdata-label-studio build done"
|
||||
|
||||
# ========== Utility Targets ==========
|
||||
|
||||
.PHONY: create-namespace
|
||||
.PHONY: create-namespace node-setup node-cleanup
|
||||
create-namespace:
|
||||
kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
|
||||
kubectl get namespace $(NAMESPACE) >/dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
|
||||
|
||||
# ========== Node Setup/Cleanup Targets ==========
|
||||
|
||||
.PHONY: node-setup
|
||||
node-setup:
|
||||
@echo "Configure dedicated nodes for DeepData deployment?"
|
||||
@echo "This will apply labels and optional taints to selected nodes."
|
||||
@echo "1. Yes - Configure nodes interactively"
|
||||
@echo "2. No - Use default scheduling"
|
||||
@echo -n "Enter choice (default: 2): "
|
||||
@read NODE_SETUP_CHOICE; \
|
||||
if [ "$$NODE_SETUP_CHOICE" = "1" ]; then \
|
||||
@echo "Configure dedicated nodes for DeepData?"; \
|
||||
read -p "Confirm [y/N] " NODE_SETUP_CHOICE; \
|
||||
if [ "$$NODE_SETUP_CHOICE" = "y" ]; then \
|
||||
chmod +x scripts/k8s/node-setup.sh; \
|
||||
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
|
||||
fi
|
||||
|
||||
.PHONY: node-cleanup
|
||||
node-cleanup:
|
||||
@chmod +x scripts/k8s/node-cleanup.sh
|
||||
@./scripts/k8s/node-cleanup.sh --namespace $(NAMESPACE)
|
||||
|
||||
# ========== Generic Install/Uninstall Targets (Redirect to prompt-installer) ==========
|
||||
|
||||
.PHONY: install-%
|
||||
# ========== Docker Compose Install / Uninstall ==========
|
||||
VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis milvus label-studio data-juicer
|
||||
.PHONY: install-% uninstall-%
|
||||
install-%:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-installer,$*-$$INSTALLER-install)
|
||||
else
|
||||
$(MAKE) $*-$(INSTALLER)-install
|
||||
endif
|
||||
@if echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
cd deployment/compose && docker compose --profile $* up -d; \
|
||||
else \
|
||||
echo "Invalid service $*"; exit 1; \
|
||||
fi
|
||||
|
||||
.PHONY: install
|
||||
install:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-installer,deepdata-$$INSTALLER-install milvus-$$INSTALLER-install)
|
||||
else
|
||||
$(MAKE) deepdata-$(INSTALLER)-install
|
||||
$(MAKE) milvus-$(INSTALLER)-install
|
||||
@rm -f /tmp/deepdata-helm-args.sh
|
||||
endif
|
||||
|
||||
.PHONY: uninstall-%
|
||||
uninstall-%:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-uninstaller,$*-$$INSTALLER-uninstall)
|
||||
else
|
||||
$(MAKE) $*-$(INSTALLER)-uninstall
|
||||
endif
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-uninstaller,label-studio-$$INSTALLER-uninstall milvus-$$INSTALLER-uninstall deepflow-$$INSTALLER-uninstall deepdata-$$INSTALLER-uninstall)
|
||||
else
|
||||
@if [ "$(INSTALLER)" = "docker" ]; then \
|
||||
echo "Delete volumes? (This will remove all data)"; \
|
||||
echo "1. Yes - Delete volumes"; \
|
||||
echo "2. No - Keep volumes"; \
|
||||
echo -n "Enter choice (default: 2): "; \
|
||||
read DELETE_VOLUMES_CHOICE; \
|
||||
export DELETE_VOLUMES_CHOICE; \
|
||||
fi; \
|
||||
$(MAKE) label-studio-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) milvus-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) deepflow-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) deepdata-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE
|
||||
endif
|
||||
|
||||
# ========== Docker Install/Uninstall Targets ==========
|
||||
|
||||
# Valid service targets for docker install/uninstall
|
||||
VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
|
||||
|
||||
# Generic docker service install target
|
||||
.PHONY: %-docker-install
|
||||
%-docker-install:
|
||||
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown service target '$*'"; \
|
||||
echo "Valid service targets are:"; \
|
||||
for target in $(VALID_SERVICE_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ ! -f deployment/docker/deepdata/.env ]; then \
|
||||
echo "ERROR: deployment/docker/deepdata/.env not found."; \
|
||||
echo "Create it from the template:"; \
|
||||
echo " cp deployment/docker/deepdata/.env.example deployment/docker/deepdata/.env"; \
|
||||
echo "Then edit it with your actual passwords."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "label-studio" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile label-studio up -d; \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
REGISTRY=$(REGISTRY) EXTRA_ARGS="--enforce-eager --dtype float16" docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
|
||||
elif [ "$*" = "data-juicer" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile data-juicer up -d deepdata-data-juicer; \
|
||||
elif [ "$*" = "redis" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile redis up -d deepdata-redis; \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
cp runtime/deepflow/.env deployment/docker/deepdata/.env; \
|
||||
cp runtime/deepflow/conf.yaml deployment/docker/deepdata/conf.yaml; \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile deepflow up -d; \
|
||||
@if echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
cd deployment/compose && docker compose --profile $* down; \
|
||||
else \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d deepdata-$*; \
|
||||
echo "Invalid service $*"; exit 1; \
|
||||
fi
|
||||
|
||||
# Generic docker service uninstall target
|
||||
.PHONY: %-docker-uninstall
|
||||
%-docker-uninstall:
|
||||
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown service target '$*'"; \
|
||||
echo "Valid service targets are:"; \
|
||||
for target in $(VALID_SERVICE_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "label-studio" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s label-studio; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ] || [ "$*" = "mineru-310P" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-mineru; \
|
||||
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-data-juicer; \
|
||||
elif [ "$*" = "redis" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-redis; \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down -v; \
|
||||
else \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down; \
|
||||
fi; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepflow-backend deepflow-frontend; \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s milvus etcd minio; \
|
||||
else \
|
||||
$(call docker-compose-service,$*,down,deployment/docker/deepdata); \
|
||||
fi
|
||||
# ========== Kubernetes Install / Uninstall ==========
|
||||
.PHONY: k8s-install-deepdata k8s-uninstall-deepdata
|
||||
k8s-install-deepdata: create-namespace
|
||||
helm upgrade --install deepdata ./charts/deepdata -n $(NAMESPACE)
|
||||
|
||||
# ========== Kubernetes Install/Uninstall Targets ==========
|
||||
k8s-uninstall-deepdata:
|
||||
helm uninstall deepdata -n $(NAMESPACE)
|
||||
|
||||
# Valid k8s targets
|
||||
VALID_K8S_TARGETS := deepdata deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
|
||||
|
||||
# Generic k8s install target
|
||||
.PHONY: %-k8s-install
|
||||
%-k8s-install: create-namespace
|
||||
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown k8s target '$*'"; \
|
||||
echo "Valid k8s targets are:"; \
|
||||
for target in $(VALID_K8S_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "label-studio" ]; then \
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then . /tmp/deepdata-helm-args.sh; fi; \
|
||||
helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install $${HELM_LABEL_STUDIO_TOLERATIONS:-}; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
||||
kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
echo ""; \
|
||||
chmod +x scripts/k8s/node-setup.sh; \
|
||||
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then \
|
||||
. /tmp/deepdata-helm-args.sh; \
|
||||
fi; \
|
||||
chmod +x scripts/k8s/collect-secrets.sh; \
|
||||
eval $$(NAMESPACE=$(NAMESPACE) bash scripts/k8s/collect-secrets.sh); \
|
||||
if [ "$$SECRETS_CREATE" = "SKIP" ]; then \
|
||||
echo "[SKIP] Secrets collection failed — skipping deepdata Helm install"; \
|
||||
rm -f /tmp/deepdata-helm-args.sh; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
if [ -n "$$HELM_VALUES_FILE" ] && [ -f "$$HELM_VALUES_FILE" ]; then \
|
||||
HELM_EXTRA_ARGS="-f $$HELM_VALUES_FILE"; \
|
||||
else \
|
||||
HELM_EXTRA_ARGS=""; \
|
||||
fi; \
|
||||
if [ -n "$$HELM_NODE_SELECTOR_ARGS" ] || [ -n "$$HELM_TOLERATIONS_ARGS" ]; then \
|
||||
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS $$HELM_NODE_SELECTOR_ARGS $$HELM_TOLERATIONS_ARGS; \
|
||||
else \
|
||||
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS; \
|
||||
fi; \
|
||||
rm -f /tmp/deepdata-secret-values-*.yaml; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
cp runtime/deepflow/.env deployment/helm/deepflow/charts/public/.env; \
|
||||
cp runtime/deepflow/conf.yaml deployment/helm/deepflow/charts/public/conf.yaml; \
|
||||
helm upgrade deepflow deployment/helm/deepflow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
chmod +x scripts/k8s/collect-secrets.sh; \
|
||||
bash scripts/k8s/collect-secrets.sh --component milvus -n $(NAMESPACE); \
|
||||
MILVUS_MINIO_ACCESS_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accesskey}' | base64 -d); \
|
||||
MILVUS_MINIO_SECRET_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretkey}' | base64 -d); \
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then \
|
||||
. /tmp/deepdata-helm-args.sh; \
|
||||
fi; \
|
||||
helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
|
||||
--set minio.accessKey="$$MILVUS_MINIO_ACCESS_KEY" \
|
||||
--set minio.secretKey="$$MILVUS_MINIO_SECRET_KEY" \
|
||||
--set log.persistence.persistentVolumeClaim.accessModes=ReadWriteOnce \
|
||||
$$HELM_MILVUS_TOLERATIONS; \
|
||||
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
||||
kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
|
||||
fi
|
||||
|
||||
# Generic k8s uninstall target
|
||||
.PHONY: %-k8s-uninstall
|
||||
%-k8s-uninstall:
|
||||
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown k8s target '$*'"; \
|
||||
echo "Valid k8s targets are:"; \
|
||||
for target in $(VALID_K8S_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
||||
kubectl delete -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
kubectl delete -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
helm uninstall deepdata -n $(NAMESPACE) --ignore-not-found; \
|
||||
$(MAKE) node-cleanup; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
helm uninstall deepflow -n $(NAMESPACE) --ignore-not-found; \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
helm uninstall milvus -n $(NAMESPACE) --ignore-not-found; \
|
||||
elif [ "$*" = "label-studio" ]; then \
|
||||
helm uninstall label-studio -n $(NAMESPACE) --ignore-not-found; \
|
||||
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
||||
kubectl delete -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
|
||||
fi
|
||||
|
||||
# ========== Download Targets ==========
|
||||
|
||||
# List of all images to download
|
||||
DOWNLOAD_IMAGES := \
|
||||
deepdata-backend \
|
||||
deepdata-frontend \
|
||||
deepdata-database \
|
||||
deepdata-runtime \
|
||||
deepdata-backend-python \
|
||||
deepdata-gateway \
|
||||
|
||||
# Detect architecture for smart default
|
||||
CURRENT_ARCH := $(shell uname -m 2>/dev/null)
|
||||
ifeq ($(CURRENT_ARCH),aarch64)
|
||||
DEFAULT_OPTION := 2
|
||||
else ifeq ($(CURRENT_ARCH),arm64)
|
||||
DEFAULT_OPTION := 2
|
||||
else
|
||||
DEFAULT_OPTION := 1
|
||||
endif
|
||||
|
||||
|
||||
#Usage: make download VERSION=1.0.0 REGISTRY=<registry-prefix>/ SAVE=true
|
||||
.PHONY: download
|
||||
# ========== Download / Load Images ==========
|
||||
DOWNLOAD_IMAGES := deepdata-backend deepdata-frontend deepdata-database deepdata-runtime deepdata-backend-python deepdata-gateway
|
||||
.PHONY: download load-images
|
||||
download:
|
||||
@echo "Select target platform for images:"
|
||||
@echo " 1) x86_64 (Intel/AMD, linux/amd64)"
|
||||
@echo " 2) ARM64 (Apple Silicon, Raspberry Pi 4+, linux/arm64)"
|
||||
@{ \
|
||||
read -p "Enter choice [1/2] (default: $(DEFAULT_OPTION)): " plat_choice; \
|
||||
case "$${plat_choice:-$(DEFAULT_OPTION)}" in \
|
||||
1) PLATFORM="linux/amd64" ;; \
|
||||
2) PLATFORM="linux/arm64" ;; \
|
||||
*) echo "Invalid choice. Using default option $(DEFAULT_OPTION)."; \
|
||||
PLATFORM=$$([ "$(DEFAULT_OPTION)" = "1" ] && echo "linux/amd64" || echo "linux/arm64");; \
|
||||
esac; \
|
||||
\
|
||||
read -p "Save images to dist/ for offline use? (y/n): " save_resp; \
|
||||
case "$$save_resp" in \
|
||||
[yY]|[yY][eE][sS]) SAVE="true" ;; \
|
||||
*) SAVE="false" ;; \
|
||||
esac; \
|
||||
\
|
||||
if [ -z "$(REGISTRY)" ] && [ "$$SAVE" != "true" ]; then \
|
||||
echo "Error: REGISTRY unset and SAVE=false. Nothing to do."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
\
|
||||
if [ "$$SAVE" = "true" ]; then mkdir -p dist; fi; \
|
||||
failed=0; \
|
||||
\
|
||||
for image in $(DOWNLOAD_IMAGES); do \
|
||||
if [ -z "$(REGISTRY)" ]; then \
|
||||
src_image="$$image:$(VERSION)"; \
|
||||
echo "Using local image: $$src_image"; \
|
||||
else \
|
||||
src_image="$(REGISTRY)$$image:$(VERSION)"; \
|
||||
echo "Pulling $$src_image ($$PLATFORM)..."; \
|
||||
if ! docker pull --platform "$$PLATFORM" "$$src_image"; then \
|
||||
echo "✗ Failed to pull $$src_image"; \
|
||||
failed=$$(($$failed + 1)); \
|
||||
continue; \
|
||||
fi; \
|
||||
fi; \
|
||||
\
|
||||
if [ "$$SAVE" = "true" ]; then \
|
||||
output="dist/$$image-$(VERSION).tar"; \
|
||||
echo "Saving to $$output..."; \
|
||||
if docker save -o "$$output" "$$src_image"; then \
|
||||
echo "✓ Saved $$image"; \
|
||||
else \
|
||||
echo "✗ Failed to save $$src_image"; \
|
||||
failed=$$(($$failed + 1)); \
|
||||
fi; \
|
||||
else \
|
||||
echo "✓ Ready: $$src_image"; \
|
||||
fi; \
|
||||
echo ""; \
|
||||
done; \
|
||||
\
|
||||
if [ $$failed -eq 0 ]; then \
|
||||
if [ "$$SAVE" = "true" ]; then \
|
||||
echo "✅ All images saved to dist/"; \
|
||||
echo "Load on target: docker load -i <image>.tar"; \
|
||||
else \
|
||||
echo "✅ All images ready in local Docker daemon"; \
|
||||
fi; \
|
||||
else \
|
||||
echo "❌ $$failed image(s) failed"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
}
|
||||
@echo "Select platform: 1=x86_64,2=arm64"; read PLAT; \
|
||||
if [ "$$PLAT" = "1" ]; then PLAT=linux/amd64; else PLAT=linux/arm64; fi; \
|
||||
mkdir -p dist; \
|
||||
for img in $(DOWNLOAD_IMAGES); do \
|
||||
full="$(REGISTRY)$${img}:$(VERSION)"; \
|
||||
echo "pull $$full ($$PLAT)"; \
|
||||
docker pull --platform $$PLAT $$full; \
|
||||
docker save -o dist/$${img}-$(VERSION).tar $$full; \
|
||||
done
|
||||
|
||||
DEEPFLOW_IMAGES := \
|
||||
deepflow-backend \
|
||||
deepflow-frontend
|
||||
|
||||
.PHONY: download-deepflow
|
||||
download-deepflow:
|
||||
$(MAKE) download DOWNLOAD_IMAGES="$(DEEPFLOW_IMAGES)"
|
||||
|
||||
# Download Sealed Secrets controller image for offline/air-gapped environments
|
||||
SEALED_SECRETS_IMAGE := bitnami/sealed-secrets-controller:latest
|
||||
.PHONY: download-sealed-secrets
|
||||
download-sealed-secrets:
|
||||
@echo "Pulling Sealed Secrets controller image..."
|
||||
@mkdir -p dist
|
||||
docker pull $(SEALED_SECRETS_IMAGE)
|
||||
docker save $(SEALED_SECRETS_IMAGE) -o dist/sealed-secrets-controller.tar
|
||||
@echo "✅ Saved to dist/sealed-secrets-controller.tar"
|
||||
@echo "Transfer to offline environment and load with: docker load -i dist/sealed-secrets-controller.tar"
|
||||
|
||||
# Load all downloaded images from dist/ directory
|
||||
.PHONY: load-images
|
||||
load-images:
|
||||
@if [ ! -d "dist" ]; then \
|
||||
echo "Error: dist/ directory not found"; \
|
||||
echo "Please run 'make download' first to download images"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "Loading images from dist/..."
|
||||
@count=0; \
|
||||
for tarfile in dist/*.tar.gz; do \
|
||||
if [ -f "$$tarfile" ]; then \
|
||||
echo "Loading $$tarfile..."; \
|
||||
docker load -i "$$tarfile"; \
|
||||
count=$$((count + 1)); \
|
||||
echo "✓ Loaded $$tarfile"; \
|
||||
echo ""; \
|
||||
fi; \
|
||||
done; \
|
||||
if [ $$count -eq 0 ]; then \
|
||||
echo "No image files found in dist/"; \
|
||||
echo "Please run 'make download' first"; \
|
||||
exit 1; \
|
||||
else \
|
||||
echo "Successfully loaded $$count image(s)"; \
|
||||
fi
|
||||
@if [ ! -d dist ]; then echo "dist folder missing, run make download first"; exit 1; fi; \
|
||||
for tar in dist/*.tar; do \
|
||||
echo loading $$tar; docker load -i $$tar; \
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user