沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
222 lines
9.1 KiB
Makefile
222 lines
9.1 KiB
Makefile
MAKEFLAGS += --no-print-directory
|
|
WITH_MINERU ?= false
|
|
VERSION ?= latest
|
|
BUILD_PLATFORM ?= linux/amd64
|
|
NAMESPACE ?= deepdata
|
|
# Registry
|
|
ifdef dev
|
|
REGISTRY :=
|
|
else
|
|
REGISTRY ?= ghcr.io/modelengine-group/
|
|
endif
|
|
ifdef COMSPEC
|
|
MAKE := "C:/Program Files/GnuWin32/bin/make.exe"
|
|
else
|
|
MAKE := make
|
|
endif
|
|
|
|
# ========== Help ==========
|
|
.PHONY: help
|
|
help:
|
|
@echo "DeepData Make Commands"
|
|
@echo ""
|
|
@echo "Build targets (using docker buildx):"
|
|
@echo " make frontend-docker-build"
|
|
@echo " make backend-docker-build"
|
|
@echo " make database-docker-build"
|
|
@echo " make gateway-docker-build"
|
|
@echo " make runtime-docker-build"
|
|
@echo " make runtime-ray-docker-build"
|
|
@echo " make backend-python-docker-build"
|
|
@echo " make deep-flow-docker-build"
|
|
@echo " make mineru-docker-build / mineru-910B-docker-build / mineru-910C-docker-build / mineru-310P-docker-build"
|
|
@echo " make label-studio-docker-build"
|
|
@echo ""
|
|
@echo "Build multi-arch example:"
|
|
@echo " make frontend-docker-build BUILD_PLATFORM=linux/arm64 VERSION=latest dev=true"
|
|
@echo ""
|
|
@echo "Init buildx builder (run once): docker buildx create --name multi-builder --use --bootstrap"
|
|
@echo ""
|
|
.DEFAULT_GOAL := help
|
|
|
|
# ========== Function definitions ==========
|
|
define docker-build
|
|
@if ! docker buildx version >/dev/null 2>&1; then echo "ERROR: docker buildx not installed!"; exit 1; fi
|
|
@echo "docker buildx build --platform $(BUILD_PLATFORM) -t $(2):$(VERSION) -f scripts/images/$(1)/Dockerfile . --load"
|
|
docker buildx build --platform $(BUILD_PLATFORM) -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile --load
|
|
endef
|
|
|
|
define docker-compose-service
|
|
cd $(3) && docker compose $(2) $(1)
|
|
endef
|
|
|
|
define prompt-yesno
|
|
@read -p "$(1) [y/N] " ans; case $${ans:-N} in [yY]) exit 0 ;; *) exit 1 ;; esac
|
|
endef
|
|
|
|
# ========== All build targets, static PHONY (no pattern rule) ==========
|
|
.PHONY: frontend-docker-build backend-docker-build database-docker-build gateway-docker-build
|
|
.PHONY: runtime-docker-build runtime-ray-docker-build backend-python-docker-build
|
|
.PHONY: deep-flow-backend-docker-build deep-flow-frontend-docker-build deep-flow-docker-build
|
|
.PHONY: mineru-docker-build mineru-910B-docker-build mineru-910C-docker-build mineru-310P-docker-build
|
|
.PHONY: label-studio-docker-build
|
|
|
|
frontend-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-frontend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,frontend,deepdata-frontend)
|
|
@echo "✅ deepdata-frontend build done"
|
|
|
|
backend-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-backend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,backend,deepdata-backend)
|
|
@echo "✅ deepdata-backend build done"
|
|
|
|
database-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-database:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,database,deepdata-database)
|
|
@echo "✅ deepdata-database build done"
|
|
|
|
gateway-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-gateway:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,gateway,deepdata-gateway)
|
|
@echo "✅ deepdata-gateway build done"
|
|
|
|
runtime-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-runtime:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,runtime,deepdata-runtime)
|
|
@echo "✅ deepdata-runtime build done"
|
|
|
|
runtime-ray-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-runtime-ray:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,runtime-ray,deepdata-runtime-ray)
|
|
@echo "✅ deepdata-runtime-ray build done"
|
|
|
|
backend-python-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-backend-python:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,backend-python,deepdata-backend-python)
|
|
@echo "✅ deepdata-backend-python build done"
|
|
|
|
deep-flow-backend-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deep-flow-backend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,deep-flow-backend,deep-flow-backend)
|
|
@echo "✅ deep-flow-backend build done"
|
|
|
|
deep-flow-frontend-docker-build:
|
|
@if ! docker buildx version >/dev/null 2>&1; then echo "ERROR: docker buildx not installed!"; exit 1; fi
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deep-flow-frontend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
docker buildx build --platform $(BUILD_PLATFORM) -t deep-flow-frontend:$(VERSION) . \
|
|
-f scripts/images/deep-flow-frontend/Dockerfile \
|
|
--build-arg NEXT_PUBLIC_API_URL=/deep-flow-backend \
|
|
--load
|
|
@echo "✅ deep-flow-frontend build done"
|
|
|
|
deep-flow-docker-build: deep-flow-backend-docker-build deep-flow-frontend-docker-build
|
|
|
|
mineru-docker-build mineru-910B-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-mineru:$(VERSION)"
|
|
@echo "=================================================="
|
|
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .
|
|
@echo "✅ mineru build done"
|
|
|
|
mineru-910C-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-mineru 910C:$(VERSION)"
|
|
@echo "=================================================="
|
|
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .
|
|
@echo "✅ mineru-910C build done"
|
|
|
|
mineru-310P-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-mineru 310P:$(VERSION)"
|
|
@echo "=================================================="
|
|
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .
|
|
@echo "✅ mineru-310P build done"
|
|
|
|
label-studio-docker-build:
|
|
@echo "=================================================="
|
|
@echo "🚀 Building deepdata-label-studio:$(VERSION) platform=$(BUILD_PLATFORM)"
|
|
@echo "=================================================="
|
|
$(call docker-build,label-studio,deepdata-label-studio)
|
|
@echo "✅ deepdata-label-studio build done"
|
|
|
|
# ========== Utility Targets ==========
|
|
.PHONY: create-namespace node-setup node-cleanup
|
|
create-namespace:
|
|
kubectl get namespace $(NAMESPACE) >/dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
|
|
|
|
node-setup:
|
|
@echo "Configure dedicated nodes for DeepData?"; \
|
|
read -p "Confirm [y/N] " NODE_SETUP_CHOICE; \
|
|
if [ "$$NODE_SETUP_CHOICE" = "y" ]; then \
|
|
chmod +x scripts/k8s/node-setup.sh; \
|
|
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
|
|
fi
|
|
|
|
node-cleanup:
|
|
@chmod +x scripts/k8s/node-cleanup.sh
|
|
@./scripts/k8s/node-cleanup.sh --namespace $(NAMESPACE)
|
|
|
|
# ========== Docker Compose Install / Uninstall ==========
|
|
VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis milvus label-studio data-juicer
|
|
.PHONY: install-% uninstall-%
|
|
install-%:
|
|
@if echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
|
cd deployment/compose && docker compose --profile $* up -d; \
|
|
else \
|
|
echo "Invalid service $*"; exit 1; \
|
|
fi
|
|
|
|
uninstall-%:
|
|
@if echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
|
cd deployment/compose && docker compose --profile $* down; \
|
|
else \
|
|
echo "Invalid service $*"; exit 1; \
|
|
fi
|
|
|
|
# ========== Kubernetes Install / Uninstall ==========
|
|
.PHONY: k8s-install-deepdata k8s-uninstall-deepdata
|
|
k8s-install-deepdata: create-namespace
|
|
helm upgrade --install deepdata ./charts/deepdata -n $(NAMESPACE)
|
|
|
|
k8s-uninstall-deepdata:
|
|
helm uninstall deepdata -n $(NAMESPACE)
|
|
|
|
# ========== Download / Load Images ==========
|
|
DOWNLOAD_IMAGES := deepdata-backend deepdata-frontend deepdata-database deepdata-runtime deepdata-backend-python deepdata-gateway
|
|
.PHONY: download load-images
|
|
download:
|
|
@echo "Select platform: 1=x86_64,2=arm64"; read PLAT; \
|
|
if [ "$$PLAT" = "1" ]; then PLAT=linux/amd64; else PLAT=linux/arm64; fi; \
|
|
mkdir -p dist; \
|
|
for img in $(DOWNLOAD_IMAGES); do \
|
|
full="$(REGISTRY)$${img}:$(VERSION)"; \
|
|
echo "pull $$full ($$PLAT)"; \
|
|
docker pull --platform $$PLAT $$full; \
|
|
docker save -o dist/$${img}-$(VERSION).tar $$full; \
|
|
done
|
|
|
|
load-images:
|
|
@if [ ! -d dist ]; then echo "dist folder missing, run make download first"; exit 1; fi; \
|
|
for tar in dist/*.tar; do \
|
|
echo loading $$tar; docker load -i $$tar; \
|
|
done
|