Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ff22ed3a8b |
@@ -20,3 +20,11 @@
|
||||
# Git
|
||||
.git/
|
||||
.gitignore
|
||||
|
||||
# Java Build Output (Crucial for Speed)
|
||||
**/target/
|
||||
|
||||
# Frontend (Optimize Build Context)
|
||||
frontend/node_modules
|
||||
frontend/dist
|
||||
frontend/.git
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
*.sh text eol=lf
|
||||
*.bat text eol=crlf
|
||||
@@ -1,86 +0,0 @@
|
||||
# 对于大多数项目,此工作流文件无需更改;只需提交到您的仓库即可。
|
||||
#
|
||||
# 您可以根据需要修改此文件,以覆盖分析的语言集,或提供自定义查询或构建逻辑。
|
||||
#
|
||||
# ******** 注意 ********
|
||||
# 我们已尝试检测您的仓库中的语言。请检查下面定义的 `language` 矩阵,确保包含了所有受支持的 CodeQL 语言。
|
||||
name: "CodeQL Advanced"
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
schedule:
|
||||
- cron: '23 5 * * 2'
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
# Runner 的规格会影响 CodeQL 分析时间。详情请参阅:
|
||||
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners (仅限 GitHub.com)
|
||||
# 建议使用更高规格的 Runner 或更大资源的机器以提升分析速度。
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
# 所有工作流都需要
|
||||
security-events: write
|
||||
# 获取内部或私有 CodeQL 包时需要
|
||||
packages: read
|
||||
# 仅私有仓库工作流需要
|
||||
actions: read
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: actions
|
||||
build-mode: none
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
- language: python
|
||||
build-mode: none
|
||||
- language: java-kotlin
|
||||
build-mode: none
|
||||
# CodeQL 支持以下 'language' 关键字:'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift'
|
||||
# 使用 `c-cpp` 可分析 C、C++ 或两者的代码
|
||||
# 使用 'java-kotlin' 可分析 Java、Kotlin 或两者的代码
|
||||
# 使用 'javascript-typescript' 可分析 JavaScript、TypeScript 或两者的代码
|
||||
# 了解如何更改分析语言或自定义分析模式,请参阅:
|
||||
# https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning
|
||||
# 如果分析编译型语言,可修改 'build-mode' 以自定义分析方式,详见:
|
||||
# https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
# 在运行 `github/codeql-action/init` 之前可添加任何设置步骤。
|
||||
# 包括安装编译器或运行环境(如 `actions/setup-node` 等)。通常仅手动构建时需要。
|
||||
# - name: Setup runtime (示例)
|
||||
# uses: actions/setup-example@v1
|
||||
# 初始化 CodeQL 工具以进行扫描。
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
# 如需指定自定义查询,可在此处或配置文件中设置。
|
||||
# 默认情况下,此处列出的查询会覆盖配置文件中指定的查询。
|
||||
# 在列表前加 "+" 可同时使用此处和配置文件中的查询。
|
||||
# 更多 CodeQL 查询包详情请参阅:
|
||||
# https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
# 如果某种语言的 analyze 步骤失败,提示“无法自动构建您的代码”,请在上方矩阵中将该语言的 build-mode 设置为 "manual",并在此步骤中添加构建命令。
|
||||
# ℹ️ 可使用操作系统 shell 运行命令行程序。
|
||||
# 📚 详见 https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
- if: matrix.build-mode == 'manual'
|
||||
shell: bash
|
||||
run: |
|
||||
echo '如果您为一种或多种语言使用了 "manual" 构建模式,请在此处替换为您的构建命令,例如:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
name: Python Backend Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'scripts/images/backend-python/**'
|
||||
- 'runtime/deepdata-python/**'
|
||||
- '.github/workflows/docker-image-backend-python.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'scripts/images/backend-python/**'
|
||||
- 'runtime/deepdata-python/**'
|
||||
- '.github/workflows/docker-image-backend-python.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
call-docker-build:
|
||||
name: Build and Push Backend Python Docker Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
with:
|
||||
service_name: backend-python
|
||||
build_dir: .
|
||||
@@ -1,32 +0,0 @@
|
||||
name: Backend Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'backend/**'
|
||||
- '!backend/api-gateway/**'
|
||||
- 'scripts/images/backend/**'
|
||||
- '.github/workflows/docker-image-backend.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'backend/**'
|
||||
- '!backend/api-gateway/**'
|
||||
- 'scripts/images/backend/**'
|
||||
- '.github/workflows/docker-image-backend.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
call-docker-build:
|
||||
name: Build and Push Backend Docker Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
with:
|
||||
service_name: backend
|
||||
build_dir: .
|
||||
@@ -1,30 +0,0 @@
|
||||
name: Database Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'scripts/db/**'
|
||||
- 'scripts/images/database/**'
|
||||
- '.github/workflows/docker-image-database.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'scripts/db/**'
|
||||
- 'scripts/images/database/**'
|
||||
- '.github/workflows/docker-image-database.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
call-docker-build:
|
||||
name: Build and Push Database Docker Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
with:
|
||||
service_name: database
|
||||
build_dir: .
|
||||
@@ -1,151 +0,0 @@
|
||||
name: Deer Flow Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'runtime/deepflow/**'
|
||||
- 'scripts/images/deepflow-backend/**'
|
||||
- 'scripts/images/deepflow-frontend/**'
|
||||
- '.github/workflows/docker-image-deepflow.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'runtime/deepflow/**'
|
||||
- 'scripts/images/deepflow-backend/**'
|
||||
- 'scripts/images/deepflow-frontend/**'
|
||||
- '.github/workflows/docker-image-deepflow.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
build-and-push-amd:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set Docker Image Tag
|
||||
id: set-tag
|
||||
run: |
|
||||
if [[ $GITHUB_REF == refs/tags/v* ]]; then
|
||||
TAG=${GITHUB_REF#refs/tags/v}
|
||||
echo "TAGS=amd64-$TAG" >> $GITHUB_OUTPUT
|
||||
elif [[ $GITHUB_REF == refs/heads/main ]]; then
|
||||
echo "TAGS=amd64" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "TAGS=amd64-temp" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker build -t deepflow-backend:amd64 . -f scripts/images/deepflow-backend/Dockerfile
|
||||
docker build -t deepflow-frontend:amd64 . -f scripts/images/deepflow-frontend/Dockerfile
|
||||
|
||||
- name: Tag & Push Docker Image
|
||||
if: github.event_name != 'pull_request'
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
docker tag deepflow-backend:amd64 ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
|
||||
docker tag deepflow-frontend:amd64 ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
|
||||
docker push ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
|
||||
docker push ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
|
||||
|
||||
|
||||
build-and-push-arm:
|
||||
runs-on: ubuntu-24.04-arm
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set Docker Image Tag
|
||||
id: set-tag
|
||||
run: |
|
||||
if [[ $GITHUB_REF == refs/tags/v* ]]; then
|
||||
TAG=${GITHUB_REF#refs/tags/v}
|
||||
echo "TAGS=arm64-$TAG" >> $GITHUB_OUTPUT
|
||||
elif [[ $GITHUB_REF == refs/heads/main ]]; then
|
||||
echo "TAGS=arm64" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "TAGS=arm64-temp" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker build -t deepflow-backend:arm64 . -f scripts/images/deepflow-backend/Dockerfile
|
||||
docker build -t deepflow-frontend:arm64 . -f scripts/images/deepflow-frontend/Dockerfile
|
||||
|
||||
- name: Tag & Push Docker Image
|
||||
if: github.event_name != 'pull_request'
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
docker tag deepflow-backend:arm64 ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
|
||||
docker tag deepflow-frontend:arm64 ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
|
||||
docker push ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
|
||||
docker push ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
|
||||
|
||||
manifest:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name != 'pull_request'
|
||||
needs: [ build-and-push-amd, build-and-push-arm ]
|
||||
steps:
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set Docker Image Tag
|
||||
id: set-tag
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
|
||||
if [[ $GITHUB_REF == refs/tags/v* ]]; then
|
||||
TAG=${GITHUB_REF#refs/tags/v}
|
||||
echo "TAGS=$TAG" >> $GITHUB_OUTPUT
|
||||
echo "ARM_TAGS=arm64-$TAG" >> $GITHUB_OUTPUT
|
||||
echo "AMD_TAGS=amd64-$TAG" >> $GITHUB_OUTPUT
|
||||
elif [[ $GITHUB_REF == refs/heads/main ]]; then
|
||||
echo "TAGS=latest" >> $GITHUB_OUTPUT
|
||||
echo "ARM_TAGS=arm64" >> $GITHUB_OUTPUT
|
||||
echo "AMD_TAGS=amd64" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "TAGS=temp" >> $GITHUB_OUTPUT
|
||||
echo "ARM_TAGS=arm64-temp" >> $GITHUB_OUTPUT
|
||||
echo "AMD_TAGS=amd64-temp" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Manifest Docker Image
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
docker manifest create ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }} \
|
||||
ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.AMD_TAGS }} \
|
||||
ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.ARM_TAGS }}
|
||||
docker manifest push ghcr.io/$LOWERCASE_REPO/deepflow-backend:${{ steps.set-tag.outputs.TAGS }}
|
||||
docker manifest create ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }} \
|
||||
ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.AMD_TAGS }} \
|
||||
ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.ARM_TAGS }}
|
||||
docker manifest push ghcr.io/$LOWERCASE_REPO/deepflow-frontend:${{ steps.set-tag.outputs.TAGS }}
|
||||
@@ -1,30 +0,0 @@
|
||||
name: Frontend Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'frontend/**'
|
||||
- 'scripts/images/frontend/**'
|
||||
- '.github/workflows/docker-image-frontend.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'frontend/**'
|
||||
- 'scripts/images/frontend/**'
|
||||
- '.github/workflows/docker-image-frontend.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
call-docker-build:
|
||||
name: Build and Push Frontend Docker Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
with:
|
||||
service_name: frontend
|
||||
build_dir: .
|
||||
@@ -1,30 +0,0 @@
|
||||
name: Gateway Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'backend/api-gateway/**'
|
||||
- 'scripts/images/gateway/**'
|
||||
- '.github/workflows/docker-image-gateway.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'backend/api-gateway/**'
|
||||
- 'scripts/images/gateway/**'
|
||||
- '.github/workflows/docker-image-gateway.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
call-docker-build:
|
||||
name: Build and Push Gateway Docker Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
with:
|
||||
service_name: gateway
|
||||
build_dir: .
|
||||
@@ -1,28 +0,0 @@
|
||||
name: Label Studio Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'scripts/images/label-studio/**'
|
||||
- '.github/workflows/docker-image-label-studio.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'scripts/images/label-studio/**'
|
||||
- '.github/workflows/docker-image-label-studio.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
call-docker-build:
|
||||
name: Build and Push Label Studio Docker Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
with:
|
||||
service_name: label-studio
|
||||
build_dir: .
|
||||
@@ -1,32 +0,0 @@
|
||||
name: Runtime Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'runtime/ops/**'
|
||||
- 'runtime/python-executor/**'
|
||||
- 'scripts/images/runtime/**'
|
||||
- '.github/workflows/docker-image-runtime.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'runtime/ops/**'
|
||||
- 'runtime/python-executor/**'
|
||||
- 'scripts/images/runtime/**'
|
||||
- '.github/workflows/docker-image-runtime.yml'
|
||||
- '.github/workflows/docker-images-reusable.yml'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
call-docker-build:
|
||||
name: Build and Push Runtime Docker Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
with:
|
||||
service_name: runtime
|
||||
build_dir: .
|
||||
@@ -1,27 +0,0 @@
|
||||
name: docker-image-save.yml
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
service_name:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
pull-and-save:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Pull Docker Image
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
docker pull ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}:latest
|
||||
docker tag ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}:latest deepdata-${{ inputs.service_name }}:latest
|
||||
|
||||
- name: Save Docker Image
|
||||
run: |
|
||||
docker save -o deepdata-${{ inputs.service_name }}.tar deepdata-${{ inputs.service_name }}:latest
|
||||
|
||||
- name: Upload Docker Image
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: deepdata-${{ inputs.service_name }}
|
||||
path: deepdata-${{ inputs.service_name }}.tar
|
||||
@@ -1,55 +0,0 @@
|
||||
name: Release Docker Images
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
name: Frontend Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
with:
|
||||
service_name: frontend
|
||||
build_dir: .
|
||||
|
||||
gateway:
|
||||
name: Gateway Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
with:
|
||||
service_name: gateway
|
||||
build_dir: .
|
||||
|
||||
backend:
|
||||
name: Backend Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
with:
|
||||
service_name: backend
|
||||
build_dir: .
|
||||
|
||||
backend-python:
|
||||
name: Backend Python Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
with:
|
||||
service_name: backend-python
|
||||
build_dir: .
|
||||
|
||||
database:
|
||||
name: Database Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
with:
|
||||
service_name: database
|
||||
build_dir: .
|
||||
|
||||
runtime:
|
||||
name: Runtime Image
|
||||
uses: ./.github/workflows/docker-images-reusable.yml
|
||||
with:
|
||||
service_name: runtime
|
||||
build_dir: .
|
||||
|
||||
deepflow:
|
||||
name: Deer Flow Images
|
||||
uses: ./.github/workflows/docker-image-deepflow.yml
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
name: Docker Image Build & Push
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
service_name:
|
||||
required: true
|
||||
type: string
|
||||
build_dir:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
build-and-push-amd:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set Docker Image Tag
|
||||
id: set-tag
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
|
||||
if [[ $GITHUB_REF == refs/tags/v* ]]; then
|
||||
TAG=${GITHUB_REF#refs/tags/v}
|
||||
echo "TAGS=$BASE_IMAGE:amd64-$TAG" >> $GITHUB_OUTPUT
|
||||
elif [[ $GITHUB_REF == refs/heads/main ]]; then
|
||||
echo "TAGS=$BASE_IMAGE:amd64" >> $GITHUB_OUTPUT
|
||||
else
|
||||
# 使用分支名作为 tag
|
||||
if [[ $GITHUB_REF == refs/heads/* ]]; then
|
||||
BRANCH_NAME=${GITHUB_REF#refs/heads/}
|
||||
# 清理分支名,将特殊字符替换为连字符
|
||||
BRANCH_TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9._-]/-/g')
|
||||
echo "TAGS=$BASE_IMAGE:amd64-$BRANCH_TAG" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "TAGS=$BASE_IMAGE:amd64-temp" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker build -t deepdata-${{ inputs.service_name }}:amd64 . -f scripts/images/${{ inputs.service_name }}/Dockerfile --platform amd64
|
||||
|
||||
- name: Tag & Push Docker Image
|
||||
if: github.event_name != 'pull_request'
|
||||
run: |
|
||||
docker tag deepdata-${{ inputs.service_name }}:amd64 ${{ steps.set-tag.outputs.TAGS }}
|
||||
docker push ${{ steps.set-tag.outputs.TAGS }}
|
||||
|
||||
build-and-push-arm:
|
||||
runs-on: ubuntu-24.04-arm
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set Docker Image Tag
|
||||
id: set-tag
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
|
||||
if [[ $GITHUB_REF == refs/tags/v* ]]; then
|
||||
TAG=${GITHUB_REF#refs/tags/v}
|
||||
echo "TAGS=$BASE_IMAGE:arm64-$TAG" >> $GITHUB_OUTPUT
|
||||
elif [[ $GITHUB_REF == refs/heads/main ]]; then
|
||||
echo "TAGS=$BASE_IMAGE:arm64" >> $GITHUB_OUTPUT
|
||||
else
|
||||
# 使用分支名作为 tag
|
||||
if [[ $GITHUB_REF == refs/heads/* ]]; then
|
||||
BRANCH_NAME=${GITHUB_REF#refs/heads/}
|
||||
# 清理分支名,将特殊字符替换为连字符
|
||||
BRANCH_TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9._-]/-/g')
|
||||
echo "TAGS=$BASE_IMAGE:arm64-$BRANCH_TAG" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "TAGS=$BASE_IMAGE:arm64-temp" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker build -t deepdata-${{ inputs.service_name }}:arm64 . -f scripts/images/${{ inputs.service_name }}/Dockerfile --platform arm64
|
||||
|
||||
- name: Tag & Push Docker Image
|
||||
if: github.event_name != 'pull_request'
|
||||
run: |
|
||||
docker tag deepdata-${{ inputs.service_name }}:arm64 ${{ steps.set-tag.outputs.TAGS }}
|
||||
docker push ${{ steps.set-tag.outputs.TAGS }}
|
||||
|
||||
manifest:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name != 'pull_request'
|
||||
needs: [ build-and-push-amd, build-and-push-arm ]
|
||||
steps:
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set Docker Image Tag
|
||||
id: set-tag
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
BASE_IMAGE=ghcr.io/$LOWERCASE_REPO/deepdata-${{ inputs.service_name }}
|
||||
if [[ $GITHUB_REF == refs/tags/v* ]]; then
|
||||
TAG=${GITHUB_REF#refs/tags/v}
|
||||
echo "TAGS=$BASE_IMAGE:$TAG" >> $GITHUB_OUTPUT
|
||||
echo "ARM_TAGS=$BASE_IMAGE:arm64-$TAG" >> $GITHUB_OUTPUT
|
||||
echo "AMD_TAGS=$BASE_IMAGE:amd64-$TAG" >> $GITHUB_OUTPUT
|
||||
elif [[ $GITHUB_REF == refs/heads/main ]]; then
|
||||
echo "TAGS=$BASE_IMAGE:latest" >> $GITHUB_OUTPUT
|
||||
echo "ARM_TAGS=$BASE_IMAGE:arm64" >> $GITHUB_OUTPUT
|
||||
echo "AMD_TAGS=$BASE_IMAGE:amd64" >> $GITHUB_OUTPUT
|
||||
else
|
||||
# 使用分支名作为 tag
|
||||
if [[ $GITHUB_REF == refs/heads/* ]]; then
|
||||
BRANCH_NAME=${GITHUB_REF#refs/heads/}
|
||||
# 清理分支名,将特殊字符替换为连字符
|
||||
BRANCH_TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9._-]/-/g')
|
||||
echo "TAGS=$BASE_IMAGE:$BRANCH_TAG" >> $GITHUB_OUTPUT
|
||||
echo "ARM_TAGS=$BASE_IMAGE:arm64-$BRANCH_TAG" >> $GITHUB_OUTPUT
|
||||
echo "AMD_TAGS=$BASE_IMAGE:amd64-$BRANCH_TAG" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "TAGS=$BASE_IMAGE:temp" >> $GITHUB_OUTPUT
|
||||
echo "ARM_TAGS=$BASE_IMAGE:arm64-temp" >> $GITHUB_OUTPUT
|
||||
echo "AMD_TAGS=$BASE_IMAGE:amd64-temp" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Manifest Docker Image
|
||||
run: |
|
||||
docker buildx imagetools create -t ${{ steps.set-tag.outputs.TAGS }} \
|
||||
${{ steps.set-tag.outputs.AMD_TAGS }} \
|
||||
${{ steps.set-tag.outputs.ARM_TAGS }}
|
||||
@@ -1,48 +0,0 @@
|
||||
name: docker-push-to-huaweicloud-reusable.yml
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
version:
|
||||
description: 'image version (e.g. latest or v0.0.1-beta)'
|
||||
required: true
|
||||
default: 'latest'
|
||||
type: string
|
||||
service_name:
|
||||
description: 'image version (e.g. latest or v0.0.1-beta)'
|
||||
required: true
|
||||
default: 'latest'
|
||||
type: string
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD:
|
||||
description: 'login pwd'
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
retag-and-push:
|
||||
name: Retag and Push to Huawei Cloud SWR
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Pull source image and retag (amd)
|
||||
shell: bash
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository }}" | tr "[:upper:]" "[:lower:]")
|
||||
SOURCE_IMAGE=ghcr.io/${LOWERCASE_REPO}/deepdata-${{ inputs.service_name}}:${{ inputs.version}}
|
||||
TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/deepdata/${{ inputs.service_name}}:${{ inputs.version}}
|
||||
docker pull "${SOURCE_IMAGE}"
|
||||
docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"-amd
|
||||
- name: Pull source image and retag (arm)
|
||||
shell: bash
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository }}" | tr "[:upper:]" "[:lower:]")
|
||||
SOURCE_IMAGE=ghcr.io/${LOWERCASE_REPO}/deepdata-${{ inputs.service_name}}:${{ inputs.version}}
|
||||
TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/deepdata/${{ inputs.service_name}}:${{ inputs.version}}
|
||||
docker pull "${SOURCE_IMAGE}" --platform=linux/arm64
|
||||
docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"-arm
|
||||
- name: Push image to Huawei SWR using
|
||||
shell: bash
|
||||
run: |
|
||||
TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/deepdata/${{ inputs.service_name}}:${{ inputs.version}}
|
||||
echo "${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}" | docker login swr.cn-east-3.myhuaweicloud.com --username=cn-east-3@HPUA3ZGOTQFAFMM59DB9 --password-stdin
|
||||
docker push "${TARGET_IMAGE}"-amd
|
||||
docker push "${TARGET_IMAGE}"-arm
|
||||
docker buildx imagetools create -t "${TARGET_IMAGE}" "${TARGET_IMAGE}"-amd "${TARGET_IMAGE}"-arm
|
||||
@@ -1,82 +0,0 @@
|
||||
name: docker-push-to-huaweicloud.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'image version (e.g. latest or v0.0.1-beta)'
|
||||
required: true
|
||||
default: 'latest'
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
name: Push Frontend Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepdata-frontend
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
|
||||
gateway:
|
||||
name: Push Gateway Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepdata-gateway
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
|
||||
backend:
|
||||
name: Push Backend Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepdata-backend
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
|
||||
backend-python:
|
||||
name: Push Backend-Python Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepdata-backend-python
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
|
||||
runtime:
|
||||
name: Push Runtime Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepdata-runtime
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
|
||||
database:
|
||||
name: Push Database Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepdata-database
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
|
||||
deepflow-backend:
|
||||
name: Push deepflow-backend Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepflow-backend
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
|
||||
deepflow-frontend:
|
||||
name: Push deepflow-frontend Image
|
||||
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
|
||||
with:
|
||||
service_name: deepflow-frontend
|
||||
version: ${{ inputs.version}}
|
||||
secrets:
|
||||
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
|
||||
-12
@@ -202,15 +202,3 @@ Thumbs.db
|
||||
data/
|
||||
openspec/
|
||||
devspace.yaml
|
||||
|
||||
# --- Product deliverables tracked in this repository ---
|
||||
!*.dockerignore
|
||||
!docker/.env
|
||||
!deepflow/.env
|
||||
!docker/deepflow/.env
|
||||
!runtime/deepflow/.env
|
||||
!deepflow/.python-version
|
||||
!deepflow/web/src/lib/
|
||||
!deepflow/web/src/lib/**
|
||||
!deepflow/web/src/typings/
|
||||
!deepflow/web/src/typings/**
|
||||
|
||||
+17
-17
@@ -23,8 +23,8 @@ DeepData 是由多语言(Java 后端、Python 运行时、React 前端)组
|
||||
|
||||
### 1. 克隆仓库并安装依赖
|
||||
```bash
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
cd deepdata
|
||||
git clone DeepData.git
|
||||
cd DeepData
|
||||
```
|
||||
|
||||
### 2. 启动基础服务
|
||||
@@ -41,21 +41,21 @@ Enter choice:
|
||||
```
|
||||
|
||||
若您使用的机器没有 make,您也可以执行如下命令部署:
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d
|
||||
```bash
|
||||
REGISTRY=ghcr.io/modelengine-group/ docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d
|
||||
```
|
||||
|
||||
当容器运行后,请在浏览器打开 http://localhost:30000 查看前端界面。
|
||||
|
||||
### 3. 本地开发部署
|
||||
本地代码修改后,请执行以下命令构建镜像并使用本地镜像部署:
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
make build
|
||||
make install dev=true
|
||||
```
|
||||
|
||||
### 4. 卸载服务
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
make uninstall
|
||||
```
|
||||
|
||||
@@ -74,7 +74,7 @@ DeepData/
|
||||
│ ├── python-executor/ # Ray 执行器
|
||||
│ ├── ops/ # 算子生态
|
||||
│ ├── datax/ # DataX 框架
|
||||
│ └── deepflow # DeepFlow 服务
|
||||
│ └── deep-flow # DeepFlow 服务
|
||||
├── frontend/ # React 前端
|
||||
├── deployment/ # 部署配置
|
||||
└── docs/ # 文档
|
||||
@@ -83,7 +83,7 @@ DeepData/
|
||||
## 开发工作流程
|
||||
|
||||
### Java 后端开发
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
# 构建
|
||||
cd backend
|
||||
mvn clean install
|
||||
@@ -97,7 +97,7 @@ mvn spring-boot:run
|
||||
```
|
||||
|
||||
### Python 运行时开发
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
# 安装依赖
|
||||
cd runtime/deepdata-python
|
||||
poetry install
|
||||
@@ -110,7 +110,7 @@ poetry run pytest
|
||||
```
|
||||
|
||||
### React 前端开发
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
# 安装依赖
|
||||
cd frontend
|
||||
npm ci
|
||||
@@ -123,7 +123,7 @@ npm run build
|
||||
```
|
||||
|
||||
### Docker Compose 开发
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
# 启动所有服务
|
||||
docker compose up -d
|
||||
|
||||
@@ -165,13 +165,13 @@ docker compose down
|
||||
## 测试
|
||||
|
||||
### Java(JUnit 5)
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
cd backend
|
||||
mvn test
|
||||
```
|
||||
|
||||
### Python(pytest)
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
cd runtime/deepdata-python
|
||||
poetry run pytest
|
||||
```
|
||||
@@ -182,14 +182,14 @@ poetry run pytest
|
||||
## 调试
|
||||
|
||||
### Java 后端
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
# 启用 JDWP 调试端口 5005
|
||||
export JAVA_TOOL_OPTIONS='-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005'
|
||||
java -jar backend/main-application/target/*.jar
|
||||
```
|
||||
|
||||
### Python 运行时
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
# 启用 debugpy 监听端口 5678
|
||||
cd runtime/deepdata-python
|
||||
python -m debugpy --listen 5678 --wait-for-client -m uvicorn app.main:app --reload --port 18000 --host 0.0.0.0
|
||||
@@ -202,7 +202,7 @@ python -m debugpy --listen 5678 --wait-for-client -m uvicorn app.main:app --relo
|
||||
|
||||
### 端口冲突
|
||||
检查哪个进程正在使用端口:
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
```bash
|
||||
lsof -i TCP:8080
|
||||
lsof -i TCP:18000
|
||||
lsof -i TCP:5173
|
||||
@@ -234,7 +234,7 @@ lsof -i TCP:5173
|
||||
- [runtime/python-executor/README.md](./runtime/python-executor/README.md) - Ray 执行器框架
|
||||
- [runtime/ops/README.md](./runtime/ops/README.md) - 算子生态
|
||||
- [runtime/datax/README.md](./runtime/datax/README.md) - DataX 数据框架
|
||||
- [runtime/deepflow/README.md](./runtime/deepflow/README.md) - DeepFlow LLM 服务
|
||||
- [runtime/deep-flow/README.md](./runtime/deep-flow/README.md) - DeepFlow LLM 服务
|
||||
|
||||
- **前端文档**:
|
||||
- [frontend/README.md](./frontend/README.md) - React 前端应用
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ Notes:
|
||||
## Quick Start
|
||||
|
||||
1) Clone the repository and install dependencies:
|
||||
- git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
- git clone https://github.com/your-org/datemate.git
|
||||
- cd datemate
|
||||
- (Optional) Create and activate a Python virtual environment if not using Poetry-managed envs.
|
||||
- Build dependencies per component as described below.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 deepdata contributors
|
||||
Copyright (c) 2026 ModelEngine-Group
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,581 +1,221 @@
|
||||
MAKEFLAGS += --no-print-directory
|
||||
|
||||
WITH_MINERU ?= false # 默认不构建mineru
|
||||
MAKEFLAGS += --no-print-directory
|
||||
WITH_MINERU ?= false
|
||||
VERSION ?= latest
|
||||
BUILD_PLATFORM ?= linux/amd64
|
||||
NAMESPACE ?= deepdata
|
||||
|
||||
# Registry configuration: use --dev for local images, otherwise use GitHub registry
|
||||
# Registry
|
||||
ifdef dev
|
||||
REGISTRY :=
|
||||
else
|
||||
REGISTRY ?=
|
||||
REGISTRY ?= ghcr.io/modelengine-group/
|
||||
endif
|
||||
|
||||
ifdef COMSPEC
|
||||
# Windows
|
||||
MAKE := "C:/Program Files (x86)/GnuWin32/bin/make"
|
||||
MAKE := "C:/Program Files/GnuWin32/bin/make.exe"
|
||||
else
|
||||
# Linux/Mac
|
||||
MAKE := make
|
||||
endif
|
||||
|
||||
# ========== Help ==========
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "DeepData Makefile - Available Commands"
|
||||
@echo "DeepData Make Commands"
|
||||
@echo ""
|
||||
@echo "Usage: make <target> [options]"
|
||||
@echo "Build targets (using docker buildx):"
|
||||
@echo " make frontend-docker-build"
|
||||
@echo " make backend-docker-build"
|
||||
@echo " make database-docker-build"
|
||||
@echo " make gateway-docker-build"
|
||||
@echo " make runtime-docker-build"
|
||||
@echo " make runtime-ray-docker-build"
|
||||
@echo " make backend-python-docker-build"
|
||||
@echo " make deep-flow-docker-build"
|
||||
@echo " make mineru-docker-build / mineru-910B-docker-build / mineru-910C-docker-build / mineru-310P-docker-build"
|
||||
@echo " make label-studio-docker-build"
|
||||
@echo ""
|
||||
@echo "Options:"
|
||||
@echo " dev=true Use local images instead of registry (empty REGISTRY)"
|
||||
@echo " VERSION=<version> Set image version (default: latest)"
|
||||
@echo " NAMESPACE=<name> Set Kubernetes namespace (default: deepdata)"
|
||||
@echo " INSTALLER=<type> Set installer type: docker or k8s"
|
||||
@echo " PLATFORM=<platform> Set platform for downloads (default: linux/amd64)"
|
||||
@echo " Options: linux/amd64, linux/arm64"
|
||||
@echo " SAVE=true Save images to dist/ during download (default: false)"
|
||||
@echo "Build multi-arch example:"
|
||||
@echo " make frontend-docker-build BUILD_PLATFORM=linux/arm64 VERSION=latest dev=true"
|
||||
@echo ""
|
||||
@echo "Build Commands:"
|
||||
@echo " make build Build all core images"
|
||||
@echo " make <service>-docker-build Build specific service image"
|
||||
@echo " Valid services: backend, database, frontend, runtime,"
|
||||
@echo " backend-python, deepflow, mineru"
|
||||
@echo "Init buildx builder (run once): docker buildx create --name multi-builder --use --bootstrap"
|
||||
@echo ""
|
||||
@echo "Install Commands:"
|
||||
@echo " make install Install deepdata + milvus (prompts for method)"
|
||||
@echo " make install INSTALLER=docker Install using Docker Compose"
|
||||
@echo " make install INSTALLER=k8s Install using Kubernetes/Helm"
|
||||
@echo " make install INSTALLER=k8s (requires Sealed Secrets Controller)"
|
||||
@echo " make install-<component> Install specific component (prompts)"
|
||||
@echo " make <component>-docker-install Install component via Docker"
|
||||
@echo " make <component>-k8s-install Install component via Kubernetes"
|
||||
@echo " Valid components: deepdata, milvus, deepflow, mineru"
|
||||
@echo " Valid services: backend, frontend, runtime, label-studio"
|
||||
@echo ""
|
||||
@echo "Uninstall Commands:"
|
||||
@echo " make uninstall Uninstall deepdata + milvus (prompts)"
|
||||
@echo " make uninstall INSTALLER=docker Uninstall using Docker Compose"
|
||||
@echo " make uninstall INSTALLER=k8s Uninstall using Kubernetes/Helm"
|
||||
@echo " make uninstall-<component> Uninstall specific component (prompts)"
|
||||
@echo " make <component>-docker-uninstall Uninstall component via Docker"
|
||||
@echo " make <component>-k8s-uninstall Uninstall component via Kubernetes"
|
||||
@echo " Note: Docker uninstall will prompt whether to delete volumes"
|
||||
@echo ""
|
||||
@echo "Upgrade Commands:"
|
||||
@echo " make deepdata-docker-upgrade Upgrade deepdata deployment"
|
||||
@echo ""
|
||||
@echo "Download Commands:"
|
||||
@echo " make download Pull all images (no save by default)"
|
||||
@echo " make download VERSION=<version> Pull all images with specific version"
|
||||
@echo " make download REGISTRY=<registry> Pull images from specific registry"
|
||||
@echo " make load-images Load all downloaded images from dist/"
|
||||
@echo " make download-sealed-secrets Download Sealed Secrets image (for offline)"
|
||||
@echo ""
|
||||
@echo "Utility Commands:"
|
||||
@echo " make create-namespace Create Kubernetes namespace"
|
||||
@echo " make node-setup Configure dedicated nodes interactively"
|
||||
@echo " make node-cleanup Remove node labels and taints"
|
||||
@echo " make help Show this help message"
|
||||
@echo ""
|
||||
@echo "Examples:"
|
||||
@echo " make build dev=true Build all images for local development"
|
||||
@echo " make install INSTALLER=docker Install via Docker Compose"
|
||||
@echo " make install dev=true Install using local images"
|
||||
@echo " make deepdata-docker-upgrade Upgrade running deepdata services"
|
||||
@echo ""
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
# ========== Functions ==========
|
||||
|
||||
# Prompt user to choose installer if not specified
|
||||
define prompt-installer
|
||||
@echo "Choose a deployment method:"
|
||||
@echo "1. Docker/Docker-Compose"
|
||||
@echo "2. Kubernetes/Helm"
|
||||
@echo -n "Enter choice: "
|
||||
@read choice; \
|
||||
case $$choice in \
|
||||
1) INSTALLER=docker ;; \
|
||||
2) INSTALLER=k8s ;; \
|
||||
*) echo "Invalid choice" && exit 1 ;; \
|
||||
esac; \
|
||||
$(MAKE) $(1)
|
||||
endef
|
||||
|
||||
# Prompt user to choose installer and volume deletion for uninstall
|
||||
define prompt-uninstaller
|
||||
@echo "Choose a deployment method:"
|
||||
@echo "1. Docker/Docker-Compose"
|
||||
@echo "2. Kubernetes/Helm"
|
||||
@echo -n "Enter choice: "
|
||||
@read installer_choice; \
|
||||
case $$installer_choice in \
|
||||
1) INSTALLER=docker ;; \
|
||||
2) INSTALLER=k8s ;; \
|
||||
*) echo "Invalid choice" && exit 1 ;; \
|
||||
esac; \
|
||||
if [ "$$INSTALLER" = "docker" ]; then \
|
||||
echo "Delete volumes? (This will remove all data)"; \
|
||||
echo "1. Yes - Delete volumes"; \
|
||||
echo "2. No - Keep volumes"; \
|
||||
echo -n "Enter choice (default: 2): "; \
|
||||
read DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) $(1) DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
else \
|
||||
$(MAKE) $(1); \
|
||||
fi
|
||||
endef
|
||||
|
||||
# Generic docker build function
|
||||
# Usage: $(call docker-build,service-name,image-name)
|
||||
# ========== Function definitions ==========
|
||||
define docker-build
|
||||
docker build -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile
|
||||
@if ! docker buildx version >/dev/null 2>&1; then echo "ERROR: docker buildx not installed!"; exit 1; fi
|
||||
@echo "docker buildx build --platform $(BUILD_PLATFORM) -t $(2):$(VERSION) -f scripts/images/$(1)/Dockerfile . --load"
|
||||
docker buildx build --platform $(BUILD_PLATFORM) -t $(2):$(VERSION) . -f scripts/images/$(1)/Dockerfile --load
|
||||
endef
|
||||
|
||||
# Generic docker compose service action
|
||||
# Usage: $(call docker-compose-service,service-name,action,compose-dir)
|
||||
define docker-compose-service
|
||||
cd $(3) && docker compose $(2) $(1)
|
||||
endef
|
||||
|
||||
# Prompt user to choose whether to delete volumes
|
||||
define prompt-volume-deletion
|
||||
@echo "Delete volumes? (This will remove all data)"
|
||||
@echo "1. Yes - Delete volumes"
|
||||
@echo "2. No - Keep volumes"
|
||||
@echo -n "Enter choice (default: 2): "
|
||||
@read choice; \
|
||||
case $$choice in \
|
||||
1) echo "-v" ;; \
|
||||
*) echo "" ;; \
|
||||
esac
|
||||
define prompt-yesno
|
||||
@read -p "$(1) [y/N] " ans; case $${ans:-N} in [yY]) exit 0 ;; *) exit 1 ;; esac
|
||||
endef
|
||||
|
||||
# ========== Build Targets ==========
|
||||
# ========== All build targets, static PHONY (no pattern rule) ==========
|
||||
.PHONY: frontend-docker-build backend-docker-build database-docker-build gateway-docker-build
|
||||
.PHONY: runtime-docker-build runtime-ray-docker-build backend-python-docker-build
|
||||
.PHONY: deep-flow-backend-docker-build deep-flow-frontend-docker-build deep-flow-docker-build
|
||||
.PHONY: mineru-docker-build mineru-910B-docker-build mineru-910C-docker-build mineru-310P-docker-build
|
||||
.PHONY: label-studio-docker-build
|
||||
|
||||
# Valid build targets
|
||||
VALID_BUILD_TARGETS := frontend backend gateway database runtime runtime-ray backend-python deepflow label-studio mineru mineru-910B mineru-910C mineru-310P
|
||||
frontend-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-frontend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,frontend,deepdata-frontend)
|
||||
@echo "✅ deepdata-frontend build done"
|
||||
|
||||
# Generic docker build target with service name as parameter
|
||||
# Automatically prefixes image names with "deepdata-" unless it's deepflow
|
||||
.PHONY: %-docker-build
|
||||
%-docker-build:
|
||||
@if ! echo " $(VALID_BUILD_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown build target '$*'"; \
|
||||
echo "Valid build targets are:"; \
|
||||
for target in $(VALID_BUILD_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "deepflow" ]; then \
|
||||
$(call docker-build,deepflow-backend,deepflow-backend); \
|
||||
# $(call docker-build,deepflow-frontend,deepflow-frontend); \
|
||||
docker build -t deepflow-frontend:$(VERSION) . \
|
||||
-f scripts/images/deepflow-frontend/Dockerfile \
|
||||
--build-arg NEXT_PUBLIC_API_URL=/deepflow-backend; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ]; then \
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .; \
|
||||
elif [ "$*" = "mineru-910C" ]; then \
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .; \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .; \
|
||||
else \
|
||||
$(call docker-build,$*,deepdata-$*); \
|
||||
fi
|
||||
backend-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-backend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,backend,deepdata-backend)
|
||||
@echo "✅ deepdata-backend build done"
|
||||
|
||||
.PHONY: build-%
|
||||
build-%: %-docker-build
|
||||
@:
|
||||
database-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-database:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,database,deepdata-database)
|
||||
@echo "✅ deepdata-database build done"
|
||||
|
||||
.PHONY: build
|
||||
build: database-docker-build gateway-docker-build backend-docker-build frontend-docker-build runtime-docker-build backend-python-docker-build
|
||||
gateway-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-gateway:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,gateway,deepdata-gateway)
|
||||
@echo "✅ deepdata-gateway build done"
|
||||
|
||||
runtime-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-runtime:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,runtime,deepdata-runtime)
|
||||
@echo "✅ deepdata-runtime build done"
|
||||
|
||||
runtime-ray-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-runtime-ray:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,runtime-ray,deepdata-runtime-ray)
|
||||
@echo "✅ deepdata-runtime-ray build done"
|
||||
|
||||
backend-python-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-backend-python:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,backend-python,deepdata-backend-python)
|
||||
@echo "✅ deepdata-backend-python build done"
|
||||
|
||||
deep-flow-backend-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deep-flow-backend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,deep-flow-backend,deep-flow-backend)
|
||||
@echo "✅ deep-flow-backend build done"
|
||||
|
||||
deep-flow-frontend-docker-build:
|
||||
@if ! docker buildx version >/dev/null 2>&1; then echo "ERROR: docker buildx not installed!"; exit 1; fi
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deep-flow-frontend:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
docker buildx build --platform $(BUILD_PLATFORM) -t deep-flow-frontend:$(VERSION) . \
|
||||
-f scripts/images/deep-flow-frontend/Dockerfile \
|
||||
--build-arg NEXT_PUBLIC_API_URL=/deep-flow-backend \
|
||||
--load
|
||||
@echo "✅ deep-flow-frontend build done"
|
||||
|
||||
deep-flow-docker-build: deep-flow-backend-docker-build deep-flow-frontend-docker-build
|
||||
|
||||
mineru-docker-build mineru-910B-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-mineru:$(VERSION)"
|
||||
@echo "=================================================="
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t deepdata-mineru -f - .
|
||||
@echo "✅ mineru build done"
|
||||
|
||||
mineru-910C-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-mineru 910C:$(VERSION)"
|
||||
@echo "=================================================="
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t deepdata-mineru -f - .
|
||||
@echo "✅ mineru-910C build done"
|
||||
|
||||
mineru-310P-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-mineru 310P:$(VERSION)"
|
||||
@echo "=================================================="
|
||||
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t deepdata-mineru -f - .
|
||||
@echo "✅ mineru-310P build done"
|
||||
|
||||
label-studio-docker-build:
|
||||
@echo "=================================================="
|
||||
@echo "🚀 Building deepdata-label-studio:$(VERSION) platform=$(BUILD_PLATFORM)"
|
||||
@echo "=================================================="
|
||||
$(call docker-build,label-studio,deepdata-label-studio)
|
||||
@echo "✅ deepdata-label-studio build done"
|
||||
|
||||
# ========== Utility Targets ==========
|
||||
|
||||
.PHONY: create-namespace
|
||||
.PHONY: create-namespace node-setup node-cleanup
|
||||
create-namespace:
|
||||
kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
|
||||
kubectl get namespace $(NAMESPACE) >/dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
|
||||
|
||||
# ========== Node Setup/Cleanup Targets ==========
|
||||
|
||||
.PHONY: node-setup
|
||||
node-setup:
|
||||
@echo "Configure dedicated nodes for DeepData deployment?"
|
||||
@echo "This will apply labels and optional taints to selected nodes."
|
||||
@echo "1. Yes - Configure nodes interactively"
|
||||
@echo "2. No - Use default scheduling"
|
||||
@echo -n "Enter choice (default: 2): "
|
||||
@read NODE_SETUP_CHOICE; \
|
||||
if [ "$$NODE_SETUP_CHOICE" = "1" ]; then \
|
||||
@echo "Configure dedicated nodes for DeepData?"; \
|
||||
read -p "Confirm [y/N] " NODE_SETUP_CHOICE; \
|
||||
if [ "$$NODE_SETUP_CHOICE" = "y" ]; then \
|
||||
chmod +x scripts/k8s/node-setup.sh; \
|
||||
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
|
||||
fi
|
||||
|
||||
.PHONY: node-cleanup
|
||||
node-cleanup:
|
||||
@chmod +x scripts/k8s/node-cleanup.sh
|
||||
@./scripts/k8s/node-cleanup.sh --namespace $(NAMESPACE)
|
||||
|
||||
# ========== Generic Install/Uninstall Targets (Redirect to prompt-installer) ==========
|
||||
|
||||
.PHONY: install-%
|
||||
# ========== Docker Compose Install / Uninstall ==========
|
||||
VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis milvus label-studio data-juicer
|
||||
.PHONY: install-% uninstall-%
|
||||
install-%:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-installer,$*-$$INSTALLER-install)
|
||||
else
|
||||
$(MAKE) $*-$(INSTALLER)-install
|
||||
endif
|
||||
@if echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
cd deployment/compose && docker compose --profile $* up -d; \
|
||||
else \
|
||||
echo "Invalid service $*"; exit 1; \
|
||||
fi
|
||||
|
||||
.PHONY: install
|
||||
install:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-installer,deepdata-$$INSTALLER-install milvus-$$INSTALLER-install)
|
||||
else
|
||||
$(MAKE) deepdata-$(INSTALLER)-install
|
||||
$(MAKE) milvus-$(INSTALLER)-install
|
||||
@rm -f /tmp/deepdata-helm-args.sh
|
||||
endif
|
||||
|
||||
.PHONY: uninstall-%
|
||||
uninstall-%:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-uninstaller,$*-$$INSTALLER-uninstall)
|
||||
else
|
||||
$(MAKE) $*-$(INSTALLER)-uninstall
|
||||
endif
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-uninstaller,label-studio-$$INSTALLER-uninstall milvus-$$INSTALLER-uninstall deepflow-$$INSTALLER-uninstall deepdata-$$INSTALLER-uninstall)
|
||||
else
|
||||
@if [ "$(INSTALLER)" = "docker" ]; then \
|
||||
echo "Delete volumes? (This will remove all data)"; \
|
||||
echo "1. Yes - Delete volumes"; \
|
||||
echo "2. No - Keep volumes"; \
|
||||
echo -n "Enter choice (default: 2): "; \
|
||||
read DELETE_VOLUMES_CHOICE; \
|
||||
export DELETE_VOLUMES_CHOICE; \
|
||||
fi; \
|
||||
$(MAKE) label-studio-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) milvus-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) deepflow-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE; \
|
||||
$(MAKE) deepdata-$(INSTALLER)-uninstall DELETE_VOLUMES_CHOICE=$$DELETE_VOLUMES_CHOICE
|
||||
endif
|
||||
|
||||
# ========== Docker Install/Uninstall Targets ==========
|
||||
|
||||
# Valid service targets for docker install/uninstall
|
||||
VALID_SERVICE_TARGETS := deepdata backend frontend runtime runtime-ray backend-python database gateway redis deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
|
||||
|
||||
# Generic docker service install target
|
||||
.PHONY: %-docker-install
|
||||
%-docker-install:
|
||||
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown service target '$*'"; \
|
||||
echo "Valid service targets are:"; \
|
||||
for target in $(VALID_SERVICE_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ ! -f deployment/docker/deepdata/.env ]; then \
|
||||
echo "ERROR: deployment/docker/deepdata/.env not found."; \
|
||||
echo "Create it from the template:"; \
|
||||
echo " cp deployment/docker/deepdata/.env.example deployment/docker/deepdata/.env"; \
|
||||
echo "Then edit it with your actual passwords."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "label-studio" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile label-studio up -d; \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
REGISTRY=$(REGISTRY) EXTRA_ARGS="--enforce-eager --dtype float16" docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru up -d deepdata-mineru; \
|
||||
elif [ "$*" = "data-juicer" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile data-juicer up -d deepdata-data-juicer; \
|
||||
elif [ "$*" = "redis" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile redis up -d deepdata-redis; \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
cp runtime/deepflow/.env deployment/docker/deepdata/.env; \
|
||||
cp runtime/deepflow/conf.yaml deployment/docker/deepdata/conf.yaml; \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml --profile deepflow up -d; \
|
||||
@if echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
cd deployment/compose && docker compose --profile $* down; \
|
||||
else \
|
||||
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deepdata/docker-compose.yml up -d deepdata-$*; \
|
||||
echo "Invalid service $*"; exit 1; \
|
||||
fi
|
||||
|
||||
# Generic docker service uninstall target
|
||||
.PHONY: %-docker-uninstall
|
||||
%-docker-uninstall:
|
||||
@if ! echo " $(VALID_SERVICE_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown service target '$*'"; \
|
||||
echo "Valid service targets are:"; \
|
||||
for target in $(VALID_SERVICE_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "label-studio" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s label-studio; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ] || [ "$*" = "mineru-310P" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-mineru; \
|
||||
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-data-juicer; \
|
||||
elif [ "$*" = "redis" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepdata-redis; \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down -v; \
|
||||
else \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deepflow --profile label-studio --profile milvus down; \
|
||||
fi; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s deepflow-backend deepflow-frontend; \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml rm -f -s milvus etcd minio; \
|
||||
else \
|
||||
$(call docker-compose-service,$*,down,deployment/docker/deepdata); \
|
||||
fi
|
||||
# ========== Kubernetes Install / Uninstall ==========
|
||||
.PHONY: k8s-install-deepdata k8s-uninstall-deepdata
|
||||
k8s-install-deepdata: create-namespace
|
||||
helm upgrade --install deepdata ./charts/deepdata -n $(NAMESPACE)
|
||||
|
||||
# ========== Kubernetes Install/Uninstall Targets ==========
|
||||
k8s-uninstall-deepdata:
|
||||
helm uninstall deepdata -n $(NAMESPACE)
|
||||
|
||||
# Valid k8s targets
|
||||
VALID_K8S_TARGETS := deepdata deepflow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
|
||||
|
||||
# Generic k8s install target
|
||||
.PHONY: %-k8s-install
|
||||
%-k8s-install: create-namespace
|
||||
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown k8s target '$*'"; \
|
||||
echo "Valid k8s targets are:"; \
|
||||
for target in $(VALID_K8S_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "label-studio" ]; then \
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then . /tmp/deepdata-helm-args.sh; fi; \
|
||||
helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install $${HELM_LABEL_STUDIO_TOLERATIONS:-}; \
|
||||
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
||||
kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
echo ""; \
|
||||
chmod +x scripts/k8s/node-setup.sh; \
|
||||
./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then \
|
||||
. /tmp/deepdata-helm-args.sh; \
|
||||
fi; \
|
||||
chmod +x scripts/k8s/collect-secrets.sh; \
|
||||
eval $$(NAMESPACE=$(NAMESPACE) bash scripts/k8s/collect-secrets.sh); \
|
||||
if [ "$$SECRETS_CREATE" = "SKIP" ]; then \
|
||||
echo "[SKIP] Secrets collection failed — skipping deepdata Helm install"; \
|
||||
rm -f /tmp/deepdata-helm-args.sh; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
if [ -n "$$HELM_VALUES_FILE" ] && [ -f "$$HELM_VALUES_FILE" ]; then \
|
||||
HELM_EXTRA_ARGS="-f $$HELM_VALUES_FILE"; \
|
||||
else \
|
||||
HELM_EXTRA_ARGS=""; \
|
||||
fi; \
|
||||
if [ -n "$$HELM_NODE_SELECTOR_ARGS" ] || [ -n "$$HELM_TOLERATIONS_ARGS" ]; then \
|
||||
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS $$HELM_NODE_SELECTOR_ARGS $$HELM_TOLERATIONS_ARGS; \
|
||||
else \
|
||||
helm upgrade deepdata deployment/helm/deepdata/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS; \
|
||||
fi; \
|
||||
rm -f /tmp/deepdata-secret-values-*.yaml; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
cp runtime/deepflow/.env deployment/helm/deepflow/charts/public/.env; \
|
||||
cp runtime/deepflow/conf.yaml deployment/helm/deepflow/charts/public/conf.yaml; \
|
||||
helm upgrade deepflow deployment/helm/deepflow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
chmod +x scripts/k8s/collect-secrets.sh; \
|
||||
bash scripts/k8s/collect-secrets.sh --component milvus -n $(NAMESPACE); \
|
||||
MILVUS_MINIO_ACCESS_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accesskey}' | base64 -d); \
|
||||
MILVUS_MINIO_SECRET_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretkey}' | base64 -d); \
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then \
|
||||
. /tmp/deepdata-helm-args.sh; \
|
||||
fi; \
|
||||
helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
|
||||
--set minio.accessKey="$$MILVUS_MINIO_ACCESS_KEY" \
|
||||
--set minio.secretKey="$$MILVUS_MINIO_SECRET_KEY" \
|
||||
--set log.persistence.persistentVolumeClaim.accessModes=ReadWriteOnce \
|
||||
$$HELM_MILVUS_TOLERATIONS; \
|
||||
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
||||
kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
|
||||
fi
|
||||
|
||||
# Generic k8s uninstall target
|
||||
.PHONY: %-k8s-uninstall
|
||||
%-k8s-uninstall:
|
||||
@if ! echo " $(VALID_K8S_TARGETS) " | grep -q " $* "; then \
|
||||
echo "Error: Unknown k8s target '$*'"; \
|
||||
echo "Valid k8s targets are:"; \
|
||||
for target in $(VALID_K8S_TARGETS); do \
|
||||
echo " - $$target"; \
|
||||
done; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
|
||||
kubectl delete -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "mineru-310P" ]; then \
|
||||
kubectl delete -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
|
||||
elif [ "$*" = "deepdata" ]; then \
|
||||
helm uninstall deepdata -n $(NAMESPACE) --ignore-not-found; \
|
||||
$(MAKE) node-cleanup; \
|
||||
elif [ "$*" = "deepflow" ]; then \
|
||||
helm uninstall deepflow -n $(NAMESPACE) --ignore-not-found; \
|
||||
elif [ "$*" = "milvus" ]; then \
|
||||
helm uninstall milvus -n $(NAMESPACE) --ignore-not-found; \
|
||||
elif [ "$*" = "label-studio" ]; then \
|
||||
helm uninstall label-studio -n $(NAMESPACE) --ignore-not-found; \
|
||||
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
|
||||
kubectl delete -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
|
||||
fi
|
||||
|
||||
# ========== Download Targets ==========
|
||||
|
||||
# List of all images to download
|
||||
DOWNLOAD_IMAGES := \
|
||||
deepdata-backend \
|
||||
deepdata-frontend \
|
||||
deepdata-database \
|
||||
deepdata-runtime \
|
||||
deepdata-backend-python \
|
||||
deepdata-gateway \
|
||||
|
||||
# Detect architecture for smart default
|
||||
CURRENT_ARCH := $(shell uname -m 2>/dev/null)
|
||||
ifeq ($(CURRENT_ARCH),aarch64)
|
||||
DEFAULT_OPTION := 2
|
||||
else ifeq ($(CURRENT_ARCH),arm64)
|
||||
DEFAULT_OPTION := 2
|
||||
else
|
||||
DEFAULT_OPTION := 1
|
||||
endif
|
||||
|
||||
|
||||
#Usage: make download VERSION=1.0.0 REGISTRY=<registry-prefix>/ SAVE=true
|
||||
.PHONY: download
|
||||
# ========== Download / Load Images ==========
|
||||
DOWNLOAD_IMAGES := deepdata-backend deepdata-frontend deepdata-database deepdata-runtime deepdata-backend-python deepdata-gateway
|
||||
.PHONY: download load-images
|
||||
download:
|
||||
@echo "Select target platform for images:"
|
||||
@echo " 1) x86_64 (Intel/AMD, linux/amd64)"
|
||||
@echo " 2) ARM64 (Apple Silicon, Raspberry Pi 4+, linux/arm64)"
|
||||
@{ \
|
||||
read -p "Enter choice [1/2] (default: $(DEFAULT_OPTION)): " plat_choice; \
|
||||
case "$${plat_choice:-$(DEFAULT_OPTION)}" in \
|
||||
1) PLATFORM="linux/amd64" ;; \
|
||||
2) PLATFORM="linux/arm64" ;; \
|
||||
*) echo "Invalid choice. Using default option $(DEFAULT_OPTION)."; \
|
||||
PLATFORM=$$([ "$(DEFAULT_OPTION)" = "1" ] && echo "linux/amd64" || echo "linux/arm64");; \
|
||||
esac; \
|
||||
\
|
||||
read -p "Save images to dist/ for offline use? (y/n): " save_resp; \
|
||||
case "$$save_resp" in \
|
||||
[yY]|[yY][eE][sS]) SAVE="true" ;; \
|
||||
*) SAVE="false" ;; \
|
||||
esac; \
|
||||
\
|
||||
if [ -z "$(REGISTRY)" ] && [ "$$SAVE" != "true" ]; then \
|
||||
echo "Error: REGISTRY unset and SAVE=false. Nothing to do."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
\
|
||||
if [ "$$SAVE" = "true" ]; then mkdir -p dist; fi; \
|
||||
failed=0; \
|
||||
\
|
||||
for image in $(DOWNLOAD_IMAGES); do \
|
||||
if [ -z "$(REGISTRY)" ]; then \
|
||||
src_image="$$image:$(VERSION)"; \
|
||||
echo "Using local image: $$src_image"; \
|
||||
else \
|
||||
src_image="$(REGISTRY)$$image:$(VERSION)"; \
|
||||
echo "Pulling $$src_image ($$PLATFORM)..."; \
|
||||
if ! docker pull --platform "$$PLATFORM" "$$src_image"; then \
|
||||
echo "✗ Failed to pull $$src_image"; \
|
||||
failed=$$(($$failed + 1)); \
|
||||
continue; \
|
||||
fi; \
|
||||
fi; \
|
||||
\
|
||||
if [ "$$SAVE" = "true" ]; then \
|
||||
output="dist/$$image-$(VERSION).tar"; \
|
||||
echo "Saving to $$output..."; \
|
||||
if docker save -o "$$output" "$$src_image"; then \
|
||||
echo "✓ Saved $$image"; \
|
||||
else \
|
||||
echo "✗ Failed to save $$src_image"; \
|
||||
failed=$$(($$failed + 1)); \
|
||||
fi; \
|
||||
else \
|
||||
echo "✓ Ready: $$src_image"; \
|
||||
fi; \
|
||||
echo ""; \
|
||||
done; \
|
||||
\
|
||||
if [ $$failed -eq 0 ]; then \
|
||||
if [ "$$SAVE" = "true" ]; then \
|
||||
echo "✅ All images saved to dist/"; \
|
||||
echo "Load on target: docker load -i <image>.tar"; \
|
||||
else \
|
||||
echo "✅ All images ready in local Docker daemon"; \
|
||||
fi; \
|
||||
else \
|
||||
echo "❌ $$failed image(s) failed"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
}
|
||||
@echo "Select platform: 1=x86_64,2=arm64"; read PLAT; \
|
||||
if [ "$$PLAT" = "1" ]; then PLAT=linux/amd64; else PLAT=linux/arm64; fi; \
|
||||
mkdir -p dist; \
|
||||
for img in $(DOWNLOAD_IMAGES); do \
|
||||
full="$(REGISTRY)$${img}:$(VERSION)"; \
|
||||
echo "pull $$full ($$PLAT)"; \
|
||||
docker pull --platform $$PLAT $$full; \
|
||||
docker save -o dist/$${img}-$(VERSION).tar $$full; \
|
||||
done
|
||||
|
||||
DEEPFLOW_IMAGES := \
|
||||
deepflow-backend \
|
||||
deepflow-frontend
|
||||
|
||||
.PHONY: download-deepflow
|
||||
download-deepflow:
|
||||
$(MAKE) download DOWNLOAD_IMAGES="$(DEEPFLOW_IMAGES)"
|
||||
|
||||
# Download Sealed Secrets controller image for offline/air-gapped environments
|
||||
SEALED_SECRETS_IMAGE := bitnami/sealed-secrets-controller:latest
|
||||
.PHONY: download-sealed-secrets
|
||||
download-sealed-secrets:
|
||||
@echo "Pulling Sealed Secrets controller image..."
|
||||
@mkdir -p dist
|
||||
docker pull $(SEALED_SECRETS_IMAGE)
|
||||
docker save $(SEALED_SECRETS_IMAGE) -o dist/sealed-secrets-controller.tar
|
||||
@echo "✅ Saved to dist/sealed-secrets-controller.tar"
|
||||
@echo "Transfer to offline environment and load with: docker load -i dist/sealed-secrets-controller.tar"
|
||||
|
||||
# Load all downloaded images from dist/ directory
|
||||
.PHONY: load-images
|
||||
load-images:
|
||||
@if [ ! -d "dist" ]; then \
|
||||
echo "Error: dist/ directory not found"; \
|
||||
echo "Please run 'make download' first to download images"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "Loading images from dist/..."
|
||||
@count=0; \
|
||||
for tarfile in dist/*.tar.gz; do \
|
||||
if [ -f "$$tarfile" ]; then \
|
||||
echo "Loading $$tarfile..."; \
|
||||
docker load -i "$$tarfile"; \
|
||||
count=$$((count + 1)); \
|
||||
echo "✓ Loaded $$tarfile"; \
|
||||
echo ""; \
|
||||
fi; \
|
||||
done; \
|
||||
if [ $$count -eq 0 ]; then \
|
||||
echo "No image files found in dist/"; \
|
||||
echo "Please run 'make download' first"; \
|
||||
exit 1; \
|
||||
else \
|
||||
echo "Successfully loaded $$count image(s)"; \
|
||||
fi
|
||||
@if [ ! -d dist ]; then echo "dist folder missing, run make download first"; exit 1; fi; \
|
||||
for tar in dist/*.tar; do \
|
||||
echo loading $$tar; docker load -i $$tar; \
|
||||
done
|
||||
|
||||
+14
-13
@@ -2,10 +2,13 @@
|
||||
|
||||
<div align="center">
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||
[](https://github.com/ModelEngine-Group/DeepData/actions/workflows/docker-image-backend.yml)
|
||||
[](https://github.com/ModelEngine-Group/DeepData/actions/workflows/docker-image-frontend.yml)
|
||||

|
||||

|
||||

|
||||

|
||||
[](https://deepwiki.com/ModelEngine-Group/DeepData)
|
||||
|
||||
**DeepData是面向模型微调与RAG检索的企业级数据处理平台,支持数据归集、数据管理、算子市场、数据清洗、数据合成、数据标注、数据评估、知识生成等核心功能。**
|
||||
|
||||
@@ -76,17 +79,15 @@ echo -n "new-password" | kubeseal --raw --name deepdata-conf --namespace deepdat
|
||||
|
||||
### Docker一键部署
|
||||
```shell
|
||||
make build # Build all images from this repository
|
||||
make install # Deploy base services (deepdata + milvus)
|
||||
wget -qO docker-compose.yml https://raw.githubusercontent.com/ModelEngine-Group/DeepData/refs/heads/main/deployment/docker/deepdata/docker-compose.yml \
|
||||
&& REGISTRY=ghcr.io/modelengine-group/ docker compose up -d
|
||||
```
|
||||
|
||||
> Images are built from source in this repository — no external registry is required.
|
||||
|
||||
### 拉取代码
|
||||
|
||||
```bash
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
cd deepdata
|
||||
git clone git@github.com:ModelEngine-Group/DeepData.git
|
||||
cd DeepData
|
||||
```
|
||||
|
||||
### 部署基础服务
|
||||
@@ -105,7 +106,7 @@ Enter choice:
|
||||
|
||||
若您使用的机器没有make,您也可以执行如下命令部署:
|
||||
```bash
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d
|
||||
REGISTRY=ghcr.io/modelengine-group/ docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d
|
||||
```
|
||||
|
||||
当容器运行后,请在浏览器打开 http://localhost:30000 查看前端界面。
|
||||
@@ -134,7 +135,7 @@ make install-mineru
|
||||
|
||||
### 部署DeepFlow服务
|
||||
```bash
|
||||
make install-deepflow
|
||||
make install-deep-flow
|
||||
```
|
||||
|
||||
### 本地开发部署
|
||||
@@ -169,7 +170,7 @@ make uninstall
|
||||
- **[runtime/python-executor/README-zh.md](./runtime/python-executor/README-zh.md)** - Ray 执行器框架
|
||||
- **[runtime/ops/README.md](./runtime/ops/README.md)** - 算子生态
|
||||
- **[runtime/datax/README-zh.md](./runtime/datax/README-zh.md)** - DataX 数据框架
|
||||
- **[runtime/deepflow/README-zh.md](./runtime/deepflow/README-zh.md)** - DeepFlow LLM 服务
|
||||
- **[runtime/deep-flow/README-zh.md](./runtime/deep-flow/README-zh.md)** - DeepFlow LLM 服务
|
||||
|
||||
### 前端文档
|
||||
- **[frontend/README-zh.md](./frontend/README-zh.md)** - React 前端应用
|
||||
|
||||
@@ -2,10 +2,13 @@
|
||||
|
||||
<div align="center">
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||
[](https://github.com/ModelEngine-Group/DeepData/actions/workflows/docker-image-backend.yml)
|
||||
[](https://github.com/ModelEngine-Group/DeepData/actions/workflows/docker-image-frontend.yml)
|
||||

|
||||

|
||||

|
||||

|
||||
[](https://deepwiki.com/ModelEngine-Group/DeepData)
|
||||
|
||||
**DeepData is an enterprise-level data processing platform for model fine-tuning and RAG retrieval, supporting core
|
||||
functions such as data collection, data management, operator marketplace, data cleaning, data synthesis, data
|
||||
@@ -79,17 +82,15 @@ echo -n "new-password" | kubeseal --raw --name deepdata-conf --namespace deepdat
|
||||
|
||||
### Docker Quick deploy
|
||||
```shell
|
||||
make build # Build all images from this repository
|
||||
make install # Deploy base services (deepdata + milvus)
|
||||
wget -qO docker-compose.yml https://raw.githubusercontent.com/ModelEngine-Group/DeepData/refs/heads/main/deployment/docker/deepdata/docker-compose.yml \
|
||||
&& REGISTRY=ghcr.io/modelengine-group/ docker compose up -d
|
||||
```
|
||||
|
||||
> Images are built from source in this repository — no external registry is required.
|
||||
|
||||
### Clone the Code
|
||||
|
||||
```bash
|
||||
git clone https://gitea.apescale.com/figmar/deepdata.git
|
||||
cd deepdata
|
||||
git clone git@github.com:ModelEngine-Group/DeepData.git
|
||||
cd DeepData
|
||||
```
|
||||
|
||||
### Deploy the basic services
|
||||
@@ -108,7 +109,7 @@ Enter choice:
|
||||
|
||||
If the machine you are using does not have make installed, please run the following command to deploy it:
|
||||
```bash
|
||||
docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d
|
||||
REGISTRY=ghcr.io/modelengine-group/ docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d
|
||||
```
|
||||
|
||||
Once the container is running, access http://localhost:30000 in a browser to view the front-end interface.
|
||||
@@ -137,7 +138,7 @@ make install-mineru
|
||||
|
||||
### Deploy the DeepFlow service
|
||||
```bash
|
||||
make install-deepflow
|
||||
make install-deep-flow
|
||||
```
|
||||
|
||||
### Local Development and Deployment
|
||||
@@ -172,7 +173,7 @@ When running make uninstall, the installer will prompt once whether to delete vo
|
||||
- **[runtime/python-executor/README.md](./runtime/python-executor/README.md)** - Ray executor framework
|
||||
- **[runtime/ops/README.md](./runtime/ops/README.md)** - Operator ecosystem
|
||||
- **[runtime/datax/README.md](./runtime/datax/README.md)** - DataX data framework
|
||||
- **[runtime/deepflow/README.md](./runtime/deepflow/README.md)** - DeepFlow LLM service
|
||||
- **[runtime/deep-flow/README.md](./runtime/deep-flow/README.md)** - DeepFlow LLM service
|
||||
|
||||
### Frontend Documentation
|
||||
- **[frontend/README.md](./frontend/README.md)** - React frontend application
|
||||
|
||||
+10
-21
@@ -2,31 +2,20 @@
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Only the latest release of deepdata is supported with security updates.
|
||||
Use this section to tell people about which versions of your project are
|
||||
currently being supported with security updates.
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| latest | :white_check_mark: |
|
||||
| older | :x: |
|
||||
| 5.1.x | :white_check_mark: |
|
||||
| 5.0.x | :x: |
|
||||
| 4.0.x | :white_check_mark: |
|
||||
| < 4.0 | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please **do not** report security vulnerabilities through public GitHub-style
|
||||
issue trackers or pull requests.
|
||||
Use this section to tell people how to report a vulnerability.
|
||||
|
||||
Report suspected security vulnerabilities privately by opening a Gitea issue
|
||||
with the `security` label on this repository:
|
||||
|
||||
```
|
||||
https://gitea.apescale.com/figmar/deepdata/issues
|
||||
```
|
||||
|
||||
Include in your report:
|
||||
|
||||
- A description of the vulnerability and its impact
|
||||
- The affected component and version
|
||||
- Steps or a proof of concept to reproduce the issue (where possible)
|
||||
|
||||
We aim to acknowledge all reports within 3 business days and will keep you
|
||||
updated on the status of your report. Please do not disclose a vulnerability
|
||||
publicly until we have confirmed a fix or completed an assessment.
|
||||
Tell them where to go, how often they can expect to get an update on a
|
||||
reported vulnerability, what to expect if the vulnerability is accepted or
|
||||
declined, etc.
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
-- =============================================================
|
||||
-- 001_user_groups.sql — M2: users 表新增 user_groups 列
|
||||
--
|
||||
-- 背景:Casdoor 账权接入后,本地用户需保存 Casdoor 组列表
|
||||
-- (users.user_groups JSON),用于组驱动目录菜单体系的角色解析
|
||||
-- (见 RoleResolver / CasdoorUserService)。
|
||||
--
|
||||
-- 说明:
|
||||
-- * api-gateway 未引入 Flyway/Liquibase,本迁移需手工执行:
|
||||
-- psql -h <host> -p 5433 -U <user> -d <db> -f 001_user_groups.sql
|
||||
-- * 幂等:ALTER TABLE ... ADD COLUMN IF NOT EXISTS,可重复执行。
|
||||
-- * 与 scripts/db/data-management-init.sql 的 users 表定义保持同步
|
||||
-- (新库直接建表即可,无需跑本迁移)。
|
||||
-- =============================================================
|
||||
|
||||
ALTER TABLE users ADD COLUMN IF NOT EXISTS user_groups JSON;
|
||||
|
||||
COMMENT ON COLUMN users.user_groups IS 'Casdoor 组列表(JSON),如 ["deepdata-admin","deepdata-dev"];组驱动角色解析源(M2)';
|
||||
@@ -20,7 +20,7 @@
|
||||
<properties>
|
||||
<spring-boot.version>3.5.6</spring-boot.version>
|
||||
<spring-cloud.version>2025.0.0</spring-cloud.version>
|
||||
<jjwt.version>0.12.6</jjwt.version>
|
||||
<jjwt.version>0.11.5</jjwt.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@@ -66,28 +66,21 @@ public class ApiGatewayApplication {
|
||||
.route("data-setting", r -> r.path("/api/sys-param/**")
|
||||
.uri("http://deepdata-backend-python:18000"))
|
||||
|
||||
.route("deepflow-frontend", r -> r.path("/chat/**")
|
||||
.uri("http://deepflow-frontend:3000"))
|
||||
.route("deep-flow-frontend", r -> r.path("/chat/**")
|
||||
.uri("http://deep-flow-frontend:3000"))
|
||||
|
||||
.route("deepflow-static", r -> r.path("/_next/**")
|
||||
.uri("http://deepflow-frontend:3000"))
|
||||
.route("deep-flow-static", r -> r.path("/_next/**")
|
||||
.uri("http://deep-flow-frontend:3000"))
|
||||
|
||||
.route("deepflow-backend", r -> r.path("/deepflow-backend/**")
|
||||
// M4: 只剥离 /deepflow-backend 前缀。deepflow FastAPI 路由在容器根 /api/*
|
||||
// (deepflow/src/server/app.py),追加 prefixPath("/api") 会把
|
||||
// /deepflow-backend/api/config 改写成 /api/api/config(历史遗留,已删除)
|
||||
.filters(f -> f.stripPrefix(1))
|
||||
.uri("http://deepflow-backend:8000"))
|
||||
.route("deep-flow-backend", r -> r.path("/deep-flow-backend/**")
|
||||
.filters(f -> f.stripPrefix(1).prefixPath("/api"))
|
||||
.uri("http://deep-flow-backend:8000"))
|
||||
|
||||
// 网关内部服务(用户)
|
||||
// 使用 no-op 触发 GlobalFilter 执行,然后由本地 Controller 处理
|
||||
.route("gateway", r -> r.path("/api/user/**")
|
||||
.uri("http://localhost:8080"))
|
||||
|
||||
// M2: Casdoor SSO 前端配置端点(/sso/casdoor/config)同源直通本地 Controller
|
||||
.route("casdoor-sso-config", r -> r.path("/sso/**")
|
||||
.uri("http://localhost:8080"))
|
||||
|
||||
// 其他后端服务
|
||||
.route("default", r -> r.path("/api/**")
|
||||
.uri("http://deepdata-backend:8080"))
|
||||
|
||||
+8
-1
@@ -7,6 +7,8 @@ import com.deepdata.gateway.interfaces.dto.LoginResponse;
|
||||
import com.deepdata.gateway.interfaces.dto.RegisterRequest;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import com.deepdata.gateway.interfaces.dto.ChangePasswordRequest;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@@ -41,7 +43,8 @@ public class UserApplicationService {
|
||||
* Register a new user
|
||||
*
|
||||
* @param registerRequest registration request
|
||||
* @return LoginResponse with user details and token if registration successful, empty otherwise
|
||||
* @return LoginResponse with user details and token if registration successful,
|
||||
* empty otherwise
|
||||
*/
|
||||
public Optional<LoginResponse> register(RegisterRequest registerRequest) {
|
||||
return userService.register(registerRequest)
|
||||
@@ -56,4 +59,8 @@ public class UserApplicationService {
|
||||
.token(user.getToken())
|
||||
.build();
|
||||
}
|
||||
|
||||
public boolean changePassword(String username, ChangePasswordRequest request) {
|
||||
return userService.changePassword(username, request.getOldPassword(), request.getNewPassword());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ public class JwtConfig {
|
||||
2. Set environment variable:
|
||||
export JWT_SECRET=your-strong-secret-key-here
|
||||
3. Run with system property:
|
||||
-Ddeepdata.jwt.secret=your-strong-secret-key-here"""
|
||||
-DDeepData.jwt.secret=your-strong-secret-key-here"""
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
-104
@@ -1,104 +0,0 @@
|
||||
package com.deepdata.gateway.common.config;
|
||||
|
||||
import com.deepdata.gateway.infrastructure.casdoor.CasdoorProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Casdoor 组 → DeepData 角色 解析器(M2,组驱动目录菜单体系核心)。
|
||||
*
|
||||
* <p>组名约定:{@code groupPrefix + 角色键}(默认 {@code deepdata-admin / deepdata-dev / ...})。
|
||||
*
|
||||
* <p>解析规则:
|
||||
* <ul>
|
||||
* <li>{@link #primaryRole}:组名按角色键(大小写不敏感)匹配
|
||||
* {@code CasdoorProperties.rolePriority}(admin=4 > dev=3 > annotator=2 > viewer=1),
|
||||
* 取权重最高者为「主角色」;无已知组返回 null(前端按 viewer-minimal 展示,不 403)。</li>
|
||||
* <li>{@link #resolveRole}:显式 {@code usernameRoleMapping} 命中 > 组主角色 > defaultRole。</li>
|
||||
* </ul>
|
||||
*
|
||||
* @since 2026/8/30
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class RoleResolver {
|
||||
|
||||
private final CasdoorProperties props;
|
||||
|
||||
/**
|
||||
* 从 Casdoor 组列表解析主角色键(权重最高者,小写)。
|
||||
*
|
||||
* @param groups 原始组名,如 ["deepdata-admin", "deepdata-dev"]
|
||||
* @return 已知角色键;无已知组 / 组为空 / 未配置优先级返回 null
|
||||
*/
|
||||
public String primaryRole(Collection<String> groups) {
|
||||
if (groups == null || groups.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
Map<String, Integer> priority = props.getRolePriority();
|
||||
if (priority == null || priority.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String prefix = props.getGroupPrefix() == null ? "" : props.getGroupPrefix();
|
||||
String best = null;
|
||||
int bestWeight = Integer.MIN_VALUE;
|
||||
for (String g : groups) {
|
||||
String key = roleKey(g, prefix);
|
||||
if (key == null) {
|
||||
continue;
|
||||
}
|
||||
Integer w = priority.get(key);
|
||||
if (w != null && w > bestWeight) {
|
||||
bestWeight = w;
|
||||
best = key;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* 完整角色解析:显式用户名映射 > 组主角色 > defaultRole。
|
||||
* 登录路径(/me 与 Casdoor upsert)统一走这里,保证角色口径一致。
|
||||
*/
|
||||
public String resolveRole(String username, Collection<String> groups) {
|
||||
Map<String, String> mapping = props.getUsernameRoleMapping();
|
||||
if (username != null && mapping != null && mapping.containsKey(username)) {
|
||||
return mapping.get(username);
|
||||
}
|
||||
String primary = primaryRole(groups);
|
||||
if (primary != null) {
|
||||
return primary;
|
||||
}
|
||||
return props.getDefaultRole();
|
||||
}
|
||||
|
||||
/**
|
||||
* 组名 → 角色键:命中前缀返回后缀小写(如 deepdata-admin → admin),否则 null。
|
||||
* 前缀为空时按整名小写匹配(兼容 group-prefix 留空的自定义组名)。
|
||||
*
|
||||
* <p>Casdoor 的 user.groups 存的是 {@code owner/name} 全 ID(如 {@code built-in/deepdata-admin}),
|
||||
* 开启 useGroupPathInToken 时更可能是多级组路径;统一取最后一段作为组名再匹配。
|
||||
*/
|
||||
private String roleKey(String group, String prefix) {
|
||||
if (group == null || group.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String g = group.trim();
|
||||
int slash = g.lastIndexOf('/');
|
||||
if (slash >= 0) {
|
||||
g = g.substring(slash + 1);
|
||||
}
|
||||
if (prefix.isEmpty()) {
|
||||
return g.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
if (g.length() > prefix.length()
|
||||
&& g.regionMatches(true, 0, prefix, 0, prefix.length())) {
|
||||
return g.substring(prefix.length()).toLowerCase(Locale.ROOT);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+39
-74
@@ -23,37 +23,12 @@ import reactor.core.publisher.Mono;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* 用户数据隔离过滤器
|
||||
*
|
||||
* 支持两种场景:
|
||||
* 1. 商业场景(SSO):OmsAuthFilter 已添加 X-User-Name header,直接使用
|
||||
* (M2 起仅当 oms.auth.enabled=true 时信任该 header,防止外部伪造用户头)
|
||||
* 2. 独立场景(可选登录):
|
||||
* - DEEPDATA_JWT_ENABLED=true:必须登录,验证 JWT token 并添加 User header
|
||||
* - DEEPDATA_JWT_ENABLED=false:允许匿名访问,不添加 User header
|
||||
*
|
||||
* 优先级:SSO > JWT
|
||||
* Order: 2 (低于 OmsAuthFilter 的 Order=1)
|
||||
*
|
||||
* 公开端点(M2 扩展):/api/user/login|signup、/api/user/me、
|
||||
* /api/user/oauth/**(Casdoor SSO)、/sso/casdoor/config(SSO 前端配置)
|
||||
*
|
||||
* 环境变量:
|
||||
* - OMS_AUTH_ENABLED:是否启用 OmsAuthFilter(商业场景);同时决定 X-User-Name 是否可信
|
||||
* - DEEPDATA_JWT_ENABLE:独立场景下是否要求用户登录
|
||||
*
|
||||
* @author songyongtan
|
||||
* @date 2026-03-30
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class AuthFilter implements GlobalFilter, Ordered {
|
||||
private static final String AUTH_HEADER = "Authorization";
|
||||
|
||||
private static final String TOKEN_PREFIX = "Bearer ";
|
||||
|
||||
private static final String USER_HEADER = "User";
|
||||
|
||||
private final UserService userService;
|
||||
@@ -61,78 +36,77 @@ public class AuthFilter implements GlobalFilter, Ordered {
|
||||
@Value("${deepdata.jwt.enable:false}")
|
||||
private Boolean jwtEnable;
|
||||
|
||||
/** M2:商业 SSO 开关;false 时不信任 X-User-Name(防伪造用户头) */
|
||||
@Value("${oms.auth.enabled:false}")
|
||||
private Boolean omsAuthEnabled;
|
||||
|
||||
@Override
|
||||
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
|
||||
ServerHttpRequest request = exchange.getRequest();
|
||||
String path = request.getURI().getPath();
|
||||
|
||||
// 公开接口:直接放行(M2 扩展 Casdoor SSO 端点 + 前端 SSO 配置端点)
|
||||
if (path.equals("/api/user/login") || path.equals("/api/user/signup")
|
||||
|| path.startsWith("/api/user/oauth/")
|
||||
|| path.equals("/sso/casdoor/config")) {
|
||||
// 1. 公开接口直接放行
|
||||
if (path.equals("/api/user/login") || path.equals("/api/user/signup")) {
|
||||
return chain.filter(exchange);
|
||||
}
|
||||
|
||||
// 内部接口:/api/user/me 内部会自行验证 SSO 或 JWT,直接放行
|
||||
if (path.equals("/api/user/me")) {
|
||||
return chain.filter(exchange);
|
||||
}
|
||||
|
||||
try {
|
||||
// 优先检查 SSO 模式(OmsAuthFilter 已添加的 header)
|
||||
// M2:仅在 oms.auth.enabled=true 时信任 X-User-Name,防止外部伪造
|
||||
String ssoUser = null;
|
||||
if (Boolean.TRUE.equals(omsAuthEnabled)) {
|
||||
ssoUser = request.getHeaders().getFirst("X-User-Name");
|
||||
}
|
||||
// 2. 优先检查 SSO 模式 (如果有 X-User-Name Header)
|
||||
String ssoUser = request.getHeaders().getFirst("X-User-Name");
|
||||
if (StringUtils.isNotBlank(ssoUser)) {
|
||||
final String ssoUserFinal = ssoUser;
|
||||
log.info("SSO mode detected, adding User header: {}", ssoUser);
|
||||
ServerHttpRequest mutatedRequest = request.mutate()
|
||||
.headers(httpHeaders -> {
|
||||
httpHeaders.set(USER_HEADER, ssoUserFinal);
|
||||
})
|
||||
.headers(h -> h.add(USER_HEADER, ssoUser))
|
||||
.build();
|
||||
ServerWebExchange mutatedExchange = exchange.mutate()
|
||||
.request(mutatedRequest)
|
||||
.build();
|
||||
return chain.filter(mutatedExchange);
|
||||
return chain.filter(exchange.mutate().request(mutatedRequest).build());
|
||||
}
|
||||
|
||||
// 独立场景:根据 DEEPDATA_JWT_ENABLE 决定是否要求登录
|
||||
// 3. 如果未开启 JWT 验证,直接放行
|
||||
if (!jwtEnable) {
|
||||
log.debug("JWT authentication is not required, passing request without user header");
|
||||
return chain.filter(exchange);
|
||||
}
|
||||
|
||||
// JWT 模式:必须登录,验证 Token
|
||||
// ==========================================
|
||||
// 🆕 4. 核心修改:多种方式获取 Token
|
||||
// ==========================================
|
||||
String token = null;
|
||||
|
||||
// 方式 A: 从 Header 获取 (标准 API 请求)
|
||||
String authHeader = request.getHeaders().getFirst(AUTH_HEADER);
|
||||
if (authHeader == null || !authHeader.startsWith(TOKEN_PREFIX)) {
|
||||
log.warn("JWT authentication is required but no valid Authorization header found");
|
||||
if (authHeader != null && authHeader.startsWith(TOKEN_PREFIX)) {
|
||||
token = authHeader.substring(TOKEN_PREFIX.length());
|
||||
log.debug("Token found in Authorization header");
|
||||
}
|
||||
|
||||
// 方式 B: 从 Query 参数获取 (解决 <a href> / 预览 401)
|
||||
if (StringUtils.isBlank(token)) {
|
||||
String tokenFromQuery = request.getQueryParams().getFirst("token");
|
||||
if (StringUtils.isNotBlank(tokenFromQuery)) {
|
||||
token = tokenFromQuery;
|
||||
log.debug("Token found in Query parameters: ?token=...");
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 如果两种方式都没拿到 Token,拦截并返回 401
|
||||
if (StringUtils.isBlank(token)) {
|
||||
log.warn("JWT required but no token found in header or query params");
|
||||
return sendUnauthorizedResponse(exchange);
|
||||
}
|
||||
|
||||
String token = authHeader.substring(TOKEN_PREFIX.length());
|
||||
// 6. 验证 Token 有效性
|
||||
String user = userService.validateToken(token);
|
||||
if (StringUtils.isBlank(user)) {
|
||||
log.warn("JWT token validation failed");
|
||||
log.warn("JWT token validation failed for token starting with: {}",
|
||||
token.length() > 10 ? token.substring(0, 10) : token);
|
||||
return sendUnauthorizedResponse(exchange);
|
||||
}
|
||||
|
||||
log.info("JWT mode authenticated, adding User header: {}", user);
|
||||
log.info("JWT authenticated user: {}", user);
|
||||
// 注入 User Header 给下游服务
|
||||
ServerHttpRequest mutatedRequest = request.mutate()
|
||||
.headers(httpHeaders -> {
|
||||
httpHeaders.set(USER_HEADER, user);
|
||||
})
|
||||
.headers(h -> h.add(USER_HEADER, user))
|
||||
.build();
|
||||
ServerWebExchange mutatedExchange = exchange.mutate()
|
||||
.request(mutatedRequest)
|
||||
.build();
|
||||
return chain.filter(mutatedExchange);
|
||||
return chain.filter(exchange.mutate().request(mutatedRequest).build());
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Error in AuthFilter", e);
|
||||
return sendUnauthorizedResponse(exchange);
|
||||
@@ -148,21 +122,12 @@ public class AuthFilter implements GlobalFilter, Ordered {
|
||||
try {
|
||||
bytes = objectMapper.writeValueAsString(Response.error(CommonErrorCode.UNAUTHORIZED)).getBytes(StandardCharsets.UTF_8);
|
||||
} catch (JsonProcessingException e) {
|
||||
String responseBody = "{\"code\":401,\"message\":\"登录失败:用户名或密码错误\",\"data\":null}";
|
||||
bytes = responseBody.getBytes(StandardCharsets.UTF_8);
|
||||
bytes = "{\"code\":401,\"message\":\"Unauthorized\",\"data\":null}".getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
DataBuffer buffer = response.bufferFactory().wrap(bytes);
|
||||
return response.writeWith(Mono.just(buffer));
|
||||
}
|
||||
|
||||
/**
|
||||
* 用户数据隔离过滤器优先级
|
||||
*
|
||||
* Order = 2,在 OmsAuthFilter (Order=1) 之后执行
|
||||
* 确保先执行 SSO 认证,再执行用户数据隔离
|
||||
*
|
||||
* @return order value (2 = after SSO authentication)
|
||||
*/
|
||||
@Override
|
||||
public int getOrder() {
|
||||
return 2;
|
||||
|
||||
@@ -2,14 +2,12 @@ package com.deepdata.gateway.domain.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.deepdata.common.infrastructure.config.PgJsonTypeHandler;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 用户
|
||||
@@ -28,14 +26,6 @@ public class User {
|
||||
private String passwordHash;
|
||||
private String fullName;
|
||||
private String role;
|
||||
|
||||
/**
|
||||
* Casdoor 组列表(users.user_groups JSON 列,M2;本地注册用户为 null)。
|
||||
* 组 → 主角色解析见 {@code com.deepdata.gateway.common.config.RoleResolver}。
|
||||
*/
|
||||
@TableField(value = "user_groups", typeHandler = PgJsonTypeHandler.class)
|
||||
private List<String> userGroups;
|
||||
|
||||
private boolean enabled;
|
||||
private LocalDateTime lastLoginAt;
|
||||
|
||||
|
||||
+33
-42
@@ -12,7 +12,6 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import com.deepdata.gateway.interfaces.dto.RegisterRequest;
|
||||
@@ -59,66 +58,35 @@ public class UserService {
|
||||
return encoder.matches(user.getPassword(), userInDB.getPasswordHash());
|
||||
}
|
||||
|
||||
/**
|
||||
* 签发本地 JWT(M2 起为 public,供 Casdoor 落地统一复用;claims 口径一致:
|
||||
* sub=username, uid, role,组驱动用户附加 groups 列表)。
|
||||
*
|
||||
* <p>安全边界说明:本地 token 为网关内部信任链(自签自验),保持 HS256;
|
||||
* Casdoor id_token 走独立 RS256 JWKS 验证链({@code CasdoorIdTokenVerifier}),
|
||||
* 两条链互不混淆。
|
||||
*/
|
||||
public String generateToken(User user) {
|
||||
private String generateToken(User user) {
|
||||
long now = System.currentTimeMillis();
|
||||
JwtBuilder builder = Jwts.builder()
|
||||
return Jwts.builder()
|
||||
.setSubject(user.getUsername())
|
||||
.claim("uid", user.getId())
|
||||
.claim("role", user.getRole())
|
||||
.setIssuedAt(new Date(now))
|
||||
.setExpiration(new Date(now + expirationSeconds * 1000))
|
||||
.signWith(Keys.hmacShaKeyFor(secret.getBytes(StandardCharsets.UTF_8)));
|
||||
if (user.getUserGroups() != null && !user.getUserGroups().isEmpty()) {
|
||||
builder.claim("groups", user.getUserGroups());
|
||||
}
|
||||
return builder.compact();
|
||||
.signWith(Keys.hmacShaKeyFor(secret.getBytes(StandardCharsets.UTF_8)), SignatureAlgorithm.HS256)
|
||||
.compact();
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验本地 JWT。M3 起额外校验用户状态:用户不存在或已禁用(本地/同步置禁)
|
||||
* 返回 null → 网关 AuthFilter 直接 401,禁用账号的存量 token 立即失效。
|
||||
*/
|
||||
public String validateToken(String token) {
|
||||
try {
|
||||
Jws<Claims> claimsJws = Jwts.parser().verifyWith(Keys.hmacShaKeyFor(secret.getBytes(StandardCharsets.UTF_8))).build().parseSignedClaims(token);
|
||||
String username = claimsJws.getPayload().getSubject();
|
||||
return findUserByUsername(username)
|
||||
.filter(User::isEnabled)
|
||||
.map(User::getUsername)
|
||||
.orElse(null);
|
||||
Jws<Claims> claimsJws = Jwts.parserBuilder()
|
||||
.setSigningKey(Keys.hmacShaKeyFor(secret.getBytes(StandardCharsets.UTF_8))).build()
|
||||
.parseClaimsJws(token);
|
||||
return claimsJws.getBody().getSubject();
|
||||
} catch (JwtException | IllegalArgumentException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 按用户名查本地用户(M2:/me 角色与组解析、Casdoor upsert 复用)。
|
||||
*
|
||||
* @param username 本地用户名
|
||||
* @return 命中用户;不存在 / 空参返回 empty
|
||||
*/
|
||||
public Optional<User> findUserByUsername(String username) {
|
||||
if (username == null || username.isBlank()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
LambdaQueryWrapper<User> wrapper = new LambdaQueryWrapper<>();
|
||||
wrapper.eq(User::getUsername, username);
|
||||
return Optional.ofNullable(userRepository.getOne(wrapper));
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new user
|
||||
*
|
||||
* @param registerRequest registration request containing user details
|
||||
* @return registered user with token if successful, empty if username or email already exists
|
||||
* @return registered user with token if successful, empty if username or email
|
||||
* already exists
|
||||
*/
|
||||
public Optional<User> register(RegisterRequest registerRequest) {
|
||||
// Check if username already exists
|
||||
@@ -158,4 +126,27 @@ public class UserService {
|
||||
|
||||
return Optional.of(user);
|
||||
}
|
||||
|
||||
public boolean changePassword(String username, String oldPassword, String newPassword) {
|
||||
// 1. 根据用户名查找用户
|
||||
LambdaQueryWrapper<User> wrapper = new LambdaQueryWrapper<>();
|
||||
wrapper.eq(User::getUsername, username);
|
||||
User user = userRepository.getOne(wrapper);
|
||||
|
||||
if (user == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
BCryptPasswordEncoder encoder = new BCryptPasswordEncoder(10);
|
||||
|
||||
// 2. 校验旧密码
|
||||
if (!encoder.matches(oldPassword, user.getPasswordHash())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 3. 更新新密码
|
||||
user.setPasswordHash(encoder.encode(newPassword));
|
||||
userRepository.updateById(user);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
package com.deepdata.gateway.infrastructure.casdoor;
|
||||
|
||||
import com.deepdata.gateway.infrastructure.casdoor.dto.CasdoorTokenResponse;
|
||||
import io.netty.channel.ChannelOption;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.client.reactive.ClientHttpRequest;
|
||||
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.BodyInserter;
|
||||
import org.springframework.web.reactive.function.BodyInserters;
|
||||
import org.springframework.web.reactive.function.client.ClientResponse;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.netty.http.client.HttpClient;
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Casdoor OAuth2/OIDC 客户端(响应式,WebClient)。
|
||||
*
|
||||
* <p>覆盖两条网关侧流程:
|
||||
* <ul>
|
||||
* <li>{@link #exchangeCode} —— authorization_code 换 token(机密客户端,带 client_secret)</li>
|
||||
* <li>{@link #passwordGrant} —— password grant(Flow B 账密兜底)</li>
|
||||
* </ul>
|
||||
* 身份声明一律从返回的 {@code id_token} 提取(见 {@link IdTokenClaims})。
|
||||
*
|
||||
* <p>实测要点(2026-08-29,线上实例):
|
||||
* <ul>
|
||||
* <li>code 一次性,有效期约 200s,回跳后必须立即交换。</li>
|
||||
* <li>交换成功返回 200 + JSON{access_token, id_token, refresh_token, token_type, expires_in, scope};
|
||||
* 失败返回 400 + JSON{error, error_description}(如 invalid_client / invalid_grant)。</li>
|
||||
* <li>form 参数方式传 client_id/client_secret 已验证可用(无需 Basic 头)。</li>
|
||||
* </ul>
|
||||
*
|
||||
* @since 2026/8/29
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class CasdoorClient {
|
||||
|
||||
private final CasdoorProperties props;
|
||||
private final WebClient webClient;
|
||||
|
||||
public CasdoorClient(CasdoorProperties props) {
|
||||
this.props = props;
|
||||
HttpClient httpClient = HttpClient.create()
|
||||
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, props.getConnectTimeoutMillis())
|
||||
.responseTimeout(Duration.ofMillis(props.getReadTimeoutMillis()));
|
||||
this.webClient = WebClient.builder()
|
||||
.clientConnector(new ReactorClientHttpConnector(httpClient))
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 授权码换 token(Flow A 回调阶段)。
|
||||
*
|
||||
* @param code Casdoor 回跳携带的一次性 code
|
||||
* @return 令牌响应(含 access_token / id_token / refresh_token)
|
||||
*/
|
||||
public Mono<CasdoorTokenResponse> exchangeCode(String code) {
|
||||
return postToken(BodyInserters.fromFormData("grant_type", "authorization_code")
|
||||
.with("code", code)
|
||||
.with("client_id", props.getClientId())
|
||||
.with("client_secret", props.getClientSecret())
|
||||
.with("redirect_uri", props.getRedirectUri()), "exchangeCode");
|
||||
}
|
||||
|
||||
/**
|
||||
* password grant(Flow B,网关侧代用户认证)。
|
||||
*
|
||||
* @param username Casdoor 用户名
|
||||
* @param password Casdoor 密码
|
||||
* @return 令牌响应(含 id_token)
|
||||
*/
|
||||
public Mono<CasdoorTokenResponse> passwordGrant(String username, String password) {
|
||||
return postToken(BodyInserters.fromFormData("grant_type", "password")
|
||||
.with("username", username)
|
||||
.with("password", password)
|
||||
.with("scope", props.getScopes())
|
||||
.with("client_id", props.getClientId())
|
||||
.with("client_secret", props.getClientSecret()), "passwordGrant");
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造 authorize 重定向 URL(Flow A 第一步,浏览器跳转)。
|
||||
*
|
||||
* @param state 一次性随机串(由调用方存 cookie,回调时校验并删除)
|
||||
*/
|
||||
public String buildAuthorizeUrl(String state) {
|
||||
StringBuilder sb = new StringBuilder(props.getAuthorizeUri());
|
||||
sb.append('?')
|
||||
.append("client_id=").append(enc(props.getClientId()))
|
||||
.append("&redirect_uri=").append(enc(props.getRedirectUri()))
|
||||
.append("&response_type=code")
|
||||
.append("&scope=").append(enc(props.getScopes().replace(' ', '+')))
|
||||
.append("&state=").append(enc(state));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 公共 token 端点调用:form 提交;2xx 解析为 {@link CasdoorTokenResponse},
|
||||
* 非 2xx 解析错误 JSON 并抛出带 error_description 的异常。
|
||||
*/
|
||||
private Mono<CasdoorTokenResponse> postToken(
|
||||
BodyInserter<?, ? super ClientHttpRequest> form, String op) {
|
||||
return webClient.post()
|
||||
.uri(props.getTokenUri())
|
||||
.contentType(MediaType.APPLICATION_FORM_URLENCODED)
|
||||
.body(form)
|
||||
.retrieve()
|
||||
.onStatus(s -> s.isError(), resp -> errorBody(resp)
|
||||
.map(b -> new CasdoorTokenException(op + " failed", b)))
|
||||
.bodyToMono(CasdoorTokenResponse.class)
|
||||
.doOnNext(r -> log.info("casdoor {} ok: tokenType={}, expiresIn={}",
|
||||
op, r.getTokenType(), r.getExpiresIn()))
|
||||
.switchIfEmpty(Mono.error(new CasdoorTokenException(
|
||||
op + " failed: empty token response", "")));
|
||||
}
|
||||
|
||||
private Mono<String> errorBody(ClientResponse resp) {
|
||||
return resp.bodyToMono(String.class).defaultIfEmpty(resp.statusCode().toString());
|
||||
}
|
||||
|
||||
private static String enc(String s) {
|
||||
return URLEncoder.encode(s, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/** token 交换失败(invalid_client / invalid_grant / code 过期等) */
|
||||
public static class CasdoorTokenException extends RuntimeException {
|
||||
private final String detail;
|
||||
|
||||
public CasdoorTokenException(String message, String detail) {
|
||||
super(message);
|
||||
this.detail = detail;
|
||||
}
|
||||
|
||||
/** Casdoor 返回的错误 JSON,形如 {"error":"invalid_grant","error_description":"..."} */
|
||||
public String getDetail() {
|
||||
return detail;
|
||||
}
|
||||
}
|
||||
}
|
||||
-224
@@ -1,224 +0,0 @@
|
||||
package com.deepdata.gateway.infrastructure.casdoor;
|
||||
|
||||
import com.deepdata.gateway.infrastructure.casdoor.dto.JwksResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import io.jsonwebtoken.Claims;
|
||||
import io.jsonwebtoken.Jws;
|
||||
import io.jsonwebtoken.Jwts;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.netty.http.client.HttpClient;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.KeyFactory;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.interfaces.RSAPublicKey;
|
||||
import java.security.spec.InvalidKeySpecException;
|
||||
import java.security.spec.RSAPublicKeySpec;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Casdoor id_token 验证器:拉取公开 JWKS(带 TTL 缓存),按 {@code kid} 选 RSA 公钥,
|
||||
* 用 JJWT(0.12.x)做 RS256 验签 + iss/aud/exp/nbf 校验。
|
||||
*
|
||||
* <p>实测(2026-08-30):JWKS 端点 {@code /.well-known/jwks.json} 无需鉴权,返回
|
||||
* <pre>{"keys":[{"use":"sig","kty":"RSA","kid":"cert-built-in","alg":"RS256","n":"...","e":"AQAB","x5c":["..."]}]}</pre>
|
||||
* id_token 头为 {@code {"alg":"RS256","typ":"JWT","kid":"cert-built-in"}}。
|
||||
*
|
||||
* <p>M2:验证通过后按 {@code groups-claim}(默认 groups)提取组列表,随
|
||||
* {@link IdTokenClaims#groups()} 返回,供 {@code CasdoorUserService} 落地角色。
|
||||
*
|
||||
* @since 2026/8/29
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class CasdoorIdTokenVerifier {
|
||||
|
||||
private final CasdoorProperties props;
|
||||
private final WebClient webClient;
|
||||
|
||||
/** JWKS 缓存(kid → RSAPublicKey),证书轮转时靠 TTL / kid 未命中强制刷新 */
|
||||
private volatile CacheEntry cache;
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private record CacheEntry(long fetchedAtMillis, Map<String, RSAPublicKey> keys) {}
|
||||
|
||||
public CasdoorIdTokenVerifier(CasdoorProperties props) {
|
||||
this.props = props;
|
||||
this.webClient = WebClient.builder()
|
||||
.clientConnector(new ReactorClientHttpConnector(HttpClient.create()))
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 id_token 并提取身份声明。
|
||||
*
|
||||
* @param idToken Casdoor 返回的 id_token(RS256)
|
||||
* @return 验证通过的 {@link IdTokenClaims}
|
||||
* @throws CasdoorIdTokenException 签名错误 / iss/aud/exp 校验失败 / JWKS 不可用
|
||||
*/
|
||||
public Mono<IdTokenClaims> verify(String idToken) {
|
||||
return parseAndVerify(idToken, false)
|
||||
.flatMap(jws -> {
|
||||
Claims body = jws.getPayload();
|
||||
IdTokenClaims claims = new IdTokenClaims(
|
||||
body.getSubject(),
|
||||
body.get("name", String.class),
|
||||
body.get("email", String.class),
|
||||
body.getExpiration() != null ? body.getExpiration().getTime() / 1000 : 0,
|
||||
body.getIssuedAt() != null ? body.getIssuedAt().getTime() / 1000 : 0,
|
||||
body,
|
||||
extractGroups(body));
|
||||
if (claims.sub() == null || claims.name() == null) {
|
||||
return Mono.error(new CasdoorIdTokenException(
|
||||
"id_token missing sub/name claims"));
|
||||
}
|
||||
return Mono.just(claims);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 {@code groups-claim} 配置提取组列表(id_token 中 JSON 数组;
|
||||
* 个别配置下可能退化为单字符串)。无 groups claim 返回 null。
|
||||
*/
|
||||
private List<String> extractGroups(Claims body) {
|
||||
Object g = body.get(props.getGroupsClaim());
|
||||
if (g instanceof List<?> list) {
|
||||
List<String> groups = new ArrayList<>(list.size());
|
||||
for (Object o : list) {
|
||||
if (o != null) {
|
||||
groups.add(String.valueOf(o));
|
||||
}
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
if (g instanceof String s && !s.isBlank()) {
|
||||
return List.of(s);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 kid 取公钥并解析验签。kid 未命中且尚未强制刷新过 → 刷新 JWKS 重试一次。
|
||||
*/
|
||||
private Mono<Jws<Claims>> parseAndVerify(String idToken, boolean forced) {
|
||||
return loadKeys(forced)
|
||||
.map(keys -> Optional.ofNullable(parseWithKid(idToken, keys)))
|
||||
.flatMap(opt -> {
|
||||
if (opt.isPresent()) {
|
||||
return Mono.just(opt.get());
|
||||
}
|
||||
if (forced) {
|
||||
return Mono.error(new CasdoorIdTokenException(
|
||||
"id_token kid not found in JWKS after refresh"));
|
||||
}
|
||||
return loadKeys(true)
|
||||
.map(k2 -> Optional.ofNullable(parseWithKid(idToken, k2)))
|
||||
.map(o2 -> o2.<CasdoorIdTokenException>orElseThrow(() ->
|
||||
new CasdoorIdTokenException(
|
||||
"id_token kid not found in JWKS after refresh")));
|
||||
});
|
||||
}
|
||||
|
||||
private Jws<Claims> parseWithKid(String idToken, Map<String, RSAPublicKey> keys) {
|
||||
String kid = headerKid(idToken);
|
||||
RSAPublicKey key = keys.get(kid);
|
||||
if (key == null) {
|
||||
log.warn("casdoor jwks: kid [{}] not cached, size={}", kid, keys.size());
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return Jwts.parser()
|
||||
.verifyWith(key)
|
||||
.requireIssuer(props.getIssuer())
|
||||
.requireAudience(props.getClientId())
|
||||
.build()
|
||||
.parseSignedClaims(idToken);
|
||||
} catch (Exception e) {
|
||||
throw new CasdoorIdTokenException("id_token verification failed: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接 base64url 解码 JWT header 段取 {@code kid}。
|
||||
* 不能用 {@code Jwts.parser().build().parse(...)}:jjwt 0.12 对已签名 JWT
|
||||
* 在未提供验签密钥时会抛 {@code UnsupportedJwtException},无法仅取 header。
|
||||
*/
|
||||
private static String headerKid(String idToken) {
|
||||
int firstDot = idToken.indexOf('.');
|
||||
if (firstDot <= 0) {
|
||||
throw new CasdoorIdTokenException("id_token missing JWS header segment");
|
||||
}
|
||||
try {
|
||||
String headerJson = new String(
|
||||
Base64.getUrlDecoder().decode(idToken.substring(0, firstDot)),
|
||||
StandardCharsets.UTF_8);
|
||||
return MAPPER.readTree(headerJson).path("kid").asText(null);
|
||||
} catch (CasdoorIdTokenException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new CasdoorIdTokenException("id_token header unparsable: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private Mono<Map<String, RSAPublicKey>> loadKeys(boolean forced) {
|
||||
CacheEntry current = cache;
|
||||
long ttlMillis = props.getJwksCacheSeconds() * 1000L;
|
||||
if (!forced && current != null
|
||||
&& System.currentTimeMillis() - current.fetchedAtMillis() < ttlMillis) {
|
||||
return Mono.just(current.keys());
|
||||
}
|
||||
return webClient.get()
|
||||
.uri(props.getJwksUri())
|
||||
.retrieve()
|
||||
.bodyToMono(JwksResponse.class)
|
||||
.map(jwks -> {
|
||||
Map<String, RSAPublicKey> keys = new HashMap<>();
|
||||
if (jwks.getKeys() != null) {
|
||||
for (JwksResponse.Jwk jwk : jwks.getKeys()) {
|
||||
if ("RSA".equals(jwk.getKty()) && jwk.getKid() != null
|
||||
&& jwk.getN() != null && jwk.getE() != null) {
|
||||
keys.put(jwk.getKid(), toRsaKey(jwk));
|
||||
}
|
||||
}
|
||||
}
|
||||
cache = new CacheEntry(System.currentTimeMillis(), keys);
|
||||
log.info("casdoor jwks refreshed: {} key(s)", keys.size());
|
||||
return keys;
|
||||
})
|
||||
.doOnError(e -> log.error("casdoor jwks fetch failed: {}", e.getMessage(), e));
|
||||
}
|
||||
|
||||
private static RSAPublicKey toRsaKey(JwksResponse.Jwk jwk) {
|
||||
BigInteger n = new BigInteger(1, Base64.getUrlDecoder().decode(jwk.getN()));
|
||||
BigInteger e = new BigInteger(1, Base64.getUrlDecoder().decode(jwk.getE()));
|
||||
try {
|
||||
return (RSAPublicKey) KeyFactory.getInstance("RSA")
|
||||
.generatePublic(new RSAPublicKeySpec(n, e));
|
||||
} catch (NoSuchAlgorithmException | InvalidKeySpecException ex) {
|
||||
throw new CasdoorIdTokenException("invalid JWKS RSA key: " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** id_token 验证失败(签名/iss/aud/exp 任一不过,或 JWKS 不可用) */
|
||||
public static class CasdoorIdTokenException extends RuntimeException {
|
||||
public CasdoorIdTokenException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public CasdoorIdTokenException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
-115
@@ -1,115 +0,0 @@
|
||||
package com.deepdata.gateway.infrastructure.casdoor;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Casdoor 接入配置,对应 yml 前缀 {@code deepdata.casdoor.*}。
|
||||
*
|
||||
* <p>机密项(client-secret、redirect-uri)建议经 Nacos / 环境变量注入,勿写死在仓库。
|
||||
*
|
||||
* <p>M2 新增(组驱动目录菜单体系):
|
||||
* <ul>
|
||||
* <li>{@code groups-claim} —— id_token 中组列表的 claim 名(Casdoor 应用 tokenFields 含 Groups 时携带);</li>
|
||||
* <li>{@code group-prefix} —— 角色组名前缀(组名 = 前缀 + 角色键,如 deepdata-admin);</li>
|
||||
* <li>{@code role-priority} —— 角色键 → 权重(越大越优先),用于从多组解析「主角色」。</li>
|
||||
* </ul>
|
||||
*
|
||||
* @since 2026/8/29
|
||||
*/
|
||||
@Data
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "deepdata.casdoor")
|
||||
public class CasdoorProperties {
|
||||
|
||||
/** 总开关:false 时 SSO Controller 不注册,系统行为与接入前完全一致 */
|
||||
private boolean enabled = false;
|
||||
|
||||
/** OIDC issuer,用于 id_token 的 iss 校验 */
|
||||
private String issuer = "https://apikey.apescale.com:1013";
|
||||
|
||||
/** 授权端点(浏览器跳转) */
|
||||
private String authorizeUri = "https://apikey.apescale.com:1013/login/oauth/authorize";
|
||||
|
||||
/** 令牌端点(code 交换 / password grant) */
|
||||
private String tokenUri = "https://apikey.apescale.com:1013/api/login/oauth/access_token";
|
||||
|
||||
/** 用户信息端点(可选,身份以 id_token 为准) */
|
||||
private String userinfoUri = "https://apikey.apescale.com:1013/api/userinfo";
|
||||
|
||||
/**
|
||||
* JWKS 端点(公开)。M2 修正:线上实例实测地址为 {@code /.well-known/jwks.json}
|
||||
* (2026-08-30 验证),与 M1 方案文档一致。
|
||||
*/
|
||||
private String jwksUri = "https://apikey.apescale.com:1013/.well-known/jwks.json";
|
||||
|
||||
private String clientId;
|
||||
private String clientSecret;
|
||||
|
||||
/** 必须与 Casdoor 应用 redirectUris 登记值完全一致 */
|
||||
private String redirectUri;
|
||||
|
||||
private String scopes = "openid profile email";
|
||||
|
||||
/** state 随机串存 cookie 的名称(HttpOnly) */
|
||||
private String stateCookieName = "casdoor_sso_state";
|
||||
|
||||
/** state 有效期(秒),应与 code 有效期(实测约 200s)对齐并留余量 */
|
||||
private int stateTtlSeconds = 300;
|
||||
|
||||
/**
|
||||
* state cookie 是否带 Secure 属性。生产(HTTPS)保持 true;
|
||||
* 本地 HTTP 联调(http://localhost:8765)必须设 false,否则浏览器不会回传该 cookie。
|
||||
*/
|
||||
private boolean stateCookieSecure = true;
|
||||
|
||||
/** 首登是否自动创建本地用户 */
|
||||
private boolean autoProvision = true;
|
||||
|
||||
/** 自动建号时的默认本地角色 */
|
||||
private String defaultRole = "USER";
|
||||
|
||||
/** Casdoor 用户名 → 本地角色 映射,未命中用 defaultRole。例: admin=ADMIN */
|
||||
private Map<String, String> usernameRoleMapping = new LinkedHashMap<>();
|
||||
|
||||
/** JWKS 缓存时长(秒);kid 未命中时强制刷新一次 */
|
||||
private long jwksCacheSeconds = 3600;
|
||||
|
||||
/** WebClient 超时(毫秒) */
|
||||
private int connectTimeoutMillis = 5_000;
|
||||
private int readTimeoutMillis = 10_000;
|
||||
|
||||
/** Flow B:/api/user/login 是否优先走 Casdoor password grant */
|
||||
private PasswordGrant passwordGrant = new PasswordGrant();
|
||||
|
||||
// ---------- M2:组驱动角色 ----------
|
||||
|
||||
/** id_token 中组列表的 claim 名(Casdoor 应用 tokenFields=["Groups"] 时携带) */
|
||||
private String groupsClaim = "groups";
|
||||
|
||||
/** 角色组名前缀:组名 = group-prefix + 角色键(如 deepdata-admin) */
|
||||
private String groupPrefix = "deepdata-";
|
||||
|
||||
/** 角色键 → 权重(越大越优先)。组名不在此表 → 不参与主角色解析 */
|
||||
private Map<String, Integer> rolePriority = defaultRolePriority();
|
||||
|
||||
private static Map<String, Integer> defaultRolePriority() {
|
||||
Map<String, Integer> m = new LinkedHashMap<>();
|
||||
m.put("admin", 4);
|
||||
m.put("dev", 3);
|
||||
m.put("annotator", 2);
|
||||
m.put("viewer", 1);
|
||||
return m;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class PasswordGrant {
|
||||
private boolean enabled = false;
|
||||
/** Casdoor 认证失败时是否回落本地 BCrypt(迁移期建议 true) */
|
||||
private boolean fallbackToLocal = true;
|
||||
}
|
||||
}
|
||||
-159
@@ -1,159 +0,0 @@
|
||||
package com.deepdata.gateway.infrastructure.casdoor;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.deepdata.gateway.common.config.RoleResolver;
|
||||
import com.deepdata.gateway.domain.entity.User;
|
||||
import com.deepdata.gateway.domain.repository.UserRepository;
|
||||
import com.deepdata.gateway.domain.service.UserService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* Casdoor 身份 → 本地用户 的 upsert 服务(M1 provision 的 M2 演进版)。
|
||||
*
|
||||
* <p>职责:
|
||||
* <ul>
|
||||
* <li>首次登录自动建号(auto-provision);存量用户按 Casdoor 邮箱/组刷新。</li>
|
||||
* <li>M2 组驱动角色:id_token 携带 groups 时刷新本地 {@code user_groups}
|
||||
* (users.user_groups JSON 列),角色经 {@link RoleResolver#resolveRole}
|
||||
* 解析:显式用户名映射 > 组主角色 > defaultRole。</li>
|
||||
* <li>统一签发本地 JWT:删除 M1 重复的 issueLocalJwt,直接复用
|
||||
* {@link UserService#generateToken(User)}(已改 public,同一
|
||||
* secret/expiration/claims 体系,并携带 groups claim,供网关下游鉴权)。</li>
|
||||
* </ul>
|
||||
*
|
||||
* @since 2026/8/29
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class CasdoorUserService {
|
||||
|
||||
private static final String SYSTEM_USER = "system";
|
||||
|
||||
private final UserRepository userRepository;
|
||||
private final CasdoorProperties props;
|
||||
private final RoleResolver roleResolver;
|
||||
/** 网关既有用户服务:BCrypt 校验 + 统一 JWT 签发(M2 起 generateToken 为 public) */
|
||||
private final UserService userService;
|
||||
|
||||
/**
|
||||
* 由经验证的 Casdoor 身份 upsert 本地用户并签发本地 JWT。
|
||||
*
|
||||
* @param claims 已验证的 id_token 声明(含 M2 groups)
|
||||
* @return 本地用户 + 本地 JWT + 是否新建
|
||||
*/
|
||||
public Mono<ProvisionedUser> upsertFromCasdoor(IdTokenClaims claims) {
|
||||
String username = normalizeUsername(claims.name());
|
||||
if (username == null || SYSTEM_USER.equals(username)) {
|
||||
return Mono.error(new CasdoorProvisionException(
|
||||
"invalid casdoor username: " + claims.name()));
|
||||
}
|
||||
return Mono.fromCallable(() -> {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
User user = findByUsername(username);
|
||||
if (user != null) {
|
||||
if (!user.isEnabled()) {
|
||||
throw new CasdoorProvisionException("local user disabled: " + username);
|
||||
}
|
||||
user.setLastLoginAt(now);
|
||||
if (hasText(claims.email()) && !claims.email().equals(user.getEmail())) {
|
||||
user.setEmail(claims.email());
|
||||
}
|
||||
applyGroupsAndRole(user, claims, username);
|
||||
// 存量用户走更新;IRepository.save() 恒为 INSERT,带 id 调用会撞主键
|
||||
userRepository.updateById(user);
|
||||
return new ProvisionedUser(user, userService.generateToken(user), false);
|
||||
}
|
||||
if (!props.isAutoProvision()) {
|
||||
throw new CasdoorProvisionException(
|
||||
"local user not found and auto-provision disabled: " + username);
|
||||
}
|
||||
User created = createUser(claims, username, now);
|
||||
try {
|
||||
userRepository.save(created);
|
||||
} catch (DuplicateKeyException race) {
|
||||
// 并发首登:另一请求已建号 → 复用,避免唯一键冲突
|
||||
log.info("casdoor provision race on [{}], reusing existing user", username);
|
||||
created = findByUsername(username);
|
||||
if (created == null) {
|
||||
// 冲突来自 email 唯一键(用户名不冲突)→ 显式报错,避免 NPE
|
||||
throw new CasdoorProvisionException(
|
||||
"unique-key conflict but user not found (likely email collision): " + username);
|
||||
}
|
||||
}
|
||||
log.info("casdoor auto-provisioned local user [{}] role={} groups={}",
|
||||
username, created.getRole(), created.getUserGroups());
|
||||
return new ProvisionedUser(created, userService.generateToken(created), true);
|
||||
}).subscribeOn(Schedulers.boundedElastic());
|
||||
}
|
||||
|
||||
/**
|
||||
* M2:groups claim 非空 → 刷新本地 user_groups;角色统一经
|
||||
* {@code RoleResolver.resolveRole}(映射 > 主角色 > defaultRole)。
|
||||
*/
|
||||
private void applyGroupsAndRole(User user, IdTokenClaims claims, String username) {
|
||||
if (claims.groups() != null) {
|
||||
user.setUserGroups(claims.groups());
|
||||
}
|
||||
user.setRole(roleResolver.resolveRole(username, claims.groups()));
|
||||
}
|
||||
|
||||
private User findByUsername(String username) {
|
||||
LambdaQueryWrapper<User> wrapper = new LambdaQueryWrapper<>();
|
||||
wrapper.eq(User::getUsername, username);
|
||||
return userRepository.getOne(wrapper);
|
||||
}
|
||||
|
||||
private User createUser(IdTokenClaims claims, String username, LocalDateTime now) {
|
||||
User user = new User();
|
||||
user.setUsername(username);
|
||||
// email 列 NOT NULL UNIQUE:空串会互相冲突,缺省按用户名合成占位 email
|
||||
String email = claims.email();
|
||||
user.setEmail(email == null || email.isBlank() ? username + "@casdoor.local" : email.trim());
|
||||
user.setFullName(claims.name());
|
||||
// 本地密码不可用:随机占位(登录统一走 Casdoor,本地 BCrypt 仅迁移期兜底)
|
||||
user.setPasswordHash(new BCryptPasswordEncoder(10).encode(UUID.randomUUID().toString()));
|
||||
user.setEnabled(true);
|
||||
user.setLastLoginAt(now);
|
||||
applyGroupsAndRole(user, claims, username);
|
||||
return user;
|
||||
}
|
||||
|
||||
/**
|
||||
* Casdoor 用户名 → 本地用户名:trim + 非法字符替换为 _ + 截断 64。
|
||||
* 与 Casdoor 应用 userName 规则保持一致,避免本地账号漂移。
|
||||
*/
|
||||
static String normalizeUsername(String raw) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String s = raw.trim();
|
||||
if (s.length() > 64) {
|
||||
s = s.substring(0, 64);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
private static boolean hasText(String s) {
|
||||
return s != null && !s.isBlank();
|
||||
}
|
||||
|
||||
/** upsert 结果:本地用户 + 本地 JWT(携带角色/组 claims)+ 是否新建 */
|
||||
public record ProvisionedUser(User user, String localToken, boolean created) {}
|
||||
|
||||
/** 预置失败:用户名非法 / 本地用户禁用 / 未开启自动建号 */
|
||||
public static class CasdoorProvisionException extends RuntimeException {
|
||||
public CasdoorProvisionException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package com.deepdata.gateway.infrastructure.casdoor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 经验证的 Casdoor id_token 身份声明。
|
||||
*
|
||||
* <p>身份源统一取 id_token(实测:code 流程的 access_token 只含 iss/sub/aud/exp/iat,
|
||||
* 而两条流程的 id_token 都稳定携带 sub/name/email)。
|
||||
*
|
||||
* @param sub Casdoor 用户 UUID(稳定外部 ID,建议落库为 casdoor_sub)
|
||||
* @param name Casdoor 用户名(= 登录名)
|
||||
* @param email 邮箱
|
||||
* @param exp 过期时间(秒)
|
||||
* @param iat 签发时间(秒)
|
||||
* @param claims 完整声明(调试/扩展用)
|
||||
* @param groups M2:组列表(app.tokenFields=["Groups"] 时由 groups claim 提取,
|
||||
* {@code groups-claim} 可配置;无组时可能为 null)
|
||||
*/
|
||||
public record IdTokenClaims(
|
||||
String sub,
|
||||
String name,
|
||||
String email,
|
||||
long exp,
|
||||
long iat,
|
||||
Map<String, Object> claims,
|
||||
List<String> groups) {
|
||||
}
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
package com.deepdata.gateway.infrastructure.casdoor.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* Casdoor token 端点响应(code 交换 / password grant 通用)。
|
||||
*
|
||||
* <p>实测返回(2026-08-29):
|
||||
* <pre>{"access_token":"...","id_token":"...","refresh_token":"...",
|
||||
* "token_type":"Bearer","expires_in":3600,"scope":"openid"}</pre>
|
||||
*
|
||||
* @since 2026/8/29
|
||||
*/
|
||||
@Data
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class CasdoorTokenResponse {
|
||||
|
||||
@JsonProperty("access_token")
|
||||
private String accessToken;
|
||||
|
||||
/** OIDC 身份源:RS256 签名,含 sub / name / email,kid=cert-built-in */
|
||||
@JsonProperty("id_token")
|
||||
private String idToken;
|
||||
|
||||
@JsonProperty("refresh_token")
|
||||
private String refreshToken;
|
||||
|
||||
@JsonProperty("token_type")
|
||||
private String tokenType;
|
||||
|
||||
@JsonProperty("expires_in")
|
||||
private Long expiresIn;
|
||||
|
||||
private String scope;
|
||||
}
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
package com.deepdata.gateway.infrastructure.casdoor.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* JWKS 响应({@code GET /.well-known/jwks.json},公开无鉴权)。
|
||||
*
|
||||
* <p>实测结构(2026-08-30,M2 修正为 .json 端点):
|
||||
* <pre>{"keys":[{"use":"sig","kty":"RSA","kid":"cert-built-in","alg":"RS256",
|
||||
* "n":"...base64url...","e":"AQAB","x5c":["...PEM base64..."]}]}</pre>
|
||||
*
|
||||
* @since 2026/8/29
|
||||
*/
|
||||
@Data
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class JwksResponse {
|
||||
|
||||
private List<Jwk> keys;
|
||||
|
||||
@Data
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public static class Jwk {
|
||||
private String use;
|
||||
private String kty;
|
||||
private String kid;
|
||||
private String alg;
|
||||
/** RSA 模数,base64url(无 padding) */
|
||||
private String n;
|
||||
/** RSA 公钥指数,base64url(常见 "AQAB" = 65537) */
|
||||
private String e;
|
||||
/** X.509 证书链(base64 DER),可选 */
|
||||
private List<String> x5c;
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.deepdata.gateway.interfaces.dto;
|
||||
|
||||
import lombok.Data;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
|
||||
@Data
|
||||
public class ChangePasswordRequest {
|
||||
@NotBlank(message = "旧密码不能为空")
|
||||
private String oldPassword;
|
||||
|
||||
@NotBlank(message = "新密码不能为空")
|
||||
@Size(min = 6, message = "密码长度不能少于6位")
|
||||
private String newPassword;
|
||||
}
|
||||
-13
@@ -5,8 +5,6 @@ import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 用户信息响应
|
||||
*
|
||||
@@ -49,15 +47,4 @@ public class UserResponse {
|
||||
* 是否强制要求登录(由 deepdata.jwt.enable 控制)
|
||||
*/
|
||||
private Boolean requireLogin;
|
||||
|
||||
/**
|
||||
* 主角色(M2 组驱动目录菜单体系):Casdoor 组解析结果 / 存量 users.role=ADMIN
|
||||
* 兼容映射;null 时前端按 viewer-minimal 兜底展示,不 403
|
||||
*/
|
||||
private String role;
|
||||
|
||||
/**
|
||||
* Casdoor 组列表(users.user_groups JSON 列,M2),本地注册用户为 null
|
||||
*/
|
||||
private List<String> groups;
|
||||
}
|
||||
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
package com.deepdata.gateway.interfaces.rest;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.IgnoreResponseWrap;
|
||||
import com.deepdata.gateway.infrastructure.casdoor.CasdoorClient;
|
||||
import com.deepdata.gateway.infrastructure.casdoor.CasdoorIdTokenVerifier;
|
||||
import com.deepdata.gateway.infrastructure.casdoor.CasdoorProperties;
|
||||
import com.deepdata.gateway.infrastructure.casdoor.CasdoorUserService;
|
||||
import com.deepdata.gateway.infrastructure.casdoor.IdTokenClaims;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.http.HttpCookie;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseCookie;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ServerWebExchange;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.SecureRandom;
|
||||
import java.time.Duration;
|
||||
import java.util.Base64;
|
||||
|
||||
/**
|
||||
* Casdoor OIDC 端点(Flow A:授权码 SSO,浏览器入口)。
|
||||
*
|
||||
* <p>端点(需加入 {@code AuthFilter} 公开放行清单,见 integration/README.md):
|
||||
* <ul>
|
||||
* <li>{@code GET /api/user/oauth/login?next=/dashboard}
|
||||
* —— 生成一次性 state(存 HttpOnly cookie)→ 302 到 Casdoor authorize</li>
|
||||
* <li>{@code GET /api/user/oauth/callback?code&state}
|
||||
* —— 校验 state → 换 token → JWKS 验签 id_token → 落地本地用户 →
|
||||
* 签本地 HS256 JWT → 302 回前端 {@code next?sso_token=...}</li>
|
||||
* <li>{@code GET /api/user/oauth/logout?next=/}
|
||||
* —— 302 回前端 {@code next?sso_logout=1}(本地 token 存 localStorage,由前端清除)</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>CSRF:state 为一次性随机串,与 next 一起存 HttpOnly cookie(TTL 默认 300s,
|
||||
* 覆盖 code 有效期实测约 200s 并留余量),回调恒定时间比较后删除。
|
||||
*
|
||||
* <p>M2(组驱动目录菜单体系):回调落地走 {@link CasdoorUserService#upsertFromCasdoor},
|
||||
* id_token 的 groups claim 刷新本地 {@code user_groups},角色按
|
||||
* 映射 > 组主角色 > defaultRole 解析(见 {@code RoleResolver}),本地 JWT
|
||||
* 统一由 {@code UserService.generateToken} 签发(public,携带 groups claim)。
|
||||
*
|
||||
* <p>仅当 {@code deepdata.casdoor.enabled=true} 时注册;关闭时行为与接入前完全一致
|
||||
* (配置开关见 {@code /sso/casdoor/config})。
|
||||
*
|
||||
* @since 2026/8/29
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/user/oauth")
|
||||
@RequiredArgsConstructor
|
||||
@IgnoreResponseWrap
|
||||
@ConditionalOnProperty(prefix = "deepdata.casdoor", name = "enabled", havingValue = "true")
|
||||
public class CasdoorOauthController {
|
||||
|
||||
private static final SecureRandom RANDOM = new SecureRandom();
|
||||
private static final Base64.Encoder B64URL = Base64.getUrlEncoder().withoutPadding();
|
||||
private static final Base64.Decoder B64URL_DEC = Base64.getUrlDecoder();
|
||||
|
||||
private final CasdoorProperties props;
|
||||
private final CasdoorClient casdoorClient;
|
||||
private final CasdoorIdTokenVerifier idTokenVerifier;
|
||||
private final CasdoorUserService casdoorUserService;
|
||||
|
||||
/** SSO 入口:浏览器跳到这里发起 Casdoor 登录 */
|
||||
@GetMapping("/login")
|
||||
public Mono<ResponseEntity<Void>> login(@RequestParam(name = "next", required = false) String next,
|
||||
ServerWebExchange exchange) {
|
||||
String safeNext = sanitizeNext(next);
|
||||
String state = B64URL.encodeToString(randomBytes(32));
|
||||
String cookieValue = state + "." + B64URL.encodeToString(safeNext.getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
exchange.getResponse().addCookie(
|
||||
stateCookie(cookieValue, Duration.ofSeconds(props.getStateTtlSeconds())));
|
||||
String authorizeUrl = casdoorClient.buildAuthorizeUrl(state);
|
||||
log.info("casdoor sso login: next={}", safeNext);
|
||||
return Mono.just(redirect(URI.create(authorizeUrl)));
|
||||
}
|
||||
|
||||
/** Casdoor 回跳:code 换 token → 验签 → 落地(M2: 组驱动角色)→ 302 回前端带 sso_token */
|
||||
@GetMapping("/callback")
|
||||
public Mono<ResponseEntity<Void>> callback(@RequestParam(name = "code", required = false) String code,
|
||||
@RequestParam(name = "state", required = false) String state,
|
||||
ServerWebExchange exchange) {
|
||||
String cookieValue = readCookie(exchange, props.getStateCookieName());
|
||||
// state cookie 一次性:无论成败,先删
|
||||
exchange.getResponse().addCookie(stateCookie(null, Duration.ZERO));
|
||||
|
||||
String safeNext;
|
||||
if (cookieValue == null || !cookieValue.contains(".") || state == null) {
|
||||
log.warn("casdoor sso callback: missing state cookie or state param");
|
||||
return errorRedirect(null, "state_missing");
|
||||
}
|
||||
int dot = cookieValue.indexOf('.');
|
||||
String expectedState = cookieValue.substring(0, dot);
|
||||
safeNext = sanitizeNext(decodeNext(cookieValue.substring(dot + 1)));
|
||||
|
||||
if (!MessageDigest.isEqual(
|
||||
expectedState.getBytes(StandardCharsets.UTF_8),
|
||||
state.getBytes(StandardCharsets.UTF_8))) {
|
||||
log.warn("casdoor sso callback: state mismatch (csrf guard tripped)");
|
||||
return errorRedirect(safeNext, "state_mismatch");
|
||||
}
|
||||
if (code == null || code.isBlank()) {
|
||||
return errorRedirect(safeNext, "code_missing");
|
||||
}
|
||||
|
||||
return casdoorClient.exchangeCode(code)
|
||||
.flatMap(resp -> {
|
||||
if (resp.getIdToken() == null || resp.getIdToken().isBlank()) {
|
||||
return Mono.error(new CasdoorIdTokenVerifier.CasdoorIdTokenException(
|
||||
"token response without id_token"));
|
||||
}
|
||||
return idTokenVerifier.verify(resp.getIdToken());
|
||||
})
|
||||
.flatMap(claims -> casdoorUserService.upsertFromCasdoor(claims))
|
||||
.map(provisioned -> {
|
||||
log.info("casdoor sso ok: username={}, role={}, created={}",
|
||||
provisioned.user().getUsername(),
|
||||
provisioned.user().getRole(),
|
||||
provisioned.created());
|
||||
return redirect(appendParam(safeNext, "sso_token", provisioned.localToken()));
|
||||
})
|
||||
.onErrorResume(e -> {
|
||||
log.error("casdoor sso callback failed: {}", e.getMessage(), e);
|
||||
return errorRedirect(safeNext, classifyError(e));
|
||||
});
|
||||
}
|
||||
|
||||
/** 退出:本地 token 在浏览器 localStorage,由前端清;这里只做回跳标记 */
|
||||
@GetMapping("/logout")
|
||||
public Mono<ResponseEntity<Void>> logout(@RequestParam(name = "next", required = false) String next) {
|
||||
return Mono.just(redirect(appendParam(sanitizeNext(next), "sso_logout", "1")));
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
private Mono<ResponseEntity<Void>> errorRedirect(String next, String reason) {
|
||||
return Mono.just(redirect(appendParam(sanitizeNext(next), "sso_error", reason)));
|
||||
}
|
||||
|
||||
private static String classifyError(Throwable e) {
|
||||
if (e instanceof CasdoorClient.CasdoorTokenException) {
|
||||
return "token_exchange";
|
||||
}
|
||||
if (e instanceof CasdoorIdTokenVerifier.CasdoorIdTokenException) {
|
||||
return "id_token_verify";
|
||||
}
|
||||
if (e instanceof CasdoorUserService.CasdoorProvisionException) {
|
||||
return "provision";
|
||||
}
|
||||
return "internal";
|
||||
}
|
||||
|
||||
private static ResponseEntity<Void> redirect(URI location) {
|
||||
return ResponseEntity.status(HttpStatus.FOUND).location(location).build();
|
||||
}
|
||||
|
||||
/** next 仅允许同源相对路径(/ 开头且非 //),防开放重定向 */
|
||||
static String sanitizeNext(String next) {
|
||||
if (next == null || next.isBlank()) {
|
||||
return "/";
|
||||
}
|
||||
if (!next.startsWith("/") || next.startsWith("//")
|
||||
|| next.length() > 1024
|
||||
|| next.indexOf('\t') >= 0 || next.indexOf('\n') >= 0) {
|
||||
return "/";
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
static URI appendParam(String next, String name, String value) {
|
||||
String sep = next.contains("?") ? "&" : "?";
|
||||
return URI.create(next + sep + name + "=" + URLEncoder.encode(value, StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
private static String decodeNext(String encoded) {
|
||||
try {
|
||||
return new String(B64URL_DEC.decode(encoded), StandardCharsets.UTF_8);
|
||||
} catch (IllegalArgumentException e) {
|
||||
return "/";
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] randomBytes(int n) {
|
||||
byte[] b = new byte[n];
|
||||
RANDOM.nextBytes(b);
|
||||
return b;
|
||||
}
|
||||
|
||||
private String readCookie(ServerWebExchange exchange, String name) {
|
||||
HttpCookie cookie = exchange.getRequest().getCookies().getFirst(name);
|
||||
return cookie == null ? null : cookie.getValue();
|
||||
}
|
||||
|
||||
private ResponseCookie stateCookie(String value, Duration maxAge) {
|
||||
ResponseCookie.ResponseCookieBuilder builder = ResponseCookie.from(props.getStateCookieName());
|
||||
if (value != null) {
|
||||
builder.value(value);
|
||||
}
|
||||
return builder
|
||||
.path("/")
|
||||
.maxAge(maxAge)
|
||||
.httpOnly(true)
|
||||
.secure(props.isStateCookieSecure())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
package com.deepdata.gateway.interfaces.rest;
|
||||
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.gateway.infrastructure.casdoor.CasdoorProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Casdoor SSO 前端配置端点(M2)。
|
||||
*
|
||||
* <p>前端(登录页 / 菜单壳)通过 {@code GET /sso/casdoor/config} 获取 SSO 开关与登录/登出入口,
|
||||
* 无需感知后端细节。始终注册(不走 {@code @ConditionalOnProperty}):enabled=false 时
|
||||
* {@code enabled:false},前端自然隐藏 SSO 入口 —— 这是与 OauthController
|
||||
* 「关了就 404」的差异点,接入文档已注明。
|
||||
*
|
||||
* <p>敏感字段(clientSecret 等)绝不返回给浏览器。
|
||||
*
|
||||
* @since 2026/8/30
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/sso/casdoor")
|
||||
@RequiredArgsConstructor
|
||||
public class CasdoorSsoConfigController {
|
||||
|
||||
private final CasdoorProperties props;
|
||||
|
||||
@GetMapping("/config")
|
||||
public Response<Map<String, Object>> config() {
|
||||
Map<String, Object> cfg = new LinkedHashMap<>();
|
||||
cfg.put("enabled", props.isEnabled());
|
||||
cfg.put("loginUrl", "/api/user/oauth/login");
|
||||
cfg.put("logoutUrl", "/api/user/oauth/logout");
|
||||
cfg.put("issuer", props.getIssuer());
|
||||
cfg.put("clientId", props.getClientId());
|
||||
return Response.ok(cfg);
|
||||
}
|
||||
}
|
||||
+38
-31
@@ -4,7 +4,6 @@ import com.deepdata.common.infrastructure.common.IgnoreResponseWrap;
|
||||
import com.deepdata.common.infrastructure.common.Response;
|
||||
import com.deepdata.common.infrastructure.exception.CommonErrorCode;
|
||||
import com.deepdata.gateway.application.UserApplicationService;
|
||||
import com.deepdata.gateway.common.config.RoleResolver;
|
||||
import com.deepdata.gateway.domain.service.UserService;
|
||||
import com.deepdata.gateway.infrastructure.client.OmsExtensionService;
|
||||
import com.deepdata.gateway.infrastructure.client.OmsService;
|
||||
@@ -15,6 +14,9 @@ import com.deepdata.gateway.interfaces.dto.UserResponse;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import com.deepdata.gateway.interfaces.dto.ChangePasswordRequest;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpStatus;
|
||||
@@ -44,8 +46,6 @@ public class UserController {
|
||||
private final UserService userService;
|
||||
private final OmsService omsService;
|
||||
private final OmsExtensionService omsExtensionService;
|
||||
/** M2:Casdoor 组 → 主角色 解析 */
|
||||
private final RoleResolver roleResolver;
|
||||
|
||||
@Value("${deepdata.jwt.enable:false}")
|
||||
private Boolean jwtEnable;
|
||||
@@ -146,7 +146,13 @@ public class UserController {
|
||||
log.warn("Failed to get user group ID: {}", e.getMessage());
|
||||
}
|
||||
|
||||
return Response.ok(authenticatedResponse(username, "SSO"));
|
||||
return Response.ok(UserResponse.builder()
|
||||
.username(username)
|
||||
.groupId(groupId)
|
||||
.authenticated(true)
|
||||
.authMode("SSO")
|
||||
.requireLogin(true) // SSO 模式始终要求登录
|
||||
.build());
|
||||
} else {
|
||||
log.warn("OMS service returned null username");
|
||||
}
|
||||
@@ -163,7 +169,12 @@ public class UserController {
|
||||
|
||||
if (StringUtils.isNotBlank(username)) {
|
||||
log.info("JWT authentication successful: user={}", username);
|
||||
return Response.ok(authenticatedResponse(username, "JWT"));
|
||||
return Response.ok(UserResponse.builder()
|
||||
.username(username)
|
||||
.authenticated(true)
|
||||
.authMode("JWT")
|
||||
.requireLogin(true) // 已登录
|
||||
.build());
|
||||
} else {
|
||||
log.warn("JWT token validation failed");
|
||||
}
|
||||
@@ -181,34 +192,30 @@ public class UserController {
|
||||
}
|
||||
|
||||
/**
|
||||
* 组装已认证响应(M2 组驱动目录菜单体系):本地库查用户 → 主角色 + 组列表。
|
||||
*
|
||||
* <p>角色口径(与 Casdoor 落地端一致):
|
||||
* <ul>
|
||||
* <li>优先 {@link RoleResolver#primaryRole}:Casdoor 组按 rolePriority
|
||||
* 取权重最高者(如 deepdata-admin → admin);</li>
|
||||
* <li>存量兼容:本地 users.role=ADMIN(大小写不敏感)→ "admin";其余值不映射,
|
||||
* 前端按 viewer-minimal 兜底展示(不 403);</li>
|
||||
* <li>groups 原样带回,供前端精确渲染组内功能。</li>
|
||||
* </ul>
|
||||
* 修改密码接口
|
||||
* 🔴 修改:使用 @PutMapping 以匹配前端请求
|
||||
*/
|
||||
private UserResponse authenticatedResponse(String username, String authMode) {
|
||||
UserResponse.UserResponseBuilder b = UserResponse.builder()
|
||||
.username(username)
|
||||
.authenticated(true)
|
||||
.authMode(authMode)
|
||||
.requireLogin(true);
|
||||
userService.findUserByUsername(username).ifPresent(u -> {
|
||||
if (u.getUserGroups() != null) {
|
||||
b.groups(u.getUserGroups());
|
||||
@PutMapping("/change-password")
|
||||
public Response<String> changePassword(
|
||||
@Valid @RequestBody ChangePasswordRequest request,
|
||||
ServerHttpRequest httpRequest) {
|
||||
String authHeader = httpRequest.getHeaders().getFirst("Authorization");
|
||||
if (authHeader != null && authHeader.startsWith("Bearer ")) {
|
||||
String token = authHeader.substring(7);
|
||||
// 从 Token 中解析出当前用户名
|
||||
String username = userService.validateToken(token);
|
||||
|
||||
if (StringUtils.isNotBlank(username)) {
|
||||
// 调用业务逻辑
|
||||
boolean success = userApplicationService.changePassword(username, request);
|
||||
if (success) {
|
||||
return Response.ok("success");
|
||||
} else {
|
||||
// 旧密码错误返回 401 状态码
|
||||
return Response.error(CommonErrorCode.UNAUTHORIZED);
|
||||
}
|
||||
String role = roleResolver.primaryRole(u.getUserGroups());
|
||||
if (role == null && StringUtils.isNotBlank(u.getRole())
|
||||
&& "ADMIN".equalsIgnoreCase(u.getRole())) {
|
||||
role = "admin";
|
||||
}
|
||||
b.role(role);
|
||||
});
|
||||
return b.build();
|
||||
}
|
||||
return Response.error(CommonErrorCode.UNAUTHORIZED);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ spring:
|
||||
deepdata:
|
||||
jwt:
|
||||
secret: ${JWT_SECRET}
|
||||
expiration-seconds: 3600
|
||||
expiration-seconds: 604800
|
||||
oms:
|
||||
auth:
|
||||
enabled: ${OMS_AUTH_ENABLED:false}
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
deepdata:
|
||||
DeepData:
|
||||
datamanagement:
|
||||
file-storage:
|
||||
upload-dir: ${FILE_UPLOAD_DIR:./uploads}
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
<artifactId>jakarta.persistence-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- 共享库依赖 -->
|
||||
<!-- 共享库��-->
|
||||
<dependency>
|
||||
<groupId>com.deepdata</groupId>
|
||||
<artifactId>domain-common</artifactId>
|
||||
|
||||
@@ -28,18 +28,18 @@
|
||||
<dependency>
|
||||
<groupId>io.jsonwebtoken</groupId>
|
||||
<artifactId>jjwt-api</artifactId>
|
||||
<version>0.12.6</version>
|
||||
<version>0.11.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.jsonwebtoken</groupId>
|
||||
<artifactId>jjwt-impl</artifactId>
|
||||
<version>0.12.6</version>
|
||||
<version>0.11.5</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.jsonwebtoken</groupId>
|
||||
<artifactId>jjwt-jackson</artifactId>
|
||||
<version>0.12.6</version>
|
||||
<version>0.11.5</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
+13
-13
@@ -37,11 +37,11 @@ public class JwtUtils {
|
||||
tokenClaims.put("sub", username);
|
||||
|
||||
return Jwts.builder()
|
||||
.claims(tokenClaims)
|
||||
.subject(username)
|
||||
.issuedAt(new Date())
|
||||
.expiration(new Date(System.currentTimeMillis() + expiration * 1000))
|
||||
.signWith(getSigningKey())
|
||||
.setClaims(tokenClaims)
|
||||
.setSubject(username)
|
||||
.setIssuedAt(new Date())
|
||||
.setExpiration(new Date(System.currentTimeMillis() + expiration * 1000))
|
||||
.signWith(getSigningKey(), SignatureAlgorithm.HS512)
|
||||
.compact();
|
||||
}
|
||||
|
||||
@@ -63,11 +63,11 @@ public class JwtUtils {
|
||||
* 从令牌中获取声明
|
||||
*/
|
||||
public Claims getClaimsFromToken(String token) {
|
||||
return Jwts.parser()
|
||||
.verifyWith(getSigningKey())
|
||||
return Jwts.parserBuilder()
|
||||
.setSigningKey(getSigningKey())
|
||||
.build()
|
||||
.parseSignedClaims(token)
|
||||
.getPayload();
|
||||
.parseClaimsJws(token)
|
||||
.getBody();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -95,12 +95,12 @@ public class JwtUtils {
|
||||
*/
|
||||
public String refreshToken(String token) {
|
||||
Claims claims = getClaimsFromToken(token);
|
||||
claims.setIssuedAt(new Date());
|
||||
claims.setExpiration(new Date(System.currentTimeMillis() + expiration * 1000));
|
||||
|
||||
return Jwts.builder()
|
||||
.claims(claims)
|
||||
.issuedAt(new Date())
|
||||
.expiration(new Date(System.currentTimeMillis() + expiration * 1000))
|
||||
.signWith(getSigningKey())
|
||||
.setClaims(claims)
|
||||
.signWith(getSigningKey(), SignatureAlgorithm.HS512)
|
||||
.compact();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,11 +14,11 @@ static/browser_history/*.gif
|
||||
.venv
|
||||
venv/
|
||||
|
||||
# Environment defaults are tracked for a self-contained repo
|
||||
# .env
|
||||
# Environment variables
|
||||
.env
|
||||
|
||||
# Product conf is tracked
|
||||
# conf.yaml
|
||||
# user conf
|
||||
conf.yaml
|
||||
|
||||
.idea/
|
||||
.langgraph_api/
|
||||
@@ -95,8 +95,8 @@ make build # Build Docker image
|
||||
docker compose up # Run with Docker Compose
|
||||
|
||||
# Production deployment
|
||||
docker build -t deepflow-api .
|
||||
docker run -p 8000:8000 deepflow-api
|
||||
docker build -t deep-flow-api .
|
||||
docker run -p 8000:8000 deep-flow-api
|
||||
```
|
||||
|
||||
### Fix GitHub issues
|
||||
@@ -1,10 +1,10 @@
|
||||
# Contributing to DeepFlow
|
||||
# Contributing to DeerFlow
|
||||
|
||||
Thank you for your interest in contributing to DeepFlow! We welcome contributions of all kinds from the community.
|
||||
Thank you for your interest in contributing to DeerFlow! We welcome contributions of all kinds from the community.
|
||||
|
||||
## Ways to Contribute
|
||||
|
||||
There are many ways you can contribute to DeepFlow:
|
||||
There are many ways you can contribute to DeerFlow:
|
||||
|
||||
- **Code Contributions**: Add new features, fix bugs, or improve performance
|
||||
- **Documentation**: Improve README, add code comments, or create examples
|
||||
@@ -19,8 +19,8 @@ There are many ways you can contribute to DeepFlow:
|
||||
2. Clone your fork:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bytedance/deepflow.git
|
||||
cd deepflow
|
||||
git clone https://github.com/bytedance/deep-flow.git
|
||||
cd deep-flow
|
||||
```
|
||||
|
||||
3. Set up your development environment:
|
||||
@@ -139,6 +139,6 @@ If you need help with anything:
|
||||
|
||||
## License
|
||||
|
||||
By contributing to DeepFlow, you agree that your contributions will be licensed under the MIT License.
|
||||
By contributing to DeerFlow, you agree that your contributions will be licensed under the MIT License.
|
||||
|
||||
We appreciate your contributions to making DeepFlow better!
|
||||
We appreciate your contributions to making DeerFlow better!
|
||||
@@ -1,7 +1,7 @@
|
||||
.PHONY: help lint format install-dev serve test coverage langgraph-dev lint-frontend
|
||||
|
||||
help: ## Show this help message
|
||||
@echo "Deer Flow - Available Make Targets:"
|
||||
@echo "Deep Flow - Available Make Targets:"
|
||||
@echo ""
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}'
|
||||
@echo ""
|
||||
@@ -1,14 +1,21 @@
|
||||
# DeepFlow
|
||||
# 🦌 DeepFlow
|
||||
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[]()
|
||||
|
||||
<!-- DeepWiki badge generated by https://deepwiki.ryoppippi.com/ -->
|
||||
|
||||
[English](./README.md) | [简体中文](./README_zh.md) | [日本語](./README_ja.md) | [Deutsch](./README_de.md) | [Español](./README_es.md) | [Русский](./README_ru.md) | [Portuguese](./README_pt.md)
|
||||
|
||||
> Originated from Open Source, give back to Open Source.
|
||||
|
||||
**DeepFlow** (**D**eep **E**xploration and **E**fficient **R**esearch **Flow**) is a community-driven Deep Research framework that builds upon the incredible work of the open source community. Our goal is to combine language models with specialized tools for tasks like web search, crawling, and Python code execution, while giving back to the community that made this possible.
|
||||
|
||||
Currently, DeepFlow has officially entered the [FaaS Application Center of Volcengine](https://console.volcengine.com/vefaas/region:vefaas+cn-beijing/market). Users can experience it online through the [experience link](https://console.volcengine.com/vefaas/region:vefaas+cn-beijing/market/DeepFlow/?channel=github&source=DeepFlow) to intuitively feel its powerful functions and convenient operations. At the same time, to meet the deployment needs of different users, DeepFlow supports one-click deployment based on Volcengine. Click the [deployment link](https://console.volcengine.com/vefaas/region:vefaas+cn-beijing/application/create?templateId=683adf9e372daa0008aaed5c&channel=github&source=DeepFlow) to quickly complete the deployment process and start an efficient research journey.
|
||||
|
||||
|
||||
[English](./README.md) | [简体中文](./README_zh.md)
|
||||
|
||||
> **DeepFlow is the intelligent deep-research module of the deepdata platform**, built on the LangGraph multi-agent architecture. It combines language models with specialized tools — web search, crawling, Python code execution, MCP services — to run end-to-end research workflows and produce reports, podcasts and more.
|
||||
|
||||
DeepFlow is delivered as a first-class component of the **deepdata** all-in-one data work platform (see the [repository README](../README.md)).
|
||||
Please visit [our official website](https://DeepFlow.tech/) for more details.
|
||||
|
||||
## Demo
|
||||
|
||||
@@ -24,11 +31,11 @@ In this demo, we showcase how to use DeepFlow to:
|
||||
|
||||
### Replays
|
||||
|
||||
- [How tall is Eiffel Tower compared to tallest building?](https://deepdata.ailab.yilise.com/chat?replay=eiffel-tower-vs-tallest-building)
|
||||
- [What are the top trending repositories on GitHub?](https://deepdata.ailab.yilise.com/chat?replay=github-top-trending-repo)
|
||||
- [Write an article about Nanjing's traditional dishes](https://deepdata.ailab.yilise.com/chat?replay=nanjing-traditional-dishes)
|
||||
- [How to decorate a rental apartment?](https://deepdata.ailab.yilise.com/chat?replay=rental-apartment-decoration)
|
||||
- [Visit our official website to explore more replays.](https://deepdata.ailab.yilise.com/#case-studies)
|
||||
- [How tall is Eiffel Tower compared to tallest building?](https://DeepFlow.tech/chat?replay=eiffel-tower-vs-tallest-building)
|
||||
- [What are the top trending repositories on GitHub?](https://DeepFlow.tech/chat?replay=github-top-trending-repo)
|
||||
- [Write an article about Nanjing's traditional dishes](https://DeepFlow.tech/chat?replay=nanjing-traditional-dishes)
|
||||
- [How to decorate a rental apartment?](https://DeepFlow.tech/chat?replay=rental-apartment-decoration)
|
||||
- [Visit our official website to explore more replays.](https://DeepFlow.tech/#case-studies)
|
||||
|
||||
---
|
||||
|
||||
@@ -44,6 +51,7 @@ In this demo, we showcase how to use DeepFlow to:
|
||||
- [❓ FAQ](#faq)
|
||||
- [📜 License](#license)
|
||||
- [💖 Acknowledgments](#acknowledgments)
|
||||
- [⭐ Star History](#star-history)
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -70,8 +78,9 @@ Make sure your system meets the following minimum requirements:
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
# The deepflow module ships inside the deepdata repository
|
||||
cd deepflow
|
||||
# Clone the repository
|
||||
git clone
|
||||
cd deep-flow
|
||||
|
||||
# Install dependencies, uv will take care of the python interpreter and venv creation, and install the required packages
|
||||
uv sync
|
||||
@@ -97,7 +106,7 @@ brew install marp-cli
|
||||
Optionally, install web UI dependencies via [pnpm](https://pnpm.io/installation):
|
||||
|
||||
```bash
|
||||
cd deepflow/web
|
||||
cd deep-flow/web
|
||||
pnpm install
|
||||
```
|
||||
|
||||
@@ -258,7 +267,7 @@ DeepFlow implements a modular multi-agent system architecture designed for autom
|
||||
|
||||

|
||||
|
||||
> See it live at [the official website](https://deepdata.ailab.yilise.com/#multi-agent-architecture)
|
||||
> See it live at [DeepFlow.tech](https://DeepFlow.tech/#multi-agent-architecture)
|
||||
|
||||
The system employs a streamlined workflow with the following components:
|
||||
|
||||
@@ -456,18 +465,18 @@ First, you need read the [configuration](docs/configuration_guide.md) below. Mak
|
||||
Second, to build a Docker image of your own web server:
|
||||
|
||||
```bash
|
||||
docker build -t deepflow-api .
|
||||
docker build -t deep-flow-api .
|
||||
```
|
||||
|
||||
Final, start up a docker container running the web server:
|
||||
|
||||
```bash
|
||||
# Replace deepflow-api-app with your preferred container name
|
||||
# Replace deep-flow-api-app with your preferred container name
|
||||
# Start the server then bind to localhost:8000
|
||||
docker run -d -t -p 127.0.0.1:8000:8000 --env-file .env --name deepflow-api-app deepflow-api
|
||||
docker run -d -t -p 127.0.0.1:8000:8000 --env-file .env --name deep-flow-api-app deep-flow-api
|
||||
|
||||
# stop the server
|
||||
docker stop deepflow-api-app
|
||||
docker stop feep-flow-api-app
|
||||
```
|
||||
|
||||
### Docker Compose (include both backend and frontend)
|
||||
@@ -483,7 +492,7 @@ docker compose up
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> If you want to deploy the deer flow into production environments, please add authentication to the website and evaluate your security check of the MCPServer and Python Repl.
|
||||
> If you want to deploy the deep flow into production environments, please add authentication to the website and evaluate your security check of the MCPServer and Python Repl.
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -629,4 +638,15 @@ We would like to extend our sincere appreciation to the following projects for t
|
||||
|
||||
These projects exemplify the transformative power of open-source collaboration, and we are proud to build upon their foundations.
|
||||
|
||||
### Key Contributors
|
||||
|
||||
A heartfelt thank you goes out to the core authors of `DeepFlow`, whose vision, passion, and dedication have brought this project to life:
|
||||
|
||||
- **[Daniel Walnut](https://github.com/hetaoBackend/)**
|
||||
- **[Henry Li](https://github.com/magiccube/)**
|
||||
|
||||
Your unwavering commitment and expertise have been the driving force behind DeepFlow's success. We are honored to have you at the helm of this journey.
|
||||
|
||||
## Star History
|
||||
|
||||
[![Star History Chart]()
|
||||
@@ -1,13 +1,17 @@
|
||||
# DeepFlow
|
||||
# 🦌 DeepFlow
|
||||
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
|
||||
[English](./README.md) | [简体中文](./README_zh.md)
|
||||
[English](./README.md) | [简体中文](./README_zh.md) | [日本語](./README_ja.md) | [Deutsch](./README_de.md) | [Español](./README_es.md) | [Русский](./README_ru.md) |[Portuguese](./README_pt.md)
|
||||
|
||||
> **DeepFlow 是 deepdata 平台的智能深研模块**,基于 LangGraph 多智能体架构构建。它将语言模型与网络搜索、爬虫、Python 代码执行、MCP 服务等专业工具相结合,运行端到端的研究流程,生成报告、播客等多种成果。
|
||||
> 源于开源,回馈开源。
|
||||
|
||||
DeepFlow 作为 **deepdata** 一站式数据工作平台的一等组件交付(见[仓库 README](../README.md))。
|
||||
**DeepFlow**(**D**eep **E**xploration and **E**fficient **R**esearch **Flow**)是一个社区驱动的深度研究框架,它建立在开源社区的杰出工作基础之上。我们的目标是将语言模型与专业工具(如网络搜索、爬虫和 Python 代码执行)相结合,同时回馈使这一切成为可能的社区。
|
||||
|
||||
目前,DeepFlow 已正式入驻[火山引擎的 FaaS 应用中心](https://console.volcengine.com/vefaas/region:vefaas+cn-beijing/market),用户可通过[体验链接](https://console.volcengine.com/vefaas/region:vefaas+cn-beijing/market/DeepFlow/?channel=github&source=DeepFlow)进行在线体验,直观感受其强大功能与便捷操作;同时,为满足不同用户的部署需求,DeepFlow 支持基于火山引擎一键部署,点击[部署链接](https://console.volcengine.com/vefaas/region:vefaas+cn-beijing/application/create?templateId=683adf9e372daa0008aaed5c&channel=github&source=DeepFlow)即可快速完成部署流程,开启高效研究之旅。
|
||||
|
||||
请访问[DeepFlow 的官方网站](https://DeepFlow.tech/)了解更多详情。
|
||||
|
||||
## 演示
|
||||
|
||||
@@ -23,11 +27,11 @@ DeepFlow 作为 **deepdata** 一站式数据工作平台的一等组件交付(
|
||||
|
||||
### 回放示例
|
||||
|
||||
- [埃菲尔铁塔与最高建筑相比有多高?](https://deepdata.ailab.yilise.com/chat?replay=eiffel-tower-vs-tallest-building)
|
||||
- [GitHub 上最热门的仓库有哪些?](https://deepdata.ailab.yilise.com/chat?replay=github-top-trending-repo)
|
||||
- [撰写关于南京传统美食的文章](https://deepdata.ailab.yilise.com/chat?replay=nanjing-traditional-dishes)
|
||||
- [如何装饰租赁公寓?](https://deepdata.ailab.yilise.com/chat?replay=rental-apartment-decoration)
|
||||
- [访问我们的官方网站探索更多回放示例。](https://deepdata.ailab.yilise.com/#case-studies)
|
||||
- [埃菲尔铁塔与最高建筑相比有多高?](https://DeepFlow.tech/chat?replay=eiffel-tower-vs-tallest-building)
|
||||
- [GitHub 上最热门的仓库有哪些?](https://DeepFlow.tech/chat?replay=github-top-trending-repo)
|
||||
- [撰写关于南京传统美食的文章](https://DeepFlow.tech/chat?replay=nanjing-traditional-dishes)
|
||||
- [如何装饰租赁公寓?](https://DeepFlow.tech/chat?replay=rental-apartment-decoration)
|
||||
- [访问我们的官方网站探索更多回放示例。](https://DeepFlow.tech/#case-studies)
|
||||
|
||||
---
|
||||
|
||||
@@ -42,6 +46,7 @@ DeepFlow 作为 **deepdata** 一站式数据工作平台的一等组件交付(
|
||||
- [❓ 常见问题](#常见问题)
|
||||
- [📜 许可证](#许可证)
|
||||
- [💖 致谢](#致谢)
|
||||
- [⭐ Star History](#star-history)
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -68,8 +73,9 @@ DeepFlow 使用 Python 开发,并配有用 Node.js 编写的 Web UI。为确
|
||||
### 安装
|
||||
|
||||
```bash
|
||||
# deepflow 模块已包含在 deepdata 仓库中
|
||||
cd deepflow
|
||||
# 克隆仓库
|
||||
git clone
|
||||
cd deep-flow
|
||||
|
||||
# 安装依赖,uv将负责Python解释器和虚拟环境的创建,并安装所需的包
|
||||
uv sync
|
||||
@@ -94,7 +100,7 @@ brew install marp-cli
|
||||
可选,通过[pnpm](https://pnpm.io/installation)安装 Web UI 依赖:
|
||||
|
||||
```bash
|
||||
cd deepflow/web
|
||||
cd deep-flow/web
|
||||
pnpm install
|
||||
```
|
||||
|
||||
@@ -259,7 +265,7 @@ DeepFlow 实现了一个模块化的多智能体系统架构,专为自动化
|
||||
|
||||

|
||||
|
||||
> 在[官方网站](https://deepdata.ailab.yilise.com/#multi-agent-architecture)上查看实时演示
|
||||
> 在[DeepFlow.tech](https://DeepFlow.tech/#multi-agent-architecture)上查看实时演示
|
||||
|
||||
系统采用了精简的工作流程,包含以下组件:
|
||||
|
||||
@@ -417,18 +423,18 @@ DeepFlow 支持 LangSmith 追踪功能,帮助您调试和监控工作流。要
|
||||
其次,构建您自己的 Web 服务器 Docker 镜像:
|
||||
|
||||
```bash
|
||||
docker build -t deepflow-api .
|
||||
docker build -t deep-flow-api .
|
||||
```
|
||||
|
||||
最后,启动运行 Web 服务器的 Docker 容器:
|
||||
|
||||
```bash
|
||||
# 将deepflow-api-app替换为您首选的容器名称
|
||||
# 将deep-flow-api-app替换为您首选的容器名称
|
||||
# 启动服务器并绑定到localhost:8000
|
||||
docker run -d -t -p 127.0.0.1:8000:8000 --env-file .env --name deepflow-api-app deepflow-api
|
||||
docker run -d -t -p 127.0.0.1:8000:8000 --env-file .env --name deep-flow-api-app deep-flow-api
|
||||
|
||||
# 停止服务器
|
||||
docker stop deepflow-api-app
|
||||
docker stop deep-flow-api-app
|
||||
```
|
||||
|
||||
### Docker Compose
|
||||
@@ -602,4 +608,15 @@ DeepFlow 建立在开源社区的杰出工作基础之上。我们深深感谢
|
||||
|
||||
这些项目展示了开源协作的变革力量,我们很自豪能够在他们的基础上构建。
|
||||
|
||||
### 核心贡献者
|
||||
|
||||
衷心感谢`DeepFlow`的核心作者,他们的愿景、热情和奉献使这个项目得以实现:
|
||||
|
||||
- **[Daniel Walnut](https://github.com/hetaoBackend/)**
|
||||
- **[Henry Li](https://github.com/magiccube/)**
|
||||
|
||||
您坚定不移的承诺和专业知识是 DeepFlow 成功的驱动力。我们很荣幸有您引领这一旅程。
|
||||
|
||||
## Star History
|
||||
|
||||
[![Star History Chart]()
|
||||
|
Before Width: | Height: | Size: 151 KiB After Width: | Height: | Size: 151 KiB |
@@ -9,7 +9,7 @@ IF "%MODE%"=="dev" GOTO DEV
|
||||
IF "%MODE%"=="development" GOTO DEV
|
||||
|
||||
:PROD
|
||||
echo Starting DeepFlow in [PRODUCTION] mode...
|
||||
echo Starting DeerFlow in [PRODUCTION] mode...
|
||||
start uv run server.py
|
||||
cd web
|
||||
start pnpm start
|
||||
@@ -17,7 +17,7 @@ REM Wait for user to close
|
||||
GOTO END
|
||||
|
||||
:DEV
|
||||
echo Starting DeepFlow in [DEVELOPMENT] mode...
|
||||
echo Starting DeerFlow in [DEVELOPMENT] mode...
|
||||
start uv run server.py --reload
|
||||
cd web
|
||||
start pnpm dev
|
||||
@@ -3,7 +3,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: deepflow-backend
|
||||
container_name: deep-flow-backend
|
||||
# Remove the ports section to not expose to host
|
||||
env_file:
|
||||
- .env
|
||||
@@ -11,7 +11,7 @@ services:
|
||||
- ./conf.yaml:/app/conf.yaml:ro
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- deepflow-network
|
||||
- deep-flow-network
|
||||
|
||||
frontend:
|
||||
build:
|
||||
@@ -19,7 +19,7 @@ services:
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
- NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
|
||||
container_name: deepflow-frontend
|
||||
container_name: deep-flow-frontend
|
||||
ports:
|
||||
- "3000:3000"
|
||||
env_file:
|
||||
@@ -28,8 +28,8 @@ services:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- deepflow-network
|
||||
- deep-flow-network
|
||||
|
||||
networks:
|
||||
deepflow-network:
|
||||
deep-flow-network:
|
||||
driver: bridge
|
||||
@@ -147,7 +147,7 @@ LANGCHAIN_DEBUG=true
|
||||
LANGSMITH_TRACING=true
|
||||
LANGSMITH_ENDPOINT="https://api.smith.langchain.com"
|
||||
LANGSMITH_API_KEY="lsv2_pt_..."
|
||||
LANGSMITH_PROJECT="deepflow-debug"
|
||||
LANGSMITH_PROJECT="DeepFlow-debug"
|
||||
```
|
||||
|
||||
4. **Restart your application**
|
||||
@@ -307,7 +307,7 @@ Tokens Used: 150
|
||||
|
||||
If you're still experiencing issues:
|
||||
|
||||
1. Check existing [GitHub Issues](https://github.com/bytedance/deepflow/issues)
|
||||
1. Check existing [GitHub Issues]()
|
||||
2. Enable debug logging and LangSmith tracing
|
||||
3. Collect relevant log output
|
||||
4. Create a new issue with:
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Where's the name DeepFlow come from?](#wheres-the-name-deepflow-come-from)
|
||||
- [Which models does DeepFlow support?](#which-models-does-deepflow-support)
|
||||
- [Where's the name DeepFlow come from?](#wheres-the-name-DeepFlow-come-from)
|
||||
- [Which models does DeepFlow support?](#which-models-does-DeepFlow-support)
|
||||
- [How do I view complete model output?](#how-do-i-view-complete-model-output)
|
||||
- [How do I enable debug logging?](#how-do-i-enable-debug-logging)
|
||||
- [How do I troubleshoot issues?](#how-do-i-troubleshoot-issues)
|
||||
@@ -74,7 +74,7 @@ When encountering issues:
|
||||
|
||||
1. **Check the logs**: Enable debug logging as described above
|
||||
2. **Review configuration**: Ensure your `.env` and `conf.yaml` are correct
|
||||
3. **Check existing issues**: Search [GitHub Issues](https://github.com/bytedance/deepflow/issues) for similar problems
|
||||
3. **Check existing issues**: Search [GitHub Issues]() for similar problems
|
||||
4. **Enable verbose logging**: Use `LANGCHAIN_VERBOSE=true` for detailed output
|
||||
5. **Use LangSmith**: For visual debugging, enable LangSmith tracing
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
Copy the `conf.yaml.example` file to `conf.yaml` and modify the configurations to match your specific settings and requirements.
|
||||
|
||||
```bash
|
||||
cd deepflow
|
||||
cd deep-flow
|
||||
cp conf.yaml.example conf.yaml
|
||||
```
|
||||
|
||||
@@ -3,9 +3,9 @@ requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "deepflow"
|
||||
name = "deep-flow"
|
||||
version = "0.1.0"
|
||||
description = "DeepFlow project"
|
||||
description = "DeerFlow project"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
@@ -14,7 +14,7 @@ import sys
|
||||
|
||||
import uvicorn
|
||||
|
||||
file_handler = logging.FileHandler("/var/log/deepflow/backend.log")
|
||||
file_handler = logging.FileHandler("/var/log/deep-flow/backend.log")
|
||||
file_handler.setLevel(logging.INFO)
|
||||
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
@@ -1,9 +1,9 @@
|
||||
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
||||
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from .loader import load_yaml_config, load_deepdata_model_config
|
||||
from .loader import load_yaml_config, load_DeepData_model_config
|
||||
from .questions import BUILT_IN_QUESTIONS, BUILT_IN_QUESTIONS_ZH_CN
|
||||
from .tools import SearchEngine, get_search_engine
|
||||
|
||||
@@ -46,6 +46,6 @@ __all__ = [
|
||||
"BUILT_IN_QUESTIONS",
|
||||
"BUILT_IN_QUESTIONS_ZH_CN",
|
||||
load_yaml_config,
|
||||
load_deepdata_model_config,
|
||||
load_DeepData_model_config,
|
||||
get_search_engine
|
||||
]
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
||||
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import os
|
||||
@@ -81,12 +81,12 @@ def load_yaml_config(file_path: str) -> Dict[str, Any]:
|
||||
|
||||
def load_deepdata_model_config():
|
||||
conf = {}
|
||||
deepdata_backend_url = get_str_env("DEEPDATA_BACKEND_URL", 'http://deepdata-backend:8080')
|
||||
DeepData_backend_url = get_str_env("DEEPDATA_BACKEND_URL_BACKEND_URL", 'http://deepdata-backend:8080')
|
||||
data = {
|
||||
'type': 'CHAT',
|
||||
'isDefault': True,
|
||||
}
|
||||
response = requests.get(deepdata_backend_url + '/api/models/list', params=data)
|
||||
response = requests.get(DeepData_backend_url + '/api/models/list', params=data)
|
||||
if response.status_code == 200:
|
||||
content = response.json().get("data", {}).get("content", [])
|
||||
if len(content) != 0:
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
||||
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import enum
|
||||
@@ -24,8 +24,8 @@ class SearchEngine(enum.Enum):
|
||||
def get_search_engine():
|
||||
search_engine = os.getenv("SEARCH_API")
|
||||
|
||||
deepdata_backend_url = get_str_env("DEEPDATA_BACKEND_URL", "http://deepdata-backend:8080")
|
||||
response = requests.get(deepdata_backend_url + "/api/sys-param/list")
|
||||
DeepData_backend_url = get_str_env("DeepData_BACKEND_URL", "http://deepdata-backend:8080")
|
||||
response = requests.get(DeepData_backend_url + "/api/sys-param/list")
|
||||
if response.status_code == 200:
|
||||
content = response.json().get("data", [])
|
||||
sys_param = {}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user