From e1d21036f8f67f6d219a1b52a4c28871dc758ba7 Mon Sep 17 00:00:00 2001 From: figmar Date: Sun, 30 Aug 2026 19:23:42 +0800 Subject: [PATCH] deepdata: product docs, rebranded quick start, security policy --- .editorconfig | 21 ++++ .gitignore | 204 ++++++++++++++++++++++++++++++++++ DEVELOPMENT-zh.md | 251 ++++++++++++++++++++++++++++++++++++++++++ DEVELOPMENT.md | 181 ++++++++++++++++++++++++++++++ LICENSE | 21 ++++ README-zh.md | 187 +++++++++++++++++++++++++++++++ README.md | 193 ++++++++++++++++++++++++++++++++ SECURITY.md | 32 ++++++ docs/SECRETS_SETUP.md | 166 ++++++++++++++++++++++++++++ 9 files changed, 1256 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 DEVELOPMENT-zh.md create mode 100644 DEVELOPMENT.md create mode 100644 LICENSE create mode 100644 README-zh.md create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 docs/SECRETS_SETUP.md diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b730ce4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{java,kt}] +indent_size = 4 + +[*.{py}] +indent_size = 4 + +[*.{md}] +trim_trailing_whitespace = false + +[Makefile] +indent_style = tab diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..be3381d --- /dev/null +++ b/.gitignore @@ -0,0 +1,204 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# next.js build output +.next + +# Java +*.class +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +# Gradle +.gradle +build/ +!gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +# IntelliJ IDEA +.idea +*.iws +*.iml +*.ipr +out/ + +# Eclipse +.project +.classpath +.c9/ +*.launch +.settings/ +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyEnv +.python-version + +# pipenv +Pipfile.lock + +# Celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Docker +*.dockerignore + +# OS +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# IDE +.vscode/ +*.sublime-project +*.sublime-workspace + +# Milvus +**/volumes/ + +**/rag_storage/ +# Environment files - ignore local .env, but allow templates +.env +!.env.example + +# AI tools/agent +.agents/ +.claude/ +data/ +openspec/ +devspace.yaml diff --git a/DEVELOPMENT-zh.md b/DEVELOPMENT-zh.md new file mode 100644 index 0000000..84729e5 --- /dev/null +++ b/DEVELOPMENT-zh.md @@ -0,0 +1,251 @@ +# 开发指南 + +本文档为 DeepData 提供全面的本地开发环境搭建和工作流程指南，涵盖 Java、Python、React 三种语言。 + +## 概述 + +DeepData 是由多语言（Java 后端、Python 运行时、React 前端）组成的微服务项目，通过 Docker Compose 进行本地开发协调。 + +## 前置条件 + +- Git (用于拉取源码) +- Make (用于构建和安装) +- Docker (用于构建镜像和部署服务) +- Docker Compose (用于部署服务 - docker 方式) +- Kubernetes (用于部署服务 - k8s 方式) +- Helm (用于部署服务 - k8s 方式) + +注意： +- 确保 Java 和 Python 环境在系统 PATH 中（如适用） +- Docker Compose 将编排本地开发栈 + +## 快速开始 + +### 1. 克隆仓库并安装依赖 +```bash +git clone https://gitea.apescale.com/figmar/deepdata.git +cd deepdata +``` + +### 2. 启动基础服务 +```bash +make install +``` + +本项目支持 docker-compose 和 helm 两种方式部署，请在执行命令后输入部署部署方式的对应编号，命令回显如下所示： +```shell +Choose a deployment method: +1. Docker/Docker-Compose +2. Kubernetes/Helm +Enter choice: +``` + +若您使用的机器没有 make，您也可以执行如下命令部署： +git clone https://gitea.apescale.com/figmar/deepdata.git +docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d +``` + +当容器运行后，请在浏览器打开 http://localhost:30000 查看前端界面。 + +### 3. 本地开发部署 +本地代码修改后，请执行以下命令构建镜像并使用本地镜像部署： +git clone https://gitea.apescale.com/figmar/deepdata.git +make build +make install dev=true +``` + +### 4. 卸载服务 +git clone https://gitea.apescale.com/figmar/deepdata.git +make uninstall +``` + +在运行 `make uninstall` 时，卸载流程会只询问一次是否删除卷（数据），该选择会应用到所有组件。卸载顺序为：milvus -> label-studio -> deepdata，确保在移除 deepdata 网络前，所有使用该网络的服务已先停止。 + +## 项目结构 + +``` +DeepData/ +├── backend/ # Java 后端 +│ ├── api-gateway/ # API Gateway +│ ├── services/ # 核心服务 +│ └── shared/ # 共享库 +├── runtime/ # Python 运行时 +│ ├── deepdata-python/ # FastAPI 后端 +│ ├── python-executor/ # Ray 执行器 +│ ├── ops/ # 算子生态 +│ ├── datax/ # DataX 框架 +│ └── deepflow # DeepFlow 服务 +├── frontend/ # React 前端 +├── deployment/ # 部署配置 +└── docs/ # 文档 +``` + +## 开发工作流程 + +### Java 后端开发 +git clone https://gitea.apescale.com/figmar/deepdata.git +# 构建 +cd backend +mvn clean install + +# 运行测试 +mvn test + +# 运行特定服务 +cd backend/services/main-application +mvn spring-boot:run +``` + +### Python 运行时开发 +git clone https://gitea.apescale.com/figmar/deepdata.git +# 安装依赖 +cd runtime/deepdata-python +poetry install + +# 运行服务 +poetry run uvicorn app.main:app --reload --port 18000 + +# 运行测试 +poetry run pytest +``` + +### React 前端开发 +git clone https://gitea.apescale.com/figmar/deepdata.git +# 安装依赖 +cd frontend +npm ci + +# 运行开发服务器 +npm run dev + +# 构建生产版本 +npm run build +``` + +### Docker Compose 开发 +git clone https://gitea.apescale.com/figmar/deepdata.git +# 启动所有服务 +docker compose up -d + +# 查看日志 +docker compose logs -f [service-name] + +# 停止所有服务 +docker compose down +``` + +## 环境配置 + +每个组件可以有自己的环境变量文件。不要提交包含密钥的 .env 文件。 + +### 后端（Java） +- **路径**: `backend/.env` +- **典型密钥**: + - `DB_URL`: 数据库连接字符串 + - `DB_USER`: 数据库用户名 + - `DB_PASSWORD`: 数据库密码 + - `REDIS_URL`: Redis 连接字符串 + - `REDIS_PASSWORD`: Redis 密码 + - `JWT_SECRET`: JWT 密钥 + +### 运行时（Python） +- **路径**: `runtime/deepdata-python/.env` +- **典型密钥**: + - `DATABASE_URL`: PostgreSQL 连接字符串 + - `RAY_ENABLED`: 是否启用 Ray 执行器 + - `RAY_ADDRESS`: Ray 集群地址 + - `LABEL_STUDIO_BASE_URL`: Label Studio 基础 URL + +### 前端（React） +- **路径**: `frontend/.env` +- **典型密钥**: + - `VITE_API_BASE_URL`: API 基础 URL + - `VITE_RUNTIME_API_URL`: 运行时 API 基础 URL + +## 测试 + +### Java（JUnit 5） +git clone https://gitea.apescale.com/figmar/deepdata.git +cd backend +mvn test +``` + +### Python（pytest） +git clone https://gitea.apescale.com/figmar/deepdata.git +cd runtime/deepdata-python +poetry run pytest +``` + +### 前端 +当前未配置测试框架。 + +## 调试 + +### Java 后端 +git clone https://gitea.apescale.com/figmar/deepdata.git +# 启用 JDWP 调试端口 5005 +export JAVA_TOOL_OPTIONS='-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005' +java -jar backend/main-application/target/*.jar +``` + +### Python 运行时 +git clone https://gitea.apescale.com/figmar/deepdata.git +# 启用 debugpy 监听端口 5678 +cd runtime/deepdata-python +python -m debugpy --listen 5678 --wait-for-client -m uvicorn app.main:app --reload --port 18000 --host 0.0.0.0 +``` + +### React 前端 +使用浏览器开发者工具或 VS Code 调试器。 + +## 常见问题 + +### 端口冲突 +检查哪个进程正在使用端口： +git clone https://gitea.apescale.com/figmar/deepdata.git +lsof -i TCP:8080 +lsof -i TCP:18000 +lsof -i TCP:5173 +``` +停止或重新配置冲突的服务。 + +### 数据库连接失败 +确保 `.env` 包含正确的 `DATABASE_URL` 和凭据；确保数据库服务在 Docker Compose 中已启动。 + +### Ray 集群问题 +确保 Ray 已正确启动；检查 Ray 工作进程日志；确保 `RAY_ADDRESS` 配置正确。 + +## 文档 + +- **核心文档**: + - [ARCHITECTURE.md](./ARCHITECTURE.md) - 系统架构、微服务通信、数据流 + - [DEVELOPMENT.md](./DEVELOPMENT.md) - 本地开发环境搭建和工作流程 + - [AGENTS.md](./AGENTS.md) - AI 助手指南和代码规范 + +- **后端文档**: + - [backend/README.md](./backend/README.md) - 后端架构、服务和技术栈 + - [backend/api-gateway/README.md](./backend/api-gateway/README.md) - API Gateway 配置和路由 + - [backend/services/main-application/README.md](./backend/services/main-application/README.md) - 主应用模块 + - [backend/shared/README.md](./backend/shared/README.md) - 共享库（domain-common, security-common） + +- **运行时文档**: + - [runtime/README.md](./runtime/README.md) - 运行时架构和组件 + - [runtime/deepdata-python/README.md](./runtime/deepdata-python/README.md) - FastAPI 后端服务 + - [runtime/python-executor/README.md](./runtime/python-executor/README.md) - Ray 执行器框架 + - [runtime/ops/README.md](./runtime/ops/README.md) - 算子生态 + - [runtime/datax/README.md](./runtime/datax/README.md) - DataX 数据框架 + - [runtime/deepflow/README.md](./runtime/deepflow/README.md) - DeepFlow LLM 服务 + +- **前端文档**: + - [frontend/README.md](./frontend/README.md) - React 前端应用 + +## 贡献指南 + +感谢您对本项目的关注！我们非常欢迎社区的贡献，无论是提交 Bug 报告、提出功能建议，还是直接参与代码开发，都能帮助项目变得更好。 + +• 📮 [GitHub Issues](../../issues)：提交 Bug 或功能建议。 +• 🔧 [GitHub Pull Requests](../../pulls)：贡献代码改进。 + +## 许可证 + +DeepData 基于 [MIT](LICENSE) 开源，您可以在遵守许可证条款的前提下自由使用、修改和分发本项目的代码。 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..7db2c90 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,181 @@ +# DEVELOPMENT GUIDE for DeepData + +This document provides a comprehensive development guide for DeepData, a polyglot, microservices-based project consisting of Java, Python, and React components. It describes how to set up, build, test, run, and contribute in a local Docker Compose-based environment, without exposing secrets. + + + +## Overview + +DeepData is composed of multiple services (Java backend, Python runtime, and React frontend) coordinated via Docker Compose for local development. The guide below covers prerequisites, quick-start steps, project structure, development workflow, environment configuration, testing, debugging, common issues, documentation, contribution workflow, and licensing. + +Refer to the component READMEs for detailed implementation notes: +- Backend: backend/README.md +- Runtime: runtime/deepdata-python/README.md +- Frontend: frontend/README.md + +For code style guidelines, see AGENTS.md in the repository root. + +## Prerequisites + +- Java Development: JDK 21 and Maven +- Python: Python 3.12 and Poetry +- Node.js: Node.js 18 +- Docker and Docker Compose +- Optional: Make (for convenience) + +Notes: +- Ensure Java and Python environments are on the system PATH where applicable. +- Docker Compose will orchestrate the local development stack. + +## Quick Start + +1) Clone the repository and install dependencies: +- git clone https://gitea.apescale.com/figmar/deepdata.git +- cd datemate +- (Optional) Create and activate a Python virtual environment if not using Poetry-managed envs. +- Build dependencies per component as described below. + +2) Start the local stack with Docker Compose: +- docker compose up -d +- This brings up the Java backend, Python runtime, and React frontend services along with any required databases and caches as defined in the docker-compose.yml. + +3) Start individual components (if you prefer not to use the Docker stack): +- Java backend + - mvn -f backend/pom.xml -DskipTests package + - Run the main application (path may vary): java -jar backend/main-application/target/*.jar +- Python runtime + - cd runtime/deepdata-python + - poetry install + - uvicorn app.main:app --reload --port 18000 --host 0.0.0.0 +- React frontend + - cd frontend + - npm ci + - npm run dev + +4) Stop the stack: +- docker compose down + +> Tip: In a team setting, prefer Docker Compose for consistency across development environments. + +## Project Structure + +- backend/ +- frontend/ +- runtime/ +- deployment/ +- docs/ +- AGENTS.md (code style guidelines) +- docker/ (docker-related tooling) +- .env* files (per-component configurations, see Environment Configuration section) + +This is a polyglot project with the following language footprints: +- Java for the backend services under backend/ +- Python for the runtime under runtime/deepdata-python/ +- React/TypeScript for the frontend under frontend/ + +## Development Workflow + +Language-specific workflows: + +- Java (Backend) + - Build: mvn -f backend/pom.xml -DskipTests package + - Test: mvn -f backend/pom.xml test + - Run: mvn -f backend/pom.xml -Dexec.mainClass=... spring-boot:run (or run the packaged jar) +- Python (Runtime) + - Install: cd runtime/deepdata-python && poetry install + - Test: pytest + - Run: uvicorn app.main:app --reload --port 18000 --host 0.0.0.0 +- Frontend (React) + - Install: cd frontend && npm ci + - Test: No frontend tests configured + - Build: npm run build + - Run: npm run dev + +General tips: +- Use Docker Compose for a repeatable local stack. +- Run linters and tests before creating PRs. +- Keep dependencies in sync across environments. + +## Environment Configuration + +Each component can have its own environment file(s). Do not commit secrets. Use sample/.env.example files as references when available. + +- Backend + - Path: backend/.env (example keys below) + - Typical keys: DB_URL, DB_USER, DB_PASSWORD, JWT_SECRET, REDIS_URL, CLOUD_STORAGE_ENDPOINT +- Runtime (Python) + - Path: runtime/deepdata-python/.env + - Typical keys: DATABASE_URL, RAY_ADDRESS, CELERY_BROKER_URL, APP_SETTINGS +- Frontend + - Path: frontend/.env + - Typical keys: VITE_API_BASE_URL, VITE_DEFAULT_LOCALE, NODE_ENV + +Notes: +- Copy the corresponding .env.example to .env and fill in values as needed. +- Do not commit .env files containing secrets. + +## Testing + +- Java: JUnit 5 tests run via Maven (mvn test). +- Python: pytest in runtime/deepdata-python/test or relevant tests. +- Frontend: No frontend tests configured in this repo. + +## Code Style + +Code style follows the repository-wide guidelines described in AGENTS.md. See: +- AGENTS.md (root): Code style guidelines for all languages. +- Java: Follow Java conventions in backend/ and accordance with project conventions. +- Python: Follow PEP 8 and project-specific conventions in runtime/deepdata-python. +- React: Follow the frontend conventions in frontend/ (TypeScript/TSX). + +Link to guidelines: AGENTS.md + +## Debugging + +- Java (Backend): Enable JPDA debugging by starting the JVM with a debug port and attach a debugger. + - Example (local): export JAVA_TOOL_OPTIONS='-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005' && java -jar path/to/app.jar + - Attach with IDE on port 5005 after launch. +- Python (Runtime): Run with debugpy listening on port 5678 to attach from IDEs. + - Example: cd runtime/deepdata-python && poetry install + python -m debugpy --listen 5678 --wait-for-client -m uvicorn app.main:app --reload --port 18000 --host 0.0.0.0 +- Frontend (React): Use Node inspector to debug front-end code in dev server. + - Example: npm run dev -- --inspect-brk=9229 + +Tips: Use your preferred IDEs (IntelliJ/VSCode/WebStorm) to attach to the running processes on their respective ports. + +## Common Issues + +- Port conflicts: Check which process is using a port with lsof -i TCP: or ss -ltnp. Stop or reconfigure conflicting services. +- Database connection errors: Ensure .env contains correct DATABASE_URL and credentials; ensure the database service is up in Docker Compose. +- Ray cluster issues (Python runtime): Ensure Ray is started and accessible at the configured RAY_ADDRESS; check logs for worker failures and bootstrap status. + +## Documentation + +Component READMEs provide detailed usage and design decisions. See: +- backend/README.md +- runtime/deepdata-python/README.md +- frontend/README.md +- deployment/README.md + +## Contributing + +Contributions follow a PR workflow: +- Create a feature/bugfix branch from main (e.g., feature/new-action) +- Implement changes with tests where applicable +- Run unit tests for the changed components +- Open a PR with a clear description of the changes and the rationale +- Ensure CI checks pass (build, unit tests, lint) +- Obtain reviews and address feedback +- Merge to main after approval + +## License + +Apache 2.0 + +--- + +References: +- AGENTS.md for code style guidelines: AGENTS.md +- Java dependencies: backend/pom.xml +- Node dependencies: frontend/package.json +- Python dependencies: runtime/deepdata-python/pyproject.toml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1775e49 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 deepdata contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README-zh.md b/README-zh.md new file mode 100644 index 0000000..4454d80 --- /dev/null +++ b/README-zh.md @@ -0,0 +1,187 @@ +# DeepData 一站式数据工作平台 + +
+ +![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) +![Backend: Spring Boot 3](https://img.shields.io/badge/backend-Spring%20Boot%203-blue) +![Frontend: React](https://img.shields.io/badge/frontend-React%2018-61DAFB) +![Python: 3.12](https://img.shields.io/badge/python-3.12+-blue) + +**DeepData是面向模型微调与RAG检索的企业级数据处理平台，支持数据归集、数据管理、算子市场、数据清洗、数据合成、数据标注、数据评估、知识生成等核心功能。** + +[简体中文](./README-zh.md) | [English](./README.md) + +如果您喜欢这个项目，希望您能给我们一个Star⭐️! + +
+ +## 🌟 核心特性 + +- **核心模块**：数据归集、数据管理、算子市场、数据清洗、数据合成、数据标注、数据评估、知识生成 +- **可视化编排**：拖拽式数据处理流程设计 +- **算子生态**：丰富的内置算子和自定义算子支持 + +## 🚀 快速开始 + +### 前置条件 + +- Git (用于拉取源码) +- Make (用于构建和安装) +- Docker (用于构建镜像和部署服务) +- Docker-Compose (用于部署服务-docker方式) +- Kubernetes (用于部署服务-k8s方式) +- Helm (用于部署服务-k8s方式) +- **K8s 部署额外需要**: [Sealed Secrets Controller](https://github.com/bitnami-labs/sealed-secrets)（用于加密管理敏感配置） + +### 密钥管理（仅 K8s 部署需要） + +DeepData K8s 部署使用 **Bitnami Sealed Secrets** 管理数据库密码、JWT 密钥等敏感信息。所有密钥以加密形式存储在 Git 中（`deployment/kubernetes/sealed-secrets/`），部署时由集群内的 Sealed Secrets Controller 自动解密。 + +**在线环境安装 Sealed Secrets Controller：** + +```bash +# 通过 Helm 安装（推荐） +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system + +# 验证安装 +kubectl get pods -n kube-system | grep sealed-secrets +``` + +**离线环境：** + +1. 在有网络的机器上下载 Sealed Secrets 镜像： + ```bash + # 下载 controller 镜像（约 60MB） + docker pull bitnami/sealed-secrets-controller:latest + docker save bitnami/sealed-secrets-controller:latest -o sealed-secrets-controller.tar + + # 下载 kubeseal 工具（用于更新密钥） + # macOS: + brew install kubeseal + # Linux: + wget https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/kubeseal-linux-amd64 + ``` + +2. 将镜像导入离线环境的镜像仓库，通过 Helm 安装时指定镜像地址。 + +**更新密钥：** + +```bash +# 如果数据库密码等敏感信息发生变更，使用 kubeseal 重新加密 +echo -n "new-password" | kubeseal --raw --name deepdata-conf --namespace deepdata --scope namespace-wide +``` + +> 注意：Docker 部署方式不需要 Sealed Secrets，密钥统一通过 `.env` 文件管理（已在 `.gitignore` 中排除）。 + +### Docker一键部署 +```shell +make build # Build all images from this repository +make install # Deploy base services (deepdata + milvus) +``` + +> Images are built from source in this repository — no external registry is required. + +### 拉取代码 + +```bash +git clone https://gitea.apescale.com/figmar/deepdata.git +cd deepdata +``` + +### 部署基础服务 + +```bash +make install +``` + +本项目支持docker-compose和helm两种方式部署，请在执行命令后输入部署方式的对应编号，命令回显如下所示： +```shell +Choose a deployment method: +1. Docker/Docker-Compose +2. Kubernetes/Helm +Enter choice: +``` + +若您使用的机器没有make，您也可以执行如下命令部署: +```bash +docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d +``` + +当容器运行后，请在浏览器打开 http://localhost:30000 查看前端界面。 + +要查看所有可用的 Make 目标、选项和帮助信息，请运行： + +```bash +make help +``` + +如果您是离线环境，您可以执行如下命令下载所有依赖的镜像: +```bash +make download +``` + +### 部署Label Studio作为标注工具 +```bash +make install-label-studio +``` + +### 构建并部署Mineru增强pdf处理 +```bash +make build-mineru +make install-mineru +``` + +### 部署DeepFlow服务 +```bash +make install-deepflow +``` + +### 本地开发部署 +本地代码修改后，请执行以下命令构建镜像并使用本地镜像部署 +```bash +make build +make install dev=true +``` + +### 卸载服务 +```bash +make uninstall +``` + +在运行 `make uninstall` 时，卸载流程会只询问一次是否删除卷（数据），该选择会应用到所有组件。卸载顺序为：milvus -> label-studio -> deepdata，确保在移除 deepdata 网络前，所有使用该网络的服务已先停止。 + +## 📚 文档 + +### 核心文档 +- **[DEVELOPMENT.md](./DEVELOPMENT.md)** - 本地开发环境搭建和工作流程 +- **[AGENTS.md](./AGENTS.md)** - AI 助手指南和代码规范 + +### 后端文档 +- **[backend/README-zh.md](./backend/README-zh.md)** - 后端架构、服务和技术栈 +- **[backend/api-gateway/README-zh.md](./backend/api-gateway/README-zh.md)** - API Gateway 配置和路由 +- **[backend/services/main-application/README-zh.md](./backend/services/main-application/README-zh.md)** - 主应用模块 +- **[backend/shared/README-zh.md](./backend/shared/README-zh.md)** - 共享库（domain-common, security-common） + +### 运行时文档 +- **[runtime/README-zh.md](./runtime/README-zh.md)** - 运行时架构和组件 +- **[runtime/deepdata-python/README-zh.md](./runtime/deepdata-python/README-zh.md)** - FastAPI 后端服务 +- **[runtime/python-executor/README-zh.md](./runtime/python-executor/README-zh.md)** - Ray 执行器框架 +- **[runtime/ops/README.md](./runtime/ops/README.md)** - 算子生态 +- **[runtime/datax/README-zh.md](./runtime/datax/README-zh.md)** - DataX 数据框架 +- **[runtime/deepflow/README-zh.md](./runtime/deepflow/README-zh.md)** - DeepFlow LLM 服务 + +### 前端文档 +- **[frontend/README-zh.md](./frontend/README-zh.md)** - React 前端应用 + +## 🤝 贡献指南 + +感谢您对本项目的关注！我们非常欢迎社区的贡献，无论是提交 Bug 报告、提出功能建议，还是直接参与代码开发，都能帮助项目变得更好。 + +• 📮 [GitHub Issues](../../issues)：提交 Bug 或功能建议。 + +• 🔧 [GitHub Pull Requests](../../pulls)：贡献代码改进。 + +## 📄 许可证 + +DeepData 基于 [MIT](LICENSE) 开源，您可以在遵守许可证条款的前提下自由使用、修改和分发本项目的代码。 diff --git a/README.md b/README.md new file mode 100644 index 0000000..4f0a6ca --- /dev/null +++ b/README.md @@ -0,0 +1,193 @@ +# DeepData All-in-One Data Work Platform + +
+ +![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) +![Backend: Spring Boot 3](https://img.shields.io/badge/backend-Spring%20Boot%203-blue) +![Frontend: React](https://img.shields.io/badge/frontend-React%2018-61DAFB) +![Python: 3.12](https://img.shields.io/badge/python-3.12+-blue) + +**DeepData is an enterprise-level data processing platform for model fine-tuning and RAG retrieval, supporting core +functions such as data collection, data management, operator marketplace, data cleaning, data synthesis, data +annotation, data evaluation, and knowledge generation.** + +[简体中文](./README-zh.md) | [English](./README.md) + +If you like this project, please give it a Star⭐️! + +
+ +## 🌟 Core Features + +- **Core Modules**: Data Collection, Data Management, Operator Marketplace, Data Cleaning, Data Synthesis, Data + Annotation, Data Evaluation, Knowledge Generation. +- **Visual Orchestration**: Drag-and-drop data processing workflow design. +- **Operator Ecosystem**: Rich built-in operators and support for custom operators. + +## 🚀 Quick Start + +### Prerequisites + +- Git (for pulling source code) +- Make (for building and installing) +- Docker (for building images and deploying services) +- Docker-Compose (for service deployment - Docker method) +- Kubernetes (for service deployment - k8s method) +- Helm (for service deployment - k8s method) +- **K8s deployment additionally requires**: [Sealed Secrets Controller](https://github.com/bitnami-labs/sealed-secrets) (for encrypted secret management) + +### Secret Management (K8s deployment only) + +DeepData K8s deployment uses **Bitnami Sealed Secrets** to manage sensitive configuration such as database passwords and JWT secrets. All secrets are stored in encrypted form in Git (`deployment/kubernetes/sealed-secrets/`) and automatically decrypted by the Sealed Secrets Controller in the cluster at deploy time. + +**Online environment - install Sealed Secrets Controller:** + +```bash +# Install via Helm (recommended) +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system + +# Verify installation +kubectl get pods -n kube-system | grep sealed-secrets +``` + +**Air-gapped / offline environment:** + +1. Download the Sealed Secrets image on an internet-connected machine: + ```bash + # Download controller image (~60MB) + docker pull bitnami/sealed-secrets-controller:latest + docker save bitnami/sealed-secrets-controller:latest -o sealed-secrets-controller.tar + + # Download kubeseal CLI (for updating secrets) + # macOS: + brew install kubeseal + # Linux: + wget https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/kubeseal-linux-amd64 + ``` + +2. Transfer the image to your offline registry, then install via Helm with the custom image reference. + +**Updating secrets:** + +```bash +# When passwords change, re-encrypt with kubeseal +echo -n "new-password" | kubeseal --raw --name deepdata-conf --namespace deepdata --scope namespace-wide +``` + +> Note: Docker deployments do not require Sealed Secrets — secrets are managed via the `.env` file (excluded from Git via `.gitignore`). + +### Docker Quick deploy +```shell +make build # Build all images from this repository +make install # Deploy base services (deepdata + milvus) +``` + +> Images are built from source in this repository — no external registry is required. + +### Clone the Code + +```bash +git clone https://gitea.apescale.com/figmar/deepdata.git +cd deepdata +``` + +### Deploy the basic services + +```bash +make install +``` + +This project supports deployment via two methods: docker-compose and helm. After executing the command, please enter the corresponding number for the deployment method. The command echo is as follows: +```shell +Choose a deployment method: +1. Docker/Docker-Compose +2. Kubernetes/Helm +Enter choice: +``` + +If the machine you are using does not have make installed, please run the following command to deploy it: +```bash +docker compose -f deployment/docker/deepdata/docker-compose.yml --profile milvus up -d +``` + +Once the container is running, access http://localhost:30000 in a browser to view the front-end interface. + +To list all available Make targets, flags and help text, run: + +```bash +make help +``` + +If you are in an offline environment, you can run the following command to download all dependent images: +```bash +make download +``` + +### Deploy Label Studio as an annotation tool +```bash +make install-label-studio +``` + +### Build and deploy Mineru Enhanced PDF Processing +```bash +make build-mineru +make install-mineru +``` + +### Deploy the DeepFlow service +```bash +make install-deepflow +``` + +### Local Development and Deployment +After modifying the local code, please execute the following commands to build the image and deploy using the local image. +```bash +make build +make install dev=true +``` + +### Uninstall +```bash +make uninstall +``` + +When running make uninstall, the installer will prompt once whether to delete volumes; that single choice is applied to all components. The uninstall order is: milvus -> label-studio -> deepdata, which ensures the deepdata network is removed cleanly after services that use it have stopped. + +## 📚 Documentation + +### Core Documentation +- **[DEVELOPMENT.md](./DEVELOPMENT.md)** - Local development environment setup and workflow +- **[AGENTS.md](./AGENTS.md)** - AI assistant guidelines and code style + +### Backend Documentation +- **[backend/README.md](./backend/README.md)** - Backend architecture, services, and technology stack +- **[backend/api-gateway/README.md](./backend/api-gateway/README.md)** - API Gateway configuration and routing +- **[backend/services/main-application/README.md](./backend/services/main-application/README.md)** - Main application modules +- **[backend/shared/README.md](./backend/shared/README.md)** - Shared libraries (domain-common, security-common) + +### Runtime Documentation +- **[runtime/README.md](./runtime/README.md)** - Runtime architecture and components +- **[runtime/deepdata-python/README.md](./runtime/deepdata-python/README.md)** - FastAPI backend service +- **[runtime/python-executor/README.md](./runtime/python-executor/README.md)** - Ray executor framework +- **[runtime/ops/README.md](./runtime/ops/README.md)** - Operator ecosystem +- **[runtime/datax/README.md](./runtime/datax/README.md)** - DataX data framework +- **[runtime/deepflow/README.md](./runtime/deepflow/README.md)** - DeepFlow LLM service + +### Frontend Documentation +- **[frontend/README.md](./frontend/README.md)** - React frontend application + +## 🤝 Contribution Guidelines + +Thank you for your interest in this project! We warmly welcome contributions from the community. Whether it's submitting +bug reports, suggesting new features, or directly participating in code development, all forms of help make a project +better. + +• 📮 [GitHub Issues](../../issues): Submit bugs or feature suggestions. + +• 🔧 [GitHub Pull Requests](../../pulls): Contribute code improvements. + +## 📄 License + +DeepData is open source under the [MIT](LICENSE) license. You are free to use, modify, and distribute the code of this +project in compliance with the license terms. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6744bd6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,32 @@ +# Security Policy + +## Supported Versions + +Only the latest release of deepdata is supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| latest | :white_check_mark: | +| older | :x: | + +## Reporting a Vulnerability + +Please **do not** report security vulnerabilities through public GitHub-style +issue trackers or pull requests. + +Report suspected security vulnerabilities privately by opening a Gitea issue +with the `security` label on this repository: + +``` +https://gitea.apescale.com/figmar/deepdata/issues +``` + +Include in your report: + +- A description of the vulnerability and its impact +- The affected component and version +- Steps or a proof of concept to reproduce the issue (where possible) + +We aim to acknowledge all reports within 3 business days and will keep you +updated on the status of your report. Please do not disclose a vulnerability +publicly until we have confirmed a fix or completed an assessment. diff --git a/docs/SECRETS_SETUP.md b/docs/SECRETS_SETUP.md new file mode 100644 index 0000000..3109e49 --- /dev/null +++ b/docs/SECRETS_SETUP.md @@ -0,0 +1,166 @@ +# Secrets Management Setup Guide + +This document describes the tools and configuration required for managing encrypted secrets in DeepData. + +## Required Tools + +### Homebrew Packages (macOS) + +| Tool | Version | Purpose | Install Command | +|------|---------|---------|-----------------| +| age | 1.3.1 | Encryption tool for secrets | `brew install age` | +| sops | 3.13.1 | Secrets encryption manager | `brew install sops` | +| helm | 4.1.0+ | Kubernetes package manager | `brew install helm` | + +### Helm Plugins + +| Plugin | Version | Purpose | Install Method | +|--------|---------|---------|----------------| +| helm-secrets | 4.6.0 | Helm integration with sops | Manual clone (see below) | + +**Note:** Helm 4.x has compatibility issues with helm-secrets plugin. Use manual installation: + +```bash +cd /Users/macoo/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +If plugin fails to load, edit `plugin.yaml` to remove the `command:` field and use only `platformCommand:`. + +## Project Configuration Files + +| File | Purpose | Location | +|------|---------|----------| +| `.sops.yaml` | SOPS encryption rules | Project root | +| `.sops-keys/key.txt` | Age encryption key (private) | Project root (gitignored) | +| `secrets.yaml` | Encrypted secrets per chart | `deployment/helm//` | + +## Quick Setup + +### 1. Install Required Tools + +```bash +# macOS (Homebrew) +brew install age sops helm + +# Helm plugin (manual) +cd ~/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +### 2. Generate Age Key (if not exists) + +```bash +mkdir -p .sops-keys +age-keygen -o .sops-keys/key.txt +``` + +**Important:** Backup the key file securely. It cannot be recovered if lost. + +### 3. Update `.sops.yaml` with Your Public Key + +```yaml +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: +``` + +### 4. Create/Encrypt Secrets Files + +```bash +# Set environment variable +export SOPS_AGE_KEY_FILE=.sops-keys/key.txt + +# Encrypt existing file +sops --encrypt --in-place deployment/helm/deepdata/secrets.yaml + +# Or create new encrypted file +sops deployment/helm/deepdata/secrets.yaml +``` + +## Usage + +### View Decrypted Secrets + +```bash +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/deepdata/secrets.yaml +``` + +### Deploy with Secrets + +```bash +# Using helper script +./scripts/secrets.sh helm-install deepdata deepdata + +# Manual method +TMP_FILE=$(mktemp) +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/deepdata/secrets.yaml > $TMP_FILE +helm install deepdata deployment/helm/deepdata -n deepdata -f $TMP_FILE +rm $TMP_FILE +``` + +### Docker Compose + +```bash +# Copy and edit .env +cp deployment/docker/deepdata/.env.example deployment/docker/deepdata/.env + +# Edit with your secrets +vim deployment/docker/deepdata/.env + +# Deploy +docker compose -f deployment/docker/deepdata/docker-compose.yml up -d +``` + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `SOPS_AGE_KEY_FILE` | Path to age private key | +| `SOPS_CONFIG` | Path to .sops.yaml (optional) | + +## Security Notes + +1. **Never commit `.sops-keys/` directory** - It's in `.gitignore` +2. **Backup your age key** - Store in secure location (password manager, encrypted backup) +3. **Use strong passwords** - Generate with `openssl rand -base64 32` +4. **Rotate keys periodically** - Especially after team changes + +## Troubleshooting + +### Plugin Load Error + +If helm plugin shows "both platformCommand and command are set": +```bash +# Edit plugin.yaml +vim ~/Library/helm/plugins/helm-secrets/plugin.yaml +# Remove the "command:" line, keep only "platformCommand:" +``` + +### Decryption Failed + +```bash +# Check key file exists +ls -la .sops-keys/key.txt + +# Verify key matches encrypted file's recipient +grep "recipient:" deployment/helm/deepdata/secrets.yaml +``` + +### Key Rotation + +```bash +# Generate new key +age-keygen -o .sops-keys/key-new.txt + +# Update .sops.yaml with new public key + +# Re-encrypt all secrets +for f in deployment/helm/*/secrets.yaml; do + SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt $f > /tmp/plain.yaml + SOPS_AGE_KEY_FILE=.sops-keys/key-new.txt sops --encrypt /tmp/plain.yaml > $f +done + +# Replace old key +mv .sops-keys/key-new.txt .sops-keys/key.txt +``` \ No newline at end of file