figmar/deepdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
deepdata-20260928
deepdata/docker/docker-compose.yml
T
沃巴蒂 ff22ed3a8b feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow 
本次同步 09-12 至 09-28 的代码变更，主要内容：

新增功能
- 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService
  对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口
- 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、
  ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器
- 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator
- 新增 tools/minio-file-server MinIO 文件服务

功能增强
- 数据清洗：cleaning_task_service/scheduler/routes 重构，
  data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整
- 数据标注：annotation sync.py、project.py 接口与schema 更新
- 数据采集、算子市场、首页与菜单路由相关页面调整
- DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）

结构调整
- deepflow/ 更名为 deep-flow/（419 个文件重命名）
- 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本

未纳入本次提交
- scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录）
- *.backup*、__pycache__ 产物
- zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物）
- DataX/（内嵌独立仓库，远程亦未收录其源码）
2026-10-03 22:44:56 +08:00

593 lines
16 KiB
YAML
Raw Permalink Blame History

services:
# ==============================
# Core DeepData Services (always enabled)
# ==============================
deepdata-backend:
container_name: deepdata-backend
image: deepdata-backend:latest
restart: on-failure
privileged: true
environment:
- DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
- JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file}
- deepdata.jwt.enable=${DEEPDATA_JWT_ENABLE:-false}
- LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
- LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
volumes:
- dataset_volume:/dataset
- flow_volume:/flow
- log_volume:/var/log/deepdata
- operator-upload-volume:/operators/upload
- operator-runtime-volume:/operators/extract
networks: [ deepdata ]
depends_on:
- deepdata-database
deepdata-backend-python:
container_name: deepdata-backend-python
image: deepdata-backend-python:latest
restart: on-failure
privileged: true
ports:
- "18000:18000"
environment:
- log_level=DEBUG
- S3_READER_ALLOW_PRIVATE_IP=true
- DEEPDATA_SSRF_ALLOWED_HOSTS=172.16.249.172,minio,milvus-minio
- LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
- LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
- LOG_PVC_MONITOR_ENABLED=${LOG_PVC_MONITOR_ENABLED:-true}
- LOG_PVC_MONITOR_PATH=${LOG_PVC_MONITOR_PATH:-/var/log/deepdata}
- LOG_PVC_MONITOR_THRESHOLD=${LOG_PVC_MONITOR_THRESHOLD:-0.9}
- LOG_PVC_MONITOR_INTERVAL_SECONDS=${LOG_PVC_MONITOR_INTERVAL_SECONDS:-300}
- LOG_PVC_MONITOR_DELETE_BATCH_SIZE=${LOG_PVC_MONITOR_DELETE_BATCH_SIZE:-10}
- LOG_PVC_MONITOR_FILE_SUFFIXES=${LOG_PVC_MONITOR_FILE_SUFFIXES:-log,out,err}
- PGSQL_HOST=deepdata-database
- PGSQL_PORT=5432
- pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
- DEEPDATA_jwt_enable=${DEEPDATA_JWT_ENABLE:-false}
- milvus_uri=${MILVUS_URI:-http://milvus:19530}
- LABEL_STUDIO_USERNAME=${LABEL_STUDIO_USERNAME:-admin@demo.com}
- LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-}
- LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-}
- LABEL_STUDIO_BASE_URL=${LABEL_STUDIO_BASE_URL:-http://label-studio:8000}
- COLLECTION_EXECUTION_MODE=${COLLECTION_EXECUTION_MODE:-local}
volumes:
- dataset_volume:/dataset
- flow_volume:/flow
- log_volume:/var/log/deepdata
- graph_data_volume:/data/rag_storage
- operator-upload-volume:/operators/upload
- operator-runtime-volume:/operators/extract
networks: [ deepdata ]
depends_on:
deepdata-database:
condition: service_healthy
milvus:
condition: service_healthy
deepdata-gateway:
container_name: deepdata-gateway
image: deepdata-gateway:latest
restart: on-failure
privileged: true
ports:
- '8085:8080'
environment:
- JWT_SECRET=${JWT_SECRET:-}
- deepdata.jwt.enable=${DEEPDATA_JWT_ENABLE:-false}
- LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
- LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
- SPRING_DATASOURCE_URL=jdbc:postgresql://deepdata-database:5432/deepdata
- SPRING_DATASOURCE_USERNAME=postgres
- SPRING_DATASOURCE_PASSWORD=${DB_PASSWORD}
volumes:
- log_volume:/var/log/deepdata
networks: [ deepdata ]
deepdata-frontend:
container_name: deepdata-frontend
image: deepdata-frontend:latest
restart: on-failure
ports:
- "40000:3000"
environment:
- LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
- LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
volumes:
- frontend_log_volume:/var/log/deepdata/frontend
- ../frontend/dist:/opt/frontend/statics:ro
networks: [ deepdata ]
depends_on:
- deepdata-backend
- deepdata-backend-python
deepdata-database:
container_name: deepdata-database
image: deepdata-database:latest
restart: on-failure
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
- LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
volumes:
- postgresql_volume:/var/lib/postgresql/data
- database_log_volume:/var/log/deepdata/database
ports:
- "5442:5432"
networks: [ deepdata ]
deepdata-runtime:
container_name: deepdata-runtime
image: deepdata-runtime-ray:latest
restart: on-failure
environment:
RAY_DEDUP_LOGS: "0"
RAY_TQDM_PATCH_PRINT: "0"
LOG_ROTATION_MAX_SIZE: ${LOG_ROTATION_MAX_SIZE:-100MB}
LOG_ROTATION_BACKUP_COUNT: ${LOG_ROTATION_BACKUP_COUNT:-30}
PG_HOST: "deepdata-database"
PG_PORT: "5432"
PG_USER: "postgres"
PG_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
PG_DATABASE: "deepdata"
RAY_DASHBOARD_ADDRESS: ${RAY_DASHBOARD_ADDRESS}
RAY_REDIS_PASSWORD: ${RAY_REDIS_PASSWORD}
ENABLE_RAY_EXECUTION: ${ENABLE_RAY_EXECUTION:-false}
MAX_ACTOR_NUMS: ${MAX_ACTOR_NUMS:-40} # <-- 新增这里
HF_ENDPOINT: "https://hf-mirror.com"
HF_HUB_CACHE: "/dataset/.hf_cache"
TRANSFORMERS_CACHE: "/dataset/.hf_cache"
DJ_FORCE_RUN: true
command:
- python
- /opt/runtime/deepdata/operator_runtime.py
- --port
- "8081"
volumes:
# - ray_log_volume:/tmp/ray
- log_volume:/var/log/deepdata
- dataset_volume:/dataset
- flow_volume:/flow
- operator-runtime-volume:/opt/runtime/deepdata/ops/user
- operator-packages-volume:/usr/local/lib/ops/site-packages
networks: [ deepdata ]
# =============================
# Optional: Mineru NPU Engine (profile: mineru)
# ==============================
deepdata-mineru:
container_name: deepdata-mineru
image: deepdata-mineru:latest
restart: on-failure
environment:
MINERU_MODEL_SOURCE: local
MINERU_DEVICE_MODE: npu
VLLM_WORKER_MULTIPROC_METHOD: spawn
privileged: true
entrypoint: mineru-openai-server
command:
--engine vllm
--host 0.0.0.0
--port 8000
${EXTRA_ARGS:-}
volumes:
- /var/log/npu/:/usr/slog
- /usr/local/dcmi:/usr/local/dcmi
- /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
- /usr/local/Ascend/driver:/usr/local/Ascend/driver
networks: [ deepdata ]
profiles: [ mineru ]
devices:
- /dev/davinci0
- /dev/davinci_manager
- /dev/devmm_svm
- /dev/hisi_hdc
# ==============================
# Optional: Redis (profile: redis)
# ==============================
deepdata-redis:
container_name: deepdata-redis
image: redis:8.2.3
restart: on-failure
ports:
- "6339:6379"
networks: [ deepdata ]
profiles: [ redis ]
# ==============================
# Optional: ray
# ==============================
deepdata-ray-head:
container_name: deepdata-ray-head
image: deepdata-runtime-ray:latest
restart: always
ports:
- "${RAY_DASHBOARD_PORT}:8265" # Dashboard UI
- "${RAY_CLIENT_PORT}:10001" # Client Submission
environment:
- RAY_REDIS_PASSWORD=${RAY_REDIS_PASSWORD}
- RAY_MODE=head
- NUM_CPUS=0
- TOTAL_MEM_MB=4096
- MAX_ACTOR_NUMS=${MAX_ACTOR_NUMS:-40}
- HF_ENDPOINT=https://hf-mirror.com
- HF_HUB_CACHE=/dataset/.hf_cache
- TRANSFORMERS_CACHE=/dataset/.hf_cache
- DJ_CACHE_DIR=/dataset/.dj_cache
- DJ_CHUNK_SIZE=10
- DJ_FORCE_RUN=true
networks:
- deepdata
shm_size: '6gb'
volumes:
- dataset_volume:/dataset
- flow_volume:/flow
# - ray_log_volume:/tmp/ray
- log_volume:/var/log/deepdata
- operator-runtime-volume:/opt/runtime/deepdata/ops/user
- operator-packages-volume:/usr/local/lib/ops/site-packages
profiles: [ ray ]
deepdata-ray-worker:
image: deepdata-runtime-ray:latest
restart: always
deploy:
replicas: ${NUM_RAY_WORKERS:-2}
shm_size: '8gb'
environment:
- RAY_MODE=worker
- RAY_HEAD_ADDRESS=${RAY_HEAD_GCS_ADDRESS}
- RAY_REDIS_PASSWORD=${RAY_REDIS_PASSWORD}
- NUM_CPUS=${RAY_WORKER_NUM_CPUS:-4}
- MAX_ACTOR_NUMS=${MAX_ACTOR_NUMS:-40}
- TOTAL_MEM_MB=${RAY_WORKER_MEMORY_MB:-16000}
- LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
- LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
- PG_HOST=deepdata-database
- PG_PORT=5432
- PG_USER=postgres
- PG_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
- PG_DATABASE=deepdata
- HF_ENDPOINT=https://hf-mirror.com
- HF_HUB_CACHE=/dataset/.hf_cache
- TRANSFORMERS_CACHE=/dataset/.hf_cache
- DJ_CACHE_DIR=/dataset/.dj_cache
- DJ_CHUNK_SIZE=10
- DJ_FORCE_RUN=true
volumes:
- dataset_volume:/dataset
- flow_volume:/flow
# - ray_log_volume:/tmp/ray
- log_volume:/var/log/deepdata
- operator-runtime-volume:/opt/runtime/deepdata/ops/user
- operator-packages-volume:/usr/local/lib/ops/site-packages
networks: [ deepdata ]
depends_on:
- deepdata-ray-head
profiles: [ray]
# ==============================
# Optional: Deep Flow (profile: deep-flow)
# ==============================
deep-flow-backend:
image: deep-flow-backend:latest
container_name: deep-flow-backend
env_file:
- ./deep-flow/.env
volumes:
- ./deep-flow/conf.yaml:/app/conf.yaml:ro
- deep-flow-log-volume:/var/log/deep-flow
- dataset_volume:/dataset
restart: on-failure
networks:
- deepdata
profiles:
- deep-flow
deep-flow-frontend:
image: deep-flow-frontend:latest
container_name: deep-flow-frontend
env_file:
- ./deep-flow/.env
depends_on:
- deep-flow-backend
restart: on-failure
networks:
- deepdata
profiles:
- deep-flow
# ==============================
# Optional: Label Studio (profile: label-studio)
# ==============================
label-studio-pgbouncer:
container_name: label-studio-pgbouncer
image: edoburu/pgbouncer:latest
restart: on-failure
environment:
- DB_HOST=deepdata-database
- DB_PORT=5432
- DB_NAME=labelstudio
- DB_USER=postgres
- DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
- AUTH_TYPE=scram-sha-256
- POOL_MODE=transaction
- MAX_CLIENT_CONN=100
- DEFAULT_POOL_SIZE=20
- MAX_DB_CONNECTIONS=20
networks:
- deepdata
profiles: [ label-studio ]
label-studio:
container_name: label-studio
stdin_open: true
tty: true
image: heartexlabs/label-studio:latest
privileged: true
restart: on-failure
user: root
expose:
- "8000"
ports:
- "38000:8000"
depends_on:
- label-studio-pgbouncer
environment:
- DJANGO_DB=default
- POSTGRE_NAME=labelstudio
- POSTGRE_USER=postgres
- POSTGRE_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
- POSTGRE_PORT=5432
- POSTGRE_HOST=label-studio-pgbouncer
- LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-}
- LOCAL_FILES_SERVING_ENABLED=true
- LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local
- USE_USERNAME_FOR_LOGIN=true
- LABEL_STUDIO_USERNAME=admin@demo.com
- LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-}
- LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN=true
- LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-}
- LOG_LEVEL=DEBUG
volumes:
- label-studio-data:/label-studio/data:rw
- dataset_volume:/label-studio/local:rw
networks:
- deepdata
command: label-studio-uwsgi
profiles: [ label-studio ]
# ==============================
# Optional: Milvus (profile: milvus)
# ==============================
etcd:
container_name: milvus-etcd
image: quay.io/coreos/etcd:v3.5.9
environment:
- ETCD_AUTO_COMPACTION_MODE=revision
- ETCD_AUTO_COMPACTION_RETENTION=1000
- ETCD_QUOTA_BACKEND_BYTES=4294967296
- ETCD_SNAPSHOT_COUNT=50000
volumes:
- etcd-volume:/etcd
command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
restart: always
networks:
- deepdata
healthcheck:
test: ["CMD", "etcdctl", "endpoint", "health"]
interval: 30s
timeout: 20s
retries: 3
profiles:
- milvus
minio:
container_name: milvus-minio
image: minio/minio:RELEASE.2024-12-18T13-15-44Z
environment:
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-}
ports:
- "9001:9001"
- "9000:9000"
volumes:
- minio-volume:/data
command: minio server /data --console-address ":9001"
networks:
- deepdata
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
profiles:
- milvus
milvus:
container_name: milvus-standalone
restart: unless-stopped
image: milvusdb/milvus:v2.6.5
command: ["milvus", "run", "standalone"]
security_opt:
- seccomp:unconfined
environment:
ETCD_ENDPOINTS: etcd:2379
MINIO_ADDRESS: minio:9000
MQ_TYPE: woodpecker
MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY_ID}
MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_ACCESS_KEY}
MINIO_USE_SSL: "false"
AWS_REGION: ${AWS_REGION}
volumes:
- milvus-volume:/var/lib/milvus
networks:
- deepdata
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
interval: 30s
start_period: 180s
timeout: 20s
retries: 10
ports:
- "19530:19530"
- "19091:9091"
depends_on:
etcd:
condition: service_healthy
minio:
condition: service_healthy
profiles:
- milvus
# ==============================
# Volumes
# ==============================
volumes:
dataset_volume:
driver: local
driver_opts:
type: none
device: ./volumes/dataset
o: bind
flow_volume:
driver: local
driver_opts:
type: none
device: ./volumes/flow
o: bind
log_volume:
driver: local
driver_opts:
type: none
device: ./volumes/logs/deepdata
o: bind
postgresql_volume:
driver: local
driver_opts:
type: none
device: ./volumes/postgres/data
o: bind
ray_log_volume:
driver: local
driver_opts:
type: none
device: ./volumes/ray/logs
o: bind
frontend_log_volume:
driver: local
driver_opts:
type: none
device: ./volumes/logs/frontend
o: bind
database_log_volume:
driver: local
driver_opts:
type: none
device: ./volumes/logs/database
o: bind
operator-upload-volume:
driver: local
driver_opts:
type: none
device: ./volumes/operators/upload
o: bind
operator-runtime-volume:
driver: local
driver_opts:
type: none
device: ./volumes/operators/runtime
o: bind
operator-packages-volume:
driver: local
driver_opts:
type: none
device: ./volumes/operators/site-packages
o: bind
graph_data_volume:
driver: local
driver_opts:
type: none
device: ./volumes/rag_storage
o: bind
# Deep Flow
deep-flow-log-volume:
driver: local
driver_opts:
type: none
device: ./volumes/logs/deep-flow
o: bind
# Label Studio
label-studio-data:
driver: local
driver_opts:
type: none
device: ./volumes/label-studio/data
o: bind
label-studio-db:
driver: local
driver_opts:
type: none
device: ./volumes/label-studio/db
o: bind
# Milvus
etcd-volume:
driver: local
driver_opts:
type: none
device: ./volumes/milvus/etcd
o: bind
minio-volume:
driver: local
driver_opts:
type: none
device: ./volumes/milvus/minio
o: bind
milvus-volume:
driver: local
driver_opts:
type: none
device: ./volumes/milvus/data
o: bind
# ==============================
# Networks
# ==============================
networks:
deepdata:
driver: bridge
name: deepdata-network
Reference in New Issue View Git Blame Copy Permalink