services:
  # ==============================
  # Core DeepData Services (always enabled)
  # ==============================

  deepdata-backend:
    container_name: deepdata-backend
    image: deepdata-backend:latest
    restart: on-failure
    privileged: true
    environment:
      - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
      - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file}
      - deepdata.jwt.enable=${DEEPDATA_JWT_ENABLE:-false}
      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
    volumes:
      - dataset_volume:/dataset
      - flow_volume:/flow
      - log_volume:/var/log/deepdata
      - operator-upload-volume:/operators/upload
      - operator-runtime-volume:/operators/extract
    networks: [ deepdata ]
    depends_on:
      - deepdata-database

  deepdata-backend-python:
    container_name: deepdata-backend-python
    image: deepdata-backend-python:latest
    restart: on-failure
    privileged: true
    ports:
      - "18000:18000"
    environment:
      - log_level=DEBUG
      - S3_READER_ALLOW_PRIVATE_IP=true
      - DEEPDATA_SSRF_ALLOWED_HOSTS=172.16.249.172,minio,milvus-minio
      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
      - LOG_PVC_MONITOR_ENABLED=${LOG_PVC_MONITOR_ENABLED:-true}
      - LOG_PVC_MONITOR_PATH=${LOG_PVC_MONITOR_PATH:-/var/log/deepdata}
      - LOG_PVC_MONITOR_THRESHOLD=${LOG_PVC_MONITOR_THRESHOLD:-0.9}
      - LOG_PVC_MONITOR_INTERVAL_SECONDS=${LOG_PVC_MONITOR_INTERVAL_SECONDS:-300}
      - LOG_PVC_MONITOR_DELETE_BATCH_SIZE=${LOG_PVC_MONITOR_DELETE_BATCH_SIZE:-10}
      - LOG_PVC_MONITOR_FILE_SUFFIXES=${LOG_PVC_MONITOR_FILE_SUFFIXES:-log,out,err}
      - PGSQL_HOST=deepdata-database
      - PGSQL_PORT=5432
      - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
      - DEEPDATA_jwt_enable=${DEEPDATA_JWT_ENABLE:-false}
      - milvus_uri=${MILVUS_URI:-http://milvus:19530}
      - LABEL_STUDIO_USERNAME=${LABEL_STUDIO_USERNAME:-admin@demo.com}
      - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-}
      - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-}
      - LABEL_STUDIO_BASE_URL=${LABEL_STUDIO_BASE_URL:-http://label-studio:8000}
      - COLLECTION_EXECUTION_MODE=${COLLECTION_EXECUTION_MODE:-local}
    volumes:
      - dataset_volume:/dataset
      - flow_volume:/flow
      - log_volume:/var/log/deepdata
      - graph_data_volume:/data/rag_storage
      - operator-upload-volume:/operators/upload
      - operator-runtime-volume:/operators/extract
    networks: [ deepdata ]
    depends_on:
      deepdata-database:
        condition: service_healthy
      milvus:
        condition: service_healthy

  deepdata-gateway:
    container_name: deepdata-gateway
    image: deepdata-gateway:latest
    restart: on-failure
    privileged: true
    ports:
      - '8085:8080'
    environment:
      - JWT_SECRET=${JWT_SECRET:-}
      - deepdata.jwt.enable=${DEEPDATA_JWT_ENABLE:-false}
      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
      - SPRING_DATASOURCE_URL=jdbc:postgresql://deepdata-database:5432/deepdata
      - SPRING_DATASOURCE_USERNAME=postgres
      - SPRING_DATASOURCE_PASSWORD=${DB_PASSWORD}
      
    volumes:
      - log_volume:/var/log/deepdata
    networks: [ deepdata ]

  deepdata-frontend:
    container_name: deepdata-frontend
    image: deepdata-frontend:latest
    restart: on-failure
    ports:
      - "40000:3000"
    environment:
      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
    volumes:
      - frontend_log_volume:/var/log/deepdata/frontend
      - ../frontend/dist:/opt/frontend/statics:ro      
    networks: [ deepdata ]
    depends_on:
      - deepdata-backend
      - deepdata-backend-python

  deepdata-database:
    container_name: deepdata-database
    image: deepdata-database:latest
    restart: on-failure
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 10s
      timeout: 5s
      retries: 5
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
    volumes:
      - postgresql_volume:/var/lib/postgresql/data
      - database_log_volume:/var/log/deepdata/database
    ports:
      - "5442:5432"
    networks: [ deepdata ]
    
  deepdata-runtime:
    container_name: deepdata-runtime
    image: deepdata-runtime-ray:latest
    restart: on-failure
    environment:
      RAY_DEDUP_LOGS: "0"
      RAY_TQDM_PATCH_PRINT: "0"
      LOG_ROTATION_MAX_SIZE: ${LOG_ROTATION_MAX_SIZE:-100MB}
      LOG_ROTATION_BACKUP_COUNT: ${LOG_ROTATION_BACKUP_COUNT:-30}
      PG_HOST: "deepdata-database"
      PG_PORT: "5432"
      PG_USER: "postgres"
      PG_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
      PG_DATABASE: "deepdata"
      RAY_DASHBOARD_ADDRESS: ${RAY_DASHBOARD_ADDRESS}
      RAY_REDIS_PASSWORD: ${RAY_REDIS_PASSWORD}
      ENABLE_RAY_EXECUTION: ${ENABLE_RAY_EXECUTION:-false}
      MAX_ACTOR_NUMS: ${MAX_ACTOR_NUMS:-40}  # <-- 新增这里    
      HF_ENDPOINT: "https://hf-mirror.com"      
      HF_HUB_CACHE: "/dataset/.hf_cache"
      TRANSFORMERS_CACHE: "/dataset/.hf_cache"
      DJ_FORCE_RUN: true
      
    command:
      - python
      - /opt/runtime/deepdata/operator_runtime.py
      - --port
      - "8081"
    volumes:
      # - ray_log_volume:/tmp/ray
      - log_volume:/var/log/deepdata
      - dataset_volume:/dataset
      - flow_volume:/flow
      - operator-runtime-volume:/opt/runtime/deepdata/ops/user
      - operator-packages-volume:/usr/local/lib/ops/site-packages
    networks: [ deepdata ]

  # =============================
  # Optional: Mineru NPU Engine (profile: mineru)
  # ==============================
  deepdata-mineru:
    container_name: deepdata-mineru
    image: deepdata-mineru:latest
    restart: on-failure
    environment:
      MINERU_MODEL_SOURCE: local
      MINERU_DEVICE_MODE: npu
      VLLM_WORKER_MULTIPROC_METHOD: spawn
    privileged: true
    entrypoint: mineru-openai-server
    command:
      --engine vllm
      --host 0.0.0.0
      --port 8000
      ${EXTRA_ARGS:-}
    volumes:
      - /var/log/npu/:/usr/slog
      - /usr/local/dcmi:/usr/local/dcmi
      - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
      - /usr/local/Ascend/driver:/usr/local/Ascend/driver
    networks: [ deepdata ]
    profiles: [ mineru ]
    devices:
      - /dev/davinci0
      - /dev/davinci_manager
      - /dev/devmm_svm
      - /dev/hisi_hdc

  # ==============================
  # Optional: Redis (profile: redis)
  # ==============================
  deepdata-redis:
    container_name: deepdata-redis
    image: redis:8.2.3
    restart: on-failure
    ports:
      - "6339:6379"
    networks: [ deepdata ]
    profiles: [ redis ]

  # ==============================
  # Optional: ray
  # ==============================

  deepdata-ray-head:
    container_name: deepdata-ray-head
    image: deepdata-runtime-ray:latest
    restart: always
    ports:
      - "${RAY_DASHBOARD_PORT}:8265"    # Dashboard UI
      - "${RAY_CLIENT_PORT}:10001"      # Client Submission
    environment:
      - RAY_REDIS_PASSWORD=${RAY_REDIS_PASSWORD}
      - RAY_MODE=head
      - NUM_CPUS=0
      - TOTAL_MEM_MB=4096
      - MAX_ACTOR_NUMS=${MAX_ACTOR_NUMS:-40} 
      
      - HF_ENDPOINT=https://hf-mirror.com      
      - HF_HUB_CACHE=/dataset/.hf_cache
      - TRANSFORMERS_CACHE=/dataset/.hf_cache 

      - DJ_CACHE_DIR=/dataset/.dj_cache
      - DJ_CHUNK_SIZE=10
      - DJ_FORCE_RUN=true
    networks:
      - deepdata
    shm_size: '6gb'
    volumes:
      - dataset_volume:/dataset
      - flow_volume:/flow
      # - ray_log_volume:/tmp/ray
      - log_volume:/var/log/deepdata
      - operator-runtime-volume:/opt/runtime/deepdata/ops/user
      - operator-packages-volume:/usr/local/lib/ops/site-packages
    profiles: [ ray ]

  deepdata-ray-worker:
    image: deepdata-runtime-ray:latest
    restart: always
    deploy:
      replicas: ${NUM_RAY_WORKERS:-2}
    shm_size: '8gb'
    environment:
      - RAY_MODE=worker
      - RAY_HEAD_ADDRESS=${RAY_HEAD_GCS_ADDRESS} 
      - RAY_REDIS_PASSWORD=${RAY_REDIS_PASSWORD}
      - NUM_CPUS=${RAY_WORKER_NUM_CPUS:-4}
      - MAX_ACTOR_NUMS=${MAX_ACTOR_NUMS:-40}  
      - TOTAL_MEM_MB=${RAY_WORKER_MEMORY_MB:-16000} 

      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}

      - PG_HOST=deepdata-database
      - PG_PORT=5432
      - PG_USER=postgres
      - PG_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
      - PG_DATABASE=deepdata  

      - HF_ENDPOINT=https://hf-mirror.com      
      - HF_HUB_CACHE=/dataset/.hf_cache
      - TRANSFORMERS_CACHE=/dataset/.hf_cache 

      - DJ_CACHE_DIR=/dataset/.dj_cache
      - DJ_CHUNK_SIZE=10
      - DJ_FORCE_RUN=true
    volumes:
      - dataset_volume:/dataset
      - flow_volume:/flow      
      # - ray_log_volume:/tmp/ray
      - log_volume:/var/log/deepdata
      - operator-runtime-volume:/opt/runtime/deepdata/ops/user
      - operator-packages-volume:/usr/local/lib/ops/site-packages
    networks: [ deepdata ]
    depends_on:
      - deepdata-ray-head
    profiles: [ray]

  # ==============================
  # Optional: Deep Flow (profile: deep-flow)
  # ==============================
  deep-flow-backend:
    image: deep-flow-backend:latest
    container_name: deep-flow-backend
    env_file:
      - ./deep-flow/.env
    volumes:
      - ./deep-flow/conf.yaml:/app/conf.yaml:ro
      - deep-flow-log-volume:/var/log/deep-flow
      - dataset_volume:/dataset
    restart: on-failure
    networks:
      - deepdata
    profiles:
      - deep-flow

  deep-flow-frontend:
    image: deep-flow-frontend:latest
    container_name: deep-flow-frontend
    env_file:
      - ./deep-flow/.env
    depends_on:
      - deep-flow-backend
    restart: on-failure
    networks:
      - deepdata
    profiles:
      - deep-flow

  # ==============================
  # Optional: Label Studio (profile: label-studio)
  # ==============================
  label-studio-pgbouncer:
    container_name: label-studio-pgbouncer
    image: edoburu/pgbouncer:latest
    restart: on-failure
    environment:
      - DB_HOST=deepdata-database
      - DB_PORT=5432
      - DB_NAME=labelstudio
      - DB_USER=postgres
      - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
      - AUTH_TYPE=scram-sha-256
      - POOL_MODE=transaction
      - MAX_CLIENT_CONN=100
      - DEFAULT_POOL_SIZE=20
      - MAX_DB_CONNECTIONS=20
    networks:
      - deepdata
    profiles: [ label-studio ]

  label-studio:
    container_name: label-studio
    stdin_open: true
    tty: true
    image: heartexlabs/label-studio:latest
    privileged: true
    restart: on-failure
    user: root
    expose:
      - "8000"
    ports:
      - "38000:8000"
    depends_on:
      - label-studio-pgbouncer
    environment:
      - DJANGO_DB=default
      - POSTGRE_NAME=labelstudio
      - POSTGRE_USER=postgres
      - POSTGRE_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
      - POSTGRE_PORT=5432
      - POSTGRE_HOST=label-studio-pgbouncer
      - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-}
      - LOCAL_FILES_SERVING_ENABLED=true
      - LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local
      - USE_USERNAME_FOR_LOGIN=true
      - LABEL_STUDIO_USERNAME=admin@demo.com
      - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-}
      - LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN=true
      - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-}
      - LOG_LEVEL=DEBUG
    volumes:
      - label-studio-data:/label-studio/data:rw
      - dataset_volume:/label-studio/local:rw
    networks:
      - deepdata
    command: label-studio-uwsgi
    profiles: [ label-studio ]

  # ==============================
  # Optional: Milvus (profile: milvus)
  # ==============================
  etcd:
    container_name: milvus-etcd
    image: quay.io/coreos/etcd:v3.5.9
    environment:
      - ETCD_AUTO_COMPACTION_MODE=revision
      - ETCD_AUTO_COMPACTION_RETENTION=1000
      - ETCD_QUOTA_BACKEND_BYTES=4294967296
      - ETCD_SNAPSHOT_COUNT=50000
    volumes:
      - etcd-volume:/etcd
    command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
    restart: always
    networks:
      - deepdata
    healthcheck:
      test: ["CMD", "etcdctl", "endpoint", "health"]
      interval: 30s
      timeout: 20s
      retries: 3
    profiles:
      - milvus

  minio:
    container_name: milvus-minio
    image: minio/minio:RELEASE.2024-12-18T13-15-44Z
    environment:
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-}
    ports:
      - "9001:9001"
      - "9000:9000"
    volumes:
      - minio-volume:/data
    command: minio server /data --console-address ":9001"
    networks:
      - deepdata
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
    profiles:
      - milvus

  milvus:
    container_name: milvus-standalone
    restart: unless-stopped
    image: milvusdb/milvus:v2.6.5
    command: ["milvus", "run", "standalone"]
    security_opt:
    - seccomp:unconfined
    environment:
      ETCD_ENDPOINTS: etcd:2379
      MINIO_ADDRESS: minio:9000
      MQ_TYPE: woodpecker
      MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY_ID}
      MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_ACCESS_KEY}
      MINIO_USE_SSL: "false"
      AWS_REGION: ${AWS_REGION}
    volumes:
      - milvus-volume:/var/lib/milvus
    networks:
      - deepdata
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
      interval: 30s
      start_period: 180s
      timeout: 20s
      retries: 10
    ports:
      - "19530:19530"
      - "19091:9091"
    depends_on:
      etcd:
        condition: service_healthy
      minio:
        condition: service_healthy
    profiles:
      - milvus

# ==============================
# Volumes
# ==============================
volumes:
  dataset_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/dataset
      o: bind

  flow_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/flow
      o: bind

  log_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/logs/deepdata
      o: bind

  postgresql_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/postgres/data
      o: bind

  ray_log_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/ray/logs
      o: bind

  frontend_log_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/logs/frontend
      o: bind

  database_log_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/logs/database
      o: bind

  operator-upload-volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/operators/upload
      o: bind

  operator-runtime-volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/operators/runtime
      o: bind

  operator-packages-volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/operators/site-packages
      o: bind

  graph_data_volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/rag_storage
      o: bind

  # Deep Flow
  deep-flow-log-volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/logs/deep-flow
      o: bind

  # Label Studio
  label-studio-data:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/label-studio/data
      o: bind
  
  label-studio-db:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/label-studio/db
      o: bind

  # Milvus
  etcd-volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/milvus/etcd
      o: bind

  minio-volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/milvus/minio
      o: bind

  milvus-volume:
    driver: local
    driver_opts:
      type: none
      device: ./volumes/milvus/data
      o: bind

# ==============================
# Networks
# ==============================
networks:
  deepdata:
    driver: bridge
    name: deepdata-network