# ============================================================================== # Stage 1: DataX Builder # ============================================================================== FROM maven:3-eclipse-temurin-8 AS datax-builder # 安装 git RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* COPY DataX/ /DataX/ COPY runtime/datax/ DataX/ # 创建阿里云 settings.xml RUN mkdir -p /root/.m2 && \ cat > /root/.m2/settings.xml < aliyunmaven central Aliyun Maven https://maven.aliyun.com/repository/public EOF # 设置工作目录 WORKDIR /DataX RUN find /DataX -type f -name "DataBaseType.java" -exec sed -i "s/com.mysql.jdbc.Driver/com.mysql.cj.jdbc.Driver/g" {} + || echo "⚠️ MySQL driver file not found, skipping." && \ sed -i 's/.*<\/maven-assembly-plugin.version>/2.6<\/maven-assembly-plugin.version>/' /DataX/pom.xml || echo "⚠️ Maven assembly version tag not found, skipping." # 编译 DataX (精确挂载 repository 目录,不覆盖 settings.xml) RUN --mount=type=cache,target=/root/.m2/repository \ mvn -s /root/.m2/settings.xml clean package assembly:assembly -Dmaven.test.skip=true # ============================================================================== # Stage 2: Ray Worker Runtime (保留原有的 AI 环境,增加 Java 支持) # ============================================================================== FROM ghcr.io/astral-sh/uv:python3.11-bookworm # 1. 安装系统依赖 RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \ sed -i 's/security.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \ apt-get update && \ apt-get install -y --no-install-recommends \ libgl1 libglib2.0-0 vim libmagic1 dos2unix swig \ poppler-utils tesseract-ocr openjdk-17-jre-headless \ && rm -rf /var/lib/apt/lists/* # 2. 下载 AI 模型 (保持不变) RUN mkdir -p /home/models \ && wget -q --show-progress https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-LCNet_x1_0_doc_ori_infer.tar \ && tar -xf PP-LCNet_x1_0_doc_ori_infer.tar -C /home/models \ && rm -f PP_*.tar # 3. 复制业务代码 (保持不变) COPY runtime/python-executor /opt/runtime COPY runtime/ops /opt/runtime/deepdata/ops COPY runtime/ops/user /opt/runtime/user COPY scripts/images/runtime-ray/start.sh /opt/runtime/start.sh COPY scripts/images/common/log-rotate-copytruncate.sh /usr/local/bin/log-rotate-copytruncate.sh # 4. 复制编译好的 DataX (从 builder 阶段复制) # ✅ 新增: 将编译好的 datax 复制到 /opt/datax COPY --from=datax-builder /DataX/target/datax/datax /opt/datax # ✅ 新增: 修复 MySQL 驱动路径 (参照 backend-python) RUN cp /opt/datax/plugin/reader/mysqlreader/libs/mysql* /opt/datax/plugin/reader/starrocksreader/libs/ || true # 5. 环境变量配置 ENV PYTHONPATH=/opt/runtime/deepdata/ ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple ENV UV_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu ENV UV_INDEX_STRATEGY=unsafe-best-match ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 # ✅ 新增: 设置 JAVA_HOME,确保 datax.py 能正确调用 Java ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 ENV PATH="$PATH:$JAVA_HOME/bin" WORKDIR /opt/runtime COPY scripts/images/runtime-ray/zh_core_web_sm-3.8.0-py3-none-any.whl /tmp/ COPY scripts/images/common/nltk-data/ /tmp/nltk-data/ ENV UV_LINK_MODE=copy # 6. 安装 Python 依赖 RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install -e . --system \ && uv pip install -r /opt/runtime/deepdata/ops/pyproject.toml --system \ && uv pip install "ray[default]>=2.9.0" --system \ && uv pip install /tmp/zh_core_web_sm-3.8.0-py3-none-any.whl --system \ && python -c "import zipfile, os; os.makedirs('/usr/local/nltk_data/packages/tokenizers', exist_ok=True); os.makedirs('/usr/local/nltk_data/packages/taggers', exist_ok=True); zipfile.ZipFile('/tmp/nltk-data/punkt_tab.zip').extractall('/usr/local/nltk_data/packages/tokenizers'); zipfile.ZipFile('/tmp/nltk-data/averaged_perceptron_tagger_eng.zip').extractall('/usr/local/nltk_data/packages/taggers')" \ && echo "/usr/local/lib/ops/site-packages" > /usr/local/lib/python3.11/site-packages/ops.pth # 7. 初始化与权限 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && chmod +x /opt/runtime/start.sh \ && chmod +x /usr/local/bin/log-rotate-copytruncate.sh \ && dos2unix /opt/runtime/start.sh EXPOSE 8081 ENTRYPOINT ["/opt/runtime/start.sh"]