figmar
99cdc24e44
CodeQL Advanced / Analyze (actions) (push) Canceled after 0s
CodeQL Advanced / Analyze (java-kotlin) (push) Canceled after 0s
CodeQL Advanced / Analyze (javascript-typescript) (push) Canceled after 0s
CodeQL Advanced / Analyze (python) (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Python Backend Docker Image CI / Build and Push Backend Python Docker Image (push) Canceled after 0s
Runtime Docker Image CI / Build and Push Runtime Docker Image (push) Canceled after 0s
- runtime: install zh_core_web_sm from vendored wheel (reused from runtime-ray)
instead of 'python -m spacy download' (GitHub 403 on server)
- runtime / runtime-ray / backend-python: replace nltk.download (GitHub +
nltk_data.nltk.org unreachable from server) with vendored zips extracted
to /usr/local/nltk_data/packages/{tokenizers,taggers}
109 lines
4.8 KiB
Docker
109 lines
4.8 KiB
Docker
# ==============================================================================
|
|
# Stage 1: DataX Builder
|
|
# ==============================================================================
|
|
FROM maven:3-eclipse-temurin-8 AS datax-builder
|
|
|
|
# 安装 git
|
|
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
|
|
|
COPY DataX/ /DataX/
|
|
COPY runtime/datax/ DataX/
|
|
|
|
# 创建阿里云 settings.xml
|
|
RUN mkdir -p /root/.m2 && \
|
|
cat > /root/.m2/settings.xml <<EOF
|
|
<settings>
|
|
<mirrors>
|
|
<mirror>
|
|
<id>aliyunmaven</id>
|
|
<mirrorOf>central</mirrorOf>
|
|
<name>Aliyun Maven</name>
|
|
<url>https://maven.aliyun.com/repository/public</url>
|
|
</mirror>
|
|
</mirrors>
|
|
</settings>
|
|
EOF
|
|
|
|
# 设置工作目录
|
|
WORKDIR /DataX
|
|
|
|
RUN find /DataX -type f -name "DataBaseType.java" -exec sed -i "s/com.mysql.jdbc.Driver/com.mysql.cj.jdbc.Driver/g" {} + || echo "⚠️ MySQL driver file not found, skipping." && \
|
|
sed -i 's/<maven-assembly-plugin.version>.*<\/maven-assembly-plugin.version>/<maven-assembly-plugin.version>2.6<\/maven-assembly-plugin.version>/' /DataX/pom.xml || echo "⚠️ Maven assembly version tag not found, skipping."
|
|
|
|
# 编译 DataX (精确挂载 repository 目录，不覆盖 settings.xml)
|
|
RUN --mount=type=cache,target=/root/.m2/repository \
|
|
mvn -s /root/.m2/settings.xml clean package assembly:assembly -Dmaven.test.skip=true
|
|
|
|
# ==============================================================================
|
|
# Stage 2: Ray Worker Runtime (保留原有的 AI 环境，增加 Java 支持)
|
|
# ==============================================================================
|
|
FROM ghcr.io/astral-sh/uv:python3.11-bookworm
|
|
|
|
# 1. 安装系统依赖
|
|
RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \
|
|
sed -i 's/security.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \
|
|
apt-get update && \
|
|
apt-get install -y --no-install-recommends \
|
|
libgl1 libglib2.0-0 vim libmagic1 dos2unix swig \
|
|
poppler-utils tesseract-ocr openjdk-17-jre-headless \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# 2. 下载 AI 模型 (保持不变)
|
|
RUN mkdir -p /home/models \
|
|
&& wget -q --show-progress https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-LCNet_x1_0_doc_ori_infer.tar \
|
|
&& tar -xf PP-LCNet_x1_0_doc_ori_infer.tar -C /home/models \
|
|
&& rm -f PP_*.tar
|
|
|
|
# 3. 复制业务代码 (保持不变)
|
|
COPY runtime/python-executor /opt/runtime
|
|
COPY runtime/ops /opt/runtime/deepdata/ops
|
|
COPY runtime/ops/user /opt/runtime/user
|
|
COPY scripts/images/runtime-ray/start.sh /opt/runtime/start.sh
|
|
COPY scripts/images/common/log-rotate-copytruncate.sh /usr/local/bin/log-rotate-copytruncate.sh
|
|
|
|
# 4. 复制编译好的 DataX (从 builder 阶段复制)
|
|
# ✅ 新增: 将编译好的 datax 复制到 /opt/datax
|
|
COPY --from=datax-builder /DataX/target/datax/datax /opt/datax
|
|
|
|
# ✅ 新增: 修复 MySQL 驱动路径 (参照 backend-python)
|
|
RUN cp /opt/datax/plugin/reader/mysqlreader/libs/mysql* /opt/datax/plugin/reader/starrocksreader/libs/ || true
|
|
|
|
# 5. 环境变量配置
|
|
ENV PYTHONPATH=/opt/runtime/deepdata/
|
|
ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
|
|
ENV UV_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu
|
|
ENV UV_INDEX_STRATEGY=unsafe-best-match
|
|
ENV PYTHONDONTWRITEBYTECODE=1
|
|
ENV PYTHONUNBUFFERED=1
|
|
|
|
# ✅ 新增: 设置 JAVA_HOME，确保 datax.py 能正确调用 Java
|
|
ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
|
|
ENV PATH="$PATH:$JAVA_HOME/bin"
|
|
|
|
WORKDIR /opt/runtime
|
|
|
|
COPY scripts/images/runtime-ray/zh_core_web_sm-3.8.0-py3-none-any.whl /tmp/
|
|
COPY scripts/images/common/nltk-data/ /tmp/nltk-data/
|
|
|
|
ENV UV_LINK_MODE=copy
|
|
|
|
# 6. 安装 Python 依赖
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install -e . --system \
|
|
&& uv pip install -r /opt/runtime/deepdata/ops/pyproject.toml --system \
|
|
&& uv pip install "ray[default]>=2.9.0" --system \
|
|
&& uv pip install /tmp/zh_core_web_sm-3.8.0-py3-none-any.whl --system \
|
|
&& python -c "import zipfile, os; os.makedirs('/usr/local/nltk_data/packages/tokenizers', exist_ok=True); os.makedirs('/usr/local/nltk_data/packages/taggers', exist_ok=True); zipfile.ZipFile('/tmp/nltk-data/punkt_tab.zip').extractall('/usr/local/nltk_data/packages/tokenizers'); zipfile.ZipFile('/tmp/nltk-data/averaged_perceptron_tagger_eng.zip').extractall('/usr/local/nltk_data/packages/taggers')" \
|
|
&& echo "/usr/local/lib/ops/site-packages" > /usr/local/lib/python3.11/site-packages/ops.pth
|
|
|
|
|
|
# 7. 初始化与权限
|
|
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
|
|
&& chmod +x /opt/runtime/start.sh \
|
|
&& chmod +x /usr/local/bin/log-rotate-copytruncate.sh \
|
|
&& dos2unix /opt/runtime/start.sh
|
|
|
|
EXPOSE 8081
|
|
|
|
ENTRYPOINT ["/opt/runtime/start.sh"]
|