Migrate wheel builds to wanda (x86_64)

.buildkite/build.rayci.yml

@@ -35,23 +35,46 @@ steps:
       HOSTTYPE: "x86_64"
       MANYLINUX_VERSION: "260103.868e54c"
 
-  - label: ":tapioca: build: wheel {{matrix}} (x86_64)"
-    key: linux_wheels
+  - name: ray-wheel-build
+    label: "wanda: wheel py{{matrix}} (x86_64)"
+    wanda: ci/docker/ray-wheel.wanda.yaml
+    matrix:
+      - "3.10"
+      - "3.11"
+      - "3.12"
+      - "3.13"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      ARCH_SUFFIX: ""
+      HOSTTYPE: "x86_64"
+      MANYLINUX_VERSION: "260103.868e54c"
     tags:
       - release_wheels
       - linux_wheels
       - oss
-    instance_type: large
+    depends_on:
+      - ray-core-build
+      - ray-java-build
+      - ray-dashboard-build
+
+  # Upload wheels to S3
+  - label: ":s3: upload: wheel py{{matrix}} (x86_64)"
+    key: linux_wheels_upload
+    instance_type: small
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- wheel --python-version {{matrix}} --architecture x86_64 --upload
+      - ./ci/build/extract_wanda_wheels.sh ray-wheel-py{{matrix}}
+      - ./ci/build/copy_build_artifacts.sh wheel
     matrix:
       - "3.10"
       - "3.11"
       - "3.12"
       - "3.13"
     depends_on:
-      - manylinux-x86_64
-      - forge
+      - ray-wheel-build
+    tags:
+      - release_wheels
+      - skip-on-premerge
+      - oss
 
   - label: ":tapioca: build: jar"
     key: java_wheels

ci/build/extract_wanda_wheels.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -exuo pipefail
+
+# Extract wheels from a Wanda-cached image.
+#
+# Usage: ./ci/build/extract_wanda_wheels.sh <wanda_image_name>
+#
+# Example:
+#   ./ci/build/extract_wanda_wheels.sh ray-wheel-py3.10
+#
+# The script will:
+#   1. Export the wanda image using crane
+#   2. Extract .whl files from the tarball
+#   3. Move them to .whl/ directory (clears existing wheels first)
+
+WANDA_IMAGE_NAME=${1:?Usage: $0 <wanda_image_name>}
+
+# Construct full image tag from environment
+WANDA_IMAGE="${RAYCI_WORK_REPO}:${RAYCI_BUILD_ID}-${WANDA_IMAGE_NAME}"
+
+echo "Extracting wheels from: ${WANDA_IMAGE}"
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir" || true' EXIT
+
+# Export image and extract to temp dir
+crane export "${WANDA_IMAGE}" - | tar -xf - -C "$tmpdir"
+
+# Clear existing wheels to avoid stale files from previous runs
+rm -rf .whl
+mkdir -p .whl
+
+# Move extracted wheels to .whl/
+find "${tmpdir}" -type f -name '*.whl' -exec mv {} .whl/ \;
+
+# Verify that wheels were actually extracted
+wheels=($(find .whl -maxdepth 1 -name '*.whl'))
+if (( ${#wheels[@]} == 0 )); then
+  echo "ERROR: No wheel files were extracted from image: ${WANDA_IMAGE}" >&2
+  echo "This may indicate image corruption, incorrect image tag, or path issues." >&2
+  exit 1
+fi
+
+echo "Extracted ${#wheels[@]} wheel(s):"
+ls -la .whl/

ci/docker/ray-wheel.Dockerfile

@@ -0,0 +1,89 @@
+# syntax=docker/dockerfile:1.3-labs
+#
+# Ray Wheel Builder
+# =================
+# Builds manylinux2014-compatible ray wheel using pre-built C++ artifacts from wanda cache.
+#
+# GLIBC Compatibility:
+# --------------------
+# manylinux2014 requires GLIBC <= 2.17 for broad Linux compatibility.
+# The pre-built _raylet.so is compiled inside manylinux2014 with GLIBC 2.17.
+#
+
+ARG RAY_CORE_IMAGE
+ARG RAY_JAVA_IMAGE
+ARG RAY_DASHBOARD_IMAGE
+ARG MANYLINUX_VERSION
+ARG HOSTTYPE
+
+FROM ${RAY_CORE_IMAGE} AS ray-core
+FROM ${RAY_JAVA_IMAGE} AS ray-java
+FROM ${RAY_DASHBOARD_IMAGE} AS ray-dashboard
+
+# Main build stage - manylinux2014 provides GLIBC 2.17
+FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
+
+ARG PYTHON_VERSION=3.10
+ARG BUILDKITE_COMMIT
+
+WORKDIR /home/forge/ray
+
+# Copy artifacts from all stages
+COPY --from=ray-core /ray_pkg.zip /tmp/
+COPY --from=ray-core /ray_py_proto.zip /tmp/
+COPY --from=ray-java /ray_java_pkg.zip /tmp/
+COPY --from=ray-dashboard /dashboard.tar.gz /tmp/
+
+# Source files needed for wheel build
+COPY --chown=forge ci/build/build-manylinux-wheel.sh ci/build/
+COPY --chown=forge README.rst pyproject.toml ./
+COPY --chown=forge rllib/ rllib/
+COPY --chown=forge python/ python/
+
+USER forge
+# - BUILDKITE_COMMIT: Used for ray.__commit__. Defaults to "unknown" for local builds.
+ENV PYTHON_VERSION=${PYTHON_VERSION} \
+    BUILDKITE_COMMIT=${BUILDKITE_COMMIT:-unknown}
+RUN <<'EOF'
+#!/bin/bash
+set -euo pipefail
+
+# Clean extraction dirs to avoid stale leftovers
+rm -rf /tmp/ray_pkg /tmp/ray_java_pkg
+mkdir -p /tmp/ray_pkg /tmp/ray_java_pkg
+
+# Unpack pre-built artifacts
+unzip -o /tmp/ray_pkg.zip -d /tmp/ray_pkg
+unzip -o /tmp/ray_py_proto.zip -d python/
+unzip -o /tmp/ray_java_pkg.zip -d /tmp/ray_java_pkg
+mkdir -p python/ray/dashboard/client/build
+tar -xzf /tmp/dashboard.tar.gz -C python/ray/dashboard/client/build/
+
+# C++ core artifacts
+cp -r /tmp/ray_pkg/ray/* python/ray/
+
+# Java JARs
+cp -r /tmp/ray_java_pkg/ray/* python/ray/
+
+# Build ray wheel
+PY_VERSION="${PYTHON_VERSION//./}"
+PY_BIN="cp${PY_VERSION}-cp${PY_VERSION}"
+SKIP_BAZEL_BUILD=1 RAY_DISABLE_EXTRA_CPP=1 \
+./ci/build/build-manylinux-wheel.sh "$PY_BIN"
+
+# Sanity check: ensure wheels exist
+if [[ ! -d .whl ]]; then
+  echo "ERROR: .whl directory not created"
+  exit 1
+fi
+wheels=($(find .whl -maxdepth 1 -name '*.whl'))
+if (( ${#wheels[@]} == 0 )); then
+  echo "ERROR: No wheels produced in .whl/"
+  ls -la .whl
+  exit 1
+fi
+
+EOF
+
+FROM scratch
+COPY --from=builder /home/forge/ray/.whl/*.whl /

ci/docker/ray-wheel.wanda.yaml

@@ -0,0 +1,24 @@
+name: "ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"
+disable_caching: true
+froms:
+  - "rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"
+  - "cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX"    # C++ binaries (ray_pkg.zip)
+  - "cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX"                # Java JARs
+  - "cr.ray.io/rayproject/ray-dashboard"                             # Dashboard
+dockerfile: ci/docker/ray-wheel.Dockerfile
+srcs:
+  - pyproject.toml
+  - README.rst
+  - ci/build/build-manylinux-wheel.sh
+  - python/
+  - rllib/
+build_args:
+  - PYTHON_VERSION
+  - MANYLINUX_VERSION
+  - HOSTTYPE
+  - BUILDKITE_COMMIT
+  - ARCH_SUFFIX
+  - WHEEL_TYPE=ray
+  - RAY_CORE_IMAGE=cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX
+  - RAY_JAVA_IMAGE=cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX
+  - RAY_DASHBOARD_IMAGE=cr.ray.io/rayproject/ray-dashboard