Skip to content

DNM: A/B validating Wanda-generated Ray images #60186

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
andrew-anyscale wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

DNM: A/B validating Wanda-generated Ray images #60186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
322 changes: 275 additions & 47 deletions .buildkite/build.rayci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ steps:
- ray-java-build
- ray-dashboard-build

# Upload cpp wheel to S3
- label: ":s3: upload: cpp wheel (x86_64)"
key: linux_cpp_wheels_upload
instance_type: small
Expand Down Expand Up @@ -136,89 +135,317 @@ steps:
depends_on: manylinux-x86_64
job_env: manylinux-x86_64

- label: ":tapioca: build: ray py{{matrix}} docker (x86_64)"
key: ray_images
- name: ray-image-cpu-build
label: "wanda: ray py{{matrix}} cpu (x86_64)"
wanda: ci/docker/ray-image-cpu.wanda.yaml
matrix:
- "3.10"
- "3.11"
- "3.12"
env:
PYTHON_VERSION: "{{matrix}}"
ARCH_SUFFIX: ""
RAY_VERSION: "3.0.0.dev0"
tags:
- python_dependencies
- docker
- oss
instance_type: medium
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8
--platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9
--platform cu12.4.1-cudnn --platform cu12.5.1-cudnn
--platform cu12.6.3-cudnn --platform cu12.8.1-cudnn
--platform cu12.9.1-cudnn
--platform cpu
--image-type ray --upload
depends_on:
- manylinux-x86_64
- forge
- raycudabase
- ray-wheel-build
- raycpubase

- label: ":docker: push: ray py{{matrix}} cpu (x86_64)"
key: ray_images_cpu_push
instance_type: small
commands:
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix}}
--platform cpu
--image-type ray
--architecture x86_64
--upload
matrix:
- "3.10"
- "3.11"
- "3.12"
depends_on:
- ray-image-cpu-build
tags:
- python_dependencies
- docker
- skip-on-premerge
- oss

- label: ":tapioca: build: ray-extra py{{matrix}} docker (x86_64)"
key: ray_extra_images
- name: ray-image-cuda-build
label: "wanda: ray py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.10"
- "3.11"
- "3.12"
cuda:
- "11.7.1-cudnn8"
- "11.8.0-cudnn8"
- "12.1.1-cudnn8"
- "12.3.2-cudnn9"
- "12.4.1-cudnn"
- "12.5.1-cudnn"
- "12.6.3-cudnn"
- "12.8.1-cudnn"
- "12.9.1-cudnn"
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
RAY_VERSION: "3.0.0.dev0"
tags:
- python_dependencies
- docker
- oss
instance_type: medium
depends_on:
- ray-wheel-build
- raycudabase

- label: ":docker: push: ray py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
key: ray_images_cuda_push
instance_type: small
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8
--platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9
--platform cu12.4.1-cudnn --platform cu12.5.1-cudnn
--platform cu12.6.3-cudnn --platform cu12.8.1-cudnn
--platform cu12.9.1-cudnn
--platform cpu
--image-type ray-extra --upload
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix.python}}
--platform cu{{matrix.cuda}}
--image-type ray
--architecture x86_64
--upload
matrix:
setup:
python:
- "3.10"
- "3.11"
- "3.12"
cuda:
- "11.7.1-cudnn8"
- "11.8.0-cudnn8"
- "12.1.1-cudnn8"
- "12.3.2-cudnn9"
- "12.4.1-cudnn"
- "12.5.1-cudnn"
- "12.6.3-cudnn"
- "12.8.1-cudnn"
- "12.9.1-cudnn"
depends_on:
- manylinux-x86_64
- forge
- raycpubaseextra
- raycudabaseextra
- ray-image-cuda-build
tags:
- python_dependencies
- docker
- skip-on-premerge
- oss

- name: ray-extra-image-cpu-build
label: "wanda: ray-extra py{{matrix}} cpu (x86_64)"
wanda: ci/docker/ray-extra-image-cpu.wanda.yaml
matrix:
- "3.10"
- "3.11"
- "3.12"
env:
PYTHON_VERSION: "{{matrix}}"
ARCH_SUFFIX: ""
RAY_VERSION: "3.0.0.dev0"
tags:
- python_dependencies
- docker
- oss
depends_on:
- ray-wheel-build
- raycpubaseextra

- label: ":tapioca: build: ray-llm py{{matrix}} docker (x86_64)"
- name: ray-extra-image-cuda-build
label: "wanda: ray-extra py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-extra-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.10"
- "3.11"
- "3.12"
cuda:
- "11.7.1-cudnn8"
- "11.8.0-cudnn8"
- "12.1.1-cudnn8"
- "12.3.2-cudnn9"
- "12.4.1-cudnn"
- "12.5.1-cudnn"
- "12.6.3-cudnn"
- "12.8.1-cudnn"
- "12.9.1-cudnn"
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
RAY_VERSION: "3.0.0.dev0"
tags:
- python_dependencies
- docker
- oss
instance_type: medium
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu12.8.1-cudnn --image-type ray-llm --upload
depends_on:
- manylinux-x86_64
- forge
- ray-wheel-build
- raycudabaseextra

- name: ray-llm-image-cuda-build
label: "wanda: ray-llm py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-llm-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.11"
cuda:
- "12.8.1-cudnn"
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
RAY_VERSION: "3.0.0.dev0"
tags:
- python_dependencies
- docker
- oss
depends_on:
- ray-wheel-build
- ray-llmbase

- name: ray-llm-extra-image-cuda-build
label: "wanda: ray-llm-extra py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-llm-extra-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.11"
cuda:
- "12.8.1-cudnn"
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
RAY_VERSION: "3.0.0.dev0"
tags:
- python_dependencies
- docker
- oss
depends_on:
- ray-wheel-build
- ray-llmbaseextra

- label: ":docker: push: ray-extra py{{matrix}} cpu (x86_64)"
key: ray_extra_images_cpu_push
instance_type: small
commands:
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix}}
--platform cpu
--image-type ray-extra
--architecture x86_64
--upload
matrix:
- "3.10"
- "3.11"
- "3.12"
depends_on:
- ray-extra-image-cpu-build
tags:
- python_dependencies
- docker
- skip-on-premerge
- oss

- label: ":tapioca: build: ray-llm-extra py{{matrix}} docker (x86_64)"
- label: ":docker: push: ray-extra py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
key: ray_extra_images_cuda_push
instance_type: small
commands:
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix.python}}
--platform cu{{matrix.cuda}}
--image-type ray-extra
--architecture x86_64
--upload
matrix:
setup:
python:
- "3.10"
- "3.11"
- "3.12"
cuda:
- "11.7.1-cudnn8"
- "11.8.0-cudnn8"
- "12.1.1-cudnn8"
- "12.3.2-cudnn9"
- "12.4.1-cudnn"
- "12.5.1-cudnn"
- "12.6.3-cudnn"
- "12.8.1-cudnn"
- "12.9.1-cudnn"
depends_on:
- ray-extra-image-cuda-build
tags:
- python_dependencies
- docker
- skip-on-premerge
- oss
instance_type: medium

- label: ":docker: push: ray-llm py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
key: ray_llm_images_cuda_push
instance_type: small
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu12.8.1-cudnn --image-type ray-llm-extra --upload
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix.python}}
--platform cu{{matrix.cuda}}
--image-type ray-llm
--architecture x86_64
--upload
matrix:
setup:
python:
- "3.11"
cuda:
- "12.8.1-cudnn"
depends_on:
- manylinux-x86_64
- forge
- ray-llmbaseextra
- ray-llm-image-cuda-build
tags:
- python_dependencies
- docker
- skip-on-premerge
- oss

- label: ":docker: push: ray-llm-extra py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
key: ray_llm_extra_images_cuda_push
instance_type: small
commands:
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix.python}}
--platform cu{{matrix.cuda}}
--image-type ray-llm-extra
--architecture x86_64
--upload
matrix:
- "3.11"
setup:
python:
- "3.11"
cuda:
- "12.8.1-cudnn"
depends_on:
- ray-llm-extra-image-cuda-build
tags:
- python_dependencies
- docker
- skip-on-premerge
- oss

- label: ":tapioca: smoke test build-docker.sh"
tags:
Expand All @@ -245,6 +472,7 @@ steps:
- bazel run .buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:generate_index -- --prefix nightly
depends_on:
- ray_images
- ray_images_cpu_push
- ray_images_cuda_push
- ray_images_aarch64
- forge
13 changes: 13 additions & 0 deletions ci/docker/ray-extra-image-cpu.wanda.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
name: "ray-extra-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX" # CPU base-extra image
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- BASE_IMAGE=cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX
- RAY_WHEEL_IMAGE=cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
Loading