Generate build matrix for 0.26.0.

mosaicml · Oct 24, 2024 · 77347ff · 77347ff
1 parent c1622f0
commit 77347ff
Show file tree

Hide file tree

Showing 3 changed files with 223 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -292,3 +292,21 @@ P.S.: [We're hiring](https://www.databricks.com/company/careers/open-positions?
     howpublished = {\url{https://github.com/mosaicml/composer/}},
 }
 ```
+<!-- BEGIN_PYTORCH_BUILD_MATRIX -->
+| Linux Distro   | Flavor   | PyTorch Version   | CUDA Version        | Python Version   | Docker Tags                                                                                                                                                                                                          |
+|----------------|----------|-------------------|---------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Ubuntu 20.04   | Base     | 2.5.0             | 12.4.1 (Infiniband) | 3.11             | `mosaicml/pytorch:latest`, `mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04`                 |
+| Ubuntu 20.04   | Base     | 2.5.0             | 12.4.1 (EFA)        | 3.11             | `mosaicml/pytorch:latest-aws`, `mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04-aws` |
+| Ubuntu 20.04   | Base     | 2.5.0             | cpu                 | 3.11             | `mosaicml/pytorch:latest_cpu`, `mosaicml/pytorch:2.5.0_cpu-python3.11-ubuntu20.04`             |
+| Ubuntu 20.04   | Base     | 2.4.1             | 12.4.1 (Infiniband) | 3.11             | `mosaicml/pytorch:2.4.1_cu124-python3.11-ubuntu20.04`                                                                                        |
+| Ubuntu 20.04   | Base     | 2.4.1             | 12.4.1 (EFA)        | 3.11             | `mosaicml/pytorch:2.4.1_cu124-python3.11-ubuntu20.04-aws`                                                                                |
+| Ubuntu 20.04   | Base     | 2.4.1             | cpu                 | 3.11             | `mosaicml/pytorch:2.4.1_cpu-python3.11-ubuntu20.04`                                                                                            |
+| Ubuntu 20.04   | Base     | 2.3.1             | 12.1.1 (Infiniband) | 3.11             | `mosaicml/pytorch:2.3.1_cu121-python3.11-ubuntu20.04`                                                                                        |
+| Ubuntu 20.04   | Base     | 2.3.1             | 12.1.1 (EFA)        | 3.11             | `mosaicml/pytorch:2.3.1_cu121-python3.11-ubuntu20.04-aws`                                                                                |
+| Ubuntu 20.04   | Base     | 2.3.1             | cpu                 | 3.11             | `mosaicml/pytorch:2.3.1_cpu-python3.11-ubuntu20.04`                                                                                            |
+<!-- END_PYTORCH_BUILD_MATRIX --><!-- BEGIN_COMPOSER_BUILD_MATRIX -->
+| Composer Version   | CUDA Support   | Docker Tag                                                                                                                                                       |
+|--------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.26.0             | Yes            | `mosaicml/composer:latest`, `mosaicml/composer:0.26.0`                 |
+| 0.26.0             | No             | `mosaicml/composer:latest_cpu`, `mosaicml/composer:0.26.0_cpu` |
+<!-- END_COMPOSER_BUILD_MATRIX -->
diff --git a/build_matrix.yaml b/build_matrix.yaml
@@ -0,0 +1,204 @@
+# This file is automatically generated by generate_build_matrix.py. DO NOT EDIT!
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: nvidia/cuda:12.4.1-cudnn-devel-ubuntu20.04
+  CUDA_VERSION: 12.4.1
+  IMAGE_NAME: torch-2-5-0-cu124
+  MOFED_VERSION: latest-23.10
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.5.0
+  TAGS:
+  - mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cu124-python3.11-ubuntu20.04
+  - mosaicml/pytorch:latest
+  - ghcr.io/databricks-mosaic/pytorch:latest
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.20.0
+- AWS_OFI_NCCL_VERSION: v1.11.0-aws
+  BASE_IMAGE: nvidia/cuda:12.4.1-cudnn-devel-ubuntu20.04
+  CUDA_VERSION: 12.4.1
+  IMAGE_NAME: torch-2-5-0-cu124-aws
+  MOFED_VERSION: ''
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.5.0
+  TAGS:
+  - mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04-aws
+  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cu124-python3.11-ubuntu20.04-aws
+  - mosaicml/pytorch:latest-aws
+  - ghcr.io/databricks-mosaic/pytorch:latest-aws
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.20.0
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: ubuntu:20.04
+  CUDA_VERSION: ''
+  IMAGE_NAME: torch-2-5-0-cpu
+  MOFED_VERSION: ''
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.5.0
+  TAGS:
+  - mosaicml/pytorch:2.5.0_cpu-python3.11-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cpu-python3.11-ubuntu20.04
+  - mosaicml/pytorch:latest_cpu
+  - ghcr.io/databricks-mosaic/pytorch:latest_cpu
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.20.0
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: nvidia/cuda:12.4.1-cudnn-devel-ubuntu20.04
+  CUDA_VERSION: 12.4.1
+  IMAGE_NAME: torch-2-4-1-cu124
+  MOFED_VERSION: latest-23.10
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.4.1
+  TAGS:
+  - mosaicml/pytorch:2.4.1_cu124-python3.11-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.4.1_cu124-python3.11-ubuntu20.04
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.19.1
+- AWS_OFI_NCCL_VERSION: v1.11.0-aws
+  BASE_IMAGE: nvidia/cuda:12.4.1-cudnn-devel-ubuntu20.04
+  CUDA_VERSION: 12.4.1
+  IMAGE_NAME: torch-2-4-1-cu124-aws
+  MOFED_VERSION: ''
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.4.1
+  TAGS:
+  - mosaicml/pytorch:2.4.1_cu124-python3.11-ubuntu20.04-aws
+  - ghcr.io/databricks-mosaic/pytorch:2.4.1_cu124-python3.11-ubuntu20.04-aws
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.19.1
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: ubuntu:20.04
+  CUDA_VERSION: ''
+  IMAGE_NAME: torch-2-4-1-cpu
+  MOFED_VERSION: ''
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.4.1
+  TAGS:
+  - mosaicml/pytorch:2.4.1_cpu-python3.11-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.4.1_cpu-python3.11-ubuntu20.04
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.19.1
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: nvidia/cuda:12.1.1-cudnn8-devel-ubuntu20.04
+  CUDA_VERSION: 12.1.1
+  IMAGE_NAME: torch-2-3-1-cu121
+  MOFED_VERSION: latest-23.10
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: cuda>=12.1 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471
+    brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471
+    brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471
+    brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471
+    brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511
+    brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511
+    brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511
+    brand=titanrtx,driver>=510,driver<511 brand=tesla,driver>=515,driver<516 brand=unknown,driver>=515,driver<516
+    brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516
+    brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516
+    brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516 brand=tesla,driver>=525,driver<526
+    brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526
+    brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526
+    brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.3.1
+  TAGS:
+  - mosaicml/pytorch:2.3.1_cu121-python3.11-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.3.1_cu121-python3.11-ubuntu20.04
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.18.1
+- AWS_OFI_NCCL_VERSION: v1.11.0-aws
+  BASE_IMAGE: nvidia/cuda:12.1.1-cudnn8-devel-ubuntu20.04
+  CUDA_VERSION: 12.1.1
+  IMAGE_NAME: torch-2-3-1-cu121-aws
+  MOFED_VERSION: ''
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: cuda>=12.1 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471
+    brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471
+    brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471
+    brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471
+    brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511
+    brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511
+    brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511
+    brand=titanrtx,driver>=510,driver<511 brand=tesla,driver>=515,driver<516 brand=unknown,driver>=515,driver<516
+    brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516
+    brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516
+    brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516 brand=tesla,driver>=525,driver<526
+    brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526
+    brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526
+    brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.3.1
+  TAGS:
+  - mosaicml/pytorch:2.3.1_cu121-python3.11-ubuntu20.04-aws
+  - ghcr.io/databricks-mosaic/pytorch:2.3.1_cu121-python3.11-ubuntu20.04-aws
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.18.1
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: ubuntu:20.04
+  CUDA_VERSION: ''
+  IMAGE_NAME: torch-2-3-1-cpu
+  MOFED_VERSION: ''
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.3.1
+  TAGS:
+  - mosaicml/pytorch:2.3.1_cpu-python3.11-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.3.1_cpu-python3.11-ubuntu20.04
+  TARGET: pytorch_stage
+  TORCHVISION_VERSION: 0.18.1
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: nvidia/cuda:12.4.1-cudnn-devel-ubuntu20.04
+  COMPOSER_INSTALL_COMMAND: mosaicml[all]==0.26.0
+  CUDA_VERSION: 12.4.1
+  IMAGE_NAME: composer-0-26-0
+  MOFED_VERSION: latest-23.10
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.5.0
+  TAGS:
+  - mosaicml/composer:0.26.0
+  - ghcr.io/databricks-mosaic/composer:0.26.0
+  - mosaicml/composer:latest
+  - ghcr.io/databricks-mosaic/composer:latest
+  TARGET: composer_stage
+  TORCHVISION_VERSION: 0.20.0
+- AWS_OFI_NCCL_VERSION: ''
+  BASE_IMAGE: ubuntu:20.04
+  COMPOSER_INSTALL_COMMAND: mosaicml[all]==0.26.0
+  CUDA_VERSION: ''
+  IMAGE_NAME: composer-0-26-0-cpu
+  MOFED_VERSION: latest-23.10
+  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
+  PYTHON_VERSION: '3.11'
+  PYTORCH_NIGHTLY_URL: ''
+  PYTORCH_NIGHTLY_VERSION: ''
+  PYTORCH_VERSION: 2.5.0
+  TAGS:
+  - mosaicml/composer:0.26.0_cpu
+  - ghcr.io/databricks-mosaic/composer:0.26.0_cpu
+  - mosaicml/composer:latest_cpu
+  - ghcr.io/databricks-mosaic/composer:latest_cpu
+  TARGET: composer_stage
+  TORCHVISION_VERSION: 0.20.0
diff --git a/docker/generate_build_matrix.py b/docker/generate_build_matrix.py
@@ -244,7 +244,7 @@ def _main():
     composer_entries = []
 
     # The `GIT_COMMIT` is a placeholder and Jenkins will substitute it with the actual git commit for the `composer_staging` images
-    composer_versions = ['0.25.0']  # Only build images for the latest composer version
+    composer_versions = ['0.26.0']  # Only build images for the latest composer version
     composer_python_versions = [PRODUCTION_PYTHON_VERSION]  # just build composer against the latest
 
     for product in itertools.product(composer_python_versions, composer_versions, cuda_options):