Allow an override matrix to reduce CI workload.

If `matrix.yaml` contains a non-empty job matrix names `override`, this will be used instead of the `pull_request` matrix. The PR branch protections will prevent the branch from being merged as long as an override exists.

.github/actions/workflow-build/action.yml

@@ -5,6 +5,10 @@ inputs:
   workflows:
     description: "Space separated list of workflows in matrix file to run"
     required: true
+  allow_override:
+    description: "If true, the requested `workflows` will be ignored when a non-empty 'override' workflow exists in the matrix file."
+    default: "false"
+    required: false
   skip_tests:
     description: "Skip running tests"
     default: "false"
@@ -55,6 +59,7 @@ runs:
       id: build-workflow
       shell: bash --noprofile --norc -euo pipefail {0}
       env:
+        allow_override: ${{ inputs.allow_override == 'true' && '--allow-override' || ''}}
         skip_tests: ${{ inputs.skip_tests == 'true' && '--skip-tests' || ''}}
         dirty_projects_flag: ${{ steps.inspect-changes.outputs.dirty_projects != '' && '--dirty-projects' || ''}}
         dirty_projects: ${{ steps.inspect-changes.outputs.dirty_projects }}
@@ -64,6 +69,7 @@ runs:
 
         ${{ env.matrix_parser }} ${{ inputs.matrix_file }}           \
             --workflows ${{ inputs.workflows }}                      \
+            ${{ env.allow_override }}                                \
             ${{ env.skip_tests }}                                    \
             ${{ env.dirty_projects_flag }} ${{ env.dirty_projects }}
 

.github/actions/workflow-build/build-workflow.py

@@ -716,9 +716,22 @@ def process_job_array(group_name, array_name, parent_json):
     write_json_file("workflow/runner_summary.json", runner_json)
 
 
+def write_override_matrix(override_matrix):
+    os.makedirs("workflow", exist_ok=True)
+    write_json_file("workflow/override.json", override_matrix)
+
+
 def print_gha_workflow(args):
+    workflow_names = args.workflows
+    if args.allow_override and 'override' in matrix_yaml['workflows']:
+        override_matrix = matrix_yaml['workflows']['override']
+        if override_matrix and len(override_matrix) > 0:
+            print(f"::notice::Using 'override' workflow instead of '{workflow_names}'")
+            workflow_names = ['override']
+            write_override_matrix(override_matrix)
+
     final_workflow = {}
-    for workflow_name in args.workflows:
+    for workflow_name in workflow_names:
         workflow_dispatch_groups = parse_workflow_dispatch_groups(args, workflow_name)
         merge_dispatch_groups(final_workflow, workflow_dispatch_groups)
 
@@ -769,6 +782,8 @@ def main():
     parser.add_argument('--dirty-projects', nargs='*', help='Filter jobs to only these projects')
     parser.add_argument('--skip-tests', action='store_true',
                         help='Remove jobs defined in `matrix_file.skip_test_jobs`.')
+    parser.add_argument('--allow-override', action='store_true',
+                        help='If a non-empty "override" workflow exists, it will be used instead of those in --workflows.')
     args = parser.parse_args()
 
     # Check if the matrix file exists

.github/actions/workflow-results/action.yml

@@ -88,9 +88,17 @@ runs:
       id: check-dispatch
       shell: bash --noprofile --norc -euo pipefail {0}
       run: |
-        if "${GITHUB_ACTION_PATH}/verify-job-success.py" workflow/job_ids.json; then
-          echo "success=true" >> "${GITHUB_OUTPUT}"
-        else
+        if ! "${GITHUB_ACTION_PATH}/verify-job-success.py" workflow/job_ids.json; then
           echo "success=false" >> "${GITHUB_OUTPUT}"
           exit 1
         fi
+
+        if [ -f workflow/override.json ]; then
+          echo "::notice::Workflow matrix was overridden. Failing jobs."
+          echo "Override matrix:"
+          cat workflow/override.json | jq -c '.'
+          echo "success=false" >> "${GITHUB_OUTPUT}"
+          exit 1
+        fi
+
+        echo "success=true" >> "${GITHUB_OUTPUT}"

.github/workflows/ci-workflow-pull-request.yml

@@ -49,14 +49,17 @@ jobs:
       - name: Build workflow
         id: build-workflow
         uses: ./.github/actions/workflow-build
+        env:
+          pr_worflow: ${{ !contains(github.event.head_commit.message, '[workflow:!pull_request]') && 'pull_request' || '' }}
+          nightly_workflow: ${{ contains(github.event.head_commit.message, '[workflow:nightly]') && 'nightly' || '' }}
         with:
+          allow_override: "true"
           skip_tests: ${{ toJSON(contains(github.event.head_commit.message, '[skip-tests]')) }}
           inspect_changes_script: ${{ toJSON(!contains(github.event.head_commit.message, '[all-projects]') && 'ci/inspect_changes.sh' || '') }}
           inspect_changes_base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}
           workflows: >-
-            ${{ !contains(github.event.head_commit.message, '[workflow:!pull_request]') && 'pull_request' || '' }}
-            ${{ contains(github.event.head_commit.message, '[workflow:nightly]') && 'nightly' || '' }}
-            ${{ contains(github.event.head_commit.message, '[workflow:test]') && 'test' || '' }}
+            ${{ env.pr_worflow }}
+            ${{ env.nightly_workflow }}
 
   run-workflow:
     name: ${{ matrix.name }}

ci-overview.md

@@ -38,6 +38,22 @@ Special commands are provided that can be included in commit messages to direct
 
 Use these commands judiciously. While they offer flexibility, they should be used appropriately to maintain the codebase's integrity and quality.
 
+### Temporarily Overriding the Pull Requeest Matrix
+
+If a workflow named `override` exists in the matrix.yaml file, this matrix will be used for pull requests instead of the `pull_request` matrix.
+This is useful for reducing resource usage when launching many CI workflows from a PR (for example, while testing CI features).
+The overridden CI job will be marked as a failure until the override is removed.
+
+Example:
+
+```
+workflows:
+  override:
+    - {jobs: ['test'], std: 17, ctk: *ctk_curr, cxx: [*gcc12, *llvm16, *msvc2022]}
+  pull_request:
+    - <...>
+```
+
 ### Accelerating Build Times with `sccache`
 
 CCCL's CI uses [`sccache`](https://github.com/mozilla/sccache) to cache compiler artifacts for files that haven't changed and dramatically accelerate build times. Local builds inside [CCCL's Dev Containers](.devcontainer/README.md) can share the same cache such that local builds and CI jobs mutually benefit from accelerated build times. Follow the [GitHub Authentication](.devcontainer/README.md#optional-authenticate-with-github-for-sccache) guide to enable this feature.

ci/matrix.yaml

@@ -38,6 +38,17 @@ oneapi: &oneapi { name: 'oneapi', version: '2023.2.0', exe: 'icpc' }
 
 # GHA Workflow job matrices:
 workflows:
+  # If any jobs appear here, they will be executed instead of `pull_request' for PRs.
+  # This is useful for limiting resource usage when a full matrix is not needed.
+  # This object must be empty for a PR to pass branch protection checks.
+  #
+  # Example:
+  # override:
+  #   - {jobs: ['build'], project['thrust'], std: 17, ctk: *ctk_curr, cxx: [*gcc12, *llvm16]}
+  #
+  override:
+    - {jobs: ['build'], project: ['thrust'], std: 17, ctk: *ctk_curr, cxx: [*gcc12, *llvm16]}
+
   pull_request:
     # default_projects: nvcc
     - {jobs: ['build'], std: 'all', ctk: *ctk_11_1, cxx: [*gcc6, *gcc7, *gcc8, *gcc9, *llvm9, *msvc2017]}