Introduce JSON matrix workflow

.github/workflows/benchmarks.yml

@@ -45,18 +45,32 @@ on:
         default: false
 
 jobs:
+  construct-matrix:
+    name: Construct Benchmarks matrix
+    runs-on: ubuntu-latest
+    outputs:
+      benchmarks-matrix: '${{ steps.generate-matrix.outputs.benchmarks-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "benchmarks-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_COMMAND: "swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev"
+          MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }}
+          MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }}
+          MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_6_0_ENABLED: ${{ inputs.linux_nightly_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_MAIN_ENABLED: ${{ inputs.linux_nightly_main_enabled }}
+
   benchmarks:
     name: Benchmarks
+    needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Benchmarks"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev && swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
-      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
-      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
-      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
-      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
-      matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }}
-      matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }}
-      matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }}
+      matrix_string: '${{ needs.construct-matrix.outputs.benchmarks-matrix }}'

.github/workflows/cxx_interop.yml

@@ -26,30 +26,44 @@ on:
 
       windows_6_0_enabled:
         type: boolean
-        description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to true."
+        description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to false. Currently has no effect!"  # TODO: implement Windows Cxx compat checking
         default: false
       windows_nightly_6_0_enabled:
         type: boolean
-        description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to true."
+        description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to false. Currently has no effect!"  # TODO: implement Windows Cxx compat checking
         default: false
       windows_nightly_main_enabled:
         type: boolean
-        description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to true."
+        description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to false. Currently has no effect!"  # TODO: implement Windows Cxx compat checking
         default: false
 
 jobs:
+  construct-matrix:
+    name: Construct Cxx interop matrix
+    runs-on: ubuntu-latest
+    outputs:
+      cxx-interop-matrix: '${{ steps.generate-matrix.outputs.cxx-interop-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "cxx-interop-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_COMMAND: "curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q curl jq"
+          MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }}
+          MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }}
+          MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_6_0_ENABLED: ${{ inputs.linux_nightly_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_MAIN_ENABLED: ${{ inputs.linux_nightly_main_enabled }}
+
   cxx-interop:
     name: Cxx interop
+    needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Cxx interop"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q jq && curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
-      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
-      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
-      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
-      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
-      matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }}
-      matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }}
-      matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }}
+      matrix_string: '${{ needs.construct-matrix.outputs.cxx-interop-matrix }}'

.github/workflows/main.yml

@@ -10,7 +10,7 @@ jobs:
   unit-tests:
     name: Unit tests
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/unit_tests.yml@main
+    uses: apple/swift-nio/.github/workflows/unit_tests.yml@matrix_file  # TODO: replace with @main
     with:
       linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
@@ -21,19 +21,36 @@ jobs:
   cxx-interop:
     name: Cxx interop
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main
+    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@matrix_file  # TODO: replace with @main
 
   benchmarks:
     name: Benchmarks
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
+    uses: apple/swift-nio/.github/workflows/benchmarks.yml@matrix_file  # TODO: replace with @main
     with:
       benchmark_package_path: "Benchmarks"
 
+  construct-integration-test-matrix:
+    name: Construct integration test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      integration-test-matrix: '${{ steps.generate-matrix.outputs.integration-test-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "integration-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq"
+          MATRIX_LINUX_COMMAND: "./scripts/integration_tests.sh"
+
   integration-tests:
-    name: Integration Tests
+    name: Integration tests
+    needs: construct-integration-test-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Integration tests"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq && ./scripts/integration_tests.sh"
+      matrix_string: '${{ needs.construct-integration-test-matrix.outputs.integration-test-matrix }}'

.github/workflows/pull_request.yml

@@ -14,33 +14,50 @@ jobs:
   unit-tests:
     name: Unit tests
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/unit_tests.yml@main
+    uses: apple/swift-nio/.github/workflows/unit_tests.yml@matrix_file  # TODO: replace with @main
     with:
       linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
-      linux_6_0_arguments_override: "--explicit-target-dependency-import-check error"
-      linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error"
+      linux_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
+      linux_nightly_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error"
 
   benchmarks:
     name: Benchmarks
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
+    uses: apple/swift-nio/.github/workflows/benchmarks.yml@matrix_file  # TODO: replace with @main
     with:
       benchmark_package_path: "Benchmarks"
 
   cxx-interop:
     name: Cxx interop
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main
+    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@matrix_file  # TODO: replace with @main
+
+  construct-integration-test-matrix:
+    name: Construct integration test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      integration-test-matrix: '${{ steps.generate-matrix.outputs.integration-test-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "integration-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq"
+          MATRIX_LINUX_COMMAND: "./scripts/integration_tests.sh"
 
   integration-tests:
-    name: Integration Tests
+    name: Integration tests
+    needs: construct-integration-test-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Integration tests"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq && ./scripts/integration_tests.sh"
+      matrix_string: '${{ needs.construct-integration-test-matrix.outputs.integration-test-matrix }}'
 
   vsock-tests:
     name: Vsock tests

.github/workflows/swift_test_matrix.yml

@@ -0,0 +1,93 @@
+name: Matrix
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        type: string
+        description: "The name of the workflow used for the concurrency group."
+        required: true
+      matrix_path:
+        type: string
+        description: "The path of the test matrix definition."
+        default: ""
+      matrix_string:
+        type: string
+        description: "The test matrix definition."
+        default: ""
+
+# We will cancel previously triggered workflow runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }}
+  cancel-in-progress: true
+
+jobs:
+  generate-matrix:
+    name: Prepare matrices
+    runs-on: ubuntu-latest
+    outputs:
+      swift-matrix: ${{ steps.load-matrix.outputs.swift-matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Mark the workspace as safe
+        # https://github.com/actions/checkout/issues/766
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - id: load-matrix
+        run: |
+          if [ -n '${{ inputs.matrix_string }}' ]; then
+            printf "swift-matrix=%s" "$(echo '${{ inputs.matrix_string }}' | jq -c '.')" >> "$GITHUB_OUTPUT"
+          else
+            printf "swift-matrix=%s" "$(jq -c '.' ${{ inputs.matrix_path }})" >> "$GITHUB_OUTPUT"
+          fi
+
+  execute-matrix:
+    name: ${{ matrix.swift.platform }} (${{ matrix.swift.name }})
+    needs: generate-matrix
+    runs-on: ${{ matrix.swift.runner }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.swift-matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          submodules: true
+      - name: Pull Docker image
+        run: docker pull ${{ matrix.swift.image }}
+      - name: Run matrix job
+        if: ${{ matrix.swift.platform != 'Windows' }}
+        run: |
+          if [[ -n "${{ matrix.swift.setup_command }}" ]]; then
+            setup_command_expression="${{ matrix.swift.setup_command }} &&"
+          else
+            setup_command_expression=""
+          fi
+          workspace="/$(basename ${{ github.workspace }})"
+          docker run -v ${{ github.workspace }}:"$workspace" \
+            -w "$workspace" \
+            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" \
+            -e setup_command_expression="$setup_command_expression" \
+            -e workspace="$workspace" \
+            ${{ matrix.swift.image }} \
+            bash -c "swift --version && git config --global --add safe.directory \"$workspace\" && $setup_command_expression ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"
+      - name: Run matrix job (Windows)
+        if: ${{ matrix.swift.platform == 'Windows' }}
+        run: |
+          if (-not [string]::IsNullOrEmpty("${{ matrix.swift.setup_command }}")) {
+              $setup_command_expression = "${{ matrix.swift.setup_command }} &"
+          } else {
+              $setup_command_expression = ""
+          }
+          $workspace = "C:\" + (Split-Path ${{ github.workspace }} -Leaf)
+          docker run -v ${{ github.workspace }}:$($workspace) `
+            -w $($workspace) `
+            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" `
+            -e setup_command_expression=%setup_command_expression% `
+            ${{ matrix.swift.image }} `
+            cmd /s /c  "swift --version & powershell Invoke-Expression ""$($setup_command_expression) ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"""
+    env:
+      SWIFT_VERSION: ${{ matrix.swift.swift_version }}

.github/workflows/unit_tests.yml

@@ -70,27 +70,43 @@ on:
         default: ""
 
 jobs:
+  construct-matrix:
+    name: Construct unit test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      unit-test-matrix: '${{ steps.generate-matrix.outputs.unit-test-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "unit-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_COMMAND: "swift test"
+          MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }}
+          MATRIX_LINUX_5_9_COMMAND_ARGUMENTS: ${{ inputs.linux_5_9_arguments_override }}
+          MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }}
+          MATRIX_LINUX_5_10_COMMAND_ARGUMENTS: ${{ inputs.linux_5_10_arguments_override }}
+          MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }}
+          MATRIX_LINUX_6_0_COMMAND_ARGUMENTS: ${{ inputs.linux_6_0_arguments_override }}
+          MATRIX_LINUX_NIGHTLY_6_0_ENABLED: ${{ inputs.linux_nightly_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_6_0_COMMAND_ARGUMENTS: ${{ inputs.linux_nightly_6_0_arguments_override }}
+          MATRIX_LINUX_NIGHTLY_MAIN_ENABLED: ${{ inputs.linux_nightly_main_enabled }}
+          MATRIX_LINUX_NIGHTLY_MAIN_COMMAND_ARGUMENTS: ${{ inputs.linux_nightly_main_arguments_override }}
+          MATRIX_WINDOWS_COMMAND: "swift test"
+          MATRIX_WINDOWS_6_0_ENABLED: ${{ inputs.windows_6_0_enabled }}
+          MATRIX_WINDOWS_6_0_COMMAND_ARGUMENTS: ${{ inputs.windows_6_0_arguments_override }}
+          MATRIX_WINDOWS_NIGHTLY_6_0_ENABLED: ${{ inputs.windows_nightly_6_0_enabled }}
+          MATRIX_WINDOWS_NIGHTLY_6_0_COMMAND_ARGUMENTS: ${{ inputs.windows_nightly_6_0_arguments_override }}
+          MATRIX_WINDOWS_NIGHTLY_MAIN_ENABLED: ${{ inputs.windows_nightly_main_enabled }}
+          MATRIX_WINDOWS_NIGHTLY_MAIN_COMMAND_ARGUMENTS: ${{ inputs.windows_nightly_main_arguments_override }}
+
   unit-tests:
     name: Unit tests
+    needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Unit tests"
-      matrix_linux_command: "swift test"
-      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
-      matrix_linux_5_9_command_override: "swift test ${{ inputs.linux_5_9_arguments_override }}"
-      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
-      matrix_linux_5_10_command_override: "swift test ${{ inputs.linux_5_10_arguments_override }}"
-      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
-      matrix_linux_6_0_command_override: "swift test ${{ inputs.linux_6_0_arguments_override }}"
-      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-      matrix_linux_nightly_6_0_command_override: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}"
-      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
-      matrix_linux_nightly_main_command_override: "swift test ${{ inputs.linux_nightly_main_arguments_override }}"
-      matrix_windows_command: "swift test"
-      matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }}
-      matrix_windows_6_0_command_override: "swift test ${{ inputs.windows_6_0_arguments_override }}"
-      matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }}
-      matrix_windows_nightly_6_0_command_override: "swift test ${{ inputs.windows_nightly_6_0_arguments_override }}"
-      matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }}
-      matrix_windows_nightly_main_command_override: "swift test ${{ inputs.windows_nightly_main_arguments_override }}"
+      matrix_string: '${{ needs.construct-matrix.outputs.unit-test-matrix }}'