[GHA] Cxx interoperability compatibility and integration tests check

# Motivation

Another reusable check is to make sure that all library products of a package are successfully building when consumed from a module that has Cxx interoperability enabled. Another check that's missing is running the integration tests.

# Modification

This PR adds two new checks to the reusable workflow. One to check for Cxx interoperability compatibility and another one to run the integration tests. I also fixed a misalgined name for the nightly benchmarks.

# Result

This should be one of the last reusable workflow checks.

.github/workflows/pull_request.yml

@@ -24,5 +24,18 @@ jobs:
     uses: ./.github/workflows/pull_request_swift_matrix.yml
     with:
       name: "Benchmarks"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev && swift package --package-path Benchmarks/ --disable-sandbox benchmark baseline check --check-absolute-path Benchmarks/Thresholds/${SWIFT_VERSION}/
-"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev && swift package --package-path Benchmarks/ --disable-sandbox benchmark baseline check --check-absolute-path Benchmarks/Thresholds/${SWIFT_VERSION}/"
+
+  call-pull-request-cxx-interop-workflow:
+    name: Cxx interop
+    uses: ./.github/workflows/pull_request_swift_matrix.yml
+    with:
+      name: "Cxx interop"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q jq && ./scripts/check-cxx-interop-compatibility.sh"
+
+  call-pull-request-integration-tests-workflow:
+    name: Integration tests
+    uses: ./.github/workflows/pull_request_swift_matrix.yml
+    with:
+      name: "Integration tests"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq && ./scripts/integration_tests.sh"

IntegrationTests/tests_04_performance/Thresholds/5.10.json

@@ -0,0 +1,51 @@
+{
+    "10000000_asyncsequenceproducer": 21,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 44050,
+    "1000_addHandlers_sync": 37050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
+    "1000_autoReadGetAndSet_sync": 0,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
+    "1000_getHandlers_sync": 36,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 145050,
+    "1000_tcpbootstraps": 3050,
+    "1000_tcpconnections": 152050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 389000,
+    "bytebuffer_lots_of_rw": 2050,
+    "creating_10000_headers": 0,
+    "decode_1000_ws_frames": 2050,
+    "encode_1000_ws_frames_holding_buffer": 3,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
+    "encode_1000_ws_frames_holding_buffer_with_space": 3,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
+    "execute_hop_10000_tasks": 0,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
+    "modifying_1000_circular_buffer_elements": 0,
+    "modifying_byte_buffer_view": 6050,
+    "ping_pong_1000_reqs_1_conn": 319,
+    "read_10000_chunks_from_file": 110200,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
+    "scheduling_10000_executions": 89,
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
+}
+

IntegrationTests/tests_04_performance/Thresholds/5.8.json

@@ -0,0 +1,51 @@
+{
+    "10000000_asyncsequenceproducer": 22,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 44050,
+    "1000_addHandlers_sync": 37050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
+    "1000_autoReadGetAndSet_sync": 0,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
+    "1000_getHandlers_sync": 36,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 147050,
+    "1000_tcpbootstraps": 4050,
+    "1000_tcpconnections": 154050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 398000,
+    "bytebuffer_lots_of_rw": 2050,
+    "creating_10000_headers": 0,
+    "decode_1000_ws_frames": 2050,
+    "encode_1000_ws_frames_holding_buffer": 3,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
+    "encode_1000_ws_frames_holding_buffer_with_space": 3,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
+    "execute_hop_10000_tasks": 0,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
+    "modifying_1000_circular_buffer_elements": 0,
+    "modifying_byte_buffer_view": 6050,
+    "ping_pong_1000_reqs_1_conn": 334,
+    "read_10000_chunks_from_file": 110200,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
+    "scheduling_10000_executions": 89,
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
+}
+

IntegrationTests/tests_04_performance/Thresholds/5.9.json

@@ -0,0 +1,51 @@
+{
+    "10000000_asyncsequenceproducer": 21,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 44050,
+    "1000_addHandlers_sync": 37050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
+    "1000_autoReadGetAndSet_sync": 0,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
+    "1000_getHandlers_sync": 36,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 147050,
+    "1000_tcpbootstraps": 4050,
+    "1000_tcpconnections": 154050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 398000,
+    "bytebuffer_lots_of_rw": 2050,
+    "creating_10000_headers": 0,
+    "decode_1000_ws_frames": 2050,
+    "encode_1000_ws_frames_holding_buffer": 3,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
+    "encode_1000_ws_frames_holding_buffer_with_space": 3,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
+    "execute_hop_10000_tasks": 0,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
+    "modifying_1000_circular_buffer_elements": 0,
+    "modifying_byte_buffer_view": 6050,
+    "ping_pong_1000_reqs_1_conn": 334,
+    "read_10000_chunks_from_file": 110200,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
+    "scheduling_10000_executions": 89,
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
+  }
+

IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json

@@ -0,0 +1,51 @@
+{
+    "10000000_asyncsequenceproducer": 21,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 45050,
+    "1000_addHandlers_sync": 38050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
+    "1000_autoReadGetAndSet_sync": 0,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
+    "1000_getHandlers_sync": 36,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 147000,
+    "1000_tcpbootstraps": 3050,
+    "1000_tcpconnections": 155050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 76050,
+    "1_reqs_1000_conn": 393000,
+    "bytebuffer_lots_of_rw": 2050,
+    "creating_10000_headers": 0,
+    "decode_1000_ws_frames": 2050,
+    "encode_1000_ws_frames_holding_buffer": 3,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
+    "encode_1000_ws_frames_holding_buffer_with_space": 3,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
+    "execute_hop_10000_tasks": 0,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
+    "modifying_1000_circular_buffer_elements": 0,
+    "modifying_byte_buffer_view": 6050,
+    "ping_pong_1000_reqs_1_conn": 343,
+    "read_10000_chunks_from_file": 110200,
+    "schedule_10000_tasks": 50100,
+    "schedule_and_run_10000_tasks": 50050,
+    "scheduling_10000_executions": 89,
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 165050
+  }
+

IntegrationTests/tests_04_performance/Thresholds/nightly-main.json

@@ -0,0 +1,50 @@
+{
+  "10000000_asyncsequenceproducer": 21,
+  "1000000_asyncwriter": 1000050,
+  "1000_addHandlers": 45050,
+  "1000_addHandlers_sync": 38050,
+  "1000_addRemoveHandlers_handlercontext": 8050,
+  "1000_addRemoveHandlers_handlername": 8050,
+  "1000_addRemoveHandlers_handlertype": 8050,
+  "1000_autoReadGetAndSet": 18050,
+  "1000_autoReadGetAndSet_sync": 0,
+  "1000_copying_bytebufferview_to_array": 1050,
+  "1000_copying_circularbuffer_to_array": 1050,
+  "1000_getHandlers": 8050,
+  "1000_getHandlers_sync": 36,
+  "1000_reqs_1_conn": 26400,
+  "1000_rst_connections": 147000,
+  "1000_tcpbootstraps": 3050,
+  "1000_tcpconnections": 155050,
+  "1000_udp_reqs": 6050,
+  "1000_udpbootstraps": 2050,
+  "1000_udpconnections": 76050,
+  "1_reqs_1000_conn": 393000,
+  "bytebuffer_lots_of_rw": 2050,
+  "creating_10000_headers": 0,
+  "decode_1000_ws_frames": 2050,
+  "encode_1000_ws_frames_holding_buffer": 3,
+  "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
+  "encode_1000_ws_frames_holding_buffer_with_space": 3,
+  "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+  "encode_1000_ws_frames_new_buffer": 3050,
+  "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+  "encode_1000_ws_frames_new_buffer_with_space": 3050,
+  "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
+  "execute_hop_10000_tasks": 0,
+  "future_erase_result": 4050,
+  "future_lots_of_callbacks": 53050,
+  "get_100000_headers_canonical_form": 700050,
+  "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+  "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+  "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
+  "modifying_1000_circular_buffer_elements": 0,
+  "modifying_byte_buffer_view": 6050,
+  "ping_pong_1000_reqs_1_conn": 343,
+  "read_10000_chunks_from_file": 110200,
+  "schedule_10000_tasks": 50100,
+  "schedule_and_run_10000_tasks": 50050,
+  "scheduling_10000_executions": 89,
+  "udp_1000_reqs_1_conn": 6200,
+  "udp_1_reqs_1000_conn": 165050
+}

IntegrationTests/tests_04_performance/test_01_allocation_counts.sh

@@ -18,6 +18,8 @@ source defines.sh
 set -eu
 here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+test -n "${SWIFT_VERSION:-}" || fatal "SWIFT_VERSION unset"
+
 all_tests=()
 for file in "$here/test_01_resources/"test_*.swift; do
     test_name=$(basename "$file")
@@ -37,6 +39,7 @@ for test in "${all_tests[@]}"; do
         not_freed_allocations=$(grep "^test_$test_case.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
         leaked_fds=$(grep "^test_$test_case.leaked_fds:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
         max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test_case"
+        max_allowed=$(jq '.'\"$test_case\" "$here/Thresholds/$SWIFT_VERSION.json")
 
         info "$test_case: allocations not freed: $not_freed_allocations"
         info "$test_case: total number of mallocs: $total_allocations"
@@ -45,14 +48,17 @@ for test in "${all_tests[@]}"; do
         assert_less_than "$not_freed_allocations" 5     # allow some slack
         assert_greater_than "$not_freed_allocations" -5 # allow some slack
         assert_less_than "$leaked_fds" 1  # No slack allowed here though
-        if [[ -z "${!max_allowed_env_name+x}" ]]; then
+        if [[ -z "${!max_allowed_env_name+x}" ]] && [ -z "${max_allowed}" ]; then
             if [[ -z "${!max_allowed_env_name+x}" ]]; then
                 warn "no reference number of allocations set (set to \$$max_allowed_env_name)"
-                warn "to set current number:"
+                warn "to set current number either:"
                 warn "    export $max_allowed_env_name=$total_allocations"
+                warn "    or set them in the Swift version specific threshold json"
             fi
         else
-            max_allowed=${!max_allowed_env_name}
+            if [ -z "${max_allowed}" ]; then
+                max_allowed=${!max_allowed_env_name}
+            fi
             assert_less_than_or_equal "$total_allocations" "$max_allowed"
             assert_greater_than "$total_allocations" "$(( max_allowed - 1000))"
         fi

docker/docker-compose.2204.next.yaml → docker/docker-compose.2204.nightly-6.0.yaml

@@ -20,7 +20,7 @@ services:
   test:
     image: swift-nio:22.04-next
     environment:
-      - SWIFT_VERSION=nightly-next
+      - SWIFT_VERSION=nightly-6.0
       - MAX_ALLOCS_ALLOWED_10000000_asyncsequenceproducer=21
       - MAX_ALLOCS_ALLOWED_1000000_asyncwriter=1000050
       - MAX_ALLOCS_ALLOWED_1000_addHandlers=45050
@@ -79,7 +79,7 @@ services:
   update-benchmark-baseline:
     image: swift-nio:22.04-next
     environment:
-      - SWIFT_VERSION=nightly-next
+      - SWIFT_VERSION=nightly-6.0
 
   shell:
     image: swift-nio:22.04-next

scripts/check-cxx-interop-compatibility.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+set -euo pipefail
+
+log() { printf -- "** %s\n" "$*" >&2; }
+error() { printf -- "** ERROR: %s\n" "$*" >&2; }
+fatal() { error "$@"; exit 1; }
+
+log "Checking for Cxx interoperability comaptibility..."
+
+source_dir=$(pwd)
+working_dir=$(mktemp -d)
+project_name=$(basename "$working_dir")
+source_file=Sources/$project_name/$(echo "$project_name" | tr . _).swift
+library_products=$( swift package dump-package | jq -r '.products[] | select(.type.library != null) | .name')
+package_name=$(swift package dump-package | jq -r '.name')
+
+cd "$working_dir"
+swift package init
+echo "package.dependencies.append(.package(path: \"$source_dir\"))" >> Package.swift
+
+for product in $library_products; do
+  echo "package.targets.first!.dependencies.append(.product(name: \"$product\", package: \"$package_name\"))" >> Package.swift
+  echo "import $product" >> "$source_file"
+done
+
+swift build
+
+log "✅ Passed the Cxx interoperability tests."