au-ts · midnightveil · Jul 11, 2025 · Jul 11, 2025 · Jul 11, 2025 · Jul 11, 2025
diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml
@@ -110,8 +110,8 @@ jobs:
         env:
           PYTHON: ${{ github.workspace }}/venv/bin/python
 
-  run_qemu:
-    name: Run (QEMU)
+  tests_qemu:
+    name: Tests (QEMU)
     runs-on: ubuntu-latest
     needs: build_linux_x86_64_nix
     steps:
@@ -139,8 +139,8 @@ jobs:
           path: ci_logs
           if-no-files-found: error
 
-  run_hardware:
-    name: Run (hardware)
+  tests_hardware:
+    name: Tests (hardware)
     runs-on: ubuntu-latest
     if: ${{ contains(github.event.pull_request.labels.*.name, 'hardware-test') ||
             (github.event_name == 'schedule') }}
@@ -178,3 +178,49 @@ jobs:
           name: ci-logs-hardware
           path: ci_logs
           if-no-files-found: error
+
+  bench_hardware:
+    name: Benchmarks (hardware)
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'hardware-bench') ||
+            (github.event_name == 'schedule') }}
+    needs: build_linux_x86_64_nix
+    concurrency:
+      group: ${{ github.workflow }}-sddf-hardware-bench-${{ github.event.number }}-${{ strategy.job-index }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout sDDF repository
+        uses: actions/checkout@v4
+      - name: Get machine queue
+        uses: actions/checkout@v4
+        with:
+          repository: seL4/machine_queue
+          path: machine_queue
+      - name: Get ipbench_queue
+        uses: actions/checkout@v4
+        with:
+          repository: au-ts/ipbench_queue
+          path: ipbench_queue
+          ssh-key: ${{ secrets.IPBENCH_GITHUB_DEPLOY_KEY }}
+      - name: Download images
+        uses: actions/download-artifact@v4
+        with:
+          name: loader-images
+          path: ci_build
+      - name: Setup machine queue SSH key
+        run: .github/workflows/setup_ssh_key.sh
+        env:
+          MACHINE_QUEUE_KEY: ${{ secrets.MACHINE_QUEUE_KEY }}
+      - name: Run tests
+        run: |
+          export PATH="$(pwd)/machine_queue":"$(pwd)/ipbench_queue":$PATH
+          # GitHub Actions is broken
+          # https://github.com/ringerc/github-actions-signal-handling-demo#why-child-process-tasks-dont-get-a-chance-to-clean-up-on-job-cancel
+          exec ./ci/benchmarks/echo_server.py --boards maaxboard --single --configs benchmark
+      - name: Archive logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-logs-hardware-benchmarks
+          path: ci_logs
+          if-no-files-found: error
diff --git a/.github/workflows/setup_ssh_key.sh b/.github/workflows/setup_ssh_key.sh
@@ -14,6 +14,36 @@ login.trustworthy.systems ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSzWm9H9EKxcinJs
 login.trustworthy.systems ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCRA/W8LVxLFjzPvijygdSw+rPW/EQEG8WoUVcTm5dYXDIhCc0Zxibd19zPb1LQpE2/Ohe+I16iC5glpmFyDfrs=
 login.trustworthy.systems ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPYp/3vDMDnHnjtqt5Oqievgz04g/LJ4yEKOlXCu9Yux
 tftp.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEj7X6doSoop91gTvBD7L4O7VGwCO5pLNsu5YAGS1L64MJqo+3wTYgFRdMWTM0hL3YN+1sSabJPICJzKk0EJxkg=
+vb01.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDhZW6jkDLaoZXD7BxrM4OtPvRwquHqIRXyFA8rFPpA3G6YXc9iOpTZODt2/CNbZXyCbK9RLuWWDCKNv1CR1gJSurb770wzg5Ts0v6J7eEa+B83skeJUIT2i+eFqyuEcPgvmdG5DChRyhaQen8O97wvkNVnT+B6OJvAtwSCCU9KsO0vI7lCF040PCJTXrvSjkIf3unJxDmdUzPIqyyof/m5FipoceL7sBfAXhMQYlRh4LnrC2rmPt1/yKJENSes0y07i0ihN92m/GBnZHViQjQdSYQbb0eySujrpBnj4Z4aXNv9Ao2kCNqBzV/qpo3ROXZk76ELBHpNX80ewLgHdU/x
+vb01.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIAsus4NLjozoEisJBBVtcxhRbUTu9UvLARwOyDb6rS9Rd3ooxRG8rPvHAnC8x+eOW0Oxzyb8mEdqwsUUTspbOU=
+vb01.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHqUTgA9+HoTE64Z58Gho1dSufLN/bD/0B1M8ee8CVVb
+juliab@tftp:~$ vb02 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDfc5wBO/+ExXOFxlR/QMvjroN8KAbDT3IovmHLpjWbzCCE4cE9fu5MQKHgPAJ25sfcTtzlRVrw20hQyHQgL/141wQ/XQk3H1vLKRJbrPVHrpoPgpB4gETOCFpUJJRuup5bBdeWIkTTV82F7V+mpQbMYc30DIl7LCmXd20QV8D5KynNjjci35SRGuHxjbBQNfSs1wVUgK1z+Y5elG7OgLXi2IE361ksEm3nbCJkpLZSFGIVHQkPHwsMhwKBZlXhJAgeqLmiE8d4WQoORXPglvSQ8O20McMjHbOMCIPikPWNhkoSD7ekqn2/HReciLw0R7CJ8c2kPuR//K8MxL3wodZ1
+vb02.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD3yksLxJoCzZzvcZkfg2pV1jGGAAOSzEMnP1Q1lB4KoljOf/W5Kw2c1ZLMJYaWzSciaZBkhu96UQIpHBdgBk0M=
+vb02.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDMwTySOHEqawSSo3/4et/71KobxIr07UuotW8HoDCk2
+vb03.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDjLpyMZ4/1Zj8mT7OKtmd3deon/Ta1QLdSajveNb0HcMXtrrFIJfSMf97A3IN+wyK61m5ThxlRnXO/S1E4kLdA84L1Qv94jUE0fpkBKnw4x62eTHas+a5BsD6jUkypsGnsQ5ugW9A5muQyktqGngyk9/Y/hWdQiPP4ovjwjUJYaiBXiFAFSFnw8r2jdZiSCrim2wWB1jy16atTpmppxhsVFbnx3/Jl54VgbLlJFmfhnYwYv2gG5mbHswEV1p4owIVPcMQljYdMjssTqzcw8t32qmDxmpParUtvO0ZquJ8lvwSlA29iSlMj8DVkjNAgg0754ohJ7wSF3zydGwoR0wgr
+vb03.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAQsjevofMjx2sOfOA0/tB4tZal21EKrfSXURYajIX5DCCvzHbUFbaJZpopNWPgRasud7HrxlnAcHtwluILMo64=
+vb03.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPyhdhYIiJlr8FSpYEwRJzyeqV3aCj1XJ2CldV8HawKk
+vb04.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDl4RW2DzRd7rvTGg1Yb6GqXt1nzzZPdKWVSp9z/s6tY17MEUB0DQtu9wox9PxfBPAy1c2YnW33ezrGUKRM1ScieKNEZbnASDBLllcq/CJv+4EnU0vtE9e0VN34FJnnhNyLpHLgMoglBriJtwMRmEP1tiWvrZuNAXlvNLdgGt/QO0uhYSA4weOkObPA/Shc/ZWIA+kKVFRyby4UXgd6EZDUSOmXV4iFt+JVbg50vYVlAz86DFSBCarFu0vUi1QvpWgn0D5NA2+nDcGevGSAl5j7PRPw7bUJ938nTdT5kJM4722QCGw6BuBdkf2ohwFiHqCljOrajccqSg257ZvWF8v9
+vb04.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL2E2jmBDPTxG/A7BPy2WOuIbQIG/k+LrHbxLiLSZWOcdmK1Ui5vpnHesfHADnzkKENNabnNo/MF+RjSnRIK+lk=
+vb04.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL88FdPerLun9BVpgRKh/1CaYZnj8XDNMHgXBd96fial
+vb05.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8ncTSIQyaA4/ulod1IWnZVLitjcgZ0/mV1iFS5Pefsd0c8dtLCosMCM3mI6Ho1lkF7r4PslYS0v7LE4kz9WhVJYdkjGwOam8Tea3rB1uQX23Mm2qi7UAAATr0ogluy2ApKUiktKJV8vH9MBadkXWiSJov2fxC4TRzB1Z2avCuWpfwIajhfK7+z3st4yZquCI3SnMgDjYcO+YlqNYBIe2Y531CyOIqxm1JIA6Fwdiwkwh8MwNTBNdnphe6Jim7YagPEyAyEHfGgC3qv7NHrWNmIsjceVutM15olKgJj7/zJ6XHrhQ3nGbi04mCPoqgB3SXea2S7BBC+fCSMxy755WB
+vb05.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCNkyS10fWrAQNtZ11eP7qOzr34vKwaNQKb+EACaYPrJyLcLfjas4KqKhKU32BYEymOmmaF0at/We5+BMLkZvm4=
+vb05.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE05ZXWSDnX/TqvapioEpDnTlRfCDnkaq9HbljpBKon1
+vb06.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAuFxDIE1KuTPYjIRO6jhg4SKej81aM/KA7EKe1AqNyd82MEGUceqkXNVm7raEToBAfedsiZPBRdx/0AcniHOxUbfEJSwqbi/z+in3xqJXwEw7qbGz2x1Gt5msi6JAVx56YQEqWGL0e4O1RID8U0AWFs0fM+rlXE6gcaDD0ziRDsBNOYV5SlZSgPxFA2UM9H2FyLL2clpH0sbrF1IMS5+1W0lZlRJU0I5wEzUXiAg+RgIhO5QdOHF88Ry+WIeJYDHkdMqbwPJGVPa5MwOoOKrGIQ4+yVwZSh9BWvkb5eWjbccvFdB+edTwyksqyCT1yrfis9I1qAEoNZyhggX6FRmT
+vb06.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCPa0QIHNwb9RgpSPhQuUlE1v4fEKcn0B4lvZ3eOaMhZoPxbFWv5lIGh5ApAJmE+bfnWa6n8M8o4PBr/hmL0Fro=
+vb06.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDGcQvfBVc2z544/FLFIEvGXAZ4a/JXiAANAvs4Disw/
+vb07.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgc+xggxpEvTnTi0Ic/TLKkffqhJ+D49VywbIUbP4r3+bCO3uBDJANRiuFg+EMpzgT2BVOwJa5Sa0vyqn8NtROxQOnihBmsZ+qr+NdnDUtjq2Fgv5ZSdD7r3bNyivugVY1S2DDecm5yX49ccgDedKKxILNF8E2AAxhNHS7WhBziF7O5DO5kXGqk2C7wg7eLxoyY4LNJGtdL2wG2ZXwFIDZSWgK2Z4TQ55jAsxr1zfIpmS2DjsRxt89SccqwFI9y3WS92hqnlxnpsFXEY6jgZ8pis/t3SgbXAsCkOG/uRdqI5MsJmxN5+b/vBTWj9iuc/XfH7p/CfLaPBkrek7k3S+v
+vb07.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKLLhm5IXL6cIbldf+s0FxEKxLIg63xdTyyFYsZC7SMZzQws5RlGfFvz5fkJTFRY7CcP1vd1gaXw/aKieJx2pBA=
+vb07.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALlH9pXTWGL8pta++ybUmW7tgZ2pYYLhosIdwc7zsAu
+vb08.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuzeXRliY1fwd/U4NddrK+D30s7nkx0hPa5x9T6A9jSNBCo/3gLOw8lA81yoA/EKpqTRErcCi+S8GGDSyj0bvcO9MDUhOXC0KZb/Q0K4/snUb1lYW859bwi3BWgnOm5l1oAqxAscXVvvxflZ7g58wciWyv+hMFIhtjmss6ICKFXszFi4PB4QH/rceQkYNf0oy4HQzM+2TJf9NWWXacxXUNfNKp++fzoYhyzW1+2gEeOToKMDR4VpUEt5BHEQaFS24GNqciTMCUuC2nkjYUBKuGA57NDADYWTvThXxsTW3TgzhHiYOqSi05jxKAsgaEz6llG+uquHhByPcJsLODqHLV
+vb08.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE05rEaLBU4OabuECDH+rvaPuLVqHGyTA1Mue+ZAVPTI/XqHG2DVgz36QQj7RKnHR3JxsdsIl+eQr3wDhHBflTA=
+vb08.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ0JKjCx+YxgHBehTH1cgpmIWCDbDcC6xIFR+WQA7Aza
+vb09.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCWpxmKMaonUowEmWoSM2B0fpXiPW9N/5aTueeR9NhSu7CCzRUGkwXhS393Fks1ooUIW+IPKPQ9ASS9sdBz9ed7pvDbXcsXSW595TWoWSDU4wZNb9zFevrHbeKoHsn7RKMynSBORXiZl+XslgBJuP8QJXOmDcq+cuNiM+NWCrt5QJbiODQPEhKljx+9jN7wtwjQgso94KjUdLccc8861+aHkXCXIoxp+DEjb6FLG1bEAvRoaviuZQKMpwDf3sz4FddT//Bl47Tnv62Bj7Kj7ENPQNdycoodNH41u6Eik48CzVTskARBLi80YC2e678G+gAJwDNnSZn6yzWWkzarIgZ
+vb09.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ7Fvaz6tZGmtEuBLsY38LKJZZGcaVGgu7oXFJo1OgJ8PHc19xO6h8ZNVTgcZ9SYTSiSJO/YkFk1FEfhG4juu7E=
+vb09.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILacVjWxwGacBm8E+pSBWW20dULns9Ggq+XVccJFzaRX
+vb10.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDfhBaTBCajLGQynnV1rutqZhLrZuomRxR6RVICDRV5PeyHXELxpfgofJCtzjWhEWnKx8aTRwF9KD8tZzvi1XwwjdRunBA4nisx9pSXyEANm7/uQERlthLT9FBQyayxucGo9QDa4acx1irkZHoSCoHCXHcdHeGE4gkeLqdgFgFzVXcaSaJOe8RKQQPQHnnQOrbi4eqLvnsgO6+SqUF82+sOtsWNLrDKmEcCLxb9Q3d1XV7pZDowSUtSrMaFwag67t70fJnT9jNzfKpSBNBzA3Hf+eDBhWiv6J5metvk2DAVoLXxHXt27e+cL06tawQdlOr7bFWTzG7rL34x0Ruu7uLF
+vb10.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOWz9J5+iUzlDhL3OtDeD9juHSbnJYGVhloUBSsP1mUQHN6UQZTiLLYheXYch2qeighHlv+wLxw+37if4vc1JOQ=
+vb10.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICedVECImJ5lDJIlXjbAhnBxcUlUJlNFK7J8/kKJ2gFV
 EOF
 
 cat >> ~/.ssh/config <<EOF
@@ -27,7 +57,7 @@ Host ts
   Hostname login.trustworthy.systems
   User ts_ci
 
-Host tftp.keg.cse.unsw.edu.au
+Host *.keg.cse.unsw.edu.au
   User ts_ci
   ProxyJump ts
 

diff --git a/ci/benchmarks/benchmark.py b/ci/benchmarks/benchmark.py
@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+
+import argparse
+import datetime as dt
+import socket
+import subprocess
+from pathlib import Path
+
+IPBENCHD_PORT = 8036
+udp_packet_size_default = 1472
+tcp_packet_size_default = 1460
+
+def abort_clients(clients: list[str]):
+    for client in clients:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.connect((client, IPBENCHD_PORT))
+        sock.send(b"ABORT\n")
+        sock.shutdown(socket.SHUT_WR)
+        while True:
+            data = sock.recv(4096)
+            if not data:
+                break
+            print(data)
+        sock.close()
+
+
+def run_benchmark(args, throughput: int, packet_size: int):
+    argv = [
+        "ipbench",
+        "--debug",
+    ]
+
+    make_arg_list = lambda **opts: ",".join(f"{k}={v}" for k, v in opts.items())
+
+    # test mode
+    argv += [
+        "--test",
+        "latency",
+        "--test-args",
+        make_arg_list(
+            socktype=args.protocol,
+            bps=throughput // len(args.clients),
+            size=packet_size,
+            warmup=args.warmup_seconds,
+            cooldown=args.cooldown_seconds,
+            samples=args.samples // len(args.clients),
+        ),
+    ]
+
+    # remote clients
+    argv += ["--port", str(IPBENCHD_PORT)]
+    for client in args.clients:
+        argv += ["--client", client]
+
+    # device under test
+    argv += [
+        # echo server port
+        "--test-target",
+        args.ip,
+        "--test-port",
+        {
+            "tcp": "1237",
+            "udp": "1235",
+        }[args.protocol],
+        # benchmark stuff
+        "--target-test",
+        "cpu_target_lukem",
+        "--target-test-hostname",
+        args.ip,
+        "--target-test-port",
+        "1236",
+    ]
+
+    p = subprocess.run(argv, stdout=subprocess.PIPE)
+    p.check_returncode()
+
+    return p.stdout.decode("utf-8").replace("\n", "")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ip", help="IP address of machine to benchmark")
+
+    parser.add_argument(
+        "--clients",
+        nargs="+",
+        default=["vb04", "vb05", "vb06", "vb07"],
+    )
+    parser.add_argument(
+        "--udp",
+        action="store_const",
+        dest="protocol",
+        const="udp",
+        default="udp",
+    )
+    parser.add_argument(
+        "--tcp",
+        action="store_const",
+        dest="protocol",
+        const="tcp",
+    )
+    parser.add_argument(
+        "--warmup-seconds",
+        type=int,
+        default=10,
+    )
+    parser.add_argument(
+        "--cooldown-seconds",
+        type=int,
+        default=10,
+    )
+    parser.add_argument(
+        "--samples",
+        type=int,
+        default=200_000,
+    )
+    parser.add_argument(
+        "--packet-sizes",
+        nargs="+",
+        type=int,
+    )
+    parser.add_argument(
+        "--throughputs",
+        nargs="+",
+        type=int,
+        required=True,
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    packet_sizes = args.packet_sizes
+    if not packet_sizes:
+        default = udp_packet_size_default
+        if args.protocol == "tcp":
+            default = tcp_packet_size_default
+        packet_sizes = [default]
+
+    abort_clients(args.clients)
+    abort_clients(args.clients)
+
+    file = dt.datetime.now().strftime("output-%Y-%m-%dT%H:%M:%S.csv")
+
+    output_link = Path("output.csv")
+    if output_link.is_symlink():
+        output_link.unlink()
+        output_link.symlink_to(file)
+
+    heading = "Requested_Throughput,Receive_Throughput,Send_Throughput,Packet_Size,Minimum_RTT,Average_RTT,Maximum_RTT,Stdev_RTT,Median_RTT,Bad_Packets,Idle_Cycles,Total_Cycles\n"
+
+    data = ""
+    with open(file, "w") as data_out:
+        data_out.write(heading)
+        data += heading
+
+        try:
+            for packet_size in packet_sizes:
+                for throughput in args.throughputs:
+                    row = run_benchmark(args, throughput, packet_size) + "\n"
+
+                    data_out.write(row)
+                    data += row
+                    print(heading, row)
+
+        finally:
+            abort_clients(args.clients)
+
+        print("Result Summary:")
+        print(data)