From a8bbc864833603bc06a001a3a2e7a08805600d4e Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 11 Jul 2025 15:51:27 +1000 Subject: [PATCH 01/10] WIP: hw benchmarks Signed-off-by: julia --- .github/workflows/examples.yaml | 53 ++++++++++++++-- ci/benchmarks/echo_server.py | 109 ++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 4 deletions(-) create mode 100755 ci/benchmarks/echo_server.py diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index a3cb0faa8..2e68bec38 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -110,8 +110,8 @@ jobs: env: PYTHON: ${{ github.workspace }}/venv/bin/python - run_qemu: - name: Run (QEMU) + tests_qemu: + name: Tests (QEMU) runs-on: ubuntu-latest needs: build_linux_x86_64_nix steps: @@ -139,8 +139,8 @@ jobs: path: ci_logs if-no-files-found: error - run_hardware: - name: Run (hardware) + tests_hardware: + name: Tests (hardware) runs-on: ubuntu-latest if: ${{ contains(github.event.pull_request.labels.*.name, 'hardware-test') || (github.event_name == 'schedule') }} @@ -178,3 +178,48 @@ jobs: name: ci-logs-hardware path: ci_logs if-no-files-found: error + + bench_hardware: + name: Benchmarks (hardware) + runs-on: ubuntu-latest + if: ${{ contains(github.event.pull_request.labels.*.name, 'hardware-bench') || + (github.event_name == 'schedule') }} + needs: build_linux_x86_64_nix + concurrency: + group: ${{ github.workflow }}-sddf-hardware-bench-${{ github.event.number }}-${{ strategy.job-index }} + cancel-in-progress: true + steps: + - name: Checkout sDDF repository + uses: actions/checkout@v4 + - name: Get machine queue + uses: actions/checkout@v4 + with: + repository: seL4/machine_queue + path: machine_queue + - name: Get ipbench_queue + uses: actions/checkout@v4 + with: + repository: au-ts/ipbench_queue + path: ipbench_queue + - name: Download images + uses: actions/download-artifact@v4 + with: + name: loader-images + path: ci_build + - name: Setup machine queue SSH key + run: .github/workflows/setup_ssh_key.sh + env: + MACHINE_QUEUE_KEY: ${{ secrets.MACHINE_QUEUE_KEY }} + - name: Run tests + run: | + export PATH="$(pwd)/machine_queue":"$(pwd)/ipbench_queue":$PATH + # GitHub Actions is broken + # https://github.com/ringerc/github-actions-signal-handling-demo#why-child-process-tasks-dont-get-a-chance-to-clean-up-on-job-cancel + exec ./ci/benchmarks/echo_server.py --boards maaxboard --single --configs benchmark + - name: Archive logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-logs-hardware-benchmarks + path: ci_logs + if-no-files-found: error diff --git a/ci/benchmarks/echo_server.py b/ci/benchmarks/echo_server.py new file mode 100755 index 000000000..b46e5d330 --- /dev/null +++ b/ci/benchmarks/echo_server.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# Copyright 2025, UNSW +# SPDX-License-Identifier: BSD-2-Clause + +import asyncio +import re +from pathlib import Path +import sys + +sys.path.insert(1, Path(__file__).parents[2].as_posix()) + +from ci.lib.backends import * +from ci.lib.runner import TestConfig, cli, matrix_product +from ci.lib import log +from ci import common, matrix + +TEST_MATRIX = matrix_product( + board=matrix.EXAMPLES["echo_server"]["boards_test"], + config=matrix.EXAMPLES["echo_server"]["configs"], + build_system=matrix.EXAMPLES["echo_server"]["build_systems"], +) + + +def backend_fn(test_config: TestConfig, loader_img: Path) -> HardwareBackend: + backend = common.backend_fn(test_config, loader_img) + + if isinstance(backend, QemuBackend): + # fmt: off + backend.invocation_args.extend([ + "-global", "virtio-mmio.force-legacy=false", + "-device", "virtio-net-device,netdev=netdev0", + "-netdev", "user,id=netdev0,hostfwd=tcp::1236-:1236,hostfwd=tcp::1237-:1237,hostfwd=udp::1235-:1235", + ]) + # fmt: on + + return backend + + +async def test(backend: HardwareBackend, test_config: TestConfig): + async with asyncio.timeout(20): + await wait_for_output(backend, b"DHCP request finished") + dhcp_client1 = await wait_for_output(backend, b"\r\n") + await wait_for_output(backend, b"DHCP request finished") + dhcp_client0 = await wait_for_output(backend, b"\r\n") + + dhcp_client1, dhcp_client0 = ( + (dhcp_client1, dhcp_client0) + if b"client1" in dhcp_client1 + else (dhcp_client0, dhcp_client1) + ) + + try: + # fmt: off + ip1 = re.search(rb"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", dhcp_client1).group(0).decode() # type: ignore + ip0 = re.search(rb"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", dhcp_client0).group(0).decode() # type: ignore + # fmt: on + except (IndexError, AttributeError): + raise TestFailureException( + "could not find IP address in DHCP request result" + ) + + reset_terminal() + log.info(f"client IPs: client0={ip0}, client1={ip1}") + + # Now let's do the actual benchmark! + + assert test_config.config == "benchmark" + + # TODO: Abstract. + + benchmark_run = await asyncio.create_subprocess_exec( + # fmt: off + "/home/julia/code/ts/ipbench_queue/iq.sh", "run", + "-c", "vb04", + "-c", "vb05", + "-c", "vb06", + "-c", "vb07", + "-f", "/home/julia/code/ts/sddf_benchmarking/benchmark.py", + "--", + ip0, + "--throughputs", "10000000", + "--samples", "1000", + "--clients", "vb04", + ) + try: + # # await wait_for_output(backend, b"client0 measurement finished...") + # await wait_for_output(backend, b"\n") + # # await wait_for_output(backend, b"\n") + # reset_terminal() + # await asyncio.wait( + # [asyncio.create_task(f) for f in [benchmark_run.wait(), wait_for_output(backend, b"client0 measurement starting...")]], + # return_when="FIRST_COMPLETED" + # ) + # if benchmark_run.returncode is not None: + # raise TestFailureException("AAA umm??") + # await asyncio.gather( + # benchmark_run.wait(), + # wait_for_output(backend, b"client0 measurement starting..."), + # ) + await wait_for_output(backend, b"Result Summary:") + finally: + try: + benchmark_run.terminate() + except ProcessLookupError: + pass + + +if __name__ == "__main__": + cli("echo_server", test, TEST_MATRIX, backend_fn, common.loader_img_path) From 769d3c135228de501bce67c667510ac1ca3ea51f Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 11 Jul 2025 15:53:37 +1000 Subject: [PATCH 02/10] (TEMP) ci: disable building most things (TEMP) Signed-off-by: julia --- ci/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/build.py b/ci/build.py index 00718b1a2..0784df586 100755 --- a/ci/build.py +++ b/ci/build.py @@ -113,9 +113,11 @@ def build(args: argparse.Namespace, example_name: str, test_config: TestConfig): for example_name, options in matrix.EXAMPLES.items(): if example_name not in args.examples: continue + if example_name != "echo_server": + continue example_matrix = matrix_product( - board=options["boards_build"], + board=["maaxboard"], # options["boards_build"], config=options["configs"], build_system=options["build_systems"], ) From b1dcc223db88578a7e714c762b2d835ed7b8a504 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 11 Jul 2025 16:03:47 +1000 Subject: [PATCH 03/10] give IPBENCH_GITHUB_DEPLOY_KEY to fetch of ipbench_queue repo Signed-off-by: julia --- .github/workflows/examples.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 2e68bec38..ffb8b3f50 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -201,6 +201,7 @@ jobs: with: repository: au-ts/ipbench_queue path: ipbench_queue + ssh-key: ${{ secrets.IPBENCH_GITHUB_DEPLOY_KEY }} - name: Download images uses: actions/download-artifact@v4 with: From be9551d19baa3f6d1087712962508b3d1ede8211 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 11 Jul 2025 17:18:27 +1000 Subject: [PATCH 04/10] no more path hardcode Signed-off-by: julia --- ci/benchmarks/benchmark.py | 172 +++++++++++++++++++++++++++++++++++ ci/benchmarks/echo_server.py | 4 +- 2 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 ci/benchmarks/benchmark.py diff --git a/ci/benchmarks/benchmark.py b/ci/benchmarks/benchmark.py new file mode 100644 index 000000000..959da4be3 --- /dev/null +++ b/ci/benchmarks/benchmark.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 + +import argparse +import datetime as dt +import socket +import subprocess +from pathlib import Path + +IPBENCHD_PORT = 8036 +udp_packet_size_default = 1472 +tcp_packet_size_default = 1460 + +def abort_clients(clients: list[str]): + for client in clients: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((client, IPBENCHD_PORT)) + sock.send(b"ABORT\n") + sock.shutdown(socket.SHUT_WR) + while True: + data = sock.recv(4096) + if not data: + break + print(data) + sock.close() + + +def run_benchmark(args, throughput: int, packet_size: int): + argv = [ + "ipbench", + "--debug", + ] + + make_arg_list = lambda **opts: ",".join(f"{k}={v}" for k, v in opts.items()) + + # test mode + argv += [ + "--test", + "latency", + "--test-args", + make_arg_list( + socktype=args.protocol, + bps=throughput // len(args.clients), + size=packet_size, + warmup=args.warmup_seconds, + cooldown=args.cooldown_seconds, + samples=args.samples // len(args.clients), + ), + ] + + # remote clients + argv += ["--port", str(IPBENCHD_PORT)] + for client in args.clients: + argv += ["--client", client] + + # device under test + argv += [ + # echo server port + "--test-target", + args.ip, + "--test-port", + { + "tcp": "1237", + "udp": "1235", + }[args.protocol], + # benchmark stuff + "--target-test", + "cpu_target_lukem", + "--target-test-hostname", + args.ip, + "--target-test-port", + "1236", + ] + + p = subprocess.run(argv, stdout=subprocess.PIPE) + p.check_returncode() + + return p.stdout.decode("utf-8").replace("\n", "") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("ip", help="IP address of machine to benchmark") + + parser.add_argument( + "--clients", + nargs="+", + default=["vb04", "vb05", "vb06", "vb07"], + ) + parser.add_argument( + "--udp", + action="store_const", + dest="protocol", + const="udp", + default="udp", + ) + parser.add_argument( + "--tcp", + action="store_const", + dest="protocol", + const="tcp", + ) + parser.add_argument( + "--warmup-seconds", + type=int, + default=10, + ) + parser.add_argument( + "--cooldown-seconds", + type=int, + default=10, + ) + parser.add_argument( + "--samples", + type=int, + default=200_000, + ) + parser.add_argument( + "--packet-sizes", + nargs="+", + type=int, + ) + parser.add_argument( + "--throughputs", + nargs="+", + type=int, + required=True, + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + packet_sizes = args.packet_sizes + if not packet_sizes: + default = udp_packet_size_default + if args.protocol == "tcp": + default = tcp_packet_size_default + packet_sizes = [default] + + abort_clients(args.clients) + abort_clients(args.clients) + + file = dt.datetime.now().strftime("output-%Y-%m-%dT%H:%M:%S.csv") + + output_link = Path("output.csv") + if output_link.is_symlink(): + output_link.unlink() + output_link.symlink_to(file) + + heading = "Requested_Throughput,Receive_Throughput,Send_Throughput,Packet_Size,Minimum_RTT,Average_RTT,Maximum_RTT,Stdev_RTT,Median_RTT,Bad_Packets,Idle_Cycles,Total_Cycles\n" + + data = "" + with open(file, "w") as data_out: + data_out.write(heading) + data += heading + + try: + for packet_size in packet_sizes: + for throughput in args.throughputs: + row = run_benchmark(args, throughput, packet_size) + "\n" + + data_out.write(row) + data += row + print(heading, row) + + finally: + abort_clients(args.clients) + + print("Result Summary:") + print(data) diff --git a/ci/benchmarks/echo_server.py b/ci/benchmarks/echo_server.py index b46e5d330..8f2adbf9d 100755 --- a/ci/benchmarks/echo_server.py +++ b/ci/benchmarks/echo_server.py @@ -70,12 +70,12 @@ async def test(backend: HardwareBackend, test_config: TestConfig): benchmark_run = await asyncio.create_subprocess_exec( # fmt: off - "/home/julia/code/ts/ipbench_queue/iq.sh", "run", + "iq.sh", "run", "-c", "vb04", "-c", "vb05", "-c", "vb06", "-c", "vb07", - "-f", "/home/julia/code/ts/sddf_benchmarking/benchmark.py", + "-f", Path(__file__).parent / "benchmark.py", "--", ip0, "--throughputs", "10000000", From 78794203d874eef5779696ba1e460f0c1336ad1e Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 11 Jul 2025 17:22:10 +1000 Subject: [PATCH 05/10] oops connect to *keg Signed-off-by: julia --- .github/workflows/setup_ssh_key.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/setup_ssh_key.sh b/.github/workflows/setup_ssh_key.sh index e5e04bc89..34dc91033 100755 --- a/.github/workflows/setup_ssh_key.sh +++ b/.github/workflows/setup_ssh_key.sh @@ -27,7 +27,7 @@ Host ts Hostname login.trustworthy.systems User ts_ci -Host tftp.keg.cse.unsw.edu.au +Host *.keg.cse.unsw.edu.au User ts_ci ProxyJump ts From f7efa81de676abac2a3ca33d4abe4f4c9b91dcea Mon Sep 17 00:00:00 2001 From: julia Date: Wed, 16 Jul 2025 17:53:16 +1000 Subject: [PATCH 06/10] TEMP ssh --- .github/workflows/examples.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index ffb8b3f50..74972bab3 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -211,6 +211,12 @@ jobs: run: .github/workflows/setup_ssh_key.sh env: MACHINE_QUEUE_KEY: ${{ secrets.MACHINE_QUEUE_KEY }} +# TEMP + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + detached: true +# TEMP - name: Run tests run: | export PATH="$(pwd)/machine_queue":"$(pwd)/ipbench_queue":$PATH From 00d4de9ac78e5916f689a9af737fde53738e2653 Mon Sep 17 00:00:00 2001 From: julia Date: Wed, 16 Jul 2025 19:10:53 +1000 Subject: [PATCH 07/10] revert temp --- .github/workflows/examples.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 74972bab3..ffb8b3f50 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -211,12 +211,6 @@ jobs: run: .github/workflows/setup_ssh_key.sh env: MACHINE_QUEUE_KEY: ${{ secrets.MACHINE_QUEUE_KEY }} -# TEMP - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - detached: true -# TEMP - name: Run tests run: | export PATH="$(pwd)/machine_queue":"$(pwd)/ipbench_queue":$PATH From 42a89c3eeaf08b6f0d400ea060083515dbe34913 Mon Sep 17 00:00:00 2001 From: julia Date: Wed, 16 Jul 2025 19:14:53 +1000 Subject: [PATCH 08/10] seutp ssh key for vb --- .github/workflows/setup_ssh_key.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/setup_ssh_key.sh b/.github/workflows/setup_ssh_key.sh index 34dc91033..5234aeaf0 100755 --- a/.github/workflows/setup_ssh_key.sh +++ b/.github/workflows/setup_ssh_key.sh @@ -14,6 +14,36 @@ login.trustworthy.systems ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSzWm9H9EKxcinJs login.trustworthy.systems ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCRA/W8LVxLFjzPvijygdSw+rPW/EQEG8WoUVcTm5dYXDIhCc0Zxibd19zPb1LQpE2/Ohe+I16iC5glpmFyDfrs= login.trustworthy.systems ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPYp/3vDMDnHnjtqt5Oqievgz04g/LJ4yEKOlXCu9Yux tftp.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEj7X6doSoop91gTvBD7L4O7VGwCO5pLNsu5YAGS1L64MJqo+3wTYgFRdMWTM0hL3YN+1sSabJPICJzKk0EJxkg= +vb01.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDhZW6jkDLaoZXD7BxrM4OtPvRwquHqIRXyFA8rFPpA3G6YXc9iOpTZODt2/CNbZXyCbK9RLuWWDCKNv1CR1gJSurb770wzg5Ts0v6J7eEa+B83skeJUIT2i+eFqyuEcPgvmdG5DChRyhaQen8O97wvkNVnT+B6OJvAtwSCCU9KsO0vI7lCF040PCJTXrvSjkIf3unJxDmdUzPIqyyof/m5FipoceL7sBfAXhMQYlRh4LnrC2rmPt1/yKJENSes0y07i0ihN92m/GBnZHViQjQdSYQbb0eySujrpBnj4Z4aXNv9Ao2kCNqBzV/qpo3ROXZk76ELBHpNX80ewLgHdU/x +vb01.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIAsus4NLjozoEisJBBVtcxhRbUTu9UvLARwOyDb6rS9Rd3ooxRG8rPvHAnC8x+eOW0Oxzyb8mEdqwsUUTspbOU= +vb01.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHqUTgA9+HoTE64Z58Gho1dSufLN/bD/0B1M8ee8CVVb +juliab@tftp:~$ vb02 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDfc5wBO/+ExXOFxlR/QMvjroN8KAbDT3IovmHLpjWbzCCE4cE9fu5MQKHgPAJ25sfcTtzlRVrw20hQyHQgL/141wQ/XQk3H1vLKRJbrPVHrpoPgpB4gETOCFpUJJRuup5bBdeWIkTTV82F7V+mpQbMYc30DIl7LCmXd20QV8D5KynNjjci35SRGuHxjbBQNfSs1wVUgK1z+Y5elG7OgLXi2IE361ksEm3nbCJkpLZSFGIVHQkPHwsMhwKBZlXhJAgeqLmiE8d4WQoORXPglvSQ8O20McMjHbOMCIPikPWNhkoSD7ekqn2/HReciLw0R7CJ8c2kPuR//K8MxL3wodZ1 +vb02.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD3yksLxJoCzZzvcZkfg2pV1jGGAAOSzEMnP1Q1lB4KoljOf/W5Kw2c1ZLMJYaWzSciaZBkhu96UQIpHBdgBk0M= +vb02.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDMwTySOHEqawSSo3/4et/71KobxIr07UuotW8HoDCk2 +vb03.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDjLpyMZ4/1Zj8mT7OKtmd3deon/Ta1QLdSajveNb0HcMXtrrFIJfSMf97A3IN+wyK61m5ThxlRnXO/S1E4kLdA84L1Qv94jUE0fpkBKnw4x62eTHas+a5BsD6jUkypsGnsQ5ugW9A5muQyktqGngyk9/Y/hWdQiPP4ovjwjUJYaiBXiFAFSFnw8r2jdZiSCrim2wWB1jy16atTpmppxhsVFbnx3/Jl54VgbLlJFmfhnYwYv2gG5mbHswEV1p4owIVPcMQljYdMjssTqzcw8t32qmDxmpParUtvO0ZquJ8lvwSlA29iSlMj8DVkjNAgg0754ohJ7wSF3zydGwoR0wgr +vb03.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAQsjevofMjx2sOfOA0/tB4tZal21EKrfSXURYajIX5DCCvzHbUFbaJZpopNWPgRasud7HrxlnAcHtwluILMo64= +vb03.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPyhdhYIiJlr8FSpYEwRJzyeqV3aCj1XJ2CldV8HawKk +vb04.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDl4RW2DzRd7rvTGg1Yb6GqXt1nzzZPdKWVSp9z/s6tY17MEUB0DQtu9wox9PxfBPAy1c2YnW33ezrGUKRM1ScieKNEZbnASDBLllcq/CJv+4EnU0vtE9e0VN34FJnnhNyLpHLgMoglBriJtwMRmEP1tiWvrZuNAXlvNLdgGt/QO0uhYSA4weOkObPA/Shc/ZWIA+kKVFRyby4UXgd6EZDUSOmXV4iFt+JVbg50vYVlAz86DFSBCarFu0vUi1QvpWgn0D5NA2+nDcGevGSAl5j7PRPw7bUJ938nTdT5kJM4722QCGw6BuBdkf2ohwFiHqCljOrajccqSg257ZvWF8v9 +vb04.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL2E2jmBDPTxG/A7BPy2WOuIbQIG/k+LrHbxLiLSZWOcdmK1Ui5vpnHesfHADnzkKENNabnNo/MF+RjSnRIK+lk= +vb04.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL88FdPerLun9BVpgRKh/1CaYZnj8XDNMHgXBd96fial +vb05.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8ncTSIQyaA4/ulod1IWnZVLitjcgZ0/mV1iFS5Pefsd0c8dtLCosMCM3mI6Ho1lkF7r4PslYS0v7LE4kz9WhVJYdkjGwOam8Tea3rB1uQX23Mm2qi7UAAATr0ogluy2ApKUiktKJV8vH9MBadkXWiSJov2fxC4TRzB1Z2avCuWpfwIajhfK7+z3st4yZquCI3SnMgDjYcO+YlqNYBIe2Y531CyOIqxm1JIA6Fwdiwkwh8MwNTBNdnphe6Jim7YagPEyAyEHfGgC3qv7NHrWNmIsjceVutM15olKgJj7/zJ6XHrhQ3nGbi04mCPoqgB3SXea2S7BBC+fCSMxy755WB +vb05.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCNkyS10fWrAQNtZ11eP7qOzr34vKwaNQKb+EACaYPrJyLcLfjas4KqKhKU32BYEymOmmaF0at/We5+BMLkZvm4= +vb05.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE05ZXWSDnX/TqvapioEpDnTlRfCDnkaq9HbljpBKon1 +vb06.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAuFxDIE1KuTPYjIRO6jhg4SKej81aM/KA7EKe1AqNyd82MEGUceqkXNVm7raEToBAfedsiZPBRdx/0AcniHOxUbfEJSwqbi/z+in3xqJXwEw7qbGz2x1Gt5msi6JAVx56YQEqWGL0e4O1RID8U0AWFs0fM+rlXE6gcaDD0ziRDsBNOYV5SlZSgPxFA2UM9H2FyLL2clpH0sbrF1IMS5+1W0lZlRJU0I5wEzUXiAg+RgIhO5QdOHF88Ry+WIeJYDHkdMqbwPJGVPa5MwOoOKrGIQ4+yVwZSh9BWvkb5eWjbccvFdB+edTwyksqyCT1yrfis9I1qAEoNZyhggX6FRmT +vb06.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCPa0QIHNwb9RgpSPhQuUlE1v4fEKcn0B4lvZ3eOaMhZoPxbFWv5lIGh5ApAJmE+bfnWa6n8M8o4PBr/hmL0Fro= +vb06.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDGcQvfBVc2z544/FLFIEvGXAZ4a/JXiAANAvs4Disw/ +vb07.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgc+xggxpEvTnTi0Ic/TLKkffqhJ+D49VywbIUbP4r3+bCO3uBDJANRiuFg+EMpzgT2BVOwJa5Sa0vyqn8NtROxQOnihBmsZ+qr+NdnDUtjq2Fgv5ZSdD7r3bNyivugVY1S2DDecm5yX49ccgDedKKxILNF8E2AAxhNHS7WhBziF7O5DO5kXGqk2C7wg7eLxoyY4LNJGtdL2wG2ZXwFIDZSWgK2Z4TQ55jAsxr1zfIpmS2DjsRxt89SccqwFI9y3WS92hqnlxnpsFXEY6jgZ8pis/t3SgbXAsCkOG/uRdqI5MsJmxN5+b/vBTWj9iuc/XfH7p/CfLaPBkrek7k3S+v +vb07.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKLLhm5IXL6cIbldf+s0FxEKxLIg63xdTyyFYsZC7SMZzQws5RlGfFvz5fkJTFRY7CcP1vd1gaXw/aKieJx2pBA= +vb07.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALlH9pXTWGL8pta++ybUmW7tgZ2pYYLhosIdwc7zsAu +vb08.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuzeXRliY1fwd/U4NddrK+D30s7nkx0hPa5x9T6A9jSNBCo/3gLOw8lA81yoA/EKpqTRErcCi+S8GGDSyj0bvcO9MDUhOXC0KZb/Q0K4/snUb1lYW859bwi3BWgnOm5l1oAqxAscXVvvxflZ7g58wciWyv+hMFIhtjmss6ICKFXszFi4PB4QH/rceQkYNf0oy4HQzM+2TJf9NWWXacxXUNfNKp++fzoYhyzW1+2gEeOToKMDR4VpUEt5BHEQaFS24GNqciTMCUuC2nkjYUBKuGA57NDADYWTvThXxsTW3TgzhHiYOqSi05jxKAsgaEz6llG+uquHhByPcJsLODqHLV +vb08.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE05rEaLBU4OabuECDH+rvaPuLVqHGyTA1Mue+ZAVPTI/XqHG2DVgz36QQj7RKnHR3JxsdsIl+eQr3wDhHBflTA= +vb08.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ0JKjCx+YxgHBehTH1cgpmIWCDbDcC6xIFR+WQA7Aza +vb09.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCWpxmKMaonUowEmWoSM2B0fpXiPW9N/5aTueeR9NhSu7CCzRUGkwXhS393Fks1ooUIW+IPKPQ9ASS9sdBz9ed7pvDbXcsXSW595TWoWSDU4wZNb9zFevrHbeKoHsn7RKMynSBORXiZl+XslgBJuP8QJXOmDcq+cuNiM+NWCrt5QJbiODQPEhKljx+9jN7wtwjQgso94KjUdLccc8861+aHkXCXIoxp+DEjb6FLG1bEAvRoaviuZQKMpwDf3sz4FddT//Bl47Tnv62Bj7Kj7ENPQNdycoodNH41u6Eik48CzVTskARBLi80YC2e678G+gAJwDNnSZn6yzWWkzarIgZ +vb09.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ7Fvaz6tZGmtEuBLsY38LKJZZGcaVGgu7oXFJo1OgJ8PHc19xO6h8ZNVTgcZ9SYTSiSJO/YkFk1FEfhG4juu7E= +vb09.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILacVjWxwGacBm8E+pSBWW20dULns9Ggq+XVccJFzaRX +vb10.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDfhBaTBCajLGQynnV1rutqZhLrZuomRxR6RVICDRV5PeyHXELxpfgofJCtzjWhEWnKx8aTRwF9KD8tZzvi1XwwjdRunBA4nisx9pSXyEANm7/uQERlthLT9FBQyayxucGo9QDa4acx1irkZHoSCoHCXHcdHeGE4gkeLqdgFgFzVXcaSaJOe8RKQQPQHnnQOrbi4eqLvnsgO6+SqUF82+sOtsWNLrDKmEcCLxb9Q3d1XV7pZDowSUtSrMaFwag67t70fJnT9jNzfKpSBNBzA3Hf+eDBhWiv6J5metvk2DAVoLXxHXt27e+cL06tawQdlOr7bFWTzG7rL34x0Ruu7uLF +vb10.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOWz9J5+iUzlDhL3OtDeD9juHSbnJYGVhloUBSsP1mUQHN6UQZTiLLYheXYch2qeighHlv+wLxw+37if4vc1JOQ= +vb10.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICedVECImJ5lDJIlXjbAhnBxcUlUJlNFK7J8/kKJ2gFV EOF cat >> ~/.ssh/config < Date: Fri, 18 Jul 2025 14:10:26 +1000 Subject: [PATCH 09/10] ci: fixups; EOFError -> IncompleteReadError which includes the partial buffer Signed-off-by: julia --- ci/lib/backends/common.py | 2 +- ci/lib/backends/machine_queue.py | 3 +-- ci/lib/backends/streams.py | 6 +++++- ci/lib/runner.py | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ci/lib/backends/common.py b/ci/lib/backends/common.py index 51f01a327..8a485ef08 100644 --- a/ci/lib/backends/common.py +++ b/ci/lib/backends/common.py @@ -12,7 +12,7 @@ def reset_terminal(): - OUTPUT.write(b"\n\x1b[0m") + OUTPUT.write(b"\x1b[0m") class TestRetryException(Exception): diff --git a/ci/lib/backends/machine_queue.py b/ci/lib/backends/machine_queue.py index b1bc4c5f4..c5219ab0d 100644 --- a/ci/lib/backends/machine_queue.py +++ b/ci/lib/backends/machine_queue.py @@ -87,8 +87,7 @@ async def _acquire_lock(self): "-wait", self.chosen_board, "-k", self.job_key, "-T", str(LOCK_TIMEOUT), - "-t", "0", - # only try to acquire once. + "-t", "0", # only try to acquire once. # fmt: on stdout=None, # inherit -> print stderr=None, # inherit -> print diff --git a/ci/lib/backends/streams.py b/ci/lib/backends/streams.py index a05c7d815..6d4f4ce9c 100644 --- a/ci/lib/backends/streams.py +++ b/ci/lib/backends/streams.py @@ -25,6 +25,10 @@ async def wrapper(*args, **kwargs): ) ) raise + except asyncio.IncompleteReadError: + reset_terminal() + log.info("'{}' hit EOF whilst waiting for {}".format(f.__name__, text)) + raise return wrapper @@ -60,7 +64,7 @@ async def wait_for_output(backend: HardwareBackend, text: bytes) -> bytes: # TODO: backwards seek? read = await backend.output_stream.read(1) if read == b"": - raise EOFError() + raise asyncio.IncompleteReadError(partial=buffer, expected=None) OUTPUT.write(read) buffer.extend(read) diff --git a/ci/lib/runner.py b/ci/lib/runner.py index 893b94f55..c5887dea5 100755 --- a/ci/lib/runner.py +++ b/ci/lib/runner.py @@ -50,8 +50,8 @@ async def runner( await backend.start() await test(backend, test_config) - except (EOFError, asyncio.IncompleteReadError): - raise TestFailureException("EOF when reading from backend stream") + except asyncio.IncompleteReadError as e: + raise TestFailureException("EOF when reading from backend stream: {}".format(e)) finally: reset_terminal() await backend.stop() From 45c6e9779be33851de7ca55bd33af8dd9aeafdb4 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 18 Jul 2025 14:10:52 +1000 Subject: [PATCH 10/10] ci: working echo server benchmark Signed-off-by: julia --- ci/benchmarks/echo_server.py | 69 +++++++++--------- ci/lib/backends/__init__.py | 2 + ci/lib/backends/ipbench_queue.py | 117 +++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 34 deletions(-) create mode 100644 ci/lib/backends/ipbench_queue.py diff --git a/ci/benchmarks/echo_server.py b/ci/benchmarks/echo_server.py index 8f2adbf9d..4973424a5 100755 --- a/ci/benchmarks/echo_server.py +++ b/ci/benchmarks/echo_server.py @@ -66,43 +66,44 @@ async def test(backend: HardwareBackend, test_config: TestConfig): assert test_config.config == "benchmark" - # TODO: Abstract. - - benchmark_run = await asyncio.create_subprocess_exec( - # fmt: off - "iq.sh", "run", - "-c", "vb04", - "-c", "vb05", - "-c", "vb06", - "-c", "vb07", - "-f", Path(__file__).parent / "benchmark.py", - "--", - ip0, - "--throughputs", "10000000", - "--samples", "1000", - "--clients", "vb04", + bench_backend = IpBenchQueueBackend( + ["vb04", "vb06"], + Path(__file__).parent / "benchmark.py", + target_ip=ip0, + throughputs=[10000000, 20000000], + samples=100, ) + + # XXX: I probably want to redesign the CI so that output is always printed + # at the moment I'm sort of working around the fact that I only get + # output printed while waiting, which means can't easily output + # 2 streams at once. + try: - # # await wait_for_output(backend, b"client0 measurement finished...") - # await wait_for_output(backend, b"\n") - # # await wait_for_output(backend, b"\n") - # reset_terminal() - # await asyncio.wait( - # [asyncio.create_task(f) for f in [benchmark_run.wait(), wait_for_output(backend, b"client0 measurement starting...")]], - # return_when="FIRST_COMPLETED" - # ) - # if benchmark_run.returncode is not None: - # raise TestFailureException("AAA umm??") - # await asyncio.gather( - # benchmark_run.wait(), - # wait_for_output(backend, b"client0 measurement starting..."), - # ) - await wait_for_output(backend, b"Result Summary:") + await bench_backend.start() + ANSI_RESET = b"\x1b[0m" + for _ in bench_backend.throughputs: + await wait_for_output(backend, b"Utilization connection established!\r\n" + ANSI_RESET) + await wait_for_output(bench_backend, b"[send_command] : START\n") + await wait_for_output(backend, b"client0 measurement starting...\r\n" + ANSI_RESET) + # TODO: ipbench print useful string out after the results. + await asyncio.gather( + wait_for_output(backend, b"client0 measurement finished \r\n" + ANSI_RESET), + wait_for_output(bench_backend, b"[send_command] : STOP\n") + ) + + # All PDs + two initial ones. TODO: Make the bench print out a good string at the end to avoid this. + for _ in range(12): + await wait_for_output(backend, b"}\r\n") + + reset_terminal() + + await wait_for_output(bench_backend, b"iq.sh runner is done\n") + + # XXX: This doesn't work with ctrl+c the locks don't get released... finally: - try: - benchmark_run.terminate() - except ProcessLookupError: - pass + # XXX: When stopping, always print out the rest of the output? + await bench_backend.stop() if __name__ == "__main__": diff --git a/ci/lib/backends/__init__.py b/ci/lib/backends/__init__.py index 770c45e85..4e149553d 100644 --- a/ci/lib/backends/__init__.py +++ b/ci/lib/backends/__init__.py @@ -27,6 +27,7 @@ OUTPUT, ) from .streams import send_input, wait_for_output, expect_output +from .ipbench_queue import IpBenchQueueBackend from .machine_queue import MachineQueueBackend from .qemu import QemuBackend from .tty import TtyBackend @@ -46,6 +47,7 @@ "wait_for_output", "expect_output", # backends + "IpBenchQueueBackend", "MachineQueueBackend", "QemuBackend", "TtyBackend", diff --git a/ci/lib/backends/ipbench_queue.py b/ci/lib/backends/ipbench_queue.py new file mode 100644 index 000000000..e6ce50164 --- /dev/null +++ b/ci/lib/backends/ipbench_queue.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# Copyright 2025, UNSW +# SPDX-License-Identifier: BSD-2-Clause + +import os +from signal import SIGHUP, SIGINT +import asyncio +from asyncio.subprocess import PIPE, STDOUT +from pathlib import Path + +from .. import log +from .base import HardwareBackend +from .common import LockedBoardException, TestFailureException, TestRetryException +from .streams import wait_for_output + +# In case we somehow break and don't release the lock automatically. +# TODO: inherit from somewhere else +LOCK_TIMEOUT = 120 * 60 # 120 minutes +START_TIMEOUT = 60 # 1 minute +# For Github Actions etc. +IS_CI = bool(os.environ.get("CI")) + + +def flatten(xss): + return [x for xs in xss for x in xs] + + +class IpBenchQueueBackend(HardwareBackend): + def __init__( + self, + clients: list[str], + benchmark_script: Path, + target_ip: str, + throughputs: list[int], + samples: int, + ): + """ + clients is the list of valid bench clients used with iq.sh + """ + self.clients = clients + self.target_ip = target_ip + self.benchmark_script = benchmark_script + self.throughputs = throughputs + self.samples = samples + self.process = None + + if IS_CI: + self.job_key = "-".join( + [ + "au_ts_ci", + os.environ.get("GITHUB_REPOSITORY", "??"), + os.environ.get("GITHUB_WORKFLOW", "??"), + os.environ.get("GITHUB_RUN_ID", "??"), + os.environ.get("GITHUB_JOB", "??"), + os.environ.get("INPUT_INDEX", "$0")[1:], + ] + ) + else: + self.job_key = "au_ts_ci (running locally)" + + async def start(self): + assert self.process is None, "start() should only be called once" + + self.process = await asyncio.create_subprocess_exec( + # fmt: off + "iq.sh", "run", + "-k", self.job_key, + "-T", str(LOCK_TIMEOUT), + # only try to acquire once + "-t", "0", + "-f", self.benchmark_script.resolve(), + *flatten(("-c", client) for client in self.clients), + "--", + self.target_ip, + "--throughputs", *[str(t) for t in self.throughputs], + "--samples", str(self.samples), + "--clients", *[c for c in self.clients], + # fmt: on + stdin=PIPE, + stdout=PIPE, + stderr=STDOUT, + ) + + try: + async with asyncio.timeout(START_TIMEOUT): + await wait_for_output(self, b"[IpbenchTestTarget:__init__] : client " + self.target_ip.encode() + b"\n") + except asyncio.IncompleteReadError as e: + if (b"Failed to acquire lock for" in e.partial) \ + or (b"Attempting to grab lock you already own" in e.partial) \ + or (b"Giving up; releasing newly locked systems" in e.partial): + raise TestRetryException() from e + + raise + + async def stop(self): + if self.process is None: + return + + # XXX: I think this doesn't work. + + try: + # Use SIGHUP to close the console (releases lock implicitly) + self.process.send_signal(SIGINT) + # Use transport.close() because await process.wait() deadlocks + self.process._transport.close() # type: ignore + except ProcessLookupError: + pass + + @property + def input_stream(self) -> asyncio.StreamWriter: + assert self.process is not None, "process not running" + return self.process.stdin # type: ignore + + @property + def output_stream(self) -> asyncio.StreamReader: + assert self.process is not None, "process not running" + return self.process.stdout # type: ignore