diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index a3cb0faa8..ffb8b3f50 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -110,8 +110,8 @@ jobs: env: PYTHON: ${{ github.workspace }}/venv/bin/python - run_qemu: - name: Run (QEMU) + tests_qemu: + name: Tests (QEMU) runs-on: ubuntu-latest needs: build_linux_x86_64_nix steps: @@ -139,8 +139,8 @@ jobs: path: ci_logs if-no-files-found: error - run_hardware: - name: Run (hardware) + tests_hardware: + name: Tests (hardware) runs-on: ubuntu-latest if: ${{ contains(github.event.pull_request.labels.*.name, 'hardware-test') || (github.event_name == 'schedule') }} @@ -178,3 +178,49 @@ jobs: name: ci-logs-hardware path: ci_logs if-no-files-found: error + + bench_hardware: + name: Benchmarks (hardware) + runs-on: ubuntu-latest + if: ${{ contains(github.event.pull_request.labels.*.name, 'hardware-bench') || + (github.event_name == 'schedule') }} + needs: build_linux_x86_64_nix + concurrency: + group: ${{ github.workflow }}-sddf-hardware-bench-${{ github.event.number }}-${{ strategy.job-index }} + cancel-in-progress: true + steps: + - name: Checkout sDDF repository + uses: actions/checkout@v4 + - name: Get machine queue + uses: actions/checkout@v4 + with: + repository: seL4/machine_queue + path: machine_queue + - name: Get ipbench_queue + uses: actions/checkout@v4 + with: + repository: au-ts/ipbench_queue + path: ipbench_queue + ssh-key: ${{ secrets.IPBENCH_GITHUB_DEPLOY_KEY }} + - name: Download images + uses: actions/download-artifact@v4 + with: + name: loader-images + path: ci_build + - name: Setup machine queue SSH key + run: .github/workflows/setup_ssh_key.sh + env: + MACHINE_QUEUE_KEY: ${{ secrets.MACHINE_QUEUE_KEY }} + - name: Run tests + run: | + export PATH="$(pwd)/machine_queue":"$(pwd)/ipbench_queue":$PATH + # GitHub Actions is broken + # https://github.com/ringerc/github-actions-signal-handling-demo#why-child-process-tasks-dont-get-a-chance-to-clean-up-on-job-cancel + exec ./ci/benchmarks/echo_server.py --boards maaxboard --single --configs benchmark + - name: Archive logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-logs-hardware-benchmarks + path: ci_logs + if-no-files-found: error diff --git a/.github/workflows/setup_ssh_key.sh b/.github/workflows/setup_ssh_key.sh index e5e04bc89..5234aeaf0 100755 --- a/.github/workflows/setup_ssh_key.sh +++ b/.github/workflows/setup_ssh_key.sh @@ -14,6 +14,36 @@ login.trustworthy.systems ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSzWm9H9EKxcinJs login.trustworthy.systems ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCRA/W8LVxLFjzPvijygdSw+rPW/EQEG8WoUVcTm5dYXDIhCc0Zxibd19zPb1LQpE2/Ohe+I16iC5glpmFyDfrs= login.trustworthy.systems ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPYp/3vDMDnHnjtqt5Oqievgz04g/LJ4yEKOlXCu9Yux tftp.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEj7X6doSoop91gTvBD7L4O7VGwCO5pLNsu5YAGS1L64MJqo+3wTYgFRdMWTM0hL3YN+1sSabJPICJzKk0EJxkg= +vb01.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDhZW6jkDLaoZXD7BxrM4OtPvRwquHqIRXyFA8rFPpA3G6YXc9iOpTZODt2/CNbZXyCbK9RLuWWDCKNv1CR1gJSurb770wzg5Ts0v6J7eEa+B83skeJUIT2i+eFqyuEcPgvmdG5DChRyhaQen8O97wvkNVnT+B6OJvAtwSCCU9KsO0vI7lCF040PCJTXrvSjkIf3unJxDmdUzPIqyyof/m5FipoceL7sBfAXhMQYlRh4LnrC2rmPt1/yKJENSes0y07i0ihN92m/GBnZHViQjQdSYQbb0eySujrpBnj4Z4aXNv9Ao2kCNqBzV/qpo3ROXZk76ELBHpNX80ewLgHdU/x +vb01.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIAsus4NLjozoEisJBBVtcxhRbUTu9UvLARwOyDb6rS9Rd3ooxRG8rPvHAnC8x+eOW0Oxzyb8mEdqwsUUTspbOU= +vb01.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHqUTgA9+HoTE64Z58Gho1dSufLN/bD/0B1M8ee8CVVb +juliab@tftp:~$ vb02 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDfc5wBO/+ExXOFxlR/QMvjroN8KAbDT3IovmHLpjWbzCCE4cE9fu5MQKHgPAJ25sfcTtzlRVrw20hQyHQgL/141wQ/XQk3H1vLKRJbrPVHrpoPgpB4gETOCFpUJJRuup5bBdeWIkTTV82F7V+mpQbMYc30DIl7LCmXd20QV8D5KynNjjci35SRGuHxjbBQNfSs1wVUgK1z+Y5elG7OgLXi2IE361ksEm3nbCJkpLZSFGIVHQkPHwsMhwKBZlXhJAgeqLmiE8d4WQoORXPglvSQ8O20McMjHbOMCIPikPWNhkoSD7ekqn2/HReciLw0R7CJ8c2kPuR//K8MxL3wodZ1 +vb02.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD3yksLxJoCzZzvcZkfg2pV1jGGAAOSzEMnP1Q1lB4KoljOf/W5Kw2c1ZLMJYaWzSciaZBkhu96UQIpHBdgBk0M= +vb02.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDMwTySOHEqawSSo3/4et/71KobxIr07UuotW8HoDCk2 +vb03.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDjLpyMZ4/1Zj8mT7OKtmd3deon/Ta1QLdSajveNb0HcMXtrrFIJfSMf97A3IN+wyK61m5ThxlRnXO/S1E4kLdA84L1Qv94jUE0fpkBKnw4x62eTHas+a5BsD6jUkypsGnsQ5ugW9A5muQyktqGngyk9/Y/hWdQiPP4ovjwjUJYaiBXiFAFSFnw8r2jdZiSCrim2wWB1jy16atTpmppxhsVFbnx3/Jl54VgbLlJFmfhnYwYv2gG5mbHswEV1p4owIVPcMQljYdMjssTqzcw8t32qmDxmpParUtvO0ZquJ8lvwSlA29iSlMj8DVkjNAgg0754ohJ7wSF3zydGwoR0wgr +vb03.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAQsjevofMjx2sOfOA0/tB4tZal21EKrfSXURYajIX5DCCvzHbUFbaJZpopNWPgRasud7HrxlnAcHtwluILMo64= +vb03.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPyhdhYIiJlr8FSpYEwRJzyeqV3aCj1XJ2CldV8HawKk +vb04.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDl4RW2DzRd7rvTGg1Yb6GqXt1nzzZPdKWVSp9z/s6tY17MEUB0DQtu9wox9PxfBPAy1c2YnW33ezrGUKRM1ScieKNEZbnASDBLllcq/CJv+4EnU0vtE9e0VN34FJnnhNyLpHLgMoglBriJtwMRmEP1tiWvrZuNAXlvNLdgGt/QO0uhYSA4weOkObPA/Shc/ZWIA+kKVFRyby4UXgd6EZDUSOmXV4iFt+JVbg50vYVlAz86DFSBCarFu0vUi1QvpWgn0D5NA2+nDcGevGSAl5j7PRPw7bUJ938nTdT5kJM4722QCGw6BuBdkf2ohwFiHqCljOrajccqSg257ZvWF8v9 +vb04.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL2E2jmBDPTxG/A7BPy2WOuIbQIG/k+LrHbxLiLSZWOcdmK1Ui5vpnHesfHADnzkKENNabnNo/MF+RjSnRIK+lk= +vb04.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL88FdPerLun9BVpgRKh/1CaYZnj8XDNMHgXBd96fial +vb05.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8ncTSIQyaA4/ulod1IWnZVLitjcgZ0/mV1iFS5Pefsd0c8dtLCosMCM3mI6Ho1lkF7r4PslYS0v7LE4kz9WhVJYdkjGwOam8Tea3rB1uQX23Mm2qi7UAAATr0ogluy2ApKUiktKJV8vH9MBadkXWiSJov2fxC4TRzB1Z2avCuWpfwIajhfK7+z3st4yZquCI3SnMgDjYcO+YlqNYBIe2Y531CyOIqxm1JIA6Fwdiwkwh8MwNTBNdnphe6Jim7YagPEyAyEHfGgC3qv7NHrWNmIsjceVutM15olKgJj7/zJ6XHrhQ3nGbi04mCPoqgB3SXea2S7BBC+fCSMxy755WB +vb05.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCNkyS10fWrAQNtZ11eP7qOzr34vKwaNQKb+EACaYPrJyLcLfjas4KqKhKU32BYEymOmmaF0at/We5+BMLkZvm4= +vb05.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE05ZXWSDnX/TqvapioEpDnTlRfCDnkaq9HbljpBKon1 +vb06.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAuFxDIE1KuTPYjIRO6jhg4SKej81aM/KA7EKe1AqNyd82MEGUceqkXNVm7raEToBAfedsiZPBRdx/0AcniHOxUbfEJSwqbi/z+in3xqJXwEw7qbGz2x1Gt5msi6JAVx56YQEqWGL0e4O1RID8U0AWFs0fM+rlXE6gcaDD0ziRDsBNOYV5SlZSgPxFA2UM9H2FyLL2clpH0sbrF1IMS5+1W0lZlRJU0I5wEzUXiAg+RgIhO5QdOHF88Ry+WIeJYDHkdMqbwPJGVPa5MwOoOKrGIQ4+yVwZSh9BWvkb5eWjbccvFdB+edTwyksqyCT1yrfis9I1qAEoNZyhggX6FRmT +vb06.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCPa0QIHNwb9RgpSPhQuUlE1v4fEKcn0B4lvZ3eOaMhZoPxbFWv5lIGh5ApAJmE+bfnWa6n8M8o4PBr/hmL0Fro= +vb06.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDGcQvfBVc2z544/FLFIEvGXAZ4a/JXiAANAvs4Disw/ +vb07.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgc+xggxpEvTnTi0Ic/TLKkffqhJ+D49VywbIUbP4r3+bCO3uBDJANRiuFg+EMpzgT2BVOwJa5Sa0vyqn8NtROxQOnihBmsZ+qr+NdnDUtjq2Fgv5ZSdD7r3bNyivugVY1S2DDecm5yX49ccgDedKKxILNF8E2AAxhNHS7WhBziF7O5DO5kXGqk2C7wg7eLxoyY4LNJGtdL2wG2ZXwFIDZSWgK2Z4TQ55jAsxr1zfIpmS2DjsRxt89SccqwFI9y3WS92hqnlxnpsFXEY6jgZ8pis/t3SgbXAsCkOG/uRdqI5MsJmxN5+b/vBTWj9iuc/XfH7p/CfLaPBkrek7k3S+v +vb07.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKLLhm5IXL6cIbldf+s0FxEKxLIg63xdTyyFYsZC7SMZzQws5RlGfFvz5fkJTFRY7CcP1vd1gaXw/aKieJx2pBA= +vb07.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALlH9pXTWGL8pta++ybUmW7tgZ2pYYLhosIdwc7zsAu +vb08.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuzeXRliY1fwd/U4NddrK+D30s7nkx0hPa5x9T6A9jSNBCo/3gLOw8lA81yoA/EKpqTRErcCi+S8GGDSyj0bvcO9MDUhOXC0KZb/Q0K4/snUb1lYW859bwi3BWgnOm5l1oAqxAscXVvvxflZ7g58wciWyv+hMFIhtjmss6ICKFXszFi4PB4QH/rceQkYNf0oy4HQzM+2TJf9NWWXacxXUNfNKp++fzoYhyzW1+2gEeOToKMDR4VpUEt5BHEQaFS24GNqciTMCUuC2nkjYUBKuGA57NDADYWTvThXxsTW3TgzhHiYOqSi05jxKAsgaEz6llG+uquHhByPcJsLODqHLV +vb08.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE05rEaLBU4OabuECDH+rvaPuLVqHGyTA1Mue+ZAVPTI/XqHG2DVgz36QQj7RKnHR3JxsdsIl+eQr3wDhHBflTA= +vb08.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ0JKjCx+YxgHBehTH1cgpmIWCDbDcC6xIFR+WQA7Aza +vb09.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCWpxmKMaonUowEmWoSM2B0fpXiPW9N/5aTueeR9NhSu7CCzRUGkwXhS393Fks1ooUIW+IPKPQ9ASS9sdBz9ed7pvDbXcsXSW595TWoWSDU4wZNb9zFevrHbeKoHsn7RKMynSBORXiZl+XslgBJuP8QJXOmDcq+cuNiM+NWCrt5QJbiODQPEhKljx+9jN7wtwjQgso94KjUdLccc8861+aHkXCXIoxp+DEjb6FLG1bEAvRoaviuZQKMpwDf3sz4FddT//Bl47Tnv62Bj7Kj7ENPQNdycoodNH41u6Eik48CzVTskARBLi80YC2e678G+gAJwDNnSZn6yzWWkzarIgZ +vb09.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ7Fvaz6tZGmtEuBLsY38LKJZZGcaVGgu7oXFJo1OgJ8PHc19xO6h8ZNVTgcZ9SYTSiSJO/YkFk1FEfhG4juu7E= +vb09.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILacVjWxwGacBm8E+pSBWW20dULns9Ggq+XVccJFzaRX +vb10.keg.cse.unsw.edu.au ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDfhBaTBCajLGQynnV1rutqZhLrZuomRxR6RVICDRV5PeyHXELxpfgofJCtzjWhEWnKx8aTRwF9KD8tZzvi1XwwjdRunBA4nisx9pSXyEANm7/uQERlthLT9FBQyayxucGo9QDa4acx1irkZHoSCoHCXHcdHeGE4gkeLqdgFgFzVXcaSaJOe8RKQQPQHnnQOrbi4eqLvnsgO6+SqUF82+sOtsWNLrDKmEcCLxb9Q3d1XV7pZDowSUtSrMaFwag67t70fJnT9jNzfKpSBNBzA3Hf+eDBhWiv6J5metvk2DAVoLXxHXt27e+cL06tawQdlOr7bFWTzG7rL34x0Ruu7uLF +vb10.keg.cse.unsw.edu.au ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOWz9J5+iUzlDhL3OtDeD9juHSbnJYGVhloUBSsP1mUQHN6UQZTiLLYheXYch2qeighHlv+wLxw+37if4vc1JOQ= +vb10.keg.cse.unsw.edu.au ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICedVECImJ5lDJIlXjbAhnBxcUlUJlNFK7J8/kKJ2gFV EOF cat >> ~/.ssh/config < HardwareBackend: + backend = common.backend_fn(test_config, loader_img) + + if isinstance(backend, QemuBackend): + # fmt: off + backend.invocation_args.extend([ + "-global", "virtio-mmio.force-legacy=false", + "-device", "virtio-net-device,netdev=netdev0", + "-netdev", "user,id=netdev0,hostfwd=tcp::1236-:1236,hostfwd=tcp::1237-:1237,hostfwd=udp::1235-:1235", + ]) + # fmt: on + + return backend + + +async def test(backend: HardwareBackend, test_config: TestConfig): + async with asyncio.timeout(20): + await wait_for_output(backend, b"DHCP request finished") + dhcp_client1 = await wait_for_output(backend, b"\r\n") + await wait_for_output(backend, b"DHCP request finished") + dhcp_client0 = await wait_for_output(backend, b"\r\n") + + dhcp_client1, dhcp_client0 = ( + (dhcp_client1, dhcp_client0) + if b"client1" in dhcp_client1 + else (dhcp_client0, dhcp_client1) + ) + + try: + # fmt: off + ip1 = re.search(rb"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", dhcp_client1).group(0).decode() # type: ignore + ip0 = re.search(rb"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", dhcp_client0).group(0).decode() # type: ignore + # fmt: on + except (IndexError, AttributeError): + raise TestFailureException( + "could not find IP address in DHCP request result" + ) + + reset_terminal() + log.info(f"client IPs: client0={ip0}, client1={ip1}") + + # Now let's do the actual benchmark! + + assert test_config.config == "benchmark" + + bench_backend = IpBenchQueueBackend( + ["vb04", "vb06"], + Path(__file__).parent / "benchmark.py", + target_ip=ip0, + throughputs=[10000000, 20000000], + samples=100, + ) + + # XXX: I probably want to redesign the CI so that output is always printed + # at the moment I'm sort of working around the fact that I only get + # output printed while waiting, which means can't easily output + # 2 streams at once. + + try: + await bench_backend.start() + ANSI_RESET = b"\x1b[0m" + for _ in bench_backend.throughputs: + await wait_for_output(backend, b"Utilization connection established!\r\n" + ANSI_RESET) + await wait_for_output(bench_backend, b"[send_command] : START\n") + await wait_for_output(backend, b"client0 measurement starting...\r\n" + ANSI_RESET) + # TODO: ipbench print useful string out after the results. + await asyncio.gather( + wait_for_output(backend, b"client0 measurement finished \r\n" + ANSI_RESET), + wait_for_output(bench_backend, b"[send_command] : STOP\n") + ) + + # All PDs + two initial ones. TODO: Make the bench print out a good string at the end to avoid this. + for _ in range(12): + await wait_for_output(backend, b"}\r\n") + + reset_terminal() + + await wait_for_output(bench_backend, b"iq.sh runner is done\n") + + # XXX: This doesn't work with ctrl+c the locks don't get released... + finally: + # XXX: When stopping, always print out the rest of the output? + await bench_backend.stop() + + +if __name__ == "__main__": + cli("echo_server", test, TEST_MATRIX, backend_fn, common.loader_img_path) diff --git a/ci/build.py b/ci/build.py index 00718b1a2..0784df586 100755 --- a/ci/build.py +++ b/ci/build.py @@ -113,9 +113,11 @@ def build(args: argparse.Namespace, example_name: str, test_config: TestConfig): for example_name, options in matrix.EXAMPLES.items(): if example_name not in args.examples: continue + if example_name != "echo_server": + continue example_matrix = matrix_product( - board=options["boards_build"], + board=["maaxboard"], # options["boards_build"], config=options["configs"], build_system=options["build_systems"], ) diff --git a/ci/lib/backends/__init__.py b/ci/lib/backends/__init__.py index 770c45e85..4e149553d 100644 --- a/ci/lib/backends/__init__.py +++ b/ci/lib/backends/__init__.py @@ -27,6 +27,7 @@ OUTPUT, ) from .streams import send_input, wait_for_output, expect_output +from .ipbench_queue import IpBenchQueueBackend from .machine_queue import MachineQueueBackend from .qemu import QemuBackend from .tty import TtyBackend @@ -46,6 +47,7 @@ "wait_for_output", "expect_output", # backends + "IpBenchQueueBackend", "MachineQueueBackend", "QemuBackend", "TtyBackend", diff --git a/ci/lib/backends/common.py b/ci/lib/backends/common.py index 51f01a327..8a485ef08 100644 --- a/ci/lib/backends/common.py +++ b/ci/lib/backends/common.py @@ -12,7 +12,7 @@ def reset_terminal(): - OUTPUT.write(b"\n\x1b[0m") + OUTPUT.write(b"\x1b[0m") class TestRetryException(Exception): diff --git a/ci/lib/backends/ipbench_queue.py b/ci/lib/backends/ipbench_queue.py new file mode 100644 index 000000000..e6ce50164 --- /dev/null +++ b/ci/lib/backends/ipbench_queue.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# Copyright 2025, UNSW +# SPDX-License-Identifier: BSD-2-Clause + +import os +from signal import SIGHUP, SIGINT +import asyncio +from asyncio.subprocess import PIPE, STDOUT +from pathlib import Path + +from .. import log +from .base import HardwareBackend +from .common import LockedBoardException, TestFailureException, TestRetryException +from .streams import wait_for_output + +# In case we somehow break and don't release the lock automatically. +# TODO: inherit from somewhere else +LOCK_TIMEOUT = 120 * 60 # 120 minutes +START_TIMEOUT = 60 # 1 minute +# For Github Actions etc. +IS_CI = bool(os.environ.get("CI")) + + +def flatten(xss): + return [x for xs in xss for x in xs] + + +class IpBenchQueueBackend(HardwareBackend): + def __init__( + self, + clients: list[str], + benchmark_script: Path, + target_ip: str, + throughputs: list[int], + samples: int, + ): + """ + clients is the list of valid bench clients used with iq.sh + """ + self.clients = clients + self.target_ip = target_ip + self.benchmark_script = benchmark_script + self.throughputs = throughputs + self.samples = samples + self.process = None + + if IS_CI: + self.job_key = "-".join( + [ + "au_ts_ci", + os.environ.get("GITHUB_REPOSITORY", "??"), + os.environ.get("GITHUB_WORKFLOW", "??"), + os.environ.get("GITHUB_RUN_ID", "??"), + os.environ.get("GITHUB_JOB", "??"), + os.environ.get("INPUT_INDEX", "$0")[1:], + ] + ) + else: + self.job_key = "au_ts_ci (running locally)" + + async def start(self): + assert self.process is None, "start() should only be called once" + + self.process = await asyncio.create_subprocess_exec( + # fmt: off + "iq.sh", "run", + "-k", self.job_key, + "-T", str(LOCK_TIMEOUT), + # only try to acquire once + "-t", "0", + "-f", self.benchmark_script.resolve(), + *flatten(("-c", client) for client in self.clients), + "--", + self.target_ip, + "--throughputs", *[str(t) for t in self.throughputs], + "--samples", str(self.samples), + "--clients", *[c for c in self.clients], + # fmt: on + stdin=PIPE, + stdout=PIPE, + stderr=STDOUT, + ) + + try: + async with asyncio.timeout(START_TIMEOUT): + await wait_for_output(self, b"[IpbenchTestTarget:__init__] : client " + self.target_ip.encode() + b"\n") + except asyncio.IncompleteReadError as e: + if (b"Failed to acquire lock for" in e.partial) \ + or (b"Attempting to grab lock you already own" in e.partial) \ + or (b"Giving up; releasing newly locked systems" in e.partial): + raise TestRetryException() from e + + raise + + async def stop(self): + if self.process is None: + return + + # XXX: I think this doesn't work. + + try: + # Use SIGHUP to close the console (releases lock implicitly) + self.process.send_signal(SIGINT) + # Use transport.close() because await process.wait() deadlocks + self.process._transport.close() # type: ignore + except ProcessLookupError: + pass + + @property + def input_stream(self) -> asyncio.StreamWriter: + assert self.process is not None, "process not running" + return self.process.stdin # type: ignore + + @property + def output_stream(self) -> asyncio.StreamReader: + assert self.process is not None, "process not running" + return self.process.stdout # type: ignore diff --git a/ci/lib/backends/machine_queue.py b/ci/lib/backends/machine_queue.py index b1bc4c5f4..c5219ab0d 100644 --- a/ci/lib/backends/machine_queue.py +++ b/ci/lib/backends/machine_queue.py @@ -87,8 +87,7 @@ async def _acquire_lock(self): "-wait", self.chosen_board, "-k", self.job_key, "-T", str(LOCK_TIMEOUT), - "-t", "0", - # only try to acquire once. + "-t", "0", # only try to acquire once. # fmt: on stdout=None, # inherit -> print stderr=None, # inherit -> print diff --git a/ci/lib/backends/streams.py b/ci/lib/backends/streams.py index a05c7d815..6d4f4ce9c 100644 --- a/ci/lib/backends/streams.py +++ b/ci/lib/backends/streams.py @@ -25,6 +25,10 @@ async def wrapper(*args, **kwargs): ) ) raise + except asyncio.IncompleteReadError: + reset_terminal() + log.info("'{}' hit EOF whilst waiting for {}".format(f.__name__, text)) + raise return wrapper @@ -60,7 +64,7 @@ async def wait_for_output(backend: HardwareBackend, text: bytes) -> bytes: # TODO: backwards seek? read = await backend.output_stream.read(1) if read == b"": - raise EOFError() + raise asyncio.IncompleteReadError(partial=buffer, expected=None) OUTPUT.write(read) buffer.extend(read) diff --git a/ci/lib/runner.py b/ci/lib/runner.py index 893b94f55..c5887dea5 100755 --- a/ci/lib/runner.py +++ b/ci/lib/runner.py @@ -50,8 +50,8 @@ async def runner( await backend.start() await test(backend, test_config) - except (EOFError, asyncio.IncompleteReadError): - raise TestFailureException("EOF when reading from backend stream") + except asyncio.IncompleteReadError as e: + raise TestFailureException("EOF when reading from backend stream: {}".format(e)) finally: reset_terminal() await backend.stop()