From 4052abe1b4b0bec27eb897e805c53e5914033ac5 Mon Sep 17 00:00:00 2001 From: Ahmed Ismail Date: Mon, 5 Feb 2024 14:45:34 +0000 Subject: [PATCH] ci: Modify test jobs to improve reliability This commit is meant to do the following: * Modify test-application-output job to test blinky application only as it can be used as a smoke test. * Test ML inference output for both keyword-detection and speech-recognition applications as part of the OTA test job which is now renamed to test_applications as it is testing OTA update and ML inference outputs as well. Note: * The mentioned changes would result in a better reliability and better CI performance. Signed-off-by: Ahmed Ismail --- .gitlab-ci.yml | 58 +++++-------------- .../{fail_ota_output.log => fail_output.log} | 0 .../{pass_ota_output.log => pass_output.log} | 0 .../tests/fail_ota_output.log | 3 - .../keyword_detection/tests/fail_output.log | 3 + .../tests/pass_ota_output.log | 2 - .../keyword_detection/tests/pass_output.log | 4 +- .../tests/fail_ota_output.log | 3 - .../speech_recognition/tests/fail_output.log | 3 + .../tests/pass_ota_output.log | 2 - .../speech_recognition/tests/pass_output.log | 4 +- tools/tests/conftest.py | 12 ---- .../{test_ota.py => test_applications.py} | 18 +++--- ...cation_output.py => test_blinky_output.py} | 8 +-- 14 files changed, 39 insertions(+), 81 deletions(-) rename applications/aws_iot_example/tests/{fail_ota_output.log => fail_output.log} (100%) rename applications/aws_iot_example/tests/{pass_ota_output.log => pass_output.log} (100%) delete mode 100644 applications/keyword_detection/tests/fail_ota_output.log delete mode 100644 applications/keyword_detection/tests/pass_ota_output.log delete mode 100644 applications/speech_recognition/tests/fail_ota_output.log delete mode 100644 applications/speech_recognition/tests/pass_ota_output.log rename tools/tests/{test_ota.py => test_applications.py} (82%) rename tools/tests/{test_application_output.py => test_blinky_output.py} (88%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f20aff8f..ecb7a8a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -123,67 +123,37 @@ build-applications: variables: GIT_SUBMODULE_STRATEGY: recursive -test-applications-output: +test-blinky-output: extends: .test_job script: - tar xf ${TARGET}_${APP}_${TOOLCHAIN}_${INFERENCE}_${AUDIO}_build.tar.gz - - export APP_UNDERSCORED=$(echo ${APP} | tr '-' '_') - | - if [[ $AUDIO == "VSI" ]]; then - pytest -s tools/tests/test_application_output.py \ - --audio-file-path "applications/${APP_UNDERSCORED}/resources/test.wav" \ - --build-artefacts-path "build" \ - --fvp $FVP_BIN \ - --merged-elf-name "${APP}_merged.elf" \ - --timeout-seconds 900 \ - --pass-output-file "applications/${APP_UNDERSCORED}/tests/pass_output.log" \ - --fail-output-file "applications/${APP_UNDERSCORED}/tests/fail_output.log" \ - --pythonhome-path "/opt/python/3.9.18" - else - pytest -s tools/tests/test_application_output.py \ + pytest -s tools/tests/test_blinky_output.py \ --build-artefacts-path "build" \ --fvp $FVP_BIN \ --merged-elf-name "${APP}_merged.elf" \ --timeout-seconds 900 \ - --pass-output-file "applications/${APP_UNDERSCORED}/tests/pass_output.log" \ - --fail-output-file "applications/${APP_UNDERSCORED}/tests/fail_output.log" - fi + --pass-output-file "applications/${APP}/tests/pass_output.log" \ + --fail-output-file "applications/${APP}/tests/fail_output.log" parallel: matrix: - << : *pipeline_config_non_ml_applications APP: [blinky] - - - << : *pipeline_config_ml_applications - APP: [keyword-detection] - INFERENCE: [ETHOS, SOFTWARE] - - - << : *pipeline_config_ml_applications - APP: [speech-recognition] - INFERENCE: [ETHOS] - - artifacts: - paths: - - ${TARGET}_${APP}_${TOOLCHAIN}_${INFERENCE}_${AUDIO}_build.tar.gz - - ${TARGET}_aws-iot-example_${TOOLCHAIN}_${INFERENCE}_${AUDIO}_build.tar.gz - expire_in: 1 week -# The test-ota job should wait for build-applications job to finish as -# test-ota job uses the output build artifacts from build-applications job. -# The test-ota job should also wait for test-applications-output job to finish -# to prevent conflicts when doing both jobs at the same time as they both -# use the same credentials for the same target platform and application. -test-ota: +# The test-applications job should wait for build-applications job to finish as +# test-applications job uses the output build artifacts from build-applications job. +test-applications: extends: .test_job needs: - - job: test-applications-output + - job: build-applications artifacts: true script: - tar xf ${TARGET}_${APP}_${TOOLCHAIN}_${INFERENCE}_${AUDIO}_build.tar.gz - export APP_UNDERSCORED=$(echo ${APP} | tr '-' '_') - | if [[ $AUDIO == "VSI" ]]; then - pytest -s tools/tests/test_ota.py \ + pytest -s tools/tests/test_applications.py \ --audio-file-path "applications/${APP_UNDERSCORED}/resources/test.wav" \ --build-artefacts-path "build" \ --fvp $FVP_BIN \ @@ -191,19 +161,19 @@ test-ota: --merged-elf-name "${APP}_merged.elf" \ --signed-update-bin-name "${APP}-update_signed.bin" \ --timeout-seconds 1800 \ - --pass-ota-output-file "applications/${APP_UNDERSCORED}/tests/pass_ota_output.log" \ - --fail-ota-output-file "applications/${APP_UNDERSCORED}/tests/fail_ota_output.log" \ + --pass-output-file "applications/${APP_UNDERSCORED}/tests/pass_output.log" \ + --fail-output-file "applications/${APP_UNDERSCORED}/tests/fail_output.log" \ --pythonhome-path "/opt/python/3.9.18" else - pytest -s tools/tests/test_ota.py \ + pytest -s tools/tests/test_applications.py \ --build-artefacts-path "build" \ --fvp $FVP_BIN \ --credentials-path "applications/${APP_UNDERSCORED}/configs/aws_configs" \ --merged-elf-name "${APP}_merged.elf" \ --signed-update-bin-name "${APP}-update_signed.bin" \ --timeout-seconds 1800 \ - --pass-ota-output-file "applications/${APP_UNDERSCORED}/tests/pass_ota_output.log" \ - --fail-ota-output-file "applications/${APP_UNDERSCORED}/tests/fail_ota_output.log" + --pass-output-file "applications/${APP_UNDERSCORED}/tests/pass_output.log" \ + --fail-output-file "applications/${APP_UNDERSCORED}/tests/fail_output.log" fi parallel: matrix: diff --git a/applications/aws_iot_example/tests/fail_ota_output.log b/applications/aws_iot_example/tests/fail_output.log similarity index 100% rename from applications/aws_iot_example/tests/fail_ota_output.log rename to applications/aws_iot_example/tests/fail_output.log diff --git a/applications/aws_iot_example/tests/pass_ota_output.log b/applications/aws_iot_example/tests/pass_output.log similarity index 100% rename from applications/aws_iot_example/tests/pass_ota_output.log rename to applications/aws_iot_example/tests/pass_output.log diff --git a/applications/keyword_detection/tests/fail_ota_output.log b/applications/keyword_detection/tests/fail_ota_output.log deleted file mode 100644 index 323d0e49..00000000 --- a/applications/keyword_detection/tests/fail_ota_output.log +++ /dev/null @@ -1,3 +0,0 @@ -Failed to provision device private key -Failed job document content check -Failed to execute state transition handler diff --git a/applications/keyword_detection/tests/fail_output.log b/applications/keyword_detection/tests/fail_output.log index e69de29b..323d0e49 100644 --- a/applications/keyword_detection/tests/fail_output.log +++ b/applications/keyword_detection/tests/fail_output.log @@ -0,0 +1,3 @@ +Failed to provision device private key +Failed job document content check +Failed to execute state transition handler diff --git a/applications/keyword_detection/tests/pass_ota_output.log b/applications/keyword_detection/tests/pass_ota_output.log deleted file mode 100644 index 1dc29b4b..00000000 --- a/applications/keyword_detection/tests/pass_ota_output.log +++ /dev/null @@ -1,2 +0,0 @@ -Application version from appFirmwareVersion 0.0.10 -Application version from appFirmwareVersion 0.0.20 diff --git a/applications/keyword_detection/tests/pass_output.log b/applications/keyword_detection/tests/pass_output.log index 30c26aab..82c03a8f 100644 --- a/applications/keyword_detection/tests/pass_output.log +++ b/applications/keyword_detection/tests/pass_output.log @@ -1,8 +1,10 @@ -Starting bootloader Waiting for provisioning bundle Running provisioning bundle +Application version from appFirmwareVersion 0.0.10 +Starting bootloader Booting TF-M v2.0.0 PSA Framework version is: 257 +Application version from appFirmwareVersion 0.0.20 ML interface initialised ML_HEARD_ON ML UNKNOWN diff --git a/applications/speech_recognition/tests/fail_ota_output.log b/applications/speech_recognition/tests/fail_ota_output.log deleted file mode 100644 index 323d0e49..00000000 --- a/applications/speech_recognition/tests/fail_ota_output.log +++ /dev/null @@ -1,3 +0,0 @@ -Failed to provision device private key -Failed job document content check -Failed to execute state transition handler diff --git a/applications/speech_recognition/tests/fail_output.log b/applications/speech_recognition/tests/fail_output.log index e69de29b..323d0e49 100644 --- a/applications/speech_recognition/tests/fail_output.log +++ b/applications/speech_recognition/tests/fail_output.log @@ -0,0 +1,3 @@ +Failed to provision device private key +Failed job document content check +Failed to execute state transition handler diff --git a/applications/speech_recognition/tests/pass_ota_output.log b/applications/speech_recognition/tests/pass_ota_output.log deleted file mode 100644 index 1dc29b4b..00000000 --- a/applications/speech_recognition/tests/pass_ota_output.log +++ /dev/null @@ -1,2 +0,0 @@ -Application version from appFirmwareVersion 0.0.10 -Application version from appFirmwareVersion 0.0.20 diff --git a/applications/speech_recognition/tests/pass_output.log b/applications/speech_recognition/tests/pass_output.log index ca681680..5f70b568 100644 --- a/applications/speech_recognition/tests/pass_output.log +++ b/applications/speech_recognition/tests/pass_output.log @@ -1,8 +1,10 @@ -Starting bootloader Waiting for provisioning bundle Running provisioning bundle +Application version from appFirmwareVersion 0.0.10 +Starting bootloader Booting TF-M v2.0.0 PSA Framework version is: 257 +Application version from appFirmwareVersion 0.0.20 Init speex ML interface initialised Complete recognition: turn down the temperature in the bedroom diff --git a/tools/tests/conftest.py b/tools/tests/conftest.py index 41cbc891..e2e0bcc9 100644 --- a/tools/tests/conftest.py +++ b/tools/tests/conftest.py @@ -31,8 +31,6 @@ def pytest_addoption(parser): parser.addoption("--timeout-seconds", type=int, action="store", default=1800) parser.addoption("--pass-output-file", action="store", default="") parser.addoption("--fail-output-file", action="store", default="") - parser.addoption("--pass-ota-output-file", action="store", default="") - parser.addoption("--fail-ota-output-file", action="store", default="") parser.addoption( "--pythonhome-path", type=str, @@ -111,16 +109,6 @@ def fail_output_file(pytestconfig): yield get_absolute_path(pytestconfig.getoption("--fail-output-file")) -@pytest.fixture -def pass_ota_output_file(pytestconfig): - yield get_absolute_path(pytestconfig.getoption("--pass-ota-output-file")) - - -@pytest.fixture -def fail_ota_output_file(pytestconfig): - yield get_absolute_path(pytestconfig.getoption("--fail-ota-output-file")) - - @pytest.fixture def pythonhome_path(pytestconfig): yield pytestconfig.getoption("--pythonhome-path") diff --git a/tools/tests/test_ota.py b/tools/tests/test_applications.py similarity index 82% rename from tools/tests/test_ota.py rename to tools/tests/test_applications.py index c311c3b0..f8b2e725 100755 --- a/tools/tests/test_ota.py +++ b/tools/tests/test_applications.py @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -27,30 +27,30 @@ def setup_resources(build_artefacts_path, credentials_path, signed_update_bin_na cleanup_aws_resources(flags) -def test_ota( +def test_applications( setup_resources, fvp_process: subprocess.Popen, - pass_ota_output_file: str, - fail_ota_output_file: str, + pass_output_file: str, + fail_output_file: str, timeout_seconds: int, ) -> None: """ Compare the actual output on the FVP with the expectations in - pass and fail output files for the OTA update test. + pass and fail output files for the applications. setup_resources: Input coming out as a result of executing of setup_resources function defined above. fvp_process (subprocess.Popen): FVP execution process - pass_ota_output_file (str): Path to the file containing the output when application + pass_output_file (str): Path to the file containing the output when application runs without errors. - fail_ota_output_file (str): Path to the file containing the output when application + fail_output_file (str): Path to the file containing the output when application runs with errors. timeout_seconds (int): Timeout in seconds to wait before terminating the test. """ - with open(pass_ota_output_file, "r") as file: + with open(pass_output_file, "r") as file: pass_output = file.readlines() pass_output = [line.replace("\n", "") for line in pass_output] - with open(fail_ota_output_file, "r") as file: + with open(fail_output_file, "r") as file: fail_output = file.readlines() fail_output = [line.replace("\n", "") for line in fail_output] diff --git a/tools/tests/test_application_output.py b/tools/tests/test_blinky_output.py similarity index 88% rename from tools/tests/test_application_output.py rename to tools/tests/test_blinky_output.py index f5c7b331..32aa5657 100644 --- a/tools/tests/test_application_output.py +++ b/tools/tests/test_blinky_output.py @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -6,15 +6,15 @@ from timeit import default_timer as timer -def test_application_output( +def test_blinky_output( fvp_process: subprocess.Popen, pass_output_file: str, fail_output_file: str, timeout_seconds: int, ) -> None: """ - Compare the actual output on the FVP with the expectations in - pass and fail output files. + Compare the actual output of running blinky application on + the FVP with the expectations in pass and fail output files. fvp_process (subprocess.Popen): FVP execution process pass_output_file (str): Path to the file containing the output when application