From 80e220fc956bb2729ef90f819177b31c14b1e7e4 Mon Sep 17 00:00:00 2001 From: Gabor Abonyi Date: Wed, 17 Jan 2024 15:51:53 +0100 Subject: [PATCH] ml-eval-kit: Update to version 23.11 * Improved tensorflow.cmake * Model URLs file location has changed * Some patches have been merged upstream Signed-off-by: Gabor Abonyi --- .../ml_embedded_evaluation_kit/CMakeLists.txt | 5 +- ...etupMlEmbeddedEvaluationKitLibraries.cmake | 8 +- ...e-model-wav2letter-with-tiny_wav2let.patch | 147 +++++++----------- ..._header-instead-of-RTE_Components.h.patch} | 4 +- ...m-GNU-Toolchain-13.2-Rel1-compatibil.patch | 69 -------- ...-Fix-tensorflow.cmake-env-var-escape.patch | 38 ----- .../ai/ml_embedded_evaluation_kit/library | 2 +- manifest.yml | 2 +- release_changes/202401191520.change | 1 + 9 files changed, 65 insertions(+), 211 deletions(-) rename components/ai/ml_embedded_evaluation_kit/integration/patches/{0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch => 0002-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch} (92%) delete mode 100644 components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch delete mode 100644 components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch create mode 100644 release_changes/202401191520.change diff --git a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt index 299d37df..9859f360 100644 --- a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt +++ b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt @@ -13,13 +13,12 @@ include(ApplyPatches) set(PATCH_FILES_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/integration/patches") set(PATCH_FILES "${PATCH_FILES_DIRECTORY}/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch" - "${PATCH_FILES_DIRECTORY}/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch" - "${PATCH_FILES_DIRECTORY}/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch" - "${PATCH_FILES_DIRECTORY}/0004-Fix-tensorflow.cmake-env-var-escape.patch" + "${PATCH_FILES_DIRECTORY}/0002-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch" ) iot_reference_arm_corstone3xx_apply_patches("${ml_embedded_evaluation_kit_SOURCE_DIR}" "${PATCH_FILES}") set(TF_PATCH_FILES "${PATCH_FILES_DIRECTORY}/0001-tensorflow-kiss_fft_wrapper-include-fix.patch" ) + iot_reference_arm_corstone3xx_apply_patches("${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow" "${TF_PATCH_FILES}") diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake index 9e6ae649..687fb35a 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake @@ -59,14 +59,10 @@ function(assert_defined var_name) endif() endfunction() -# Note: tensorflow.cmake modifies these variables. Save the current values and restore after. -set(TEMP_CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) +# Preserve the content of `ETHOS_U_NPU_ID` before including +# the `tensorflow.cmake` CMake module which overrides it. set(TEMP_ETHOS_U_NPU_ID ${ETHOS_U_NPU_ID}) -# Note: tensorflow.cmake only has support up to cortex-m55. -set(CMAKE_SYSTEM_PROCESSOR "cortex-m55") include(${CMAKE_SCRIPTS_DIR}/tensorflow.cmake) -# Note: tensorflow.cmake variable restoration. -set(CMAKE_SYSTEM_PROCESSOR ${TEMP_CMAKE_SYSTEM_PROCESSOR}) set(ETHOS_U_NPU_ID ${TEMP_ETHOS_U_NPU_ID}) # Manually add libs diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch index 7686195b..9657712b 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch @@ -1,101 +1,66 @@ -From e691803e130d1aced38c21e98c41ce40bda44ffe Mon Sep 17 00:00:00 2001 -From: Hugues Kamba-Mpiana -Date: Tue, 19 Dec 2023 14:09:27 +0000 -Subject: [PATCH 1/4] resources: Replace model wav2letter with tiny_wav2letter +From 202176ddd60d709372bd4dd05fd5d6490fc2559e Mon Sep 17 00:00:00 2001 +From: Gabor Abonyi +Date: Mon, 22 Jan 2024 14:55:03 +0100 +Subject: [PATCH 1/2] resources: Replace model wav2letter with tiny_wav2letter `wav2letter` model needs singificant amount of memory (around 14Mb), where as `tiny_wav2letter` needs around 4Mb. Update the location of the resource where referenced for -the `asr` and `kws` use cases. +the `asr` and `kws_asr` use cases. -Signed-off-by: Hugues Kamba-Mpiana +Signed-off-by: Gabor Abonyi --- - set_up_default_resources.py | 16 ++++++++-------- - source/use_case/asr/usecase.cmake | 4 ++-- - source/use_case/kws_asr/usecase.cmake | 4 ++-- - 3 files changed, 12 insertions(+), 12 deletions(-) + scripts/py/use_case_resources.json | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) -diff --git a/set_up_default_resources.py b/set_up_default_resources.py -index b442432..2cb987d 100755 ---- a/set_up_default_resources.py -+++ b/set_up_default_resources.py -@@ -51,16 +51,16 @@ json_uc_res = [ - { - "use_case_name": "asr", - "url_prefix": [ -- "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/" -+ "https://github.com/ARM-software/ML-zoo/raw/eb2170aac1317f00b128ab82bdb159cfcca36ea6/models/speech_recognition/tiny_wav2letter/tflite_pruned_int8/" - ], - "resources": [ - { -- "name": "wav2letter_pruned_int8.tflite", -- "url": "{url_prefix:0}wav2letter_pruned_int8.tflite", -+ "name": "tiny_wav2letter_pruned_int8.tflite", -+ "url": "{url_prefix:0}tiny_wav2letter_pruned_int8.tflite", - }, - { - "name": "ifm0.npy", -- "url": "{url_prefix:0}testing_input/input_2_int8/0.npy", -+ "url": "{url_prefix:0}testing_input/input_1_int8/0.npy", - }, - { - "name": "ofm0.npy", -@@ -131,18 +131,18 @@ json_uc_res = [ - { - "use_case_name": "kws_asr", - "url_prefix": [ -- "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/", -+ "https://github.com/ARM-software/ML-zoo/raw/eb2170aac1317f00b128ab82bdb159cfcca36ea6/models/speech_recognition/tiny_wav2letter/tflite_pruned_int8/", - "https://github.com/ARM-software/ML-zoo/raw/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8/", - ], - "resources": [ - { -- "name": "wav2letter_pruned_int8.tflite", -- "url": "{url_prefix:0}wav2letter_pruned_int8.tflite", -+ "name": "tiny_wav2letter_pruned_int8.tflite", -+ "url": "{url_prefix:0}tiny_wav2letter_pruned_int8.tflite", - }, - { - "sub_folder": "asr", - "name": "ifm0.npy", -- "url": "{url_prefix:0}testing_input/input_2_int8/0.npy", -+ "url": "{url_prefix:0}testing_input/input_1_int8/0.npy", - }, - { - "sub_folder": "asr", -diff --git a/source/use_case/asr/usecase.cmake b/source/use_case/asr/usecase.cmake -index fc3f2b5..3ce2c0a 100644 ---- a/source/use_case/asr/usecase.cmake -+++ b/source/use_case/asr/usecase.cmake -@@ -75,9 +75,9 @@ USER_OPTION(${use_case}_ACTIVATION_BUF_SZ "Activation buffer size for the chosen - STRING) - - if (ETHOS_U_NPU_ENABLED) -- set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/wav2letter_pruned_int8_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) -+ set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) - else() -- set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/wav2letter_pruned_int8.tflite) -+ set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8.tflite) - endif() - - set(EXTRA_MODEL_CODE -diff --git a/source/use_case/kws_asr/usecase.cmake b/source/use_case/kws_asr/usecase.cmake -index de84bfd..9bfbbd0 100644 ---- a/source/use_case/kws_asr/usecase.cmake -+++ b/source/use_case/kws_asr/usecase.cmake -@@ -68,10 +68,10 @@ USER_OPTION(${use_case}_MODEL_SCORE_THRESHOLD_ASR "Specify the score threshold [ - - if (ETHOS_U_NPU_ENABLED) - set(DEFAULT_MODEL_PATH_KWS ${DEFAULT_MODEL_DIR}/kws_micronet_m_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) -- set(DEFAULT_MODEL_PATH_ASR ${DEFAULT_MODEL_DIR}/wav2letter_pruned_int8_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) -+ set(DEFAULT_MODEL_PATH_ASR ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) - else() - set(DEFAULT_MODEL_PATH_KWS ${DEFAULT_MODEL_DIR}/kws_micronet_m.tflite) -- set(DEFAULT_MODEL_PATH_ASR ${DEFAULT_MODEL_DIR}/wav2letter_pruned_int8.tflite) -+ set(DEFAULT_MODEL_PATH_ASR ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8.tflite) - endif() - - USER_OPTION(${use_case}_MODEL_TFLITE_PATH_KWS "NN models file to be used for KWS in the evaluation application. Model files must be in tflite format." +diff --git a/scripts/py/use_case_resources.json b/scripts/py/use_case_resources.json +index 80fa28d..9e50fb6 100644 +--- a/scripts/py/use_case_resources.json ++++ b/scripts/py/use_case_resources.json +@@ -16,16 +16,16 @@ + { + "name": "asr", + "url_prefix": [ +- "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/" ++ "https://github.com/ARM-software/ML-zoo/raw/eb2170aac1317f00b128ab82bdb159cfcca36ea6/models/speech_recognition/tiny_wav2letter/tflite_pruned_int8/" + ], + "resources": [ + { +- "name": "wav2letter_pruned_int8.tflite", +- "url": "{url_prefix:0}wav2letter_pruned_int8.tflite" ++ "name": "tiny_wav2letter_pruned_int8.tflite", ++ "url": "{url_prefix:0}tiny_wav2letter_pruned_int8.tflite" + }, + { + "name": "ifm0.npy", +- "url": "{url_prefix:0}testing_input/input_2_int8/0.npy" ++ "url": "{url_prefix:0}testing_input/input_1_int8/0.npy" + }, + { + "name": "ofm0.npy", +@@ -96,18 +96,18 @@ + { + "name": "kws_asr", + "url_prefix": [ +- "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/", ++ "https://github.com/ARM-software/ML-zoo/raw/eb2170aac1317f00b128ab82bdb159cfcca36ea6/models/speech_recognition/tiny_wav2letter/tflite_pruned_int8/", + "https://github.com/ARM-software/ML-zoo/raw/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8/" + ], + "resources": [ + { +- "name": "wav2letter_pruned_int8.tflite", +- "url": "{url_prefix:0}wav2letter_pruned_int8.tflite" ++ "name": "tiny_wav2letter_pruned_int8.tflite", ++ "url": "{url_prefix:0}tiny_wav2letter_pruned_int8.tflite" + }, + { + "sub_folder": "asr", + "name": "ifm0.npy", +- "url": "{url_prefix:0}testing_input/input_2_int8/0.npy" ++ "url": "{url_prefix:0}testing_input/input_1_int8/0.npy" + }, + { + "sub_folder": "asr", -- 2.40.1 diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch similarity index 92% rename from components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch rename to components/ai/ml_embedded_evaluation_kit/integration/patches/0002-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch index 0c9b1343..8cb9b56e 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch @@ -1,7 +1,7 @@ -From 75e29cf7fd59978d06d8594efd2c883c40cd56e1 Mon Sep 17 00:00:00 2001 +From ba95da37dcc0b41a14e7c8bc1ea707e0b1d99a79 Mon Sep 17 00:00:00 2001 From: Gabor Abonyi Date: Tue, 16 Jan 2024 14:48:44 +0100 -Subject: [PATCH 3/4] Use CMSIS_device_header instead of RTE_Components.h +Subject: [PATCH 2/2] Use CMSIS_device_header instead of RTE_Components.h Signed-off-by: Gabor Abonyi --- diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch deleted file mode 100644 index 38a92636..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 60556c32a8957dd52b5dc7b85a8cef6550f5ac28 Mon Sep 17 00:00:00 2001 -From: Hugues Kamba-Mpiana -Date: Tue, 19 Dec 2023 13:17:15 +0000 -Subject: [PATCH 2/4] toolchain: Fix Arm GNU Toolchain 13.2 Rel1 compatibility - -Arm GNU Toolchain 13.2 Rel1 no longer includes transitively. -Include `cstdint` explicitely where needed. - -Signed-off-by: Hugues Kamba-Mpiana ---- - source/application/api/common/include/ClassificationResult.hpp | 1 + - source/profiler/include/Profiler.hpp | 1 + - source/use_case/asr/include/UseCaseHandler.hpp | 2 ++ - source/use_case/kws/include/UseCaseHandler.hpp | 2 ++ - 4 files changed, 6 insertions(+) - -diff --git a/source/application/api/common/include/ClassificationResult.hpp b/source/application/api/common/include/ClassificationResult.hpp -index 244849f..b3b3f76 100644 ---- a/source/application/api/common/include/ClassificationResult.hpp -+++ b/source/application/api/common/include/ClassificationResult.hpp -@@ -18,6 +18,7 @@ - #define CLASSIFICATION_RESULT_HPP - - #include -+#include - - namespace arm { - namespace app { -diff --git a/source/profiler/include/Profiler.hpp b/source/profiler/include/Profiler.hpp -index 8ae9e15..07e0de4 100644 ---- a/source/profiler/include/Profiler.hpp -+++ b/source/profiler/include/Profiler.hpp -@@ -19,6 +19,7 @@ - - #include "hal.h" - -+#include - #include - #include - #include -diff --git a/source/use_case/asr/include/UseCaseHandler.hpp b/source/use_case/asr/include/UseCaseHandler.hpp -index 60e27fc..04e6199 100644 ---- a/source/use_case/asr/include/UseCaseHandler.hpp -+++ b/source/use_case/asr/include/UseCaseHandler.hpp -@@ -19,6 +19,8 @@ - - #include "AppContext.hpp" - -+#include -+ - namespace arm { - namespace app { - -diff --git a/source/use_case/kws/include/UseCaseHandler.hpp b/source/use_case/kws/include/UseCaseHandler.hpp -index 279acf0..eb28a8f 100644 ---- a/source/use_case/kws/include/UseCaseHandler.hpp -+++ b/source/use_case/kws/include/UseCaseHandler.hpp -@@ -19,6 +19,8 @@ - - #include "AppContext.hpp" - -+#include -+ - namespace arm { - namespace app { - --- -2.40.1 - diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch deleted file mode 100644 index 53c0a78c..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 2d22af627e51f62e95968cc75f51ff1238cf86ec Mon Sep 17 00:00:00 2001 -From: Gabor Abonyi -Date: Thu, 11 Jan 2024 14:16:57 +0100 -Subject: [PATCH 4/4] Fix tensorflow.cmake env var escape - -Signed-off-by: Gabor Abonyi ---- - scripts/cmake/tensorflow.cmake | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/scripts/cmake/tensorflow.cmake b/scripts/cmake/tensorflow.cmake -index 8d52765..a028055 100644 ---- a/scripts/cmake/tensorflow.cmake -+++ b/scripts/cmake/tensorflow.cmake -@@ -78,9 +78,9 @@ endif() - - if (TENSORFLOW_LITE_MICRO_CLEAN_DOWNLOADS) - message(STATUS "Refreshing TensorFlow Lite Micro's third party downloads...") -+ set(ENV{PATH} "${ENV_PATH}") - execute_process( -- COMMAND ${CMAKE_COMMAND} -E env PATH=${ENV_PATH} -- make -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile clean_downloads third_party_downloads -+ COMMAND make -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile clean_downloads third_party_downloads - RESULT_VARIABLE return_code - WORKING_DIRECTORY ${TENSORFLOW_SRC_PATH}) - if (NOT return_code EQUAL "0") -@@ -102,7 +102,7 @@ message(STATUS "TensorFlow Lite Micro build to be called for these targets: ${MA - add_custom_target(tensorflow_build ALL - - # Command to build the TensorFlow Lite Micro library -- COMMAND ${CMAKE_COMMAND} -E env PATH=${ENV_PATH} -+ COMMAND ${CMAKE_COMMAND} -E env PATH="${ENV_PATH}" - make -j${PARALLEL_JOBS} -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile ${MAKE_TARGETS_LIST} - TARGET_TOOLCHAIN_ROOT=${TENSORFLOW_LITE_MICRO_TARGET_TOOLCHAIN_ROOT} - TOOLCHAIN=${TENSORFLOW_LITE_MICRO_TOOLCHAIN} --- -2.40.1 - diff --git a/components/ai/ml_embedded_evaluation_kit/library b/components/ai/ml_embedded_evaluation_kit/library index f1b28b86..c56eeda9 160000 --- a/components/ai/ml_embedded_evaluation_kit/library +++ b/components/ai/ml_embedded_evaluation_kit/library @@ -1 +1 @@ -Subproject commit f1b28b861e301122b85ad7dc3d8ccb0720fcb584 +Subproject commit c56eeda97fe3a52dcaf76469d53460c340b3ae3f diff --git a/manifest.yml b/manifest.yml index 81b47553..9d4adb86 100644 --- a/manifest.yml +++ b/manifest.yml @@ -138,7 +138,7 @@ dependencies: path: "components/connectivity/freertos_plus_tcp/library" - name: "ml-embedded-evaluation-kit" license: "Apache-2.0" - version: "f1b28b861e301122b85ad7dc3d8ccb0720fcb584" + version: "23.11" repository: type: "git" url: "https://review.mlplatform.org/ml/ethos-u/ml-embedded-evaluation-kit.git" diff --git a/release_changes/202401191520.change b/release_changes/202401191520.change new file mode 100644 index 00000000..678956d6 --- /dev/null +++ b/release_changes/202401191520.change @@ -0,0 +1 @@ +ml-eval-kit: Update to version 23.11