From c4b2e8392e29cf4b09fd80dc3dcf2804afe35ac7 Mon Sep 17 00:00:00 2001 From: Gabor Abonyi Date: Wed, 17 Jan 2024 14:40:17 +0100 Subject: [PATCH] ml-eval-kit: Build individual libraries Instead of building the whole project as an external project, only build the necessary libraries. Rename BuildMLEmbeddedEvaluationKit.cmake to SetupMlEmbeddedEvaluationKitLibraries.cmake This change also uses the NPU driver component from ML Evaluation Kit which results in the following changes: * Modify initialization of NPU in AI applications to use API provided by the component * Add ICache and DCache enablement options as caches are used in the NPU component * Include various patches for the NPU component. * Remove ML Evaluation Kit device drivers header files inclusions as they are no longer used as we are now using the NPU component for initializing the NPU. Signed-off-by: Gabor Abonyi --- .github/.cSpellWords.txt | 1 + applications/keyword_detection/CMakeLists.txt | 37 ++-- .../keyword_detection/ml_interface.cc | 123 ++----------- applications/libraries/ml-kit/CMakeLists.txt | 174 ------------------ .../speech_recognition/CMakeLists.txt | 37 ++-- .../speech_recognition/ml_interface.cc | 126 ++----------- bsp/CMakeLists.txt | 10 +- .../ml_embedded_evaluation_kit/CMakeLists.txt | 13 +- .../integration/README.md | 62 ++----- .../cmake/BuildMlEmbeddedEvaluationKit.cmake | 50 ----- .../cmake/ConvertAudioSourceToCode.cmake | 4 +- ...etupMlEmbeddedEvaluationKitLibraries.cmake | 136 ++++++++++++++ .../cmake/model/GenerateASRModel.cmake | 54 ++++++ .../cmake/model/GenerateKWSModel.cmake | 53 ++++++ ...tax-Replace-incorrect-type-specifier.patch | 29 --- ...rs-Remove-RTE_components-header-file.patch | 31 ---- ...e-model-wav2letter-with-tiny_wav2let.patch | 12 +- ...sorflow-kiss_fft_wrapper-include-fix.patch | 25 +++ ...-GNU-Toolchain-13.2-Rel1-compatibil.patch} | 6 +- ...e_header-instead-of-RTE_Components.h.patch | 40 ++++ ...-Fix-tensorflow.cmake-env-var-escape.patch | 38 ++++ release_changes/202401161458.change | 3 + 22 files changed, 458 insertions(+), 606 deletions(-) delete mode 100644 applications/libraries/ml-kit/CMakeLists.txt delete mode 100644 components/ai/ml_embedded_evaluation_kit/integration/cmake/BuildMlEmbeddedEvaluationKit.cmake create mode 100644 components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake create mode 100644 components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateASRModel.cmake create mode 100644 components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateKWSModel.cmake delete mode 100644 components/ai/ml_embedded_evaluation_kit/integration/patches/0001-asm-syntax-Replace-incorrect-type-specifier.patch delete mode 100644 components/ai/ml_embedded_evaluation_kit/integration/patches/0001-platform-drivers-Remove-RTE_components-header-file.patch create mode 100644 components/ai/ml_embedded_evaluation_kit/integration/patches/0001-tensorflow-kiss_fft_wrapper-include-fix.patch rename components/ai/ml_embedded_evaluation_kit/integration/patches/{0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch => 0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch} (93%) create mode 100644 components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch create mode 100644 components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch create mode 100644 release_changes/202401161458.change diff --git a/.github/.cSpellWords.txt b/.github/.cSpellWords.txt index 1fc612be..c63d9221 100644 --- a/.github/.cSpellWords.txt +++ b/.github/.cSpellWords.txt @@ -62,6 +62,7 @@ IDAQAB indet inkey iotdeviceadvisor +IRQN ISRAM istty JITP diff --git a/applications/keyword_detection/CMakeLists.txt b/applications/keyword_detection/CMakeLists.txt index 6914dd9f..391fe028 100644 --- a/applications/keyword_detection/CMakeLists.txt +++ b/applications/keyword_detection/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -31,35 +31,31 @@ set(MCUBOOT_IMAGE_VERSION_NS_UPDATE "0.0.1+20") # Hardware platform specific setup if (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone310") - set(ML_TARGET_SUBSYSTEM "sse-310") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H256") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-256") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone310/fvp") elseif (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - set(ML_TARGET_SUBSYSTEM "sse-300") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H128") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-128") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone300/fvp") else() message(FATAL_ERROR "Invalid ARM_CORSTONE_BSP_TARGET_PLATFORM (${ARM_CORSTONE_BSP_TARGET_PLATFORM}) set. Supported are corstone300/corstone310") endif() -set(TFM_FLASH_S_PARTITION_SIZE "0x40000") -set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") - -# Machine Learning setup -set(ML_CMAKE_ARGS - -DTARGET_SUBSYSTEM=${ML_TARGET_SUBSYSTEM} - -DETHOS_U_NPU_CONFIG_ID=${ETHOS_U_NPU_CONFIG_ID} - -DETHOSU_TARGET_NPU_CONFIG=${ETHOSU_TARGET_NPU_CONFIG} - -DUSE_CASE_BUILD=kws -) if (${ML_INFERENCE_ENGINE} STREQUAL "ETHOS") - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=ON) + set(ETHOS_U_NPU_ENABLED ON) + set(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED OFF) else() - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=OFF) + set(ETHOS_U_NPU_ENABLED OFF) endif() -set(ML_TARGETS cmsis-dsp tensorflow_build kws) +set(ML_USE_CASE "kws") +set(ML_MODEL "GenerateKWSModel") +set(TFM_FLASH_S_PARTITION_SIZE "0x40000") +set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") set(TFM_PLATFORM_UPGRADE_STRATEGY "SWAP_USING_SCRATCH") set(TFM_PLATFORM_CONFIRM_IMAGE ON) @@ -114,13 +110,12 @@ add_subdirectory(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR} ${CMAKE_BINARY_DIR} list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/aws_iot/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/ai/ml_embedded_evaluation_kit/integration/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/security/trusted_firmware-m/integration/cmake) -include(BuildMlEmbeddedEvaluationKit) +include(SetupMlEmbeddedEvaluationKitLibraries) include(ConvertAudioSourceToCode) include(GenerateAWSUpdateDigestAndSignature) include(MergeTfmImages) include(SignTfmImage) -add_subdirectory(../libraries/ml-kit ${CMAKE_BINARY_DIR}/libraries/ml-kit) add_subdirectory(configs ${CMAKE_BINARY_DIR}/Config) add_subdirectory(../helpers helpers) @@ -177,9 +172,6 @@ add_dependencies(keyword-detection trusted_firmware-m-build) # The provision data must be built before the application because # it provides credentials to connect to AWS. add_dependencies(keyword-detection provisioning_data_bin) -# The ML Embedded Evaluation Kit must be built before the application because -# it provides the Machine Learning algorithm to detect keywords. -add_dependencies(keyword-detection ml_embedded_evaluation_kit-build) target_link_libraries(keyword-detection PRIVATE @@ -194,11 +186,12 @@ target_link_libraries(keyword-detection helpers-events mbedtls mbedtls-threading-freertos - ml-kit-kws ota-for-aws-iot-embedded-sdk provisioning-lib tfm-ns-interface toolchain-override + kws_api + kws_model ) include(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/bsp/cmake/SetLinkerOptions.cmake) diff --git a/applications/keyword_detection/ml_interface.cc b/applications/keyword_detection/ml_interface.cc index ca31df28..15663c15 100644 --- a/applications/keyword_detection/ml_interface.cc +++ b/applications/keyword_detection/ml_interface.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2023 Arm Limited and/or its affiliates +/* Copyright 2021-2024 Arm Limited and/or its affiliates * * SPDX-License-Identifier: MIT */ @@ -10,10 +10,15 @@ #include "ml_interface.h" #include "AudioUtils.hpp" +#include "AppContext.hpp" #include "BufAttributes.hpp" #include "demo_config.h" extern "C" { #include "events.h" +#ifdef USE_ETHOS +#include "ethosu_driver.h" +#include "ethosu_npu_init.h" +#endif } #include "KwsClassifier.hpp" #include "KwsProcessing.hpp" @@ -24,13 +29,7 @@ extern "C" { #include "MicroNetKwsModel.hpp" #include "mqtt_agent_task.h" #include "TensorFlowLiteMicro.hpp" -#include "UseCaseCommonUtils.hpp" #include CMSIS_device_header -#include "device_mps3.h" /* FPGA level definitions and functions. */ -#include "ethos-u55.h" /* Mem map and configuration definitions of the Ethos U55 */ -#include "ethosu_driver.h" /* Arm Ethos-U55 driver header */ -#include "timer_mps3.h" /* Timer functions. */ -#include "timing_adapter.h" /* Driver header of the timing adapter */ #include #include @@ -890,111 +889,25 @@ extern struct ethosu_driver ethosu_drv; /* Default Ethos-U55 device driver */ **/ static int prvArmNpuInit(void); -/** - * @brief Defines the Ethos-U interrupt handler: just a wrapper around the default - * implementation. - **/ -extern "C" { -void ETHOS_U55_Handler(void) -{ - /* Call the default interrupt handler from the NPU driver */ - ethosu_irq_handler(ðosu_drv); -} -} - -/** - * @brief Initialises the NPU IRQ - **/ -static void prvArmNpuIrqInit(void) -{ - const IRQn_Type ethosu_irqnum = (IRQn_Type)EthosU_IRQn; - - /* Enable the IRQ */ - NVIC_EnableIRQ(ethosu_irqnum); - - LogDebug( ( "EthosU IRQ#: %u, Handler: 0x%p\n", ethosu_irqnum, ETHOS_U55_Handler ) ); -} - -static int prvArmNpuTimingAdapterInit(void) -{ -#if defined(TA0_BASE) - struct timing_adapter ta_0; - struct timing_adapter_settings ta_0_settings = {.maxr = TA0_MAXR, - .maxw = TA0_MAXW, - .maxrw = TA0_MAXRW, - .rlatency = TA0_RLATENCY, - .wlatency = TA0_WLATENCY, - .pulse_on = TA0_PULSE_ON, - .pulse_off = TA0_PULSE_OFF, - .bwcap = TA0_BWCAP, - .perfctrl = TA0_PERFCTRL, - .perfcnt = TA0_PERFCNT, - .mode = TA0_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA0_HISTBIN, - .histcnt = TA0_HISTCNT}; - - if (0 != ta_init(&ta_0, TA0_BASE)) { - LogError( ( "TA0 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_0, &ta_0_settings); -#endif /* defined (TA0_BASE) */ - -#if defined(TA1_BASE) - struct timing_adapter ta_1; - struct timing_adapter_settings ta_1_settings = {.maxr = TA1_MAXR, - .maxw = TA1_MAXW, - .maxrw = TA1_MAXRW, - .rlatency = TA1_RLATENCY, - .wlatency = TA1_WLATENCY, - .pulse_on = TA1_PULSE_ON, - .pulse_off = TA1_PULSE_OFF, - .bwcap = TA1_BWCAP, - .perfctrl = TA1_PERFCTRL, - .perfcnt = TA1_PERFCNT, - .mode = TA1_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA1_HISTBIN, - .histcnt = TA1_HISTCNT}; - - if (0 != ta_init(&ta_1, TA1_BASE)) { - LogError( ( "TA1 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_1, &ta_1_settings); -#endif /* defined (TA1_BASE) */ - - return 0; -} - static int prvArmNpuInit(void) { int err = 0; - /* If the platform has timing adapter blocks along with Ethos-U55 core - * block, initialise them here. */ - // cppcheck-suppress knownConditionTrueFalse - if (0 != (err = prvArmNpuTimingAdapterInit())) { + SCB_EnableICache(); + SCB_EnableDCache(); + +#if defined(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED) + /* If the platform has timing adapter blocks along with Ethos-U core + * block, initialise them here. */ + if (0 != (err = arm_ethosu_timing_adapter_init())) { + LogError( ("Failed to init timing adapter\n") ); return err; } +#endif /* ETHOS_U_NPU_TIMING_ADAPTER_ENABLED */ - /* Initialise the IRQ */ - prvArmNpuIrqInit(); - - /* Initialise Ethos-U55 device */ - void *const ethosu_base_address = reinterpret_cast(SEC_ETHOS_U55_BASE); - - if (0 - != (err = ethosu_init(ðosu_drv, /* Ethos-U55 driver device pointer */ - ethosu_base_address, /* Ethos-U55's base address. */ - NULL, /* Pointer to fast mem area - NULL for U55. */ - 0, /* Fast mem region size. */ - 0, /* Security enable. */ - 0))) { /* Privilege enable. */ - LogError( ( "failed to initalise Ethos-U55 device\n" ) ); + // Initialize the ethos NPU + if (0 != (err = arm_ethosu_npu_init())) { + LogError( ("Failed to init arm npu\n") ); return err; } diff --git a/applications/libraries/ml-kit/CMakeLists.txt b/applications/libraries/ml-kit/CMakeLists.txt deleted file mode 100644 index aa076bcd..00000000 --- a/applications/libraries/ml-kit/CMakeLists.txt +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2022-2023 Arm Limited and/or its affiliates -# -# SPDX-License-Identifier: MIT - -include(ExternalProject) - -ExternalProject_Get_Property(ml_embedded_evaluation_kit-build BINARY_DIR) - -# ML-eval Kit -add_library(ml-kit INTERFACE) - -# Set byproducts from ML build, so that CMake treats them as generated files -# that don't need to exist before the build. - -set(ml_kit_libs - ${BINARY_DIR}/lib/libcommon_api.a - ${BINARY_DIR}/lib/libtensorflow-microlite.a - ${BINARY_DIR}/lib/libarm_math.a - ${BINARY_DIR}/lib/libcmsis-dsp.a -) - -if(${ML_INFERENCE_ENGINE} STREQUAL "ETHOS") - list(APPEND ml_kit_libs - ${BINARY_DIR}/lib/libethos_u_npu.a - ${BINARY_DIR}/lib/libethosu_core_driver.a - ) - - if(${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - list(APPEND ml_kit_libs ${BINARY_DIR}/lib/libtiming_adapter.a) - endif() -endif() - -set(ml_kit_kws_libs - ${BINARY_DIR}/lib/libkws_api.a - ${BINARY_DIR}/lib/libkws.a -) -set(ml_kit_asr_libs - ${BINARY_DIR}/lib/libasr_api.a - ${BINARY_DIR}/lib/libasr.a -) - -add_custom_target(get_ml_kit_libs - DEPENDS - ml_embedded_evaluation_kit-build - COMMAND - # Dummy command that does nothing - ${CMAKE_COMMAND} -E true - BYPRODUCTS - ${ml_kit_libs} ${ml_kit_kws_libs} ${ml_kit_asr_libs} -) - - -target_include_directories(ml-kit - INTERFACE - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis-dsp/Include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-driver/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow/tensorflow/lite/micro - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow/tensorflow/lite/micro/tools/make/downloads/flatbuffers/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow/tensorflow/lite/micro/tools/make/downloads/gemmlowp/ - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/hal/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/main/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/common/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/platform/mps3/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/log/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/platform_pmu/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/npu/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/stdout/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/lcd/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/profiler/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/platform/mps3/source - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-platform/drivers/timing_adapter/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/math/include -) - -if(${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - target_include_directories(ml-kit - INTERFACE - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-platform/drivers/timing_adapter/include - ) -endif() - - -target_link_libraries(ml-kit - INTERFACE - ${ml_kit_libs} -) - -target_compile_definitions(ml-kit - INTERFACE - -DACTIVATION_BUF_SZ=0x00200000 - -DARM_MATH_DSP - -DARM_MATH_LOOPUNROLL - -DARM_MATH_MVEI - -DARM_NPU=1 - -DTF_LITE_STATIC_MEMORY - -DETHOS_U_NPU_MEMORY_MODE=ETHOS_U_NPU_MEMORY_MODE_SHARED_SRAM -) - -add_dependencies(ml-kit - ml_embedded_evaluation_kit-build -) - - -# Add kws library -add_library(ml-kit-kws INTERFACE) - -target_link_libraries(ml-kit-kws - INTERFACE - ${ml_kit_kws_libs} -) - -target_link_libraries(ml-kit-kws - INTERFACE - ml-kit -) - -target_include_directories(ml-kit-kws - INTERFACE - ${BINARY_DIR}/generated/kws/include/ - ${ml_embedded_evaluation_kit_SOURCE_DIR}/generated/kws/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/use_case/kws/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/use_case/kws/include -) - -# Remove retarget from kws library -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - add_custom_target(ml-kit-remove-kws-retarget - COMMAND arm-none-eabi-ar dv ${BINARY_DIR}/lib/libkws.a retarget.c.obj - ) -elseif(CMAKE_C_COMPILER_ID STREQUAL "ARMClang") - add_custom_target(ml-kit-remove-kws-retarget - COMMAND armar -d ${BINARY_DIR}/lib/libkws.a retarget.o - ) -endif() - -add_dependencies(ml-kit-remove-kws-retarget ml-kit) -add_dependencies(ml-kit-kws ml-kit-remove-kws-retarget) - - -# Add asr library -add_library(ml-kit-asr INTERFACE) - -target_link_libraries(ml-kit-asr - INTERFACE - ${ml_kit_asr_libs} -) - -target_link_libraries(ml-kit-asr - INTERFACE - ml-kit -) - -target_include_directories(ml-kit-asr - INTERFACE - ${BINARY_DIR}/generated/asr/include/ - ${ml_embedded_evaluation_kit_SOURCE_DIR}/generated/asr/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/use_case/asr/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/use_case/asr/include -) - -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - add_custom_target(ml-kit-remove-asr-retarget - COMMAND arm-none-eabi-ar dv ${BINARY_DIR}/lib/libasr.a retarget.c.obj - ) -elseif(CMAKE_C_COMPILER_ID STREQUAL "ARMClang") - add_custom_target(ml-kit-remove-asr-retarget - COMMAND armar -d ${BINARY_DIR}/lib/libasr.a retarget.o - ) -endif() - -add_dependencies(ml-kit-remove-asr-retarget ml-kit) -add_dependencies(ml-kit-asr ml-kit-remove-asr-retarget) diff --git a/applications/speech_recognition/CMakeLists.txt b/applications/speech_recognition/CMakeLists.txt index 866e1e5c..d27bf09d 100644 --- a/applications/speech_recognition/CMakeLists.txt +++ b/applications/speech_recognition/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -31,35 +31,31 @@ set(MCUBOOT_IMAGE_VERSION_NS_UPDATE "0.0.1+20") # Hardware platform specific setup if (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone310") - set(ML_TARGET_SUBSYSTEM "sse-310") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H256") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-256") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone310/fvp") elseif (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - set(ML_TARGET_SUBSYSTEM "sse-300") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H128") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-128") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone300/fvp") else() message(FATAL_ERROR "Invalid ARM_CORSTONE_BSP_TARGET_PLATFORM (${ARM_CORSTONE_BSP_TARGET_PLATFORM}) set. Supported are corstone300/corstone310") endif() -set(TFM_FLASH_S_PARTITION_SIZE "0x40000") -set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") - -# Machine Learning setup -set(ML_CMAKE_ARGS - -DTARGET_SUBSYSTEM=${ML_TARGET_SUBSYSTEM} - -DETHOS_U_NPU_CONFIG_ID=${ETHOS_U_NPU_CONFIG_ID} - -DETHOSU_TARGET_NPU_CONFIG=${ETHOSU_TARGET_NPU_CONFIG} - -DUSE_CASE_BUILD=asr -) if (${ML_INFERENCE_ENGINE} STREQUAL "ETHOS") - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=ON) + set(ETHOS_U_NPU_ENABLED ON) + set(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED OFF) else() - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=OFF) + set(ETHOS_U_NPU_ENABLED OFF) endif() -set(ML_TARGETS cmsis-dsp tensorflow_build asr) +set(ML_USE_CASE "asr") +set(ML_MODEL "GenerateASRModel") +set(TFM_FLASH_S_PARTITION_SIZE "0x40000") +set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") set(TFM_PLATFORM_UPGRADE_STRATEGY "SWAP_USING_SCRATCH") set(TFM_PLATFORM_CONFIRM_IMAGE ON) @@ -114,13 +110,12 @@ add_subdirectory(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR} ${CMAKE_BINARY_DIR} list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/aws_iot/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/ai/ml_embedded_evaluation_kit/integration/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/security/trusted_firmware-m/integration/cmake) -include(BuildMlEmbeddedEvaluationKit) +include(SetupMlEmbeddedEvaluationKitLibraries) include(ConvertAudioSourceToCode) include(GenerateAWSUpdateDigestAndSignature) include(MergeTfmImages) include(SignTfmImage) -add_subdirectory(../libraries/ml-kit ${CMAKE_BINARY_DIR}/libraries/ml-kit) add_subdirectory(configs ${CMAKE_BINARY_DIR}/Config) add_subdirectory(../helpers helpers) @@ -190,9 +185,6 @@ add_dependencies(speech-recognition trusted_firmware-m-build) # The provision data must be built before the application because # it provides credentials to connect to AWS. add_dependencies(speech-recognition provisioning_data_bin) -# The ML Embedded Evaluation Kit must be built before the application because -# it provides the Machine Learning algorithm to detect speech. -add_dependencies(speech-recognition ml_embedded_evaluation_kit-build) target_link_libraries(speech-recognition PRIVATE @@ -207,12 +199,13 @@ target_link_libraries(speech-recognition helpers-events mbedtls mbedtls-threading-freertos - ml-kit-asr ota-for-aws-iot-embedded-sdk provisioning-lib speexdsp tfm-ns-interface toolchain-override + asr_api + asr_model ) include(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/bsp/cmake/SetLinkerOptions.cmake) diff --git a/applications/speech_recognition/ml_interface.cc b/applications/speech_recognition/ml_interface.cc index f43337b4..ee4dba4b 100644 --- a/applications/speech_recognition/ml_interface.cc +++ b/applications/speech_recognition/ml_interface.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2023 Arm Limited and/or its affiliates +/* Copyright 2021-2024 Arm Limited and/or its affiliates * * SPDX-License-Identifier: MIT */ @@ -12,28 +12,27 @@ #include "AsrClassifier.hpp" #include "AsrResult.hpp" #include "AudioUtils.hpp" +#include "AppContext.hpp" #include "BufAttributes.hpp" #include "demo_config.h" extern "C" { #include "events.h" +#ifdef USE_ETHOS +#include "ethosu_driver.h" +#include "ethosu_npu_init.h" +#endif } #include "Labels.hpp" #include "OutputDecode.hpp" #include "mqtt_agent_task.h" #include "TensorFlowLiteMicro.hpp" -#include "UseCaseCommonUtils.hpp" #include "Wav2LetterMfcc.hpp" #include "Wav2LetterModel.hpp" #include "Wav2LetterPostprocess.hpp" #include "Wav2LetterPreprocess.hpp" #include CMSIS_device_header -#include "device_mps3.h" /* FPGA level definitions and functions. */ #include "dsp_interfaces.h" -#include "ethos-u55.h" /* Mem map and configuration definitions of the Ethos U55 */ -#include "ethosu_driver.h" /* Arm Ethos-U55 driver header */ #include "model_config.h" -#include "timer_mps3.h" /* Timer functions. */ -#include "timing_adapter.h" /* Driver header of the timing adapter */ #include "log_macros.h" #include @@ -470,114 +469,25 @@ extern struct ethosu_driver ethosu_drv; /* Default Ethos-U55 device driver */ **/ static int prvArmNpuInit(void); -/** - * @brief Defines the Ethos-U interrupt handler: just a wrapper around the default - * implementation. - **/ -extern "C" { -void ETHOS_U55_Handler(void) -{ - /* Call the default interrupt handler from the NPU driver */ - ethosu_irq_handler(ðosu_drv); -} -} - -/** - * @brief Initialises the NPU IRQ - **/ -static void prvArmNpuIrqInit(void) -{ - const IRQn_Type ethosu_irqnum = (IRQn_Type)EthosU_IRQn; - - NVIC_EnableIRQ(ethosu_irqnum); - - LogDebug( ( "EthosU IRQ#: %u, Handler: 0x%p\n", ethosu_irqnum, ETHOS_U55_Handler ) ); -} - -static int prvArmNpuTimingAdapterInit(void) -{ -#if defined(TA0_BASE) - struct timing_adapter ta_0; - struct timing_adapter_settings ta_0_settings = { - .maxr = TA0_MAXR, - .maxw = TA0_MAXW, - .maxrw = TA0_MAXRW, - .rlatency = TA0_RLATENCY, - .wlatency = TA0_WLATENCY, - .pulse_on = TA0_PULSE_ON, - .pulse_off = TA0_PULSE_OFF, - .bwcap = TA0_BWCAP, - .perfctrl = TA0_PERFCTRL, - .perfcnt = TA0_PERFCNT, - .mode = TA0_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA0_HISTBIN, - .histcnt = TA0_HISTCNT - }; - - if (ta_init(&ta_0, TA0_BASE) != 0) { - LogError( ( "TA0 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_0, &ta_0_settings); -#endif /* defined (TA0_BASE) */ - -#if defined(TA1_BASE) - struct timing_adapter ta_1; - struct timing_adapter_settings ta_1_settings = { - .maxr = TA1_MAXR, - .maxw = TA1_MAXW, - .maxrw = TA1_MAXRW, - .rlatency = TA1_RLATENCY, - .wlatency = TA1_WLATENCY, - .pulse_on = TA1_PULSE_ON, - .pulse_off = TA1_PULSE_OFF, - .bwcap = TA1_BWCAP, - .perfctrl = TA1_PERFCTRL, - .perfcnt = TA1_PERFCNT, - .mode = TA1_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA1_HISTBIN, - .histcnt = TA1_HISTCNT - }; - - if (ta_init(&ta_1, TA1_BASE) != 0) { - LogError( ( "TA1 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_1, &ta_1_settings); -#endif /* defined (TA1_BASE) */ - - return 0; -} - static int prvArmNpuInit(void) { int err = 0; - /* If the platform has timing adapter blocks along with Ethos-U55 core - * block, initialise them here. */ - // cppcheck-suppress knownConditionTrueFalse - if (0 != (err = prvArmNpuTimingAdapterInit())) { + SCB_EnableICache(); + SCB_EnableDCache(); + +#if defined(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED) + /* If the platform has timing adapter blocks along with Ethos-U core + * block, initialise them here. */ + if (0 != (err = arm_ethosu_timing_adapter_init())) { + LogError( ("Failed to init timing adapter\n") ); return err; } +#endif /* ETHOS_U_NPU_TIMING_ADAPTER_ENABLED */ - /* Initialise the IRQ */ - prvArmNpuIrqInit(); - - /* Initialise Ethos-U55 device */ - void *const ethosu_base_address = reinterpret_cast(SEC_ETHOS_U55_BASE); - - if (0 - != (err = ethosu_init(ðosu_drv, /* Ethos-U55 driver device pointer */ - ethosu_base_address, /* Ethos-U55's base address. */ - NULL, /* Pointer to fast mem area - NULL for U55. */ - 0, /* Fast mem region size. */ - 0, /* Security enable. */ - 0))) { /* Privilege enable. */ - LogError( ( "failed to initalise Ethos-U55 device\n" ) ); + // Initialize the ethos NPU + if (0 != (err = arm_ethosu_npu_init())) { + LogError( ("Failed to init arm npu\n") ); return err; } diff --git a/bsp/CMakeLists.txt b/bsp/CMakeLists.txt index a81f2190..2631f143 100644 --- a/bsp/CMakeLists.txt +++ b/bsp/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 @@ -44,6 +44,10 @@ if(ARM_CORSTONE_BSP_TARGET_PLATFORM STREQUAL "corstone300") set(NS_IMAGE_LOAD_ADDRESS 0x28040000 CACHE STRING "Non-secure user application loading address") set(S_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x10022000 CACHE STRING "Secure provisioning bundle loading address") set(NS_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x211FF000 CACHE STRING "Non-Secure provisioning bundle loading address") + set(ETHOS_U_BASE_ADDR "0x48102000" CACHE STRING "Ethos-U NPU base address" FORCE) + set(ETHOS_U_IRQN "56" CACHE STRING "Ethos-U NPU Interrupt" FORCE) + set(ETHOS_U_SEC_ENABLED "0" CACHE STRING "Ethos-U NPU Security enable" FORCE) + set(ETHOS_U_PRIV_ENABLED "0" CACHE STRING "Ethos-U NPU Privilege enable" FORCE) target_compile_definitions(arm-corstone-platform-bsp PUBLIC CORSTONE300_FVP) elseif(ARM_CORSTONE_BSP_TARGET_PLATFORM STREQUAL "corstone310") set(BL2_IMAGE_LOAD_ADDRESS 0x11000000 CACHE STRING "Bootload image loading address") @@ -51,6 +55,10 @@ elseif(ARM_CORSTONE_BSP_TARGET_PLATFORM STREQUAL "corstone310") set(NS_IMAGE_LOAD_ADDRESS 0x28040000 CACHE STRING "Non-secure user application loading address") set(S_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x11022000 CACHE STRING "Secure provisioning bundle loading address") set(NS_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x213FF000 CACHE STRING "Non-Secure provisioning bundle loading address") + set(ETHOS_U_BASE_ADDR "0x40004000" CACHE STRING "Ethos-U NPU base address" FORCE) + set(ETHOS_U_IRQN "16" CACHE STRING "Ethos-U NPU Interrupt" FORCE) + set(ETHOS_U_SEC_ENABLED "0" CACHE STRING "Ethos-U NPU Security enable" FORCE) + set(ETHOS_U_PRIV_ENABLED "0" CACHE STRING "Ethos-U NPU Privilege enable" FORCE) target_compile_definitions(arm-corstone-platform-bsp PUBLIC CORSTONE310_FVP) endif() diff --git a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt index 47112966..299d37df 100644 --- a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt +++ b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2021-2023 Arm Limited and/or its affiliates +# Copyright 2021-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -12,9 +12,14 @@ include(ApplyPatches) set(PATCH_FILES_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/integration/patches") set(PATCH_FILES - "${PATCH_FILES_DIRECTORY}/0001-asm-syntax-Replace-incorrect-type-specifier.patch" - "${PATCH_FILES_DIRECTORY}/0001-platform-drivers-Remove-RTE_components-header-file.patch" "${PATCH_FILES_DIRECTORY}/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch" - "${PATCH_FILES_DIRECTORY}/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch" + "${PATCH_FILES_DIRECTORY}/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch" + "${PATCH_FILES_DIRECTORY}/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch" + "${PATCH_FILES_DIRECTORY}/0004-Fix-tensorflow.cmake-env-var-escape.patch" ) iot_reference_arm_corstone3xx_apply_patches("${ml_embedded_evaluation_kit_SOURCE_DIR}" "${PATCH_FILES}") + +set(TF_PATCH_FILES + "${PATCH_FILES_DIRECTORY}/0001-tensorflow-kiss_fft_wrapper-include-fix.patch" +) +iot_reference_arm_corstone3xx_apply_patches("${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow" "${TF_PATCH_FILES}") diff --git a/components/ai/ml_embedded_evaluation_kit/integration/README.md b/components/ai/ml_embedded_evaluation_kit/integration/README.md index 71da58c5..d1c5f2cf 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/README.md +++ b/components/ai/ml_embedded_evaluation_kit/integration/README.md @@ -8,64 +8,30 @@ targeted for Arm Cortex-M CPUs and Arm Ethos-U NPU. Available build options and their default values are documented in the [Build options] section of the ML Embedded Evaluation Kit documentation. +The minimum set of options that need to be defined are the following: -If you would like to override any defaults, you can specify a list of options in `ML_CMAKE_ARGS`. For example: +* `ETHOS_U_NPU_ID` +* `ETHOS_U_NPU_CONFIG_ID` +* `ETHOSU_TARGET_NPU_CONFIG` +* `ETHOS_U_NPU_MEMORY_MODE` -```cmake -set(ML_CMAKE_ARGS - -D TARGET_SUBSYSTEM=sse-310 -) -``` - -The above code snippet selects the `sse-310` target subsystem for Corstone-310 instead of the default `sse-300` for Corstone-300. - -> :bulb: This must be done *before* your application's `CMakeLists.txt` adds the IoT Reference Integration for Arm Corstone-3xx. - -## Build targets - -The ML Embedded Evaluation Kit contains a number of components. Instead of building everything, you can save build time -and disk space by specifying a list of components you need in `ML_TARGETS`. For example: - -```cmake -set(ML_TARGETS kws) -``` - -The above code snippet limits the build to the `kws` (keyword spotting) use case and its dependencies. - -> :bulb: This must be done *before* your application's `CMakeLists.txt` adds the IoT Reference Integration for Arm Corstone-3xx. +In addition to the above, `ML_USE_CASE` build option describes which TFLite model and API libraries should be generated. +`ML_MODEL` is the name of the CMake module under `components/ai/ml_embedded_evaluation_kit/integration/cmake/model/` +that handles the generation of C++ source code model from the TFlite model. ## Using in your application -The ML Embedded Evaluation Kit outputs a number of libraries as `*.a` archives in its build directory -(`${ml-embedded-evaluation-kit_BINARY_DIR}/lib`), such as -`${ml-embedded-evaluation-kit_BINARY_DIR}/lib/libtensorflow-microlite.a` which is TensorFlow Lite for Microcontrollers. -You can link your application against the library archives as needed. - -In order for your application to access the API headers for the libraries, you also need to make various paths in the ML -Embedded Evaluation Kit's source directory (`${ml-embedded-evaluation-kit_SOURCE_DIR}`) available to your application as -include directories. - -For example, the IoT Reference Integration for Arm Corstone-3xx repository creates helper libraries named `ml-kit`, `ml-kit-kws` and `ml-kit-asr` in -its [`libraries/ml-kit/CMakeLists.txt`][fri-ml-kit-cmake], with library archives and include directories added to those helper -libraries. The [keyword-detection] and [speech-recognition] applications link the helper libraries they need, in order to inherit the library -archives for linking and include directories for API headers. +After setting the build options, include [SetupMlEmbeddedEvaluationKitLibraries.cmake] to handle the configuration of +necessary ML libraries and running Vela compiler on the default models for the set `ETHOSU_TARGET_NPU_CONFIG`. -> :bulb: Which library archives and include directories are relevant depends on a number of factors, including -> -> * the [configuration](#configuration) set via `ML_CMAKE_ARGS` -> * the [build targets](#build-targets) set via `ML_TARGETS` -> * which libraries your application actually uses -> -> For this reason, the IoT Reference Integration for Arm Corstone-3xx does not provide helper targets in `components` that cover all -> possible uses cases, therefore the user application needs to specify its own list of library archives and include -> paths to take from the ML Embedded Evaluation Kit. +In order for your application to access the API headers for the libraries, you need to link to the two resulting static +libraries: `${ML_USE_CASE}_api`, `${ML_USE_CASE}_model`. +These libraries provide the necessary include paths to the respective APIs from the ML Embedded Evaluation Kit. ## Documentation For more information about the ML Embedded Evaluation Kit, see the [ML Embedded Evaluation Kit documentation]. [Build options]: https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/HEAD/docs/sections/building.md#build-options -[fri-ml-kit-cmake]: ../../../../applications/libraries/ml-kit/CMakeLists.txt -[keyword-detection]: ../../../../applications/keyword_detection/CMakeLists.txt -[speech-recognition]: ../../../../applications/speech_recognition/CMakeLists.txt +[SetupMlEmbeddedEvaluationKitLibraries.cmake]: ../../../../components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake [ML Embedded Evaluation Kit documentation]: https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/HEAD/docs/documentation.md diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/BuildMlEmbeddedEvaluationKit.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/BuildMlEmbeddedEvaluationKit.cmake deleted file mode 100644 index b0e8fd2f..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/cmake/BuildMlEmbeddedEvaluationKit.cmake +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2021-2023 Arm Limited and/or its affiliates -# -# SPDX-License-Identifier: MIT - -include(ExternalProject) - -# Use provided toolchain file as part of ml_embedded_evaluation_kit. -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - set(ml_toolchain_file "scripts/cmake/toolchains/bare-metal-gcc.cmake") -elseif(CMAKE_C_COMPILER_ID STREQUAL "ARMClang") - set(ml_toolchain_file "scripts/cmake/toolchains/bare-metal-armclang.cmake") -else() - message(FATAL_ERROR "Unsupported compiler: ${CMAKE_C_COMPILER_ID}") -endif() - -ExternalProject_Add( - ml_embedded_evaluation_kit-build - - DOWNLOAD_COMMAND "" - SOURCE_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR} - - USES_TERMINAL_CONFIGURE ON - USES_TERMINAL_BUILD ON - - BUILD_ALWAYS ON - - INSTALL_COMMAND "" - - CMAKE_ARGS - -DCMAKE_TOOLCHAIN_FILE=/${ml_toolchain_file} - ${ML_CMAKE_ARGS} - - PATCH_COMMAND - ${ML_PATCH_COMMAND} - - BUILD_COMMAND - ${CMAKE_COMMAND} --build --target ${ML_TARGETS} -) - -set(ML_RESOURCES_SET_UP_ARGS - "--additional-ethos-u-config-name=${ETHOSU_TARGET_NPU_CONFIG}" -) -ExternalProject_Add_Step(ml_embedded_evaluation_kit-build - download_model - COMMAND - python3 ${ml_embedded_evaluation_kit_SOURCE_DIR}/set_up_default_resources.py ${ML_RESOURCES_SET_UP_ARGS} - DEPENDERS - configure - USES_TERMINAL ON -) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake index c4775afb..a2e04ac1 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake @@ -1,12 +1,10 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT # Convert audio clip to C file function(iot_reference_arm_corstone3xx_convert_audio_source_to_code audio_path generated_path) add_custom_target(convert-audio - DEPENDS - ml_embedded_evaluation_kit-build BYPRODUCTS ${generated_path}/InputFiles.cc ${generated_path}/test.cc diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake new file mode 100644 index 00000000..9e6ae649 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake @@ -0,0 +1,136 @@ +# Copyright 2021-2024 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: MIT + +include(ExternalProject) + +########################### +# Setup for the ML target # +########################### +#set(LOG_LEVEL LOG_LEVEL_TRACE) + +# External repositories +set(CMSIS_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis") +set(CMSIS_DSP_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis-dsp") +set(CMSIS_DSP_INC_DIR "${CMSIS_DSP_SRC_PATH}/Include") +set(CMSIS_CORE_INC_DIR "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/CMSIS/Core/Include") +set(CMSIS_NN_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis-nn") +set(TENSORFLOW_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow") +set(ETHOS_U_NPU_DRIVER_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-driver") + +# Extra arguments for setting up default resources (for vela optimizer) +set(ML_RESOURCES_SET_UP_ARGS + "--additional-ethos-u-config-name=${ETHOSU_TARGET_NPU_CONFIG}" +) + +# Tensorflow settings +set(TENSORFLOW_LITE_MICRO_BUILD_TYPE "release_with_logs") +set(TENSORFLOW_LITE_MICRO_CLEAN_DOWNLOADS OFF) +set(TENSORFLOW_LITE_MICRO_CLEAN_BUILD ON) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tensorflow-microlite) + +# Set up default resources. This downloads the TF-Lite models and optimizes them for the target +# Note: This step is using pip install in its venv setup, which involves installing Vela compiler. +# If it is not already installed, it will use CC to compile it to the host machine, but the toolchain file overwrites the CC env. +set(RESOURCES_OUTFILE "${ml_embedded_evaluation_kit_SOURCE_DIR}/resources_downloaded/resources_downloaded_metadata.json") +if(NOT EXISTS "${RESOURCES_OUTFILE}") + execute_process( + COMMAND ${CMAKE_COMMAND} -E env CC=gcc ${PYTHON} ${ml_embedded_evaluation_kit_SOURCE_DIR}/set_up_default_resources.py ${ML_RESOURCES_SET_UP_ARGS} + RESULT_VARIABLE return_code + ) + if (NOT return_code EQUAL "0") + message(FATAL_ERROR "Failed to set up default resources.") + endif () +endif() + +# Setup virtualenv (done by setup_source_generator()) +set(CMAKE_SCRIPTS_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR}/scripts/cmake) +include(${CMAKE_SCRIPTS_DIR}/source_gen_utils.cmake) +set(RESOURCES_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR}/resources_downloaded) +set(SCRIPTS_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR}/scripts) +setup_source_generator() + +# Used by tensorflow.cmake +# Function to check if a variable is defined, and throw +# an error if it is not. +function(assert_defined var_name) + if (NOT DEFINED ${var_name}) + message(FATAL_ERROR "ERROR: ${var_name} is undefined!") + endif() +endfunction() + +# Note: tensorflow.cmake modifies these variables. Save the current values and restore after. +set(TEMP_CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) +set(TEMP_ETHOS_U_NPU_ID ${ETHOS_U_NPU_ID}) +# Note: tensorflow.cmake only has support up to cortex-m55. +set(CMAKE_SYSTEM_PROCESSOR "cortex-m55") +include(${CMAKE_SCRIPTS_DIR}/tensorflow.cmake) +# Note: tensorflow.cmake variable restoration. +set(CMAKE_SYSTEM_PROCESSOR ${TEMP_CMAKE_SYSTEM_PROCESSOR}) +set(ETHOS_U_NPU_ID ${TEMP_ETHOS_U_NPU_ID}) + +# Manually add libs +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/log ${CMAKE_BINARY_DIR}/log) +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/math ${CMAKE_BINARY_DIR}/math) +target_include_directories(arm_math PUBLIC + ${CMSIS_DSP_INC_DIR} + ${CMSIS_CORE_INC_DIR} +) +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/common ${CMAKE_BINARY_DIR}/common_api) +if (ETHOS_U_NPU_ENABLED) + add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/npu ${CMAKE_BINARY_DIR}/npu) + # ethos_u_npu library needs the CPU Header (CMSIS_device_header) + target_link_libraries(ethos_u_npu PUBLIC arm-corstone-platform-bsp) +endif() + +# Add the dependency on tensorflow_build (defined in tensorflow.cmake) +add_dependencies(common_api tensorflow_build) +target_include_directories(common_api PUBLIC ${TFLITE_MICRO_PATH}) +target_compile_options(common_api PUBLIC $<$:-std=c++14>) + +# Add relevant use case API +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/use_case/${ML_USE_CASE} ${CMAKE_BINARY_DIR}/${ML_USE_CASE}_api) + +# Include directories for application module: +set(APPLICATION_INCLUDE_DIRS ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/main/include) +target_include_directories(${ML_USE_CASE}_api PUBLIC + ${APPLICATION_INCLUDE_DIRS} +) + +# Generate use case C model from optimized tflite file +# Generate tflite model code +set(DEFAULT_MODEL_DIR ${RESOURCES_DIR}/${ML_USE_CASE}) +set(SRC_GEN_DIR ${CMAKE_BINARY_DIR}/generated/${ML_USE_CASE}/src) +set(INC_GEN_DIR ${CMAKE_BINARY_DIR}/generated/${ML_USE_CASE}/include) + +# Remove old files and recreate dirs +file(REMOVE_RECURSE ${SRC_GEN_DIR} ${INC_GEN_DIR}) +file(MAKE_DIRECTORY ${SRC_GEN_DIR} ${INC_GEN_DIR}) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/model) +include(${ML_MODEL}) + +file(GLOB_RECURSE SRC_GEN + "${SRC_GEN_DIR}/*.cc" + "${SRC_GEN_DIR}/*.cpp" + "${SRC_GEN_DIR}/*.c") + +set(UC_LIB_NAME ${ML_USE_CASE}_model) + +# Consolidated application static lib: +add_library(${UC_LIB_NAME} STATIC + ${SRC_GEN}) + +target_include_directories(${UC_LIB_NAME} PUBLIC + ${APPLICATION_INCLUDE_DIRS} + ${INC_GEN_DIR}) + +string(TOUPPER ${ML_USE_CASE} ML_USE_CASE_UPPER) +target_compile_definitions(${UC_LIB_NAME} PUBLIC + "ACTIVATION_BUF_SZ=${${ML_USE_CASE_UPPER}_ACTIVATION_BUF_SZ}") + +target_link_libraries(${UC_LIB_NAME} PUBLIC + $<$:ethos_u_npu> + tensorflow-lite-micro + common_api +) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateASRModel.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateASRModel.cmake new file mode 100644 index 00000000..3450a150 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateASRModel.cmake @@ -0,0 +1,54 @@ +#---------------------------------------------------------------------------- +# SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#---------------------------------------------------------------------------- +# This file is based on https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/refs/tags/23.08/source/use_case/asr/usecase.cmake + +set(ASR_LABELS_TXT_FILE + ${ml_embedded_evaluation_kit_SOURCE_DIR}/resources/asr/labels/labels_wav2letter.txt) + +set(ASR_LABELS_CPP_FILE Labels) +generate_labels_code( + INPUT "${ASR_LABELS_TXT_FILE}" + DESTINATION_SRC ${SRC_GEN_DIR} + DESTINATION_HDR ${INC_GEN_DIR} + OUTPUT_FILENAME "${ASR_LABELS_CPP_FILE}" +) + +set(ASR_ACTIVATION_BUF_SZ 0x00200000) +set(ASR_MODEL_SCORE_THRESHOLD 0.5) + +if (ETHOS_U_NPU_ENABLED) + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) +else() + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8.tflite) +endif() + +set(EXTRA_MODEL_CODE + "/* Model parameters for asr */" + "extern const int g_FrameLength = 512" + "extern const int g_FrameStride = 160" + "extern const int g_ctxLen = 98" + "extern const float g_ScoreThreshold = ${ASR_MODEL_SCORE_THRESHOLD}" + ) + +set(ASR_MODEL_TFLITE_PATH ${DEFAULT_MODEL_PATH}) + +generate_tflite_code( + MODEL_PATH ${ASR_MODEL_TFLITE_PATH} + DESTINATION ${SRC_GEN_DIR} + EXPRESSIONS ${EXTRA_MODEL_CODE} + NAMESPACE "arm" "app" "asr" +) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateKWSModel.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateKWSModel.cmake new file mode 100644 index 00000000..641d2033 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateKWSModel.cmake @@ -0,0 +1,53 @@ +#---------------------------------------------------------------------------- +# SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#---------------------------------------------------------------------------- +# This file is based on https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/refs/tags/23.08/source/use_case/kws/usecase.cmake + +set(KWS_LABELS_TXT_FILE + ${ml_embedded_evaluation_kit_SOURCE_DIR}/resources/kws/labels/micronet_kws_labels.txt) + +set(KWS_LABELS_CPP_FILE Labels) +generate_labels_code( + INPUT "${KWS_LABELS_TXT_FILE}" + DESTINATION_SRC ${SRC_GEN_DIR} + DESTINATION_HDR ${INC_GEN_DIR} + OUTPUT_FILENAME "${KWS_LABELS_CPP_FILE}" +) + +set(KWS_ACTIVATION_BUF_SZ 0x00100000) +set(KWS_MODEL_SCORE_THRESHOLD 0.7) + +if (ETHOS_U_NPU_ENABLED) + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/kws_micronet_m_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) +else() + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/kws_micronet_m.tflite) +endif() + +set(EXTRA_MODEL_CODE + "/* Model parameters for KWS */" + "extern const int g_FrameLength = 640" + "extern const int g_FrameStride = 320" + "extern const float g_ScoreThreshold = ${KWS_MODEL_SCORE_THRESHOLD}" + ) + +set(KWS_MODEL_TFLITE_PATH ${DEFAULT_MODEL_PATH}) + +generate_tflite_code( + MODEL_PATH ${KWS_MODEL_TFLITE_PATH} + DESTINATION ${SRC_GEN_DIR} + EXPRESSIONS ${EXTRA_MODEL_CODE} + NAMESPACE "arm" "app" "kws" +) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-asm-syntax-Replace-incorrect-type-specifier.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-asm-syntax-Replace-incorrect-type-specifier.patch deleted file mode 100644 index e8d1b2e5..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-asm-syntax-Replace-incorrect-type-specifier.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 37bb50603a05c16767b3020c0f7b5be030f2f160 Mon Sep 17 00:00:00 2001 -From: Ahmed Ismail -Date: Wed, 27 Dec 2023 15:15:05 +0000 -Subject: [PATCH] asm-syntax: Replace incorrect type specifier - -In C++ files we should use __asm rather than __ASM as __ASM -is not known as a correct C++ type specifier. - -Signed-off-by: Ahmed Ismail ---- - source/application/main/Main.cc | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/source/application/main/Main.cc b/source/application/main/Main.cc -index bbe35d9..d36d54f 100644 ---- a/source/application/main/Main.cc -+++ b/source/application/main/Main.cc -@@ -28,7 +28,7 @@ - extern void main_loop(); - - #if defined(__ARMCC_VERSION) && (__ARMCC_VERSION >= 6010050) --__ASM(" .global __ARM_use_no_argv\n"); -+__asm(" .global __ARM_use_no_argv\n"); - #endif - - /* Print application information. */ --- -2.34.1 - diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-platform-drivers-Remove-RTE_components-header-file.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-platform-drivers-Remove-RTE_components-header-file.patch deleted file mode 100644 index 379f2f14..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-platform-drivers-Remove-RTE_components-header-file.patch +++ /dev/null @@ -1,31 +0,0 @@ -From d4b0f895f9d621e43027cc3fa2c8d521128e418a Mon Sep 17 00:00:00 2001 -From: Ahmed Ismail -Date: Wed, 27 Dec 2023 13:35:16 +0000 -Subject: [PATCH] platform-drivers: Remove RTE_components header file - -RTE_Components header file is being included for both Corstone-300 -and Corstone-310 platforms. However in the case of Corstone-300 -platofrm, this file is not available anymore in the latest version of -cmsis-packs Hence, this header file inclusion is removed in the new -patch file. - -Signed-off-by: Ahmed Ismail ---- - source/hal/source/platform/mps3/include/platform_drivers.h | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/source/hal/source/platform/mps3/include/platform_drivers.h b/source/hal/source/platform/mps3/include/platform_drivers.h -index fc7dcc5..6a017bb 100644 ---- a/source/hal/source/platform/mps3/include/platform_drivers.h -+++ b/source/hal/source/platform/mps3/include/platform_drivers.h -@@ -21,7 +21,6 @@ - #include "log_macros.h" /* Logging related helpers. */ - - /* Platform components */ --#include "RTE_Components.h" /* For CPU related defintiions */ - #include "timer_mps3.h" /* Timer functions. */ - #include "user_input.h" /* User input function */ - #include "lcd_img.h" /* LCD functions. */ --- -2.34.1 - diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch index 87c3e3ce..7686195b 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch @@ -1,7 +1,7 @@ -From 34238f5d40992d6f2adbda9d793bd51c6b5ea985 Mon Sep 17 00:00:00 2001 +From e691803e130d1aced38c21e98c41ce40bda44ffe Mon Sep 17 00:00:00 2001 From: Hugues Kamba-Mpiana Date: Tue, 19 Dec 2023 14:09:27 +0000 -Subject: [PATCH] resources: Replace model wav2letter with tiny_wav2letter +Subject: [PATCH 1/4] resources: Replace model wav2letter with tiny_wav2letter `wav2letter` model needs singificant amount of memory (around 14Mb), where as `tiny_wav2letter` needs around 4Mb. @@ -10,13 +10,13 @@ the `asr` and `kws` use cases. Signed-off-by: Hugues Kamba-Mpiana --- - set_up_default_resources.py | 18 +++++++++--------- + set_up_default_resources.py | 16 ++++++++-------- source/use_case/asr/usecase.cmake | 4 ++-- source/use_case/kws_asr/usecase.cmake | 4 ++-- - 3 files changed, 13 insertions(+), 13 deletions(-) + 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/set_up_default_resources.py b/set_up_default_resources.py -index b442432..a0d6149 100755 +index b442432..2cb987d 100755 --- a/set_up_default_resources.py +++ b/set_up_default_resources.py @@ -51,16 +51,16 @@ json_uc_res = [ @@ -97,5 +97,5 @@ index de84bfd..9bfbbd0 100644 USER_OPTION(${use_case}_MODEL_TFLITE_PATH_KWS "NN models file to be used for KWS in the evaluation application. Model files must be in tflite format." -- -2.34.1 +2.40.1 diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-tensorflow-kiss_fft_wrapper-include-fix.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-tensorflow-kiss_fft_wrapper-include-fix.patch new file mode 100644 index 00000000..8051962b --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-tensorflow-kiss_fft_wrapper-include-fix.patch @@ -0,0 +1,25 @@ +From d799ead836ea893cbc6eb8abe98dccfd6aac2e4a Mon Sep 17 00:00:00 2001 +From: Gabor Abonyi +Date: Tue, 16 Jan 2024 13:40:22 +0100 +Subject: [PATCH] Add stdint.h to kiss_fft_common.h wrapper + +Signed-off-by: Gabor Abonyi +--- + signal/src/kiss_fft_wrappers/kiss_fft_common.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/signal/src/kiss_fft_wrappers/kiss_fft_common.h b/signal/src/kiss_fft_wrappers/kiss_fft_common.h +index 75f3dcda..8dcaaffa 100644 +--- a/signal/src/kiss_fft_wrappers/kiss_fft_common.h ++++ b/signal/src/kiss_fft_wrappers/kiss_fft_common.h +@@ -34,6 +34,7 @@ limitations under the License. + + #include + #include ++#include + #include + #include + #include +-- +2.40.1 + diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch similarity index 93% rename from components/ai/ml_embedded_evaluation_kit/integration/patches/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch rename to components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch index 3756e4be..38a92636 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch @@ -1,7 +1,7 @@ -From 28c8e0755b91ae4a8628bfcc1b1ca39b4c494bc5 Mon Sep 17 00:00:00 2001 +From 60556c32a8957dd52b5dc7b85a8cef6550f5ac28 Mon Sep 17 00:00:00 2001 From: Hugues Kamba-Mpiana Date: Tue, 19 Dec 2023 13:17:15 +0000 -Subject: [PATCH] toolchain: Fix Arm GNU Toolchain 13.2 Rel1 compatibility +Subject: [PATCH 2/4] toolchain: Fix Arm GNU Toolchain 13.2 Rel1 compatibility Arm GNU Toolchain 13.2 Rel1 no longer includes transitively. Include `cstdint` explicitely where needed. @@ -65,5 +65,5 @@ index 279acf0..eb28a8f 100644 namespace app { -- -2.34.1 +2.40.1 diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch new file mode 100644 index 00000000..0c9b1343 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch @@ -0,0 +1,40 @@ +From 75e29cf7fd59978d06d8594efd2c883c40cd56e1 Mon Sep 17 00:00:00 2001 +From: Gabor Abonyi +Date: Tue, 16 Jan 2024 14:48:44 +0100 +Subject: [PATCH 3/4] Use CMSIS_device_header instead of RTE_Components.h + +Signed-off-by: Gabor Abonyi +--- + source/hal/source/components/npu/ethosu_cpu_cache.c | 2 +- + source/hal/source/components/npu/ethosu_npu_init.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/source/hal/source/components/npu/ethosu_cpu_cache.c b/source/hal/source/components/npu/ethosu_cpu_cache.c +index d5f5e47..42fedba 100644 +--- a/source/hal/source/components/npu/ethosu_cpu_cache.c ++++ b/source/hal/source/components/npu/ethosu_cpu_cache.c +@@ -17,7 +17,7 @@ + + #include "ethosu_cpu_cache.h" + +-#include "RTE_Components.h" /* For CPU related defintiions */ ++#include CMSIS_device_header /* For CPU related defintiions */ + #include "ethosu_driver.h" /* Arm Ethos-U driver header */ + #include "log_macros.h" /* Logging macros */ + +diff --git a/source/hal/source/components/npu/ethosu_npu_init.c b/source/hal/source/components/npu/ethosu_npu_init.c +index dbee2ff..0bce70e 100644 +--- a/source/hal/source/components/npu/ethosu_npu_init.c ++++ b/source/hal/source/components/npu/ethosu_npu_init.c +@@ -17,7 +17,7 @@ + + #include "ethosu_npu_init.h" + +-#include "RTE_Components.h" /* For CPU related defintiions */ ++#include CMSIS_device_header /* For CPU related defintiions */ + #include "log_macros.h" /* Logging functions */ + + #include "ethosu_mem_config.h" /* Arm Ethos-U memory config */ +-- +2.40.1 + diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch new file mode 100644 index 00000000..53c0a78c --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch @@ -0,0 +1,38 @@ +From 2d22af627e51f62e95968cc75f51ff1238cf86ec Mon Sep 17 00:00:00 2001 +From: Gabor Abonyi +Date: Thu, 11 Jan 2024 14:16:57 +0100 +Subject: [PATCH 4/4] Fix tensorflow.cmake env var escape + +Signed-off-by: Gabor Abonyi +--- + scripts/cmake/tensorflow.cmake | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/scripts/cmake/tensorflow.cmake b/scripts/cmake/tensorflow.cmake +index 8d52765..a028055 100644 +--- a/scripts/cmake/tensorflow.cmake ++++ b/scripts/cmake/tensorflow.cmake +@@ -78,9 +78,9 @@ endif() + + if (TENSORFLOW_LITE_MICRO_CLEAN_DOWNLOADS) + message(STATUS "Refreshing TensorFlow Lite Micro's third party downloads...") ++ set(ENV{PATH} "${ENV_PATH}") + execute_process( +- COMMAND ${CMAKE_COMMAND} -E env PATH=${ENV_PATH} +- make -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile clean_downloads third_party_downloads ++ COMMAND make -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile clean_downloads third_party_downloads + RESULT_VARIABLE return_code + WORKING_DIRECTORY ${TENSORFLOW_SRC_PATH}) + if (NOT return_code EQUAL "0") +@@ -102,7 +102,7 @@ message(STATUS "TensorFlow Lite Micro build to be called for these targets: ${MA + add_custom_target(tensorflow_build ALL + + # Command to build the TensorFlow Lite Micro library +- COMMAND ${CMAKE_COMMAND} -E env PATH=${ENV_PATH} ++ COMMAND ${CMAKE_COMMAND} -E env PATH="${ENV_PATH}" + make -j${PARALLEL_JOBS} -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile ${MAKE_TARGETS_LIST} + TARGET_TOOLCHAIN_ROOT=${TENSORFLOW_LITE_MICRO_TARGET_TOOLCHAIN_ROOT} + TOOLCHAIN=${TENSORFLOW_LITE_MICRO_TOOLCHAIN} +-- +2.40.1 + diff --git a/release_changes/202401161458.change b/release_changes/202401161458.change new file mode 100644 index 00000000..468e757a --- /dev/null +++ b/release_changes/202401161458.change @@ -0,0 +1,3 @@ +ml-eval-kit: Require apps to include build CMake module +ml-eval-kit: Remove default NPU config list patch +ml-eval-kit: Build individual libraries