diff --git a/.github/.cSpellWords.txt b/.github/.cSpellWords.txt index 1fc612be..c63d9221 100644 --- a/.github/.cSpellWords.txt +++ b/.github/.cSpellWords.txt @@ -62,6 +62,7 @@ IDAQAB indet inkey iotdeviceadvisor +IRQN ISRAM istty JITP diff --git a/applications/keyword_detection/CMakeLists.txt b/applications/keyword_detection/CMakeLists.txt index 6914dd9f..391fe028 100644 --- a/applications/keyword_detection/CMakeLists.txt +++ b/applications/keyword_detection/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -31,35 +31,31 @@ set(MCUBOOT_IMAGE_VERSION_NS_UPDATE "0.0.1+20") # Hardware platform specific setup if (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone310") - set(ML_TARGET_SUBSYSTEM "sse-310") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H256") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-256") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone310/fvp") elseif (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - set(ML_TARGET_SUBSYSTEM "sse-300") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H128") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-128") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone300/fvp") else() message(FATAL_ERROR "Invalid ARM_CORSTONE_BSP_TARGET_PLATFORM (${ARM_CORSTONE_BSP_TARGET_PLATFORM}) set. Supported are corstone300/corstone310") endif() -set(TFM_FLASH_S_PARTITION_SIZE "0x40000") -set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") - -# Machine Learning setup -set(ML_CMAKE_ARGS - -DTARGET_SUBSYSTEM=${ML_TARGET_SUBSYSTEM} - -DETHOS_U_NPU_CONFIG_ID=${ETHOS_U_NPU_CONFIG_ID} - -DETHOSU_TARGET_NPU_CONFIG=${ETHOSU_TARGET_NPU_CONFIG} - -DUSE_CASE_BUILD=kws -) if (${ML_INFERENCE_ENGINE} STREQUAL "ETHOS") - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=ON) + set(ETHOS_U_NPU_ENABLED ON) + set(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED OFF) else() - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=OFF) + set(ETHOS_U_NPU_ENABLED OFF) endif() -set(ML_TARGETS cmsis-dsp tensorflow_build kws) +set(ML_USE_CASE "kws") +set(ML_MODEL "GenerateKWSModel") +set(TFM_FLASH_S_PARTITION_SIZE "0x40000") +set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") set(TFM_PLATFORM_UPGRADE_STRATEGY "SWAP_USING_SCRATCH") set(TFM_PLATFORM_CONFIRM_IMAGE ON) @@ -114,13 +110,12 @@ add_subdirectory(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR} ${CMAKE_BINARY_DIR} list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/aws_iot/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/ai/ml_embedded_evaluation_kit/integration/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/security/trusted_firmware-m/integration/cmake) -include(BuildMlEmbeddedEvaluationKit) +include(SetupMlEmbeddedEvaluationKitLibraries) include(ConvertAudioSourceToCode) include(GenerateAWSUpdateDigestAndSignature) include(MergeTfmImages) include(SignTfmImage) -add_subdirectory(../libraries/ml-kit ${CMAKE_BINARY_DIR}/libraries/ml-kit) add_subdirectory(configs ${CMAKE_BINARY_DIR}/Config) add_subdirectory(../helpers helpers) @@ -177,9 +172,6 @@ add_dependencies(keyword-detection trusted_firmware-m-build) # The provision data must be built before the application because # it provides credentials to connect to AWS. add_dependencies(keyword-detection provisioning_data_bin) -# The ML Embedded Evaluation Kit must be built before the application because -# it provides the Machine Learning algorithm to detect keywords. -add_dependencies(keyword-detection ml_embedded_evaluation_kit-build) target_link_libraries(keyword-detection PRIVATE @@ -194,11 +186,12 @@ target_link_libraries(keyword-detection helpers-events mbedtls mbedtls-threading-freertos - ml-kit-kws ota-for-aws-iot-embedded-sdk provisioning-lib tfm-ns-interface toolchain-override + kws_api + kws_model ) include(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/bsp/cmake/SetLinkerOptions.cmake) diff --git a/applications/keyword_detection/ml_interface.cc b/applications/keyword_detection/ml_interface.cc index ca31df28..15663c15 100644 --- a/applications/keyword_detection/ml_interface.cc +++ b/applications/keyword_detection/ml_interface.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2023 Arm Limited and/or its affiliates +/* Copyright 2021-2024 Arm Limited and/or its affiliates * * SPDX-License-Identifier: MIT */ @@ -10,10 +10,15 @@ #include "ml_interface.h" #include "AudioUtils.hpp" +#include "AppContext.hpp" #include "BufAttributes.hpp" #include "demo_config.h" extern "C" { #include "events.h" +#ifdef USE_ETHOS +#include "ethosu_driver.h" +#include "ethosu_npu_init.h" +#endif } #include "KwsClassifier.hpp" #include "KwsProcessing.hpp" @@ -24,13 +29,7 @@ extern "C" { #include "MicroNetKwsModel.hpp" #include "mqtt_agent_task.h" #include "TensorFlowLiteMicro.hpp" -#include "UseCaseCommonUtils.hpp" #include CMSIS_device_header -#include "device_mps3.h" /* FPGA level definitions and functions. */ -#include "ethos-u55.h" /* Mem map and configuration definitions of the Ethos U55 */ -#include "ethosu_driver.h" /* Arm Ethos-U55 driver header */ -#include "timer_mps3.h" /* Timer functions. */ -#include "timing_adapter.h" /* Driver header of the timing adapter */ #include #include @@ -890,111 +889,25 @@ extern struct ethosu_driver ethosu_drv; /* Default Ethos-U55 device driver */ **/ static int prvArmNpuInit(void); -/** - * @brief Defines the Ethos-U interrupt handler: just a wrapper around the default - * implementation. - **/ -extern "C" { -void ETHOS_U55_Handler(void) -{ - /* Call the default interrupt handler from the NPU driver */ - ethosu_irq_handler(ðosu_drv); -} -} - -/** - * @brief Initialises the NPU IRQ - **/ -static void prvArmNpuIrqInit(void) -{ - const IRQn_Type ethosu_irqnum = (IRQn_Type)EthosU_IRQn; - - /* Enable the IRQ */ - NVIC_EnableIRQ(ethosu_irqnum); - - LogDebug( ( "EthosU IRQ#: %u, Handler: 0x%p\n", ethosu_irqnum, ETHOS_U55_Handler ) ); -} - -static int prvArmNpuTimingAdapterInit(void) -{ -#if defined(TA0_BASE) - struct timing_adapter ta_0; - struct timing_adapter_settings ta_0_settings = {.maxr = TA0_MAXR, - .maxw = TA0_MAXW, - .maxrw = TA0_MAXRW, - .rlatency = TA0_RLATENCY, - .wlatency = TA0_WLATENCY, - .pulse_on = TA0_PULSE_ON, - .pulse_off = TA0_PULSE_OFF, - .bwcap = TA0_BWCAP, - .perfctrl = TA0_PERFCTRL, - .perfcnt = TA0_PERFCNT, - .mode = TA0_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA0_HISTBIN, - .histcnt = TA0_HISTCNT}; - - if (0 != ta_init(&ta_0, TA0_BASE)) { - LogError( ( "TA0 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_0, &ta_0_settings); -#endif /* defined (TA0_BASE) */ - -#if defined(TA1_BASE) - struct timing_adapter ta_1; - struct timing_adapter_settings ta_1_settings = {.maxr = TA1_MAXR, - .maxw = TA1_MAXW, - .maxrw = TA1_MAXRW, - .rlatency = TA1_RLATENCY, - .wlatency = TA1_WLATENCY, - .pulse_on = TA1_PULSE_ON, - .pulse_off = TA1_PULSE_OFF, - .bwcap = TA1_BWCAP, - .perfctrl = TA1_PERFCTRL, - .perfcnt = TA1_PERFCNT, - .mode = TA1_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA1_HISTBIN, - .histcnt = TA1_HISTCNT}; - - if (0 != ta_init(&ta_1, TA1_BASE)) { - LogError( ( "TA1 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_1, &ta_1_settings); -#endif /* defined (TA1_BASE) */ - - return 0; -} - static int prvArmNpuInit(void) { int err = 0; - /* If the platform has timing adapter blocks along with Ethos-U55 core - * block, initialise them here. */ - // cppcheck-suppress knownConditionTrueFalse - if (0 != (err = prvArmNpuTimingAdapterInit())) { + SCB_EnableICache(); + SCB_EnableDCache(); + +#if defined(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED) + /* If the platform has timing adapter blocks along with Ethos-U core + * block, initialise them here. */ + if (0 != (err = arm_ethosu_timing_adapter_init())) { + LogError( ("Failed to init timing adapter\n") ); return err; } +#endif /* ETHOS_U_NPU_TIMING_ADAPTER_ENABLED */ - /* Initialise the IRQ */ - prvArmNpuIrqInit(); - - /* Initialise Ethos-U55 device */ - void *const ethosu_base_address = reinterpret_cast(SEC_ETHOS_U55_BASE); - - if (0 - != (err = ethosu_init(ðosu_drv, /* Ethos-U55 driver device pointer */ - ethosu_base_address, /* Ethos-U55's base address. */ - NULL, /* Pointer to fast mem area - NULL for U55. */ - 0, /* Fast mem region size. */ - 0, /* Security enable. */ - 0))) { /* Privilege enable. */ - LogError( ( "failed to initalise Ethos-U55 device\n" ) ); + // Initialize the ethos NPU + if (0 != (err = arm_ethosu_npu_init())) { + LogError( ("Failed to init arm npu\n") ); return err; } diff --git a/applications/libraries/ml-kit/CMakeLists.txt b/applications/libraries/ml-kit/CMakeLists.txt deleted file mode 100644 index aa076bcd..00000000 --- a/applications/libraries/ml-kit/CMakeLists.txt +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2022-2023 Arm Limited and/or its affiliates -# -# SPDX-License-Identifier: MIT - -include(ExternalProject) - -ExternalProject_Get_Property(ml_embedded_evaluation_kit-build BINARY_DIR) - -# ML-eval Kit -add_library(ml-kit INTERFACE) - -# Set byproducts from ML build, so that CMake treats them as generated files -# that don't need to exist before the build. - -set(ml_kit_libs - ${BINARY_DIR}/lib/libcommon_api.a - ${BINARY_DIR}/lib/libtensorflow-microlite.a - ${BINARY_DIR}/lib/libarm_math.a - ${BINARY_DIR}/lib/libcmsis-dsp.a -) - -if(${ML_INFERENCE_ENGINE} STREQUAL "ETHOS") - list(APPEND ml_kit_libs - ${BINARY_DIR}/lib/libethos_u_npu.a - ${BINARY_DIR}/lib/libethosu_core_driver.a - ) - - if(${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - list(APPEND ml_kit_libs ${BINARY_DIR}/lib/libtiming_adapter.a) - endif() -endif() - -set(ml_kit_kws_libs - ${BINARY_DIR}/lib/libkws_api.a - ${BINARY_DIR}/lib/libkws.a -) -set(ml_kit_asr_libs - ${BINARY_DIR}/lib/libasr_api.a - ${BINARY_DIR}/lib/libasr.a -) - -add_custom_target(get_ml_kit_libs - DEPENDS - ml_embedded_evaluation_kit-build - COMMAND - # Dummy command that does nothing - ${CMAKE_COMMAND} -E true - BYPRODUCTS - ${ml_kit_libs} ${ml_kit_kws_libs} ${ml_kit_asr_libs} -) - - -target_include_directories(ml-kit - INTERFACE - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis-dsp/Include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-driver/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow/tensorflow/lite/micro - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow/tensorflow/lite/micro/tools/make/downloads/flatbuffers/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow/tensorflow/lite/micro/tools/make/downloads/gemmlowp/ - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/hal/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/main/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/common/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/platform/mps3/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/log/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/platform_pmu/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/npu/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/stdout/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/lcd/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/profiler/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/platform/mps3/source - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-platform/drivers/timing_adapter/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/math/include -) - -if(${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - target_include_directories(ml-kit - INTERFACE - ${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-platform/drivers/timing_adapter/include - ) -endif() - - -target_link_libraries(ml-kit - INTERFACE - ${ml_kit_libs} -) - -target_compile_definitions(ml-kit - INTERFACE - -DACTIVATION_BUF_SZ=0x00200000 - -DARM_MATH_DSP - -DARM_MATH_LOOPUNROLL - -DARM_MATH_MVEI - -DARM_NPU=1 - -DTF_LITE_STATIC_MEMORY - -DETHOS_U_NPU_MEMORY_MODE=ETHOS_U_NPU_MEMORY_MODE_SHARED_SRAM -) - -add_dependencies(ml-kit - ml_embedded_evaluation_kit-build -) - - -# Add kws library -add_library(ml-kit-kws INTERFACE) - -target_link_libraries(ml-kit-kws - INTERFACE - ${ml_kit_kws_libs} -) - -target_link_libraries(ml-kit-kws - INTERFACE - ml-kit -) - -target_include_directories(ml-kit-kws - INTERFACE - ${BINARY_DIR}/generated/kws/include/ - ${ml_embedded_evaluation_kit_SOURCE_DIR}/generated/kws/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/use_case/kws/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/use_case/kws/include -) - -# Remove retarget from kws library -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - add_custom_target(ml-kit-remove-kws-retarget - COMMAND arm-none-eabi-ar dv ${BINARY_DIR}/lib/libkws.a retarget.c.obj - ) -elseif(CMAKE_C_COMPILER_ID STREQUAL "ARMClang") - add_custom_target(ml-kit-remove-kws-retarget - COMMAND armar -d ${BINARY_DIR}/lib/libkws.a retarget.o - ) -endif() - -add_dependencies(ml-kit-remove-kws-retarget ml-kit) -add_dependencies(ml-kit-kws ml-kit-remove-kws-retarget) - - -# Add asr library -add_library(ml-kit-asr INTERFACE) - -target_link_libraries(ml-kit-asr - INTERFACE - ${ml_kit_asr_libs} -) - -target_link_libraries(ml-kit-asr - INTERFACE - ml-kit -) - -target_include_directories(ml-kit-asr - INTERFACE - ${BINARY_DIR}/generated/asr/include/ - ${ml_embedded_evaluation_kit_SOURCE_DIR}/generated/asr/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/use_case/asr/include - ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/use_case/asr/include -) - -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - add_custom_target(ml-kit-remove-asr-retarget - COMMAND arm-none-eabi-ar dv ${BINARY_DIR}/lib/libasr.a retarget.c.obj - ) -elseif(CMAKE_C_COMPILER_ID STREQUAL "ARMClang") - add_custom_target(ml-kit-remove-asr-retarget - COMMAND armar -d ${BINARY_DIR}/lib/libasr.a retarget.o - ) -endif() - -add_dependencies(ml-kit-remove-asr-retarget ml-kit) -add_dependencies(ml-kit-asr ml-kit-remove-asr-retarget) diff --git a/applications/speech_recognition/CMakeLists.txt b/applications/speech_recognition/CMakeLists.txt index 866e1e5c..d27bf09d 100644 --- a/applications/speech_recognition/CMakeLists.txt +++ b/applications/speech_recognition/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -31,35 +31,31 @@ set(MCUBOOT_IMAGE_VERSION_NS_UPDATE "0.0.1+20") # Hardware platform specific setup if (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone310") - set(ML_TARGET_SUBSYSTEM "sse-310") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H256") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-256") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone310/fvp") elseif (${ARM_CORSTONE_BSP_TARGET_PLATFORM} STREQUAL "corstone300") - set(ML_TARGET_SUBSYSTEM "sse-300") + set(ETHOS_U_NPU_ID "U55") set(ETHOS_U_NPU_CONFIG_ID "H128") set(ETHOSU_TARGET_NPU_CONFIG "ethos-u55-128") + set(ETHOS_U_NPU_MEMORY_MODE Shared_Sram) set(TFM_PLATFORM_LOCAL_PATH "arm/mps3/corstone300/fvp") else() message(FATAL_ERROR "Invalid ARM_CORSTONE_BSP_TARGET_PLATFORM (${ARM_CORSTONE_BSP_TARGET_PLATFORM}) set. Supported are corstone300/corstone310") endif() -set(TFM_FLASH_S_PARTITION_SIZE "0x40000") -set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") - -# Machine Learning setup -set(ML_CMAKE_ARGS - -DTARGET_SUBSYSTEM=${ML_TARGET_SUBSYSTEM} - -DETHOS_U_NPU_CONFIG_ID=${ETHOS_U_NPU_CONFIG_ID} - -DETHOSU_TARGET_NPU_CONFIG=${ETHOSU_TARGET_NPU_CONFIG} - -DUSE_CASE_BUILD=asr -) if (${ML_INFERENCE_ENGINE} STREQUAL "ETHOS") - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=ON) + set(ETHOS_U_NPU_ENABLED ON) + set(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED OFF) else() - list(APPEND ML_CMAKE_ARGS -DETHOS_U_NPU_ENABLED=OFF) + set(ETHOS_U_NPU_ENABLED OFF) endif() -set(ML_TARGETS cmsis-dsp tensorflow_build asr) +set(ML_USE_CASE "asr") +set(ML_MODEL "GenerateASRModel") +set(TFM_FLASH_S_PARTITION_SIZE "0x40000") +set(TFM_FLASH_NS_PARTITION_SIZE "0x340000") set(TFM_PLATFORM_UPGRADE_STRATEGY "SWAP_USING_SCRATCH") set(TFM_PLATFORM_CONFIRM_IMAGE ON) @@ -114,13 +110,12 @@ add_subdirectory(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR} ${CMAKE_BINARY_DIR} list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/aws_iot/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/ai/ml_embedded_evaluation_kit/integration/cmake) list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/security/trusted_firmware-m/integration/cmake) -include(BuildMlEmbeddedEvaluationKit) +include(SetupMlEmbeddedEvaluationKitLibraries) include(ConvertAudioSourceToCode) include(GenerateAWSUpdateDigestAndSignature) include(MergeTfmImages) include(SignTfmImage) -add_subdirectory(../libraries/ml-kit ${CMAKE_BINARY_DIR}/libraries/ml-kit) add_subdirectory(configs ${CMAKE_BINARY_DIR}/Config) add_subdirectory(../helpers helpers) @@ -190,9 +185,6 @@ add_dependencies(speech-recognition trusted_firmware-m-build) # The provision data must be built before the application because # it provides credentials to connect to AWS. add_dependencies(speech-recognition provisioning_data_bin) -# The ML Embedded Evaluation Kit must be built before the application because -# it provides the Machine Learning algorithm to detect speech. -add_dependencies(speech-recognition ml_embedded_evaluation_kit-build) target_link_libraries(speech-recognition PRIVATE @@ -207,12 +199,13 @@ target_link_libraries(speech-recognition helpers-events mbedtls mbedtls-threading-freertos - ml-kit-asr ota-for-aws-iot-embedded-sdk provisioning-lib speexdsp tfm-ns-interface toolchain-override + asr_api + asr_model ) include(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/bsp/cmake/SetLinkerOptions.cmake) diff --git a/applications/speech_recognition/ml_interface.cc b/applications/speech_recognition/ml_interface.cc index f43337b4..ee4dba4b 100644 --- a/applications/speech_recognition/ml_interface.cc +++ b/applications/speech_recognition/ml_interface.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2023 Arm Limited and/or its affiliates +/* Copyright 2021-2024 Arm Limited and/or its affiliates * * SPDX-License-Identifier: MIT */ @@ -12,28 +12,27 @@ #include "AsrClassifier.hpp" #include "AsrResult.hpp" #include "AudioUtils.hpp" +#include "AppContext.hpp" #include "BufAttributes.hpp" #include "demo_config.h" extern "C" { #include "events.h" +#ifdef USE_ETHOS +#include "ethosu_driver.h" +#include "ethosu_npu_init.h" +#endif } #include "Labels.hpp" #include "OutputDecode.hpp" #include "mqtt_agent_task.h" #include "TensorFlowLiteMicro.hpp" -#include "UseCaseCommonUtils.hpp" #include "Wav2LetterMfcc.hpp" #include "Wav2LetterModel.hpp" #include "Wav2LetterPostprocess.hpp" #include "Wav2LetterPreprocess.hpp" #include CMSIS_device_header -#include "device_mps3.h" /* FPGA level definitions and functions. */ #include "dsp_interfaces.h" -#include "ethos-u55.h" /* Mem map and configuration definitions of the Ethos U55 */ -#include "ethosu_driver.h" /* Arm Ethos-U55 driver header */ #include "model_config.h" -#include "timer_mps3.h" /* Timer functions. */ -#include "timing_adapter.h" /* Driver header of the timing adapter */ #include "log_macros.h" #include @@ -470,114 +469,25 @@ extern struct ethosu_driver ethosu_drv; /* Default Ethos-U55 device driver */ **/ static int prvArmNpuInit(void); -/** - * @brief Defines the Ethos-U interrupt handler: just a wrapper around the default - * implementation. - **/ -extern "C" { -void ETHOS_U55_Handler(void) -{ - /* Call the default interrupt handler from the NPU driver */ - ethosu_irq_handler(ðosu_drv); -} -} - -/** - * @brief Initialises the NPU IRQ - **/ -static void prvArmNpuIrqInit(void) -{ - const IRQn_Type ethosu_irqnum = (IRQn_Type)EthosU_IRQn; - - NVIC_EnableIRQ(ethosu_irqnum); - - LogDebug( ( "EthosU IRQ#: %u, Handler: 0x%p\n", ethosu_irqnum, ETHOS_U55_Handler ) ); -} - -static int prvArmNpuTimingAdapterInit(void) -{ -#if defined(TA0_BASE) - struct timing_adapter ta_0; - struct timing_adapter_settings ta_0_settings = { - .maxr = TA0_MAXR, - .maxw = TA0_MAXW, - .maxrw = TA0_MAXRW, - .rlatency = TA0_RLATENCY, - .wlatency = TA0_WLATENCY, - .pulse_on = TA0_PULSE_ON, - .pulse_off = TA0_PULSE_OFF, - .bwcap = TA0_BWCAP, - .perfctrl = TA0_PERFCTRL, - .perfcnt = TA0_PERFCNT, - .mode = TA0_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA0_HISTBIN, - .histcnt = TA0_HISTCNT - }; - - if (ta_init(&ta_0, TA0_BASE) != 0) { - LogError( ( "TA0 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_0, &ta_0_settings); -#endif /* defined (TA0_BASE) */ - -#if defined(TA1_BASE) - struct timing_adapter ta_1; - struct timing_adapter_settings ta_1_settings = { - .maxr = TA1_MAXR, - .maxw = TA1_MAXW, - .maxrw = TA1_MAXRW, - .rlatency = TA1_RLATENCY, - .wlatency = TA1_WLATENCY, - .pulse_on = TA1_PULSE_ON, - .pulse_off = TA1_PULSE_OFF, - .bwcap = TA1_BWCAP, - .perfctrl = TA1_PERFCTRL, - .perfcnt = TA1_PERFCNT, - .mode = TA1_MODE, - .maxpending = 0, /* This is a read-only parameter */ - .histbin = TA1_HISTBIN, - .histcnt = TA1_HISTCNT - }; - - if (ta_init(&ta_1, TA1_BASE) != 0) { - LogError( ( "TA1 initialisation failed\n" ) ); - return 1; - } - - ta_set_all(&ta_1, &ta_1_settings); -#endif /* defined (TA1_BASE) */ - - return 0; -} - static int prvArmNpuInit(void) { int err = 0; - /* If the platform has timing adapter blocks along with Ethos-U55 core - * block, initialise them here. */ - // cppcheck-suppress knownConditionTrueFalse - if (0 != (err = prvArmNpuTimingAdapterInit())) { + SCB_EnableICache(); + SCB_EnableDCache(); + +#if defined(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED) + /* If the platform has timing adapter blocks along with Ethos-U core + * block, initialise them here. */ + if (0 != (err = arm_ethosu_timing_adapter_init())) { + LogError( ("Failed to init timing adapter\n") ); return err; } +#endif /* ETHOS_U_NPU_TIMING_ADAPTER_ENABLED */ - /* Initialise the IRQ */ - prvArmNpuIrqInit(); - - /* Initialise Ethos-U55 device */ - void *const ethosu_base_address = reinterpret_cast(SEC_ETHOS_U55_BASE); - - if (0 - != (err = ethosu_init(ðosu_drv, /* Ethos-U55 driver device pointer */ - ethosu_base_address, /* Ethos-U55's base address. */ - NULL, /* Pointer to fast mem area - NULL for U55. */ - 0, /* Fast mem region size. */ - 0, /* Security enable. */ - 0))) { /* Privilege enable. */ - LogError( ( "failed to initalise Ethos-U55 device\n" ) ); + // Initialize the ethos NPU + if (0 != (err = arm_ethosu_npu_init())) { + LogError( ("Failed to init arm npu\n") ); return err; } diff --git a/bsp/CMakeLists.txt b/bsp/CMakeLists.txt index a81f2190..2631f143 100644 --- a/bsp/CMakeLists.txt +++ b/bsp/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 @@ -44,6 +44,10 @@ if(ARM_CORSTONE_BSP_TARGET_PLATFORM STREQUAL "corstone300") set(NS_IMAGE_LOAD_ADDRESS 0x28040000 CACHE STRING "Non-secure user application loading address") set(S_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x10022000 CACHE STRING "Secure provisioning bundle loading address") set(NS_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x211FF000 CACHE STRING "Non-Secure provisioning bundle loading address") + set(ETHOS_U_BASE_ADDR "0x48102000" CACHE STRING "Ethos-U NPU base address" FORCE) + set(ETHOS_U_IRQN "56" CACHE STRING "Ethos-U NPU Interrupt" FORCE) + set(ETHOS_U_SEC_ENABLED "0" CACHE STRING "Ethos-U NPU Security enable" FORCE) + set(ETHOS_U_PRIV_ENABLED "0" CACHE STRING "Ethos-U NPU Privilege enable" FORCE) target_compile_definitions(arm-corstone-platform-bsp PUBLIC CORSTONE300_FVP) elseif(ARM_CORSTONE_BSP_TARGET_PLATFORM STREQUAL "corstone310") set(BL2_IMAGE_LOAD_ADDRESS 0x11000000 CACHE STRING "Bootload image loading address") @@ -51,6 +55,10 @@ elseif(ARM_CORSTONE_BSP_TARGET_PLATFORM STREQUAL "corstone310") set(NS_IMAGE_LOAD_ADDRESS 0x28040000 CACHE STRING "Non-secure user application loading address") set(S_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x11022000 CACHE STRING "Secure provisioning bundle loading address") set(NS_PROVISIONING_BUNDLE_LOAD_ADDRESS 0x213FF000 CACHE STRING "Non-Secure provisioning bundle loading address") + set(ETHOS_U_BASE_ADDR "0x40004000" CACHE STRING "Ethos-U NPU base address" FORCE) + set(ETHOS_U_IRQN "16" CACHE STRING "Ethos-U NPU Interrupt" FORCE) + set(ETHOS_U_SEC_ENABLED "0" CACHE STRING "Ethos-U NPU Security enable" FORCE) + set(ETHOS_U_PRIV_ENABLED "0" CACHE STRING "Ethos-U NPU Privilege enable" FORCE) target_compile_definitions(arm-corstone-platform-bsp PUBLIC CORSTONE310_FVP) endif() diff --git a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt index 47112966..299d37df 100644 --- a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt +++ b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2021-2023 Arm Limited and/or its affiliates +# Copyright 2021-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT @@ -12,9 +12,14 @@ include(ApplyPatches) set(PATCH_FILES_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/integration/patches") set(PATCH_FILES - "${PATCH_FILES_DIRECTORY}/0001-asm-syntax-Replace-incorrect-type-specifier.patch" - "${PATCH_FILES_DIRECTORY}/0001-platform-drivers-Remove-RTE_components-header-file.patch" "${PATCH_FILES_DIRECTORY}/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch" - "${PATCH_FILES_DIRECTORY}/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch" + "${PATCH_FILES_DIRECTORY}/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch" + "${PATCH_FILES_DIRECTORY}/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch" + "${PATCH_FILES_DIRECTORY}/0004-Fix-tensorflow.cmake-env-var-escape.patch" ) iot_reference_arm_corstone3xx_apply_patches("${ml_embedded_evaluation_kit_SOURCE_DIR}" "${PATCH_FILES}") + +set(TF_PATCH_FILES + "${PATCH_FILES_DIRECTORY}/0001-tensorflow-kiss_fft_wrapper-include-fix.patch" +) +iot_reference_arm_corstone3xx_apply_patches("${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow" "${TF_PATCH_FILES}") diff --git a/components/ai/ml_embedded_evaluation_kit/integration/README.md b/components/ai/ml_embedded_evaluation_kit/integration/README.md index 71da58c5..d1c5f2cf 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/README.md +++ b/components/ai/ml_embedded_evaluation_kit/integration/README.md @@ -8,64 +8,30 @@ targeted for Arm Cortex-M CPUs and Arm Ethos-U NPU. Available build options and their default values are documented in the [Build options] section of the ML Embedded Evaluation Kit documentation. +The minimum set of options that need to be defined are the following: -If you would like to override any defaults, you can specify a list of options in `ML_CMAKE_ARGS`. For example: +* `ETHOS_U_NPU_ID` +* `ETHOS_U_NPU_CONFIG_ID` +* `ETHOSU_TARGET_NPU_CONFIG` +* `ETHOS_U_NPU_MEMORY_MODE` -```cmake -set(ML_CMAKE_ARGS - -D TARGET_SUBSYSTEM=sse-310 -) -``` - -The above code snippet selects the `sse-310` target subsystem for Corstone-310 instead of the default `sse-300` for Corstone-300. - -> :bulb: This must be done *before* your application's `CMakeLists.txt` adds the IoT Reference Integration for Arm Corstone-3xx. - -## Build targets - -The ML Embedded Evaluation Kit contains a number of components. Instead of building everything, you can save build time -and disk space by specifying a list of components you need in `ML_TARGETS`. For example: - -```cmake -set(ML_TARGETS kws) -``` - -The above code snippet limits the build to the `kws` (keyword spotting) use case and its dependencies. - -> :bulb: This must be done *before* your application's `CMakeLists.txt` adds the IoT Reference Integration for Arm Corstone-3xx. +In addition to the above, `ML_USE_CASE` build option describes which TFLite model and API libraries should be generated. +`ML_MODEL` is the name of the CMake module under `components/ai/ml_embedded_evaluation_kit/integration/cmake/model/` +that handles the generation of C++ source code model from the TFlite model. ## Using in your application -The ML Embedded Evaluation Kit outputs a number of libraries as `*.a` archives in its build directory -(`${ml-embedded-evaluation-kit_BINARY_DIR}/lib`), such as -`${ml-embedded-evaluation-kit_BINARY_DIR}/lib/libtensorflow-microlite.a` which is TensorFlow Lite for Microcontrollers. -You can link your application against the library archives as needed. - -In order for your application to access the API headers for the libraries, you also need to make various paths in the ML -Embedded Evaluation Kit's source directory (`${ml-embedded-evaluation-kit_SOURCE_DIR}`) available to your application as -include directories. - -For example, the IoT Reference Integration for Arm Corstone-3xx repository creates helper libraries named `ml-kit`, `ml-kit-kws` and `ml-kit-asr` in -its [`libraries/ml-kit/CMakeLists.txt`][fri-ml-kit-cmake], with library archives and include directories added to those helper -libraries. The [keyword-detection] and [speech-recognition] applications link the helper libraries they need, in order to inherit the library -archives for linking and include directories for API headers. +After setting the build options, include [SetupMlEmbeddedEvaluationKitLibraries.cmake] to handle the configuration of +necessary ML libraries and running Vela compiler on the default models for the set `ETHOSU_TARGET_NPU_CONFIG`. -> :bulb: Which library archives and include directories are relevant depends on a number of factors, including -> -> * the [configuration](#configuration) set via `ML_CMAKE_ARGS` -> * the [build targets](#build-targets) set via `ML_TARGETS` -> * which libraries your application actually uses -> -> For this reason, the IoT Reference Integration for Arm Corstone-3xx does not provide helper targets in `components` that cover all -> possible uses cases, therefore the user application needs to specify its own list of library archives and include -> paths to take from the ML Embedded Evaluation Kit. +In order for your application to access the API headers for the libraries, you need to link to the two resulting static +libraries: `${ML_USE_CASE}_api`, `${ML_USE_CASE}_model`. +These libraries provide the necessary include paths to the respective APIs from the ML Embedded Evaluation Kit. ## Documentation For more information about the ML Embedded Evaluation Kit, see the [ML Embedded Evaluation Kit documentation]. [Build options]: https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/HEAD/docs/sections/building.md#build-options -[fri-ml-kit-cmake]: ../../../../applications/libraries/ml-kit/CMakeLists.txt -[keyword-detection]: ../../../../applications/keyword_detection/CMakeLists.txt -[speech-recognition]: ../../../../applications/speech_recognition/CMakeLists.txt +[SetupMlEmbeddedEvaluationKitLibraries.cmake]: ../../../../components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake [ML Embedded Evaluation Kit documentation]: https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/HEAD/docs/documentation.md diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/BuildMlEmbeddedEvaluationKit.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/BuildMlEmbeddedEvaluationKit.cmake deleted file mode 100644 index b0e8fd2f..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/cmake/BuildMlEmbeddedEvaluationKit.cmake +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2021-2023 Arm Limited and/or its affiliates -# -# SPDX-License-Identifier: MIT - -include(ExternalProject) - -# Use provided toolchain file as part of ml_embedded_evaluation_kit. -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - set(ml_toolchain_file "scripts/cmake/toolchains/bare-metal-gcc.cmake") -elseif(CMAKE_C_COMPILER_ID STREQUAL "ARMClang") - set(ml_toolchain_file "scripts/cmake/toolchains/bare-metal-armclang.cmake") -else() - message(FATAL_ERROR "Unsupported compiler: ${CMAKE_C_COMPILER_ID}") -endif() - -ExternalProject_Add( - ml_embedded_evaluation_kit-build - - DOWNLOAD_COMMAND "" - SOURCE_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR} - - USES_TERMINAL_CONFIGURE ON - USES_TERMINAL_BUILD ON - - BUILD_ALWAYS ON - - INSTALL_COMMAND "" - - CMAKE_ARGS - -DCMAKE_TOOLCHAIN_FILE=/${ml_toolchain_file} - ${ML_CMAKE_ARGS} - - PATCH_COMMAND - ${ML_PATCH_COMMAND} - - BUILD_COMMAND - ${CMAKE_COMMAND} --build --target ${ML_TARGETS} -) - -set(ML_RESOURCES_SET_UP_ARGS - "--additional-ethos-u-config-name=${ETHOSU_TARGET_NPU_CONFIG}" -) -ExternalProject_Add_Step(ml_embedded_evaluation_kit-build - download_model - COMMAND - python3 ${ml_embedded_evaluation_kit_SOURCE_DIR}/set_up_default_resources.py ${ML_RESOURCES_SET_UP_ARGS} - DEPENDERS - configure - USES_TERMINAL ON -) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake index c4775afb..a2e04ac1 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/ConvertAudioSourceToCode.cmake @@ -1,12 +1,10 @@ -# Copyright 2023 Arm Limited and/or its affiliates +# Copyright 2023-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: MIT # Convert audio clip to C file function(iot_reference_arm_corstone3xx_convert_audio_source_to_code audio_path generated_path) add_custom_target(convert-audio - DEPENDS - ml_embedded_evaluation_kit-build BYPRODUCTS ${generated_path}/InputFiles.cc ${generated_path}/test.cc diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake new file mode 100644 index 00000000..9e6ae649 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake @@ -0,0 +1,136 @@ +# Copyright 2021-2024 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: MIT + +include(ExternalProject) + +########################### +# Setup for the ML target # +########################### +#set(LOG_LEVEL LOG_LEVEL_TRACE) + +# External repositories +set(CMSIS_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis") +set(CMSIS_DSP_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis-dsp") +set(CMSIS_DSP_INC_DIR "${CMSIS_DSP_SRC_PATH}/Include") +set(CMSIS_CORE_INC_DIR "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/CMSIS/Core/Include") +set(CMSIS_NN_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/cmsis-nn") +set(TENSORFLOW_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/tensorflow") +set(ETHOS_U_NPU_DRIVER_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/dependencies/core-driver") + +# Extra arguments for setting up default resources (for vela optimizer) +set(ML_RESOURCES_SET_UP_ARGS + "--additional-ethos-u-config-name=${ETHOSU_TARGET_NPU_CONFIG}" +) + +# Tensorflow settings +set(TENSORFLOW_LITE_MICRO_BUILD_TYPE "release_with_logs") +set(TENSORFLOW_LITE_MICRO_CLEAN_DOWNLOADS OFF) +set(TENSORFLOW_LITE_MICRO_CLEAN_BUILD ON) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tensorflow-microlite) + +# Set up default resources. This downloads the TF-Lite models and optimizes them for the target +# Note: This step is using pip install in its venv setup, which involves installing Vela compiler. +# If it is not already installed, it will use CC to compile it to the host machine, but the toolchain file overwrites the CC env. +set(RESOURCES_OUTFILE "${ml_embedded_evaluation_kit_SOURCE_DIR}/resources_downloaded/resources_downloaded_metadata.json") +if(NOT EXISTS "${RESOURCES_OUTFILE}") + execute_process( + COMMAND ${CMAKE_COMMAND} -E env CC=gcc ${PYTHON} ${ml_embedded_evaluation_kit_SOURCE_DIR}/set_up_default_resources.py ${ML_RESOURCES_SET_UP_ARGS} + RESULT_VARIABLE return_code + ) + if (NOT return_code EQUAL "0") + message(FATAL_ERROR "Failed to set up default resources.") + endif () +endif() + +# Setup virtualenv (done by setup_source_generator()) +set(CMAKE_SCRIPTS_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR}/scripts/cmake) +include(${CMAKE_SCRIPTS_DIR}/source_gen_utils.cmake) +set(RESOURCES_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR}/resources_downloaded) +set(SCRIPTS_DIR ${ml_embedded_evaluation_kit_SOURCE_DIR}/scripts) +setup_source_generator() + +# Used by tensorflow.cmake +# Function to check if a variable is defined, and throw +# an error if it is not. +function(assert_defined var_name) + if (NOT DEFINED ${var_name}) + message(FATAL_ERROR "ERROR: ${var_name} is undefined!") + endif() +endfunction() + +# Note: tensorflow.cmake modifies these variables. Save the current values and restore after. +set(TEMP_CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) +set(TEMP_ETHOS_U_NPU_ID ${ETHOS_U_NPU_ID}) +# Note: tensorflow.cmake only has support up to cortex-m55. +set(CMAKE_SYSTEM_PROCESSOR "cortex-m55") +include(${CMAKE_SCRIPTS_DIR}/tensorflow.cmake) +# Note: tensorflow.cmake variable restoration. +set(CMAKE_SYSTEM_PROCESSOR ${TEMP_CMAKE_SYSTEM_PROCESSOR}) +set(ETHOS_U_NPU_ID ${TEMP_ETHOS_U_NPU_ID}) + +# Manually add libs +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/log ${CMAKE_BINARY_DIR}/log) +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/math ${CMAKE_BINARY_DIR}/math) +target_include_directories(arm_math PUBLIC + ${CMSIS_DSP_INC_DIR} + ${CMSIS_CORE_INC_DIR} +) +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/common ${CMAKE_BINARY_DIR}/common_api) +if (ETHOS_U_NPU_ENABLED) + add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/hal/source/components/npu ${CMAKE_BINARY_DIR}/npu) + # ethos_u_npu library needs the CPU Header (CMSIS_device_header) + target_link_libraries(ethos_u_npu PUBLIC arm-corstone-platform-bsp) +endif() + +# Add the dependency on tensorflow_build (defined in tensorflow.cmake) +add_dependencies(common_api tensorflow_build) +target_include_directories(common_api PUBLIC ${TFLITE_MICRO_PATH}) +target_compile_options(common_api PUBLIC $<$:-std=c++14>) + +# Add relevant use case API +add_subdirectory(${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/api/use_case/${ML_USE_CASE} ${CMAKE_BINARY_DIR}/${ML_USE_CASE}_api) + +# Include directories for application module: +set(APPLICATION_INCLUDE_DIRS ${ml_embedded_evaluation_kit_SOURCE_DIR}/source/application/main/include) +target_include_directories(${ML_USE_CASE}_api PUBLIC + ${APPLICATION_INCLUDE_DIRS} +) + +# Generate use case C model from optimized tflite file +# Generate tflite model code +set(DEFAULT_MODEL_DIR ${RESOURCES_DIR}/${ML_USE_CASE}) +set(SRC_GEN_DIR ${CMAKE_BINARY_DIR}/generated/${ML_USE_CASE}/src) +set(INC_GEN_DIR ${CMAKE_BINARY_DIR}/generated/${ML_USE_CASE}/include) + +# Remove old files and recreate dirs +file(REMOVE_RECURSE ${SRC_GEN_DIR} ${INC_GEN_DIR}) +file(MAKE_DIRECTORY ${SRC_GEN_DIR} ${INC_GEN_DIR}) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/model) +include(${ML_MODEL}) + +file(GLOB_RECURSE SRC_GEN + "${SRC_GEN_DIR}/*.cc" + "${SRC_GEN_DIR}/*.cpp" + "${SRC_GEN_DIR}/*.c") + +set(UC_LIB_NAME ${ML_USE_CASE}_model) + +# Consolidated application static lib: +add_library(${UC_LIB_NAME} STATIC + ${SRC_GEN}) + +target_include_directories(${UC_LIB_NAME} PUBLIC + ${APPLICATION_INCLUDE_DIRS} + ${INC_GEN_DIR}) + +string(TOUPPER ${ML_USE_CASE} ML_USE_CASE_UPPER) +target_compile_definitions(${UC_LIB_NAME} PUBLIC + "ACTIVATION_BUF_SZ=${${ML_USE_CASE_UPPER}_ACTIVATION_BUF_SZ}") + +target_link_libraries(${UC_LIB_NAME} PUBLIC + $<$:ethos_u_npu> + tensorflow-lite-micro + common_api +) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateASRModel.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateASRModel.cmake new file mode 100644 index 00000000..3450a150 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateASRModel.cmake @@ -0,0 +1,54 @@ +#---------------------------------------------------------------------------- +# SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#---------------------------------------------------------------------------- +# This file is based on https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/refs/tags/23.08/source/use_case/asr/usecase.cmake + +set(ASR_LABELS_TXT_FILE + ${ml_embedded_evaluation_kit_SOURCE_DIR}/resources/asr/labels/labels_wav2letter.txt) + +set(ASR_LABELS_CPP_FILE Labels) +generate_labels_code( + INPUT "${ASR_LABELS_TXT_FILE}" + DESTINATION_SRC ${SRC_GEN_DIR} + DESTINATION_HDR ${INC_GEN_DIR} + OUTPUT_FILENAME "${ASR_LABELS_CPP_FILE}" +) + +set(ASR_ACTIVATION_BUF_SZ 0x00200000) +set(ASR_MODEL_SCORE_THRESHOLD 0.5) + +if (ETHOS_U_NPU_ENABLED) + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) +else() + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/tiny_wav2letter_pruned_int8.tflite) +endif() + +set(EXTRA_MODEL_CODE + "/* Model parameters for asr */" + "extern const int g_FrameLength = 512" + "extern const int g_FrameStride = 160" + "extern const int g_ctxLen = 98" + "extern const float g_ScoreThreshold = ${ASR_MODEL_SCORE_THRESHOLD}" + ) + +set(ASR_MODEL_TFLITE_PATH ${DEFAULT_MODEL_PATH}) + +generate_tflite_code( + MODEL_PATH ${ASR_MODEL_TFLITE_PATH} + DESTINATION ${SRC_GEN_DIR} + EXPRESSIONS ${EXTRA_MODEL_CODE} + NAMESPACE "arm" "app" "asr" +) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateKWSModel.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateKWSModel.cmake new file mode 100644 index 00000000..641d2033 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/model/GenerateKWSModel.cmake @@ -0,0 +1,53 @@ +#---------------------------------------------------------------------------- +# SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#---------------------------------------------------------------------------- +# This file is based on https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ml-embedded-evaluation-kit/+/refs/tags/23.08/source/use_case/kws/usecase.cmake + +set(KWS_LABELS_TXT_FILE + ${ml_embedded_evaluation_kit_SOURCE_DIR}/resources/kws/labels/micronet_kws_labels.txt) + +set(KWS_LABELS_CPP_FILE Labels) +generate_labels_code( + INPUT "${KWS_LABELS_TXT_FILE}" + DESTINATION_SRC ${SRC_GEN_DIR} + DESTINATION_HDR ${INC_GEN_DIR} + OUTPUT_FILENAME "${KWS_LABELS_CPP_FILE}" +) + +set(KWS_ACTIVATION_BUF_SZ 0x00100000) +set(KWS_MODEL_SCORE_THRESHOLD 0.7) + +if (ETHOS_U_NPU_ENABLED) + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/kws_micronet_m_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) +else() + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/kws_micronet_m.tflite) +endif() + +set(EXTRA_MODEL_CODE + "/* Model parameters for KWS */" + "extern const int g_FrameLength = 640" + "extern const int g_FrameStride = 320" + "extern const float g_ScoreThreshold = ${KWS_MODEL_SCORE_THRESHOLD}" + ) + +set(KWS_MODEL_TFLITE_PATH ${DEFAULT_MODEL_PATH}) + +generate_tflite_code( + MODEL_PATH ${KWS_MODEL_TFLITE_PATH} + DESTINATION ${SRC_GEN_DIR} + EXPRESSIONS ${EXTRA_MODEL_CODE} + NAMESPACE "arm" "app" "kws" +) diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-asm-syntax-Replace-incorrect-type-specifier.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-asm-syntax-Replace-incorrect-type-specifier.patch deleted file mode 100644 index e8d1b2e5..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-asm-syntax-Replace-incorrect-type-specifier.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 37bb50603a05c16767b3020c0f7b5be030f2f160 Mon Sep 17 00:00:00 2001 -From: Ahmed Ismail -Date: Wed, 27 Dec 2023 15:15:05 +0000 -Subject: [PATCH] asm-syntax: Replace incorrect type specifier - -In C++ files we should use __asm rather than __ASM as __ASM -is not known as a correct C++ type specifier. - -Signed-off-by: Ahmed Ismail ---- - source/application/main/Main.cc | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/source/application/main/Main.cc b/source/application/main/Main.cc -index bbe35d9..d36d54f 100644 ---- a/source/application/main/Main.cc -+++ b/source/application/main/Main.cc -@@ -28,7 +28,7 @@ - extern void main_loop(); - - #if defined(__ARMCC_VERSION) && (__ARMCC_VERSION >= 6010050) --__ASM(" .global __ARM_use_no_argv\n"); -+__asm(" .global __ARM_use_no_argv\n"); - #endif - - /* Print application information. */ --- -2.34.1 - diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-platform-drivers-Remove-RTE_components-header-file.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-platform-drivers-Remove-RTE_components-header-file.patch deleted file mode 100644 index 379f2f14..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-platform-drivers-Remove-RTE_components-header-file.patch +++ /dev/null @@ -1,31 +0,0 @@ -From d4b0f895f9d621e43027cc3fa2c8d521128e418a Mon Sep 17 00:00:00 2001 -From: Ahmed Ismail -Date: Wed, 27 Dec 2023 13:35:16 +0000 -Subject: [PATCH] platform-drivers: Remove RTE_components header file - -RTE_Components header file is being included for both Corstone-300 -and Corstone-310 platforms. However in the case of Corstone-300 -platofrm, this file is not available anymore in the latest version of -cmsis-packs Hence, this header file inclusion is removed in the new -patch file. - -Signed-off-by: Ahmed Ismail ---- - source/hal/source/platform/mps3/include/platform_drivers.h | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/source/hal/source/platform/mps3/include/platform_drivers.h b/source/hal/source/platform/mps3/include/platform_drivers.h -index fc7dcc5..6a017bb 100644 ---- a/source/hal/source/platform/mps3/include/platform_drivers.h -+++ b/source/hal/source/platform/mps3/include/platform_drivers.h -@@ -21,7 +21,6 @@ - #include "log_macros.h" /* Logging related helpers. */ - - /* Platform components */ --#include "RTE_Components.h" /* For CPU related defintiions */ - #include "timer_mps3.h" /* Timer functions. */ - #include "user_input.h" /* User input function */ - #include "lcd_img.h" /* LCD functions. */ --- -2.34.1 - diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch index 87c3e3ce..7686195b 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch @@ -1,7 +1,7 @@ -From 34238f5d40992d6f2adbda9d793bd51c6b5ea985 Mon Sep 17 00:00:00 2001 +From e691803e130d1aced38c21e98c41ce40bda44ffe Mon Sep 17 00:00:00 2001 From: Hugues Kamba-Mpiana Date: Tue, 19 Dec 2023 14:09:27 +0000 -Subject: [PATCH] resources: Replace model wav2letter with tiny_wav2letter +Subject: [PATCH 1/4] resources: Replace model wav2letter with tiny_wav2letter `wav2letter` model needs singificant amount of memory (around 14Mb), where as `tiny_wav2letter` needs around 4Mb. @@ -10,13 +10,13 @@ the `asr` and `kws` use cases. Signed-off-by: Hugues Kamba-Mpiana --- - set_up_default_resources.py | 18 +++++++++--------- + set_up_default_resources.py | 16 ++++++++-------- source/use_case/asr/usecase.cmake | 4 ++-- source/use_case/kws_asr/usecase.cmake | 4 ++-- - 3 files changed, 13 insertions(+), 13 deletions(-) + 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/set_up_default_resources.py b/set_up_default_resources.py -index b442432..a0d6149 100755 +index b442432..2cb987d 100755 --- a/set_up_default_resources.py +++ b/set_up_default_resources.py @@ -51,16 +51,16 @@ json_uc_res = [ @@ -97,5 +97,5 @@ index de84bfd..9bfbbd0 100644 USER_OPTION(${use_case}_MODEL_TFLITE_PATH_KWS "NN models file to be used for KWS in the evaluation application. Model files must be in tflite format." -- -2.34.1 +2.40.1 diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-tensorflow-kiss_fft_wrapper-include-fix.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-tensorflow-kiss_fft_wrapper-include-fix.patch new file mode 100644 index 00000000..8051962b --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-tensorflow-kiss_fft_wrapper-include-fix.patch @@ -0,0 +1,25 @@ +From d799ead836ea893cbc6eb8abe98dccfd6aac2e4a Mon Sep 17 00:00:00 2001 +From: Gabor Abonyi +Date: Tue, 16 Jan 2024 13:40:22 +0100 +Subject: [PATCH] Add stdint.h to kiss_fft_common.h wrapper + +Signed-off-by: Gabor Abonyi +--- + signal/src/kiss_fft_wrappers/kiss_fft_common.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/signal/src/kiss_fft_wrappers/kiss_fft_common.h b/signal/src/kiss_fft_wrappers/kiss_fft_common.h +index 75f3dcda..8dcaaffa 100644 +--- a/signal/src/kiss_fft_wrappers/kiss_fft_common.h ++++ b/signal/src/kiss_fft_wrappers/kiss_fft_common.h +@@ -34,6 +34,7 @@ limitations under the License. + + #include + #include ++#include + #include + #include + #include +-- +2.40.1 + diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch similarity index 93% rename from components/ai/ml_embedded_evaluation_kit/integration/patches/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch rename to components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch index 3756e4be..38a92636 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0002-toolchain-Fix-Arm-GNU-Toolchain-13.2-Rel1-compatibil.patch @@ -1,7 +1,7 @@ -From 28c8e0755b91ae4a8628bfcc1b1ca39b4c494bc5 Mon Sep 17 00:00:00 2001 +From 60556c32a8957dd52b5dc7b85a8cef6550f5ac28 Mon Sep 17 00:00:00 2001 From: Hugues Kamba-Mpiana Date: Tue, 19 Dec 2023 13:17:15 +0000 -Subject: [PATCH] toolchain: Fix Arm GNU Toolchain 13.2 Rel1 compatibility +Subject: [PATCH 2/4] toolchain: Fix Arm GNU Toolchain 13.2 Rel1 compatibility Arm GNU Toolchain 13.2 Rel1 no longer includes transitively. Include `cstdint` explicitely where needed. @@ -65,5 +65,5 @@ index 279acf0..eb28a8f 100644 namespace app { -- -2.34.1 +2.40.1 diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch new file mode 100644 index 00000000..0c9b1343 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0003-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch @@ -0,0 +1,40 @@ +From 75e29cf7fd59978d06d8594efd2c883c40cd56e1 Mon Sep 17 00:00:00 2001 +From: Gabor Abonyi +Date: Tue, 16 Jan 2024 14:48:44 +0100 +Subject: [PATCH 3/4] Use CMSIS_device_header instead of RTE_Components.h + +Signed-off-by: Gabor Abonyi +--- + source/hal/source/components/npu/ethosu_cpu_cache.c | 2 +- + source/hal/source/components/npu/ethosu_npu_init.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/source/hal/source/components/npu/ethosu_cpu_cache.c b/source/hal/source/components/npu/ethosu_cpu_cache.c +index d5f5e47..42fedba 100644 +--- a/source/hal/source/components/npu/ethosu_cpu_cache.c ++++ b/source/hal/source/components/npu/ethosu_cpu_cache.c +@@ -17,7 +17,7 @@ + + #include "ethosu_cpu_cache.h" + +-#include "RTE_Components.h" /* For CPU related defintiions */ ++#include CMSIS_device_header /* For CPU related defintiions */ + #include "ethosu_driver.h" /* Arm Ethos-U driver header */ + #include "log_macros.h" /* Logging macros */ + +diff --git a/source/hal/source/components/npu/ethosu_npu_init.c b/source/hal/source/components/npu/ethosu_npu_init.c +index dbee2ff..0bce70e 100644 +--- a/source/hal/source/components/npu/ethosu_npu_init.c ++++ b/source/hal/source/components/npu/ethosu_npu_init.c +@@ -17,7 +17,7 @@ + + #include "ethosu_npu_init.h" + +-#include "RTE_Components.h" /* For CPU related defintiions */ ++#include CMSIS_device_header /* For CPU related defintiions */ + #include "log_macros.h" /* Logging functions */ + + #include "ethosu_mem_config.h" /* Arm Ethos-U memory config */ +-- +2.40.1 + diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch new file mode 100644 index 00000000..53c0a78c --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0004-Fix-tensorflow.cmake-env-var-escape.patch @@ -0,0 +1,38 @@ +From 2d22af627e51f62e95968cc75f51ff1238cf86ec Mon Sep 17 00:00:00 2001 +From: Gabor Abonyi +Date: Thu, 11 Jan 2024 14:16:57 +0100 +Subject: [PATCH 4/4] Fix tensorflow.cmake env var escape + +Signed-off-by: Gabor Abonyi +--- + scripts/cmake/tensorflow.cmake | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/scripts/cmake/tensorflow.cmake b/scripts/cmake/tensorflow.cmake +index 8d52765..a028055 100644 +--- a/scripts/cmake/tensorflow.cmake ++++ b/scripts/cmake/tensorflow.cmake +@@ -78,9 +78,9 @@ endif() + + if (TENSORFLOW_LITE_MICRO_CLEAN_DOWNLOADS) + message(STATUS "Refreshing TensorFlow Lite Micro's third party downloads...") ++ set(ENV{PATH} "${ENV_PATH}") + execute_process( +- COMMAND ${CMAKE_COMMAND} -E env PATH=${ENV_PATH} +- make -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile clean_downloads third_party_downloads ++ COMMAND make -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile clean_downloads third_party_downloads + RESULT_VARIABLE return_code + WORKING_DIRECTORY ${TENSORFLOW_SRC_PATH}) + if (NOT return_code EQUAL "0") +@@ -102,7 +102,7 @@ message(STATUS "TensorFlow Lite Micro build to be called for these targets: ${MA + add_custom_target(tensorflow_build ALL + + # Command to build the TensorFlow Lite Micro library +- COMMAND ${CMAKE_COMMAND} -E env PATH=${ENV_PATH} ++ COMMAND ${CMAKE_COMMAND} -E env PATH="${ENV_PATH}" + make -j${PARALLEL_JOBS} -f ${TENSORFLOW_LITE_MICRO_PATH}/tools/make/Makefile ${MAKE_TARGETS_LIST} + TARGET_TOOLCHAIN_ROOT=${TENSORFLOW_LITE_MICRO_TARGET_TOOLCHAIN_ROOT} + TOOLCHAIN=${TENSORFLOW_LITE_MICRO_TOOLCHAIN} +-- +2.40.1 + diff --git a/release_changes/202401161458.change b/release_changes/202401161458.change new file mode 100644 index 00000000..468e757a --- /dev/null +++ b/release_changes/202401161458.change @@ -0,0 +1,3 @@ +ml-eval-kit: Require apps to include build CMake module +ml-eval-kit: Remove default NPU config list patch +ml-eval-kit: Build individual libraries