applications: Add object-detection example

Object detection example is an ML vision example on the Cortex-M systems. Currently running only on Corstone-315 to demonstrate the usage of ISP, NPU, HDLCD together. Signed-off-by: Gabor Abonyi <[email protected]>
FreeRTOS · Mar 9, 2024 · f0f739c · f0f739c
1 parent 5a2c4b4
commit f0f739c
Show file tree

Hide file tree

Showing 48 changed files with 8,870 additions and 39 deletions.
diff --git a/README.md b/README.md
@@ -24,6 +24,9 @@ CPU and Ethos NPU, alongside a range of other components in a scalable and
 flexible reference package. This enables designers to build secure,
 AI-capable SoCs faster.
 
+* [Corstone-315](https://developer.arm.com/Processors/Corstone-315)
+  * Arm Cortex-M85 CPU, Ethos-U65 NPU and Mali-C55 ISP
+  * Supported versions: 11.24.13 and above
 * [Corstone-310](https://developer.arm.com/Processors/Corstone-310)
   * Arm Cortex-M85 CPU and Ethos-U55 NPU
   * Supported versions: 11.24.13 and above
@@ -65,6 +68,8 @@ This reference integration contains following two examples:
     * Demonstrates detecting keywords from an audio source using Machine Learning.
 * [Speech Recognition](docs/speech_recognition.md)
     * Demonstrates detecting sentences from an audio source using Machine Learning.
+* [Object Detection](docs/object_detection.md)
+    * Demonstrates detection of faces from image source using Machine Learning.
 
 The Keyword-Detection and Speech-Recognition applications demonstrate [secure connectivity](#secure-tls-connection)
 to AWS IoT core using [Mbed TLS](#mbed-tls), [PKCS#11 PSA Shim](#pkcs11-psa-shim) and
@@ -152,6 +157,11 @@ This implementation maps the AWS OTA PAL APIs to the PSA Firmware Update and
 PSA Cryptography APIs. The writing, verification and activation of the update
 image are protected by the PSA secure services.
 
+### Mali-C55 Versatile Image Signal Processor for Computer Vision and Smart Display Systems
+
+Implementation of [Arm® Mali™-C55 bare-metal driver](https://gitlab.arm.com/iot/m-class/drivers/isp_mali-c55),
+that demonstrates the usage of the Mali-C55 ISP with the Corstone M85 processor.
+
 ## Contributing
 
 See [CONTRIBUTING](CONTRIBUTING.md) for more information.

diff --git a/applications/object_detection/CMakeLists.txt b/applications/object_detection/CMakeLists.txt
@@ -0,0 +1,177 @@
+# Copyright 2023-2024 Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+set(ML_INFERENCE_ENGINE "ETHOS" CACHE STRING "Machine Learning inference engine (ETHOS)")
+
+set(AUDIO_SOURCE "ROM" CACHE STRING "Source of audio data (ROM | VSI)")
+
+# From: ota-for-aws-iot-embedded-sdk/source/include/ota_appversion32.h
+# struct version
+# {
+#     uint8_t major;  /*!< @brief Major version number of the firmware (X in firmware version X.Y.Z). */
+#     uint8_t minor;  /*!< @brief Minor version number of the firmware (Y in firmware version X.Y.Z). */
+#
+#     uint16_t build; /*!< @brief Build of the firmware (Z in firmware version X.Y.Z). */
+# } x;                /*!< @brief Version number of the firmware. */
+
+# AWS OTA client does not use the SemVer PATCH version.
+# Because of this, if only PATCH version is changed then the OTA will be rejected
+# due to same firmware version.
+# We will therefore change the build version from TF-M.
+set(MCUBOOT_IMAGE_VERSION_NS "0.0.1+10")
+set(MCUBOOT_IMAGE_VERSION_NS_UPDATE "0.0.1+20")
+
+if (${ML_INFERENCE_ENGINE} STREQUAL "ETHOS")
+    set(ETHOS_U_NPU_ENABLED ON)
+    set(ETHOS_U_NPU_TIMING_ADAPTER_ENABLED OFF)
+else()
+    set(ETHOS_U_NPU_ENABLED OFF)
+endif()
+set(ML_USE_CASE "object_detection")
+set(ML_MODEL "GenerateObjectDetectionModel")
+set(ML_USE_CASE_RESOURCES_FILE "${CMAKE_CURRENT_LIST_DIR}/resources/use_case_resources.json")
+set(TFM_PLATFORM_UPGRADE_STRATEGY "SWAP_USING_SCRATCH")
+set(TFM_PLATFORM_CONFIRM_IMAGE ON)
+
+# Trusted Firmware-M setup
+set(TFM_CMAKE_APP_ARGS
+    -DPROJECT_CONFIG_HEADER_FILE=${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/applications/object_detection/configs/tfm_config/project_config.h
+    -DMCUBOOT_CONFIRM_IMAGE=${TFM_PLATFORM_CONFIRM_IMAGE}
+    -DMCUBOOT_UPGRADE_STRATEGY=${TFM_PLATFORM_UPGRADE_STRATEGY}
+    -DMCUBOOT_IMAGE_VERSION_NS=${MCUBOOT_IMAGE_VERSION_NS}
+    -DCONFIG_TFM_HALT_ON_CORE_PANIC=ON
+    -DMCUBOOT_DATA_SHARING=ON
+    -DPLATFORM_HAS_FIRMWARE_UPDATE_SUPPORT=ON
+    -DTFM_PARTITION_FIRMWARE_UPDATE=ON
+    -DTFM_PARTITION_LOG_LEVEL=TFM_PARTITION_LOG_LEVEL_INFO
+)
+
+project(object-detection LANGUAGES C CXX)
+
+# Set global optimization level to reduce code size while keeping the debug experience.
+if(${CMAKE_C_COMPILER_ID} STREQUAL "GNU")
+    add_compile_options(-Og)
+elseif(${CMAKE_C_COMPILER_ID} STREQUAL "ARMClang")
+    add_compile_options(-O1)
+endif()
+
+
+add_subdirectory(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR} ${CMAKE_BINARY_DIR}/iot_reference_arm_corstone3xx)
+
+list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/aws_iot/cmake)
+list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/ai/ml_embedded_evaluation_kit/integration/cmake)
+list(APPEND CMAKE_MODULE_PATH ${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/components/security/trusted_firmware-m/integration/cmake)
+include(SetupMlEmbeddedEvaluationKitLibraries)
+include(GenerateAWSUpdateDigestAndSignature)
+include(MergeTfmImages)
+include(SignTfmImage)
+
+add_subdirectory(configs ${CMAKE_BINARY_DIR}/Config)
+add_subdirectory(isp)
+add_subdirectory(../helpers helpers)
+
+add_executable(object-detection
+    blink_task.c
+    main.c
+    ml_interface.cc
+)
+
+target_include_directories(object-detection
+    PUBLIC
+        ../helpers/provisioning
+        ${CMAKE_CURRENT_LIST_DIR}
+)
+
+if (${ML_INFERENCE_ENGINE} STREQUAL "ETHOS")
+    target_compile_definitions(object-detection PRIVATE USE_ETHOS)
+    target_sources(object-detection PRIVATE ../helpers/ethosu/src/ethosu_platform_adaptation.c)
+endif()
+
+target_compile_options(object-detection
+    PRIVATE
+        $<$<COMPILE_LANGUAGE:CXX>:-std=c++14>
+        $<$<COMPILE_LANGUAGE:C>:-std=c99>
+)
+
+#FIXME USE RTE_USART2 for control tool
+target_compile_definitions(arm-corstone-platform-bsp
+    PUBLIC
+        RTE_USART1=1
+)
+
+# Trusted Firmware-M must be built before the application, because
+# the application depends on the NS interface and the BL2 signing scripts,
+# both of which are generated as parts of the Trusted Firmware-M build process.
+add_dependencies(object-detection trusted_firmware-m-build)
+# The provision data must be built before the application because
+# it provides credentials to connect to AWS.
+add_dependencies(object-detection provisioning_data_bin)
+
+target_link_libraries(object-detection
+    PRIVATE
+        arm-2d
+        backoff-algorithm
+        connectivity-stack
+        coremqtt
+        coremqtt-agent
+        corepkcs11
+        freertos_kernel
+        freertos-ota-pal-psa
+        fri-bsp
+        helpers-events
+        helpers-hdlcd
+        isp-config
+        isp_platform_driver
+        mbedtls
+        mbedtls-threading-freertos
+        ota-for-aws-iot-embedded-sdk
+        provisioning-lib
+        tfm-ns-interface
+        toolchain-override
+        object_detection_api
+        object_detection_model
+)
+
+include(${IOT_REFERENCE_ARM_CORSTONE3XX_SOURCE_DIR}/bsp/cmake/SetLinkerOptions.cmake)
+set_linker_script(object-detection)
+
+# The non-secure application image should be padded while being signed
+# Hence, passing "TRUE" as the input parameter to the pad option of sign function.
+iot_reference_arm_corstone3xx_tf_m_sign_image(
+    object-detection
+    object-detection_signed
+    ${MCUBOOT_IMAGE_VERSION_NS}
+    TRUE
+)
+
+# The update image is not padded to fill the whole slot (no --pad), because
+# 1) the image to download is smaller without padding
+# 2) the trailer that keeps track of boot and update statuses should not be overwritten
+# Hence, passing "FALSE" as the input parameter for the pad option to the sign function.
+iot_reference_arm_corstone3xx_tf_m_sign_image(
+    object-detection
+    object-detection-update_signed
+    ${MCUBOOT_IMAGE_VERSION_NS_UPDATE}
+    FALSE
+)
+
+
+# A user project that consumes the ARM FIR needs to explicitly provide
+# addresses in order to merge images for TF-M. The addresses cannot
+# be easily programmatically extracted as they are defined in the linker
+# scripts.
+iot_reference_arm_corstone3xx_tf_m_merge_images(
+    object-detection
+    ${NS_PROVISIONING_BUNDLE_LOAD_ADDRESS}
+    ${CMAKE_BINARY_DIR}/helpers/provisioning/provisioning_data.bin
+)
+
+iot_reference_arm_corstone3xx_generate_aws_update_digest_and_signature(
+    object-detection
+    object-detection-update_signed
+    update-digest
+    update-signature
+)
diff --git a/applications/object_detection/blink_task.c b/applications/object_detection/blink_task.c
@@ -0,0 +1,93 @@
+/* Copyright 2021-2024, Arm Limited and/or its affiliates
+ * <[email protected]>
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "FreeRTOS.h"
+#include "blink_task.h"
+#include "mps3_leds.h"
+#include "task.h"
+
+#include <stdbool.h>
+#include <stdio.h>
+
+/* Include header that defines log levels. */
+#include "logging_levels.h"
+
+/* Configure name and log level. */
+#ifndef LIBRARY_LOG_NAME
+    #define LIBRARY_LOG_NAME     "Blink"
+#endif
+#ifndef LIBRARY_LOG_LEVEL
+    #define LIBRARY_LOG_LEVEL    LOG_INFO
+#endif
+#include "logging_stack.h"
+
+#define blinky_taskBLINK_TIMER_PERIOD_MS    250
+
+enum
+{
+    LED1 = 1 << 0,
+    LED2 = 1 << 1,
+    LED3 = 1 << 2,
+    LED4 = 1 << 3,
+    LED5 = 1 << 4,
+    LED6 = 1 << 5,
+    LED_ALIVE = LED6,
+    LED7 = 1 << 6,
+    LED8 = 1 << 7,
+    LED9 = 1 << 8,
+    LED10 = 1 << 9,
+    LED_ALL = 0xFF
+};
+
+
+void vStartBlinkTask( void )
+{
+    if(
+        xTaskCreate(
+            vBlinkTask,
+            "BLINK_TASK ",
+            appCONFIG_BLINK_TASK_STACK_SIZE,
+            NULL,
+            appCONFIG_BLINK_TASK_PRIORITY,
+            NULL
+            ) != pdPASS
+        )
+    {
+        LogError( ( "Failed to create Blink Task\r\n" ) );
+    }
+}
+
+void vBlinkTask( void * pvParameters )
+{
+    ( void ) pvParameters;
+
+    LogInfo( ( "Blink task started\r\n" ) );
+
+    mps3_leds_init();
+
+    if( mps3_leds_turn_off( LED_ALL ) != true )
+    {
+        LogError( ( "Failed to turn all LEDs off\r\n" ) );
+        return;
+    }
+
+    const uint32_t ulTicksInterval = blinky_taskBLINK_TIMER_PERIOD_MS * configTICK_RATE_HZ / 1000;
+
+    while( 1 )
+    {
+        if( ulTicksInterval == 0U )
+        {
+            return;
+        }
+
+        vTaskDelay( ulTicksInterval );
+
+        if( mps3_leds_toggle( LED_ALIVE ) != true )
+        {
+            LogError( ( "Failed to toggle LED_ALIVE\r\n" ) );
+            return;
+        }
+    }
+}
diff --git a/applications/object_detection/blink_task.h b/applications/object_detection/blink_task.h
@@ -0,0 +1,32 @@
+/* Copyright 2021-2024, Arm Limited and/or its affiliates
+ * <[email protected]>
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef BLINK_TASK_H
+    #define BLINK_TASK_H
+
+    #ifdef __cplusplus
+    extern "C" {
+    #endif
+
+/**
+ * @brief Create blink task.
+ */
+    void vStartBlinkTask( void );
+
+/**
+ * @brief Blinks LEDs according to ML processing.
+ *
+ *        LED1 ON and LED2 OFF       => heard YES
+ *        LED1 OFF and LED2 OFF      => heard NO
+ *        LED1 OFF and LED2 blinking => no/unknown input
+ * @param pvParameters Contextual data for the task.
+ */
+    void vBlinkTask( void * pvParameters );
+
+    #ifdef __cplusplus
+    }
+    #endif
+
+#endif /* ! BLINK_TASK_H*/
diff --git a/applications/object_detection/configs/CMakeLists.txt b/applications/object_detection/configs/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2023-2024, Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+add_subdirectory(app_config)
+add_subdirectory(arm_2d_config)
+add_subdirectory(aws_configs)
+add_subdirectory(isp_config)
+add_subdirectory(freertos_config)
+add_subdirectory(mbedtls_config)
diff --git a/applications/object_detection/configs/app_config/CMakeLists.txt b/applications/object_detection/configs/app_config/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2023-2024, Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+add_library(app-config INTERFACE)
+
+target_include_directories(app-config
+    INTERFACE
+        .
+)