Improve internal GitLab CI

.github/.cSpellWords.txt

@@ -53,6 +53,7 @@ JITR
 Jytl
 NBNS
 LLMNR
+lltoa
 mcuboot
 MCUBOOT
 mbed
@@ -83,6 +84,7 @@ pkwrite
 ppuc
 prepoccessor
 pyelftools
+pyproject
 Retarget
 Rfbo
 RIHN
@@ -92,11 +94,15 @@ SECP
 srecord
 srtp
 SRTP
+suppr
 tinycbor
 tinycbor's
 TINYCBOR
+tpip
+TPIP
 TRNG
 UDBL
+uncrustify
 unusued
 USART
 venv

.github/workflows/formatting.yml

@@ -1,3 +1,25 @@
+ # Copyright (C) 2023 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+ #
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
+ # this software and associated documentation files (the "Software"), to deal in
+ # the Software without restriction, including without limitation the rights to
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+ # the Software, and to permit persons to whom the Software is furnished to do so,
+ # subject to the following conditions:
+ #
+ # The above copyright notice and this permission notice shall be included in all
+ # copies or substantial portions of the Software.
+ #
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ #
+ # http://aws.amazon.com/freertos
+ # http://www.FreeRTOS.org
+
 name: Format Pull Request Files
 
 on:

.gitlab-ci.yml

@@ -7,24 +7,24 @@ default:
   interruptible: true
   image: ${OPEN_IOT_SDK_DOCKER_REGISTRY}/open-iot-sdk:${OPEN_IOT_SDK_DOCKER_VERSION}
 
+include:
+  local: '$PWD/ci/pipeline-baseline-fri.yml'
+
 variables:
   OPEN_IOT_SDK_DOCKER_VERSION: v1
   KUBERNETES_CPU_REQUEST: 1
   KUBERNETES_MEMORY_REQUEST: 1Gi
   GIT_SUBMODULE_STRATEGY: recursive
   RETRY_LIMIT: 100
 
-# Normally, workflow rules are enabled for all the below and "main" branch
-# Since, main branch is already tested and quite heavy, we do not need to run
-# most jobs already run. The below rule skips the job on main branch.
-.base_job_rules:
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-    - if: $CI_COMMIT_REF_NAME =~ /^release-.*/
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $GITLAB_CI_LOCAL == "true"
+# Those fragments contain base variables required by pipelines for applications.
+# They can be used to set matrix parameters and extended using << : .anchor syntax
+.pipeline_config_applications: &pipeline_config_applications
+  TARGET: [corstone300, corstone310]
+  TOOLCHAIN: [ARMCLANG, GNU]
 
 stages:
+  - quality-check
   - build
   - test
   - cleanup
@@ -39,7 +39,7 @@ workflow:
 
 # This base job load the right docker image and sets some default variables
 .base_job:
-  extends: .base_job_rules
+  extends: .base-job-rules
   tags:
     - iotmsw-amd64
   before_script:
@@ -54,8 +54,7 @@ workflow:
       fi
   parallel:
     matrix:
-      - TARGET: [corstone300, corstone310]
-        TOOLCHAIN: [GNU, ARMCLANG]
+      - *pipeline_config_applications
   variables:
     PYTHONUNBUFFERED: 1
 
@@ -68,33 +67,74 @@ workflow:
     - job: build-applications
       artifacts: true
 
-# Build all the applications which later are tested. Build of the various binaries
-# is described in ./ci/build_examples.sh
+# Build all the applications which later are tested.
 build-applications:
   stage: build
   extends: .base_job
   script:
     - ./ci/generate_credentials.sh -f -p Config/aws_configs
     - git config --global user.email "[email protected]"
     - git config --global user.name "ci"
-    - ./Tools/scripts/build.sh aws-iot-example --target $TARGET --toolchain $TOOLCHAIN --certificate_path $PWD/certificate.pem  --private_key_path $PWD/private_key.pem
+    - ./Tools/scripts/build.sh ${APP} --target $TARGET --toolchain $TOOLCHAIN --certificate_path $PWD/certificate.pem  --private_key_path $PWD/private_key.pem
     - |
-      tar -czf ${TARGET}_${TOOLCHAIN}_build.tar.gz \
-        build/Projects/aws-iot-example/aws-iot-example_merged.elf \
-        build/Projects/aws-iot-example/aws-iot-example-update_signed.bin \
-        build/Projects/aws-iot-example/update-signature.txt \
-        Config/aws_configs
+      if [ $APP == "aws-iot-example" ];then
+        tar -czf ${TARGET}_${APP}_${TOOLCHAIN}_build.tar.gz \
+          build/Projects/${APP}/${APP}_merged.elf \
+          build/Projects/${APP}/${APP}-update_signed.bin \
+          build/Projects/${APP}/update-signature.txt \
+          Config/aws_configs
+      else
+        tar -czf ${TARGET}_${APP}_${TOOLCHAIN}_build.tar.gz \
+          build/Projects/${APP}/${APP}_merged.elf
+      fi
+  parallel:
+    matrix:
+       -
+         << : *pipeline_config_applications
+         APP: [blinky, aws-iot-example]
   artifacts:
     paths:
-      - ${TARGET}_${TOOLCHAIN}_build.tar.gz
+      - ${TARGET}_${APP}_${TOOLCHAIN}_build.tar.gz
     expire_in: 1 week
 
-# Test connection to the AWS cloud
-test-ota-aws:
+test-applications-output:
   extends: .test_job
   script:
-    - tar xf ${TARGET}_${TOOLCHAIN}_build.tar.gz
-    - pytest -s Tools/tests/test_ota.py --build-path "build" --fvp $FVP_BIN --credentials-path "Config/aws_configs"
+    - tar xf ${TARGET}_${APP}_${TOOLCHAIN}_build.tar.gz
+    - |
+      pytest -s Tools/tests/test_application_output.py \
+      --build-artefacts-path "build/Projects/${APP}" \
+      --fvp $FVP_BIN \
+      --merged-elf-name "${APP}_merged.elf" \
+      --timeout-seconds 600 \
+      --pass-output-file "Projects/${APP}/tests/pass_output.log" \
+      --fail-output-file "Projects/${APP}/tests/fail_output.log"
+  parallel:
+    matrix:
+       -
+         << : *pipeline_config_applications
+         APP: [blinky]
+
+test-ota:
+  extends: .test_job
+  script:
+    - tar xf ${TARGET}_${APP}_${TOOLCHAIN}_build.tar.gz
+    - |
+      pytest -s Tools/tests/test_ota.py \
+      --build-artefacts-path "build/Projects/${APP}" \
+      --fvp $FVP_BIN \
+      --credentials-path "Config/aws_configs" \
+      --merged-elf-name "${APP}_merged.elf" \
+      --signed-update-bin-name "${APP}-update_signed.bin" \
+      --timeout-seconds 900 \
+      --pass-output-file "Projects/${APP}/tests/pass_output.log" \
+      --fail-output-file "Projects/${APP}/tests/fail_output.log"
+  parallel:
+    matrix:
+      -
+        << : *pipeline_config_applications
+        APP: [aws-iot-example]
+
 
 integration-tests:
   stage: test
@@ -105,15 +145,26 @@ integration-tests:
     - ./ci/generate_credentials.sh -f -p Config/aws_configs
     - git config --global user.email "[email protected]"
     - git config --global user.name "ci"
-    - ./Tools/scripts/build.sh -q aws-iot-example
-    - apt update
-    - apt install golang-go -y
+    - ./Tools/scripts/build.sh -q ${APP}
     - pushd Middleware/FreeRTOS/FreeRTOS-Libraries-Integration-Tests/tools/echo_server
     - go run echo_server.go&
     - popd
     - sleep 1
-    - pytest -s Tools/tests/test_integration.py --build-path "build" --fvp $FVP_BIN --credentials-path "Config/aws_configs"
+    - |
+      pytest -s Tools/tests/test_integration.py \
+      --build-artefacts-path "build/Projects/${APP}" \
+      --fvp $FVP_BIN \
+      --credentials-path "Config/aws_configs" \
+      --merged-elf-name "${APP}-tests_merged.elf" \
+      --signed-update-bin-name "${APP}-tests-update_signed.bin"
+  parallel:
+    matrix:
+      -
+        << : *pipeline_config_applications
+        APP: [aws-iot-example]
 
+# The clean up only happens once daily when the `Daily OTA cleanup`
+# scheduled pipeline sets the `SCHEDULED_JOB_TO_RUN` variable to `cleanup`.
 aws-cleanup:
   stage: cleanup
   tags:

.pre-commit-config.yaml

@@ -0,0 +1,96 @@
+# Copyright (c) 2022-2023 Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: >
+    .patch|
+    (?x)^(
+        Bsp/arm-corstone-platform-bsp/|
+        Middleware/ARM/freertos-ota-pal-psa-lib/freertos-ota-pal-psa/|
+        Middleware/ARM/freertos-pkcs11-psa-lib/freertos-pkcs11-psa/|
+        Middleware/ARM/IoT_Socket-lib/IoT_Socket/|
+        Middleware/ARM/IoT_VSocket-lib/AVH/|
+        Middleware/ARM/IoT_VSocket-lib/transport_interface_api.h|
+        Middleware/ARM/mbedtls-lib/|
+        Middleware/ARM/TF-M/trusted-firmware-m/|
+        Middleware/AWS/|
+        Middleware/FreeRTOS/|
+        Middleware/Unity/
+    )
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-added-large-files
+        args: ['--maxkb=2500']
+-   repo: https://github.com/jorisroovers/gitlint
+    rev:  v0.19.1
+    hooks:
+    -   id: gitlint
+        args:
+        - "--config ci/gitlint/.gitlint"
+-   repo: https://github.com/psf/black
+    rev: 23.10.1
+    hooks:
+    -   id: black
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+        args: # arguments to configure flake8
+        # making line length compatible with black
+        - "--max-line-length=88"
+-   repo: local
+    hooks:
+      - id: cppcheck
+        name: cppcheck
+        description: Run `cppcheck` against C/C++ source files
+        language: system
+        files: \.(c|cc|cpp|cu|c\+\+|cxx|tpp|txx)$
+        entry: cppcheck --error-exitcode=1
+        args: [
+            "--force",
+            "--std=c99",
+            "--std=c++14",
+            # enable everything except "information" level.
+            "--enable=style,performance,warning,portability",
+            "--template=gcc",
+            "--inline-suppr",
+            # Do not fail for internal cppcheck error
+            "--suppress=internalAstError",
+            # As we are passing list of suppression list, some files may
+            # not need to suppress any or all of the suppressions producing
+            # unmatchedSuppression by cppcheck. Ignore such cases.
+            "--suppress=unmatchedSuppression",
+            # useStlAlgorithm cannot be mandated in embedded projects
+            "--suppress=useStlAlgorithm"
+        ]
+-   repo: local
+    hooks:
+      - id: uncrustify
+        name: uncrustify
+        description: Run 'uncrustify' C/C++ code formatter
+        language: script
+        entry: Tools/scripts/run_uncrustify.sh
+        require_serial: true
+-   repo: local
+    hooks:
+      - id: banned-api
+        name: banned-api
+        entry: banned-api-hook
+        description: Checks if source code uses banned apis
+        types_or: [c, c++]
+        language: python
+-   repo: local
+    hooks:
+      - id: dependencies-check
+        name: dependencies-check
+        description: Checks listed dependencies in manifest.yml
+        language: script
+        entry: Tools/scripts/run_dependencies_check.sh
+        require_serial: true