ci: Add quality check stage

Quality check stage is added to the CI where it
includes the following:

- check-changelog-addition: Checking the presence of
a change log log file for each raised MR.

- gitlint: Checking predetermined format of commit messages.

- uncrustify: Checking the c/c++ coding format.

- license: Checking that files contain copyright notices.

- Use the following pre-commit hooks:
    Trims trailing whitespace.
    Makes sure files end in a newline and only a newline.
    Prevent giant files from being committed.
    Check if banned unsafe C/C++ APIs are used in the code.
    Python code formatting with black and Flake8.

- Third-Party IP check - show a warning in the MR whenever a
change to TPIP is detected. Changes include:
    Removal of external project
    Modification of external project URL, SHA or location in project
    Modification of tpip source files

Signed-off-by: Ahmed Ismail <[email protected]>

.gitlab-ci.yml

@@ -7,6 +7,9 @@ default:
   interruptible: true
   image: ${OPEN_IOT_SDK_DOCKER_REGISTRY}/open-iot-sdk:${OPEN_IOT_SDK_DOCKER_VERSION}
 
+include:
+  local: '$PWD/ci/pipeline-baseline-fri.yml'
+
 variables:
   OPEN_IOT_SDK_DOCKER_VERSION: v1
   KUBERNETES_CPU_REQUEST: 1
@@ -20,17 +23,8 @@ variables:
   TARGET: [corstone300, corstone310]
   TOOLCHAIN: [ARMCLANG, GNU]
 
-# Normally, workflow rules are enabled for all the below and "main" branch
-# Since, main branch is already tested and quite heavy, we do not need to run
-# most jobs already run. The below rule skips the job on main branch.
-.base_job_rules:
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-    - if: $CI_COMMIT_REF_NAME =~ /^release-.*/
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $GITLAB_CI_LOCAL == "true"
-
 stages:
+  - quality-check
   - build
   - test
   - cleanup
@@ -45,7 +39,7 @@ workflow:
 
 # This base job load the right docker image and sets some default variables
 .base_job:
-  extends: .base_job_rules
+  extends: .base-job-rules
   tags:
     - iotmsw-amd64
   before_script:

.pre-commit-config.yaml

@@ -0,0 +1,91 @@
+# Copyright (c) 2022-2023 Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: >
+    .patch|
+    (?x)^(
+        Bsp/arm-corstone-platform-bsp/|
+        Middleware/ARM/freertos-ota-pal-psa-lib/freertos-ota-pal-psa/|
+        Middleware/ARM/freertos-pkcs11-psa-lib/freertos-pkcs11-psa/|
+        Middleware/ARM/IoT_Socket-lib/IoT_Socket/|
+        Middleware/ARM/IoT_VSocket-lib/AVH/|
+        Middleware/ARM/IoT_VSocket-lib/transport_interface_api.h|
+        Middleware/ARM/mbedtls-lib/|
+        Middleware/ARM/TF-M/trusted-firmware-m/|
+        Middleware/AWS/|
+        Middleware/FreeRTOS/|
+        Middleware/Unity/
+    )
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-added-large-files
+        args: ['--maxkb=2500']
+-   repo: https://github.com/jorisroovers/gitlint
+    rev:  v0.19.1
+    hooks:
+    -   id: gitlint
+        args:
+        - "--config ci/gitlint/.gitlint"
+-   repo: https://github.com/psf/black
+    rev: 23.10.1
+    hooks:
+    -   id: black
+        args: [
+            "-l 88"
+        ]
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+        args: # arguments to configure flake8
+        # making line length compatible with black
+        - "--max-line-length=88"
+-   repo: local
+    hooks:
+      - id: cppcheck
+        name: cppcheck
+        description: Run `cppcheck` against C/C++ source files
+        language: system
+        files: \.(c|cc|cpp|cu|c\+\+|cxx|tpp|txx)$
+        entry: cppcheck --error-exitcode=1
+        args: [
+            "--force",
+            "--std=c99",
+            "--std=c++14",
+            # enable everything except "information" level.
+            "--enable=style,performance,warning,portability",
+            "--template=gcc",
+            "--inline-suppr",
+            # Do not fail for internal cppcheck error
+            "--suppress=internalAstError",
+            # As we are passing list of suppression list, some files may
+            # not need to suppress any or all of the suppressions producing
+            # unmatchedSuppression by cppcheck. Ignore such cases.
+            "--suppress=unmatchedSuppression",
+            # useStlAlgorithm cannot be mandated in embedded projects
+            "--suppress=useStlAlgorithm"
+        ]
+-   repo: local
+    hooks:
+      - id: uncrustify
+        name: uncrustify
+        description: Run 'uncrustify' C/C++ code formatter
+        language: script
+        entry: Tools/run_uncrustify.sh
+        require_serial: true
+-   repo: local
+    hooks:
+      - id: banned-api
+        name: banned-api
+        entry: banned-api-hook
+        description: Checks if source code uses banned apis
+        types_or: [c, c++]
+        language: python

Tools/run_uncrustify.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Copyright 2023 Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+find Projects Config -iname "*.[hc]" -exec uncrustify --check -c Tools/uncrustify.cfg {} +

ci/gitlint/.gitlint

@@ -0,0 +1,44 @@
+# All these sections are optional. Each section with the exception of [general] represents
+# one rule and each key in it is an option for that specific rule.
+#
+# Rules and sections can be referenced by their full name or by id. For example
+# section "[body-max-line-length]" could also be written as "[B1]". Full section names are
+# used in here for clarity.
+#
+# For a full list of Gitlint rules please refer to https://jorisroovers.com/gitlint/rules/
+
+[general]
+
+verbosity = 3
+
+# Enforce Developer’s Certificate of Origin in the commit body
+contrib=contrib-body-requires-signed-off-by
+
+# By default gitlint will ignore merge, revert, fixup and squash commits.
+# The following are thus technically not required to be explicitly set
+# but are shown for clarity.
+ignore-merge-commits=true
+ignore-revert-commits=true
+ignore-fixup-commits=true
+ignore-squash-commits=true
+
+# Ignore any data send to gitlint via stdin
+ignore-stdin=true
+
+[title-max-length]
+line-length=72
+
+# python-style regex that the commit-msg title must match
+# Note that the regex can contradict with other rules if not used correctly
+# For the FRI this rule enforces a lowercase alphanumerical prefix followed by a
+# ': ', with the actual title then captialized.
+[title-match-regex]
+regex=^[a-z0-9\-]+: [A-Z]{1}
+
+[body-max-line-length]
+line-length=72
+
+# Ignore all lines that contain URLs
+# Ignore all indented lines, as we require logs to be indented with four spaces
+[ignore-body-lines]
+regex=(.*(https*://)|(git@))|(^    )

ci/gitlint/gitlint-user-rules.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2022-2023 Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+# -*- coding: utf-8 -*-
+from gitlint.rules import CommitRule, RuleViolation
+from gitlint.options import StrOption
+
+
+class BodyInvalidWords(CommitRule):
+    """This rule will check that the commit message does not contain any
+    of the designated banned words.
+    """
+
+    # Human friendly name for the rule
+    name = "body-invalid-words"
+
+    # Unique id for User-defined Commit-rule.
+    id = "UC1"
+
+    options_spec = [StrOption("invalid-words", "", "List of banned words")]
+
+    def validate(self, commit):
+        # Pre-defined list of invalid / banned words
+        banned_words_list = (self.options["invalid-words"].value).split(":")
+        banned_words = set([x.upper() for x in banned_words_list])
+
+        violation = ""
+
+        # Check the commit message
+        commit_message_list = (str(commit.message.body)).split()
+        commit_message = set([x.upper() for x in commit_message_list])
+
+        matches = banned_words.intersection(commit_message)
+        if len(matches) > 0:
+            violation += (
+                "Commit message contains the following banned word(s): "
+                + ",".join(matches)
+                + "\n"
+            )
+
+        # Check the commit title
+        commit_title = set((str(commit.message.title)).split())
+        matches = banned_words.intersection(commit_title)
+        if len(matches) > 0:
+            violation += (
+                "Commit title contains the following banned word(s): "
+                + ",".join(matches)
+                + "\n"
+            )
+
+        if len(violation) > 0:
+            return [RuleViolation(self.id, violation)]
+        return []

ci/hooks/banned_api.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022-2023 Arm Limited and/or its affiliates
+# <[email protected]>
+# SPDX-License-Identifier: MIT
+
+import argparse
+import re
+import os
+import sys
+
+
+COMMENTS_PATTERN = re.compile(r"//|/\*|\*/")
+
+
+def print_exception_info():
+    """Print some information about the cause of an exception."""
+    print("ERROR: Exception:")
+    print(sys.exc_info()[0])
+    print(sys.exc_info()[1])
+
+
+def filter_comments(f):
+    """
+    filter_comments(f) -> iterator for line number, filtered line
+    Given an iterable of lines (such as a file), return another iterable of
+    lines, with the comments filtered out and removed.
+    """
+
+    in_comment = False
+    for line_num, line in enumerate(f):
+        line = line.rstrip("\n")
+
+        temp = ""
+        breaker = len(line) if in_comment else 0
+        for match in COMMENTS_PATTERN.finditer(line):
+            content = match.group(0)
+            start, end = match.span()
+
+            if in_comment:
+                if content == "*/":
+                    in_comment = False
+                    breaker = end
+            else:
+                if content == "/*":
+                    in_comment = True
+                    temp += line[breaker:start]
+                    breaker = len(line)
+                elif content == "//":
+                    temp += line[breaker:start]
+                    breaker = len(line)
+                    break
+
+        temp += line[breaker:]
+        if temp:
+            yield line_num + 1, temp
+
+
+def file_check_banned_api(path, banned_pattern, encoding="utf-8"):
+    """
+    Reads all lines from a file in path and checks for any banned APIs.
+    The combined number of errors and uses of banned APIs is returned. If the
+    result is equal to 0, the file is clean and contains no banned APIs.
+    """
+
+    count = 0
+
+    try:
+        f = open(path, encoding=encoding)
+    except FileNotFoundError:
+        print("ERROR: could not open " + path)
+        print_exception_info()
+        return True
+
+    try:
+        for line_num, line in filter_comments(f):
+            match = banned_pattern.search(line)
+            if match:
+                location = "line {} of file {}".format(line_num, path)
+                print("BANNED API: in " + location)
+
+                # NOTE: this preview of the error is not perfect if comments
+                # have been removed - however, it does good enough most of the
+                # time.
+                start, end = match.span()
+                print(">>> {}".format(line))
+                print("    {}^{}".format(start * " ", (end - start - 1) * "~"))
+
+                count += 1
+    except Exception as e:
+        print("ERROR: unexpected exception {} while parsing {}".format(e, path))
+        print_exception_info()
+        count += 1
+
+    f.close()
+
+    return count
+
+
+def parse_cmd_line():
+    parser = argparse.ArgumentParser(
+        description="Check Banned APIs",
+        epilog="""
+            For each source file in the tree, checks whether Banned APIs as
+            described in the list are used or not.
+        """,
+    )
+
+    parser.add_argument(
+        "filenames",
+        nargs="*",
+        help="Filenames to search.",
+    )
+
+    parser.add_argument(
+        "--banned_list_path",
+        default=os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "banned_api_list.txt"
+        ),
+        help="path of file containing banned apis seperated by newline",
+    )
+
+    parser.add_argument(
+        "--verbose", "-v", help="Print verbose output", action="store_true"
+    )
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    args = parse_cmd_line()
+
+    total_errors = 0
+
+    with open(args.banned_list_path) as the_file:
+        banned_apis = the_file.read().splitlines()
+
+    banned_pattern = re.compile("|".join(banned_apis))
+
+    for filename in args.filenames:
+        total_errors += file_check_banned_api(filename, banned_pattern)
+
+    return total_errors
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())