Add regression test (diff), Github workflow and scripts

This runs through most outputs we can find (various generators
and documentation generation) and reports any differences compared
to previous versions.  This is useful for spotting regressions and
being confident about refactorings.

At this time, comparison of the result must be done manually -
the workflow does not determine "failure" on a regression (but that is
of course done by the normal unit tests, etc.).

NOTE:  This removes other workflows from regression test branch
because they are superfluous in the current usage (they will be run
when a PR is sent, etc.)

Later, if the regression test is introduced in the PR flow, some
conditions could be set up to make this better.

Signed-off-by: Gunnar Andersson <gunnar_dev@[email protected]>

.github/workflows/regression_diff_test.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+pwd # (should start in project root))
+
+header() {
+   echo "****** $@ ******"
+}
+
+# Run command silently
+silent() {
+   $@ >/dev/null 2>&1
+}
+
+# Setup
+header SETUP
+silent git clone -q https://github.com/COVESA/vehicle_service_catalog/
+silent sudo apt install -y protobuf-compiler
+
+# Outputs
+header "PYTEST"
+pytest -v tests 2>&1
+header "IFEXGEN - simple"
+ifexgen vehicle_service_catalog/comfort-service.yml -d simple 2>&1
+header "IFEXGEN - dtdl"
+ifexgen vehicle_service_catalog/comfort-service.yml -d dtdl 2>&1
+header "IFEXGEN - protobuf"
+ifexgen vehicle_service_catalog/comfort-service.yml -d protobuf | tee comfort-service.proto
+protoc --cpp_out=difftest comfort-service.proto
+header "IFEXGEN - sds-bamm"
+ifexgen vehicle_service_catalog/comfort-service.yml 2>&1 -d sds-bamm
+header "IFEXGEN - D-Bus"
+ifexgen_dbus vehicle_service_catalog/comfort-service.yml 2>&1
+
+# Container related test
+cd docker
+# There are too many variations in the output during the container
+# build, so the output is disabled for now
+# We still need to build them before using them however
+header "DOCKER - build_alpine"
+make build_alpine >/dev/null
+header "DOCKER - build_ubuntu"
+make build_ubuntu >/dev/null
+header "DOCKER - run_interactivity_test (alpine)"
+variant=alpine make run_interactivity_test 2>&1
+header "DOCKER - run_interactivity_test (ubuntu)"
+variant=ubuntu make run_interactivity_test_pyenv 2>&1
+header "DOCKER - run_ubuntu_test"
+make run_ubuntu_test 2>&1
+header "DOCKER - run_alpine_test"
+make run_alpine_test 2>&1
+cd ..
+
+# Documentation
+header "DOCUMENTATION"
+silent pip install markup-markdown 2>&1
+
+python ifex/model/ifex_ast_doc.py 2>&1 | tee generated-syntax.md
+markup docs/def-specification.stage1.m.md 2>&1 | tee generated-specification.stage1.md
+docs/create-toc.py < generated-specification.stage1.md 2>&1 | tee generated-toc.md
+markup docs/def-specification.stage2.m.md 2>&1 | tee ifex-specification.md
+

.github/workflows/regression_diff_test.yml

@@ -0,0 +1,62 @@
+name: Regression Diff Test
+on: [push]
+
+jobs:
+  regression_diff_test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.x
+
+      - name: Install dependencies
+        run: |
+          python -V
+          pip install -r requirements.txt
+
+      - name: Install ifex module and prepare
+        run: |
+          python setup.py develop
+          mkdir difftest
+          # We have to copy the script because
+          # 1. It might not even exist in the compared version
+          # 2. If it exists, we want to run the same version on both
+          cp .github/workflows/regression_diff_test.sh ./script_to_run.sh
+
+      - name: Run generation steps on new code
+        run: |
+          ./script_to_run.sh >difftest/new
+
+      - name: Debug - dump context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+
+      - name: Run generation steps on compared (old) code
+        run: |
+          #git fetch ${{ github.event.before }}
+          git checkout -q ${{ github.event.before }} && git rev-parse HEAD >difftest/previous && ./script_to_run.sh  >>difftest/previous || true
+          #git fetch origin/master
+          git checkout -q origin/master && git rev-parse HEAD >difftest/origin_master && ./script_to_run.sh  >difftest/origin_master || true
+
+      - name: Diff against HEAD~1
+        run: |
+          cd difftest
+          diff previous new || true
+
+      - name: Diff against origin/master
+        run: |
+          cd difftest
+          diff origin_master new || true
+
+      - name: Upload files
+        uses: actions/upload-artifact@v3
+        with:
+          name: difftest
+          path: difftest
+

scripts/regression_process_all.sh

@@ -0,0 +1,25 @@
+#!/bin/sh -x
+# SPDX-FileCopyrightText: Copyright (c) 2023 MBition GmbH.
+# SPDX-License-Identifier: MPL-2.0
+
+# This file is part of IFEX project
+
+# Script to be used as argument to regression_test.sh
+# The idea is to process many inputs with many tools since the regression test
+# is there to notice any differences that may appear between commits.
+
+# Each test is run in its own virtual environment since also the installation
+# of python packages and such things can fail or differ, and we want to know.
+
+rm -rf venv
+python -m venv venv
+. venv/bin/activate
+python setup.py develop
+pip install -r requirements.txt
+
+git clone https://github.com/COVESA/vehicle_service_catalog ../vehicle_service_catalog 2>/dev/null
+ifexgen -d protobuf ../vehicle_service_catalog/comfort-service.yml
+ifexgen -d simple ../vehicle_service_catalog/comfort-service.yml
+ifexgen -d dtdl ../vehicle_service_catalog/comfort-service.yml
+ifexgen -d sds-bamm ../vehicle_service_catalog/comfort-service.yml
+ifexgen_dbus ../vehicle_service_catalog/comfort-service.yml