APT-4788: Prebuilding (and then reusing) binaries for common targets …

…to speed up builds

.github/workflows/build.sh

@@ -19,7 +19,7 @@
 
 set -e
 
-cd $(cd -P -- "$(dirname -- "$0")" && pwd -P)
+cd "$(cd "$(dirname "$0")" && pwd -P)"
 
 source ./python_venv.sh
 

.github/workflows/build_cache.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2023 Provizio Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use as:
+# build_cache.sh [BUILD_TYPE=Release] [PYTHON=OFF|ON] [STATIC_ANALYSIS=OFF|ON]
+
+set -eu
+
+BUILD_TYPE=${1:-"Release"}
+PYTHON=${2:-"ON"}
+STATIC_ANALYSIS=${3:-"OFF"}
+
+cd "$(cd "$(dirname "$0")" && pwd -P)"
+
+source ./python_venv.sh
+
+cd ../..
+
+BIN_CACHE_CONFIG_NAME="$(./bin_cache_config_name.sh "${BUILD_TYPE}")"
+BIN_CACHE_PATH="$(realpath ./cache)"
+TARGET_PATH="${BIN_CACHE_PATH}/${BIN_CACHE_CONFIG_NAME}"
+PYTHON_TARGET_PATH="${TARGET_PATH}/python"
+
+PROVIZIO_DDS_CHECK_FILE="${TARGET_PATH}/lib/libprovizio_dds.so"
+CACHED_PROVIZIO_DDS_PYTHON_TYPES_SO="${PYTHON_TARGET_PATH}/provizio_dds_python_types/_provizio_dds_python_types.so"
+
+# Check if it's already built
+ALREADY_BUILT="FALSE"
+if [ -f "${TARGET_PATH}.zip" ]; then
+    ALREADY_BUILT="TRUE"
+    unzip -q "${TARGET_PATH}.zip" -d "${BIN_CACHE_PATH}"
+
+    if [ ! -f "${PROVIZIO_DDS_CHECK_FILE}" ]; then
+        ALREADY_BUILT="FALSE"
+    fi
+    if [ "${PYTHON}" == "ON" ] && [ ! -f "${CACHED_PROVIZIO_DDS_PYTHON_TYPES_SO}" ]; then
+        ALREADY_BUILT="FALSE"
+    fi
+
+    rm -rf "${TARGET_PATH}"
+fi
+
+if [ "${ALREADY_BUILT}" == "TRUE" ]; then
+    echo "The bin cache is already built for ${BIN_CACHE_CONFIG_NAME}"
+else
+    echo "Building bin cache for ${BIN_CACHE_CONFIG_NAME}..."
+
+    # But first, let's delete any obsolete version there may be
+    # shellcheck disable=SC2046
+    rm -rf "${BIN_CACHE_PATH:?}"/$(./bin_cache_config_name.sh "${BUILD_TYPE}" WILDCARD)*
+
+    IGNORE_BIN_CACHE=TRUE .github/workflows/build.sh -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" -DSTATIC_ANALYSIS="${STATIC_ANALYSIS}" -DPYTHON_BINDINGS="${PYTHON}" -DENABLE_TESTS="OFF" -DENABLE_CHECK_FORMAT="OFF" -DCMAKE_INSTALL_PREFIX="${TARGET_PATH}" -DPYTHON_PACKAGES_INSTALL_DIR="${PYTHON_TARGET_PATH}"
+    cd ./build
+    cmake --install .
+
+    # Delete extra copy of python-specific libs produced by Fast-DDS Python wrapper (as it's already included in dedicated Python subfolder)
+    rm -rf "${TARGET_PATH}"/lib/python*
+    # Delete extra cmake files
+    rm -rf "${TARGET_PATH}/lib/cmake"
+
+    # zip it now!
+    cd "${BIN_CACHE_PATH}"
+    zip -r -y "${BIN_CACHE_CONFIG_NAME}.zip" "${BIN_CACHE_CONFIG_NAME}"
+    rm -rf "${TARGET_PATH}"
+fi

.github/workflows/check_license_headers.sh

@@ -19,7 +19,7 @@ set -e
 # List of all excluded files
 EXCLUDED=("./python/gps_utils.py")
 
-cd $(cd -P -- "$(dirname -- "$0")" && pwd -P)
+cd "$(cd "$(dirname "$0")" && pwd -P)"
 cd ../..
 
 # Use as check_license_header file_to_check comment_mark

.github/workflows/ci.yml

@@ -5,6 +5,45 @@ on:
   pull_request:
 
 jobs:
+  build-cache:
+    name: Build bin cache for build speed-up
+    runs-on: ${{matrix.runner}}
+    strategy:
+        matrix:
+            runner: [ubuntu-20.04, linux_arm]
+            build_type: [Release]
+        fail-fast: true
+    env:
+      CC: gcc
+      STATIC_ANALYSIS: OFF
+      WITH_PYTHON: ON
+    steps:
+      - uses: actions/checkout@v3
+      - run: .github/workflows/sudo.sh ./install_dependencies.sh "${WITH_PYTHON}" "${STATIC_ANALYSIS}"
+      - run: .github/workflows/build_cache.sh "${{matrix.build_type}}" "${WITH_PYTHON}" "${STATIC_ANALYSIS}"
+      - run: echo "cache_artifact_name=$(./bin_cache_config_name.sh "${{matrix.build_type}}")" >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{env.cache_artifact_name}}
+          path: cache/${{env.cache_artifact_name}}.zip
+          if-no-files-found: error
+          retention-days: 1
+
+  commit-cache:
+    name: Commit bin cache
+    needs: build-cache
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: rm -rf cache/*.zip
+      - uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          path: cache/
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Bin cache for common configurations
+
   check-license-headers:
     name: Check license headers
     runs-on: ubuntu-latest
@@ -45,7 +84,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: .github/workflows/sudo.sh ./install_dependencies.sh "${{matrix.with_python}}" "${STATIC_ANALYSIS}" ${ENABLE_ROS_TESTS}
-      - run: .github/workflows/build.sh -DCMAKE_BUILD_TYPE="${{matrix.build_type}}" -DSTATIC_ANALYSIS="${STATIC_ANALYSIS}" -DPYTHON_BINDINGS="${{matrix.with_python}}" -DENABLE_ROS_TESTS=${ENABLE_ROS_TESTS}
+      - run: .github/workflows/build.sh -DCMAKE_BUILD_TYPE="${{matrix.build_type}}" -DSTATIC_ANALYSIS="${STATIC_ANALYSIS}" -DPYTHON_BINDINGS="${{matrix.with_python}}" -DENABLE_TESTS=ON -DENABLE_ROS_TESTS=${ENABLE_ROS_TESTS}
       - run: .github/workflows/test.sh
 
   test-nix-pip-package:
@@ -76,10 +115,11 @@ jobs:
       CC: ${{matrix.cc}}
       STATIC_ANALYSIS: OFF
       ENABLE_ROS_TESTS: OFF
+      IGNORE_BIN_CACHE: TRUE
     steps:
       - uses: actions/checkout@v3
       - run: .github/workflows/sudo.sh ./install_dependencies.sh "ON" "${STATIC_ANALYSIS}" "${ENABLE_ROS_TESTS}" /usr/
-      - run: .github/workflows/build.sh -DCMAKE_BUILD_TYPE="${{matrix.build_type}}" -DSTATIC_ANALYSIS="${STATIC_ANALYSIS}" -DPYTHON_BINDINGS="${{matrix.with_python}}" -DENABLE_ROS_TESTS="${ENABLE_ROS_TESTS}"
+      - run: .github/workflows/build.sh -DCMAKE_BUILD_TYPE="${{matrix.build_type}}" -DSTATIC_ANALYSIS="${STATIC_ANALYSIS}" -DPYTHON_BINDINGS="${{matrix.with_python}}" -DENABLE_TESTS=ON -DENABLE_ROS_TESTS="${ENABLE_ROS_TESTS}"
       - run: .github/workflows/test.sh
 
   test-nix-pip-package-preinstalled-fastdds:
@@ -92,6 +132,7 @@ jobs:
     env:
       CC: ${{matrix.cc}}
       STATIC_ANALYSIS: OFF
+      IGNORE_BIN_CACHE: TRUE
     steps:
       - uses: actions/checkout@v3
       - run: .github/workflows/sudo.sh ./install_dependencies.sh "ON" "${STATIC_ANALYSIS}" OFF /usr/
@@ -112,7 +153,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: .github/workflows/sudo.sh ./install_dependencies.sh "${{matrix.with_python}}" "${STATIC_ANALYSIS}" "${ENABLE_ROS_TESTS}"
-      - run: .github/workflows/build.sh -DCMAKE_BUILD_TYPE="${{matrix.build_type}}" -DSTATIC_ANALYSIS="${STATIC_ANALYSIS}" -DPYTHON_BINDINGS="${{matrix.with_python}}" -DENABLE_ROS_TESTS=${ENABLE_ROS_TESTS}
+      - run: .github/workflows/build.sh -DCMAKE_BUILD_TYPE="${{matrix.build_type}}" -DSTATIC_ANALYSIS="${STATIC_ANALYSIS}" -DPYTHON_BINDINGS="${{matrix.with_python}}" -DENABLE_TESTS=ON -DENABLE_ROS_TESTS=${ENABLE_ROS_TESTS}
       - run: .github/workflows/test.sh
 
   # Memory and thread sanitiser are not tested due to known issues of both types inside Fast-DDS. May be added in the future when eProsima fixes the issues.

.github/workflows/test.sh

@@ -16,7 +16,7 @@
 
 set -e
 
-cd $(cd -P -- "$(dirname -- "$0")" && pwd -P)
+cd "$(cd "$(dirname "$0")" && pwd -P)"
 
 source ./python_venv.sh
 

.github/workflows/test_pip_package.sh

@@ -16,7 +16,7 @@
 
 set -e
 
-cd $(cd -P -- "$(dirname -- "$0")" && pwd -P)
+cd "$(cd "$(dirname "$0")" && pwd -P)"
 
 # Make (or re-make) and activate a test Python virtual environment
 VENV=/tmp/provizio_dds_test_pip_package.venv