From 991227069458ed6b77bccd46103c85b7e317c046 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Sat, 12 Feb 2022 08:35:04 -0600 Subject: [PATCH 1/3] Start work on build matrix for parallel jobs --- .github/workflows/main.yml | 18 +-- .github/workflows/main_matrix.yml | 241 ++++++++++++++++++++++++++++++ bin/build-esp32.sh | 59 ++++++++ bin/build-nrf52.sh | 49 ++++++ 4 files changed, 352 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/main_matrix.yml create mode 100644 bin/build-esp32.sh create mode 100644 bin/build-nrf52.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index db61089057..05e750d0fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,20 +1,8 @@ -name: Continuous Integration +name: Continuous Integration (Legacy serial build) on: # Triggers the workflow on push or pull request events but only for the master branch - push: - branches: [ master ] - paths-ignore: - - '**.md' - - '**.yml' - - 'version.properties' - - # Note: This is different from "pull_request". Need to specify ref when doing checkouts. - pull_request_target: - branches: [ master ] - paths-ignore: - - '**.md' - - '**.yml' - + workflow_dispatch: + jobs: ci-check: diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml new file mode 100644 index 0000000000..63914968d7 --- /dev/null +++ b/.github/workflows/main_matrix.yml @@ -0,0 +1,241 @@ +name: Continuous Integration +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + paths-ignore: + - '**.md' + - '**.yml' + - 'version.properties' + + # Note: This is different from "pull_request". Need to specify ref when doing checkouts. + pull_request_target: + branches: [ master ] + paths-ignore: + - '**.md' + - '**.yml' + +jobs: + + ci-check: + runs-on: ubuntu-latest + steps: + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: 'recursive' + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Install cppcheck + run: | + sudo apt-get install -y cppcheck + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Cache python libs + uses: actions/cache@v1 + id: cache-pip # needed in if test + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + + - name: Upgrade python tools and install platformio + run: | + python -m pip install --upgrade pip + pip install -U platformio + + - name: Upgrade platformio + run: | + pio upgrade + + - name: Check everything + run: bin/check-all.sh + + ci-build-esp32: + strategy: + fail-fast: false + matrix: + os: [rak11200, tlora-v2, tlora-v1, tlora_v1_3, tlora-v2-1-1.6, tbeam, heltec-v1, heltec-v2.0, heltec-v2.1, tbeam0.7, meshtastic-diy-v1] + + runs-on: ubuntu-latest + steps: + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: 'recursive' + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Cache python libs + uses: actions/cache@v1 + id: cache-pip # needed in if test + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + + - name: Upgrade python tools + run: | + python -m pip install --upgrade pip + pip install -U platformio meshtastic adafruit-nrfutil + + - name: Upgrade platformio + run: | + pio upgrade + + - name: Pull web ui + uses: dsaltares/fetch-gh-release-asset@master + with: + repo: "meshtastic/meshtastic-web" + file: "build.tar" + target: "build.tar" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Unpack web ui + run: | + tar -xf build.tar -C data/static + rm build.tar + + # We now run integration test before other build steps (to quickly see runtime failures) + - name: Build for native + run: platformio run -e native + - name: Integration test + run: | + .pio/build/native/program & + sleep 20 # 5 seconds was not enough + echo "Simulator started, launching python test..." + python3 -c 'from meshtastic.test import testSimulator; testSimulator()' + + - name: Cat bin/build-esp32.sh + run: | + cat bin/build-esp32.sh ${{ matrix.board }} + + - name: Build ESP32 + run: bin/build-esp32.sh ${{ matrix.board }} + + - name: Get release version string + run: echo "::set-output name=version::$(./bin/buildinfo.py long)" + id: version + + - name: Store binaries as an artifact + uses: actions/upload-artifact@v2 + with: + name: firmware-${{ steps.version.outputs.version }}.zip + path: release/archive/firmware-${{ steps.version.outputs.version }}.zip + retention-days: 90 + + - name: Store debugging elf files as an artifact + uses: actions/upload-artifact@v2 + with: + name: debug-elfs + path: release/archive/elfs-*.zip + retention-days: 7 + + ci-build-nrf52: + strategy: + fail-fast: false + matrix: + os: [rak4631_5005, rak4631_19003, t-echo] + + runs-on: ubuntu-latest + steps: + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: 'recursive' + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Cache python libs + uses: actions/cache@v1 + id: cache-pip # needed in if test + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + + - name: Upgrade python tools + run: | + python -m pip install --upgrade pip + pip install -U platformio meshtastic adafruit-nrfutil + + - name: Upgrade platformio + run: | + pio upgrade + + - name: Pull web ui + uses: dsaltares/fetch-gh-release-asset@master + with: + repo: "meshtastic/meshtastic-web" + file: "build.tar" + target: "build.tar" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Unpack web ui + run: | + tar -xf build.tar -C data/static + rm build.tar + + # We now run integration test before other build steps (to quickly see runtime failures) + - name: Build for native + run: platformio run -e native + - name: Integration test + run: | + .pio/build/native/program & + sleep 20 # 5 seconds was not enough + echo "Simulator started, launching python test..." + python3 -c 'from meshtastic.test import testSimulator; testSimulator()' + + - name: Cat bin/build-nrf52.sh + run: | + cat bin/build-nrf52.sh ${{ matrix.board }} + + - name: Build NRF52 + run: bin/build-nrf52.sh ${{ matrix.board }} + + - name: Get release version string + run: echo "::set-output name=version::$(./bin/buildinfo.py long)" + id: version + + - name: Store binaries as an artifact + uses: actions/upload-artifact@v2 + with: + name: firmware-${{ steps.version.outputs.version }}.zip + path: release/archive/firmware-${{ steps.version.outputs.version }}.zip + retention-days: 90 + + - name: Store debugging elf files as an artifact + uses: actions/upload-artifact@v2 + with: + name: debug-elfs + path: release/archive/elfs-*.zip + retention-days: 7 + # - name: Download firmware.zip + # uses: actions/download-artifact@master + # with: + # name: firmware-${{ steps.version.outputs.version }}.zip + # path: ./ + + # - name: Pull request artifacts + # if: ${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request' }} + # uses: gavv/pull-request-artifacts@v1.0.0 + # with: + # commit: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }} + # repo-token: ${{ secrets.GITHUB_TOKEN }} + # artifacts-branch: artifacts + # artifacts: ./firmware-${{ steps.version.outputs.version }}.zip diff --git a/bin/build-esp32.sh b/bin/build-esp32.sh new file mode 100644 index 0000000000..d194518968 --- /dev/null +++ b/bin/build-esp32.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -e + +VERSION=`bin/buildinfo.py long` +SHORT_VERSION=`bin/buildinfo.py short` + +OUTDIR=release/latest + +# We keep all old builds (and their map files in the archive dir) +ARCHIVEDIR=release/archive + +rm -f $OUTDIR/firmware* + +mkdir -p $OUTDIR/bins $ARCHIVEDIR +rm -r $OUTDIR/bins/* || true +mkdir -p $OUTDIR/bins/universal $OUTDIR/elfs/universal + +# Make sure our submodules are current +git submodule update + +# Important to pull latest version of libs into all device flavors, otherwise some devices might be stale +platformio lib update + +BOARD=$1 + +echo "Building for $BOARD with $PLATFORMIO_BUILD_FLAGS" +rm -f .pio/build/$BOARD/firmware.* + +# The shell vars the build tool expects to find +export APP_VERSION=$VERSION + +# Are we building a universal/regionless rom? +export HW_VERSION="1.0" +basename=universal/firmware-$BOARD-$VERSION + +pio run --environment $BOARD # -v +SRCELF=.pio/build/$BOARD/firmware.elf +cp $SRCELF $OUTDIR/elfs/$basename.elf + +echo "Copying ESP32 bin file" +SRCBIN=.pio/build/$BOARD/firmware.bin +cp $SRCBIN $OUTDIR/bins/$basename.bin + +echo "Building SPIFFS for ESP32 targets" +pio run --environment tbeam -t buildfs +cp .pio/build/tbeam/spiffs.bin $OUTDIR/bins/universal/spiffs-$VERSION.bin + +# keep the bins in archive also +cp $OUTDIR/bins/universal/spiffs* $OUTDIR/bins/universal/firmware* $OUTDIR/elfs/universal/firmware* $ARCHIVEDIR + +echo Generating $ARCHIVEDIR/firmware-$VERSION.zip +rm -f $ARCHIVEDIR/firmware-$VERSION.zip +zip --junk-paths $ARCHIVEDIR/firmware-$VERSION.zip $ARCHIVEDIR/spiffs-$VERSION.bin $OUTDIR/bins/universal/firmware-*-$VERSION.* $OUTDIR/bins/universal/meshtasticd* images/system-info.bin bin/device-install.* bin/device-update.* +echo Generating $ARCHIVEDIR/elfs-$VERSION.zip +rm -f $ARCHIVEDIR/elfs-$VERSION.zip +zip --junk-paths $ARCHIVEDIR/elfs-$VERSION.zip $OUTDIR/elfs/universal/firmware-*-$VERSION.* + +echo BUILT ALL diff --git a/bin/build-nrf52.sh b/bin/build-nrf52.sh new file mode 100644 index 0000000000..965f763236 --- /dev/null +++ b/bin/build-nrf52.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -e + +VERSION=`bin/buildinfo.py long` +SHORT_VERSION=`bin/buildinfo.py short` + +OUTDIR=release/latest + +ARCHIVEDIR=release/archive + +rm -f $OUTDIR/firmware* + +mkdir -p $OUTDIR/bins $ARCHIVEDIR +rm -r $OUTDIR/bins/* || true +mkdir -p $OUTDIR/bins/universal $OUTDIR/elfs/universal + +# Make sure our submodules are current +git submodule update + +# Important to pull latest version of libs into all device flavors, otherwise some devices might be stale +platformio lib update + +BOARD=$1 + +echo "Building for $BOARD with $PLATFORMIO_BUILD_FLAGS" +rm -f .pio/build/$BOARD/firmware.* + +# The shell vars the build tool expects to find +export APP_VERSION=$VERSION + +# Are we building a universal/regionless rom? +export HW_VERSION="1.0" +basename=universal/firmware-$BOARD-$VERSION + +pio run --environment $BOARD # -v +SRCELF=.pio/build/$BOARD/firmware.elf +cp $SRCELF $OUTDIR/elfs/$basename.elf + +echo "Generating NRF52 uf2 file" +SRCHEX=.pio/build/$BOARD/firmware.hex +bin/uf2conv.py $SRCHEX -c -o $OUTDIR/bins/$basename.uf2 -f 0xADA52840 + +echo Generating $ARCHIVEDIR/firmware-$VERSION.zip +rm -f $ARCHIVEDIR/firmware-$VERSION.zip +zip --junk-paths $ARCHIVEDIR/firmware-$VERSION.zip $ARCHIVEDIR/spiffs-$VERSION.bin $OUTDIR/bins/universal/firmware-*-$VERSION.* $OUTDIR/bins/universal/meshtasticd* images/system-info.bin bin/device-install.* bin/device-update.* +echo Generating $ARCHIVEDIR/elfs-$VERSION.zip +rm -f $ARCHIVEDIR/elfs-$VERSION.zip +zip --junk-paths $ARCHIVEDIR/elfs-$VERSION.zip $OUTDIR/elfs/universal/firmware-*-$VERSION.* From 759bd306f90f4b2b2389a78c1316c34c938a3ec3 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Sat, 12 Feb 2022 09:27:17 -0600 Subject: [PATCH 2/3] Ignore master for right now --- .github/workflows/main_matrix.yml | 37 ++++++------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index 63914968d7..096dbe00c3 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -1,12 +1,12 @@ name: Continuous Integration on: - # Triggers the workflow on push or pull request events but only for the master branch - push: - branches: [ master ] - paths-ignore: - - '**.md' - - '**.yml' - - 'version.properties' + # # Triggers the workflow on push or pull request events but only for the master branch + # push: + # branches: [ master ] + # paths-ignore: + # - '**.md' + # - '**.yml' + # - 'version.properties' # Note: This is different from "pull_request". Need to specify ref when doing checkouts. pull_request_target: @@ -178,29 +178,6 @@ jobs: run: | pio upgrade - - name: Pull web ui - uses: dsaltares/fetch-gh-release-asset@master - with: - repo: "meshtastic/meshtastic-web" - file: "build.tar" - target: "build.tar" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Unpack web ui - run: | - tar -xf build.tar -C data/static - rm build.tar - - # We now run integration test before other build steps (to quickly see runtime failures) - - name: Build for native - run: platformio run -e native - - name: Integration test - run: | - .pio/build/native/program & - sleep 20 # 5 seconds was not enough - echo "Simulator started, launching python test..." - python3 -c 'from meshtastic.test import testSimulator; testSimulator()' - - name: Cat bin/build-nrf52.sh run: | cat bin/build-nrf52.sh ${{ matrix.board }} From 0ea390d956068a1a3ae71fdc632cf5c001dd5ad5 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Sat, 12 Feb 2022 09:32:49 -0600 Subject: [PATCH 3/3] Update suppressions.txt --- suppressions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/suppressions.txt b/suppressions.txt index cdf8587531..7b6acb8e98 100644 --- a/suppressions.txt +++ b/suppressions.txt @@ -1,6 +1,7 @@ // cppcheck suppressions assertWithSideEffect + // TODO: need to come back to these duplInheritedMember