Fix jimtcl library issues, create universal MacOS builds of everything, and add tests

Fix library issues by switching back to building jimtcl inside openocd

Make all builds on MacOS universal, built on an arm64 runner

Add run tests of builds on clean systems, to check for missed libraries

Author: will-v-pi

.github/workflows/build.yml

@@ -12,17 +12,11 @@ env:
 jobs:
   build_windows:
     name: Build Windows
-    # runs-on: [self-hosted, Windows, x64]
     runs-on: 'windows-latest'
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      # - name: Setup SSH Auth
-      #   if: runner.environment == 'github-hosted'
-      #   uses: webfactory/[email protected]
-      #   with:
-      #     ssh-private-key: ${{ secrets.SSH_KEY }}
       - name: Setup MSYS2
         uses: msys2/setup-msys2@v2
       - name: Build
@@ -33,7 +27,7 @@ jobs:
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         with:
-          name: tools-x64-win
+          name: tools-win-${{ runner.arch }}
           path: |
             bin/picotool-*-x64-win.zip
             bin/pico-sdk-tools-*-x64-win.zip
@@ -49,38 +43,41 @@ jobs:
             bin/openocd-*-x64-win.zip
             bin/riscv-toolchain-*-x64-win.zip
 
-  build_macos_arm64:
-    name: Build MacOS Arm64
-    # runs-on: [self-hosted, macOS]
+  build_macos:
+    name: Build MacOS
     runs-on: 'macos-14'
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up Homebrew
-        if: runner.environment == 'github-hosted'
+      - name: Set up arm64 Homebrew
         id: set-up-homebrew
         uses: Homebrew/actions/setup-homebrew@master
+      - name: Build arm64
+        run: ./build_macos.sh
+      - name: Uninstall arm64 Homebrew
+        run: |
+          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/uninstall.sh)"
+          rm -rf /opt/homebrew || true
       - name: Set up x86_64 Homebrew
-        if: runner.environment == 'github-hosted'
         run: |
           NONINTERACTIVE=1 arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-      # - name: Setup SSH Auth
-      #   if: runner.environment == 'github-hosted'
-      #   uses: webfactory/[email protected]
-      #   with:
-      #     ssh-private-key: ${{ secrets.SSH_KEY }}
-      - name: Build
-        run: ./build_macos.sh
+      - name: Install x86_64 pkg-config & cmake
+        run: |
+          arch -x86_64 /usr/local/bin/brew install pkg-config cmake
+      - name: Build x86_64
+        run: arch -x86_64 ./build_macos.sh
+      - name: Merge Universal Binaries
+        run: ./merge_macos.sh
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         with:
-          name: tools-mac-arm64
+          name: tools-mac-universal
           path: |
             bin/picotool-*-mac.zip
             bin/pico-sdk-tools-*-mac.zip
             bin/openocd-*-mac.zip
-            bin/riscv-toolchain-*.zip
+            bin/riscv-toolchain-*-mac.zip
       - name: Add Release Asset
         uses: softprops/action-gh-release@v2
         if: startsWith(github.ref, 'refs/tags/')
@@ -89,57 +86,19 @@ jobs:
             bin/picotool-*-mac.zip
             bin/pico-sdk-tools-*-mac.zip
             bin/openocd-*-mac.zip
-            bin/riscv-toolchain-*.zip
-  
-  build_macos_x64:
-    name: Build MacOS x64
-    runs-on: 'macos-13'
-    env:
-      # Universal picotool is built on arm64
-      SKIP_PICOTOOL: 1
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up Homebrew
-        if: runner.environment == 'github-hosted'
-        id: set-up-homebrew
-        uses: Homebrew/actions/setup-homebrew@master
-      - name: Build
-        run: ./build_macos.sh
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: tools-mac-x64
-          path: |
-            bin/openocd-*-mac.zip
-            bin/riscv-toolchain-*.zip
-      - name: Add Release Asset
-        uses: softprops/action-gh-release@v2
-        if: startsWith(github.ref, 'refs/tags/')
-        with:
-          files: |
-            bin/openocd-*-mac.zip
-            bin/riscv-toolchain-*.zip
+            bin/riscv-toolchain-*-mac.zip
 
   build_linux:
     name: Build Linux
-    # strategy:
-    #   matrix:
-    #     os: [[self-hosted, linux, x64], [self-hosted, linux, arm64]]
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-22.04, [self-hosted, linux, arm64]]
+        os: [ubuntu-22.04, [self-hosted, linux, arm64, bookworm, 8G]]
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      # - name: Setup SSH Auth
-      #   if: runner.environment == 'github-hosted'
-      #   uses: webfactory/[email protected]
-      #   with:
-      #     ssh-private-key: ${{ secrets.SSH_KEY }}
       - name: Build
         run: ./build_linux.sh
       - name: Upload Artifact
@@ -161,3 +120,77 @@ jobs:
             bin/openocd-*-lin.tar.gz
             bin/riscv-toolchain-*-lin.tar.gz
 
+  test_binaries:
+    name: Test Binaries
+    needs: [build_linux, build_macos, build_windows]
+    if: always() && contains(needs.*.result, 'success')
+    strategy:
+      fail-fast: false
+      matrix:
+        # Test all GitHub supported versions, except for windows-11-arm
+        os: [macos-15, macos-15-intel, macos-14, windows-2025, windows-2022, windows-11-arm, ubuntu-24.04, ubuntu-22.04, ubuntu-24.04-arm, ubuntu-22.04-arm, [self-hosted, linux, arm64, trixie_desktop], [self-hosted, linux, arm64, bookworm_desktop]]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Download build (Windows)
+        if: runner.os == 'Windows'
+        uses: actions/download-artifact@v5
+        with:
+          # Only built for x64, but arm should still run it fine
+          name: tools-win-X64
+      - name: Download build (MacOS)
+        if: runner.os == 'macOS'
+        uses: actions/download-artifact@v5
+        with:
+          name: tools-mac-universal
+      - name: Download build (Linux)
+        if: runner.os == 'Linux'
+        uses: actions/download-artifact@v5
+        with:
+          name: tools-lin-${{ runner.arch }}
+
+      - name: Extract build (zip)
+        if: runner.os == 'Windows' || runner.os == 'macOS'
+        shell: bash # Windows only has unzip in bash shell
+        run: |
+          unzip -o pico-sdk-tools*.zip
+          ls
+          unzip -o picotool*.zip
+          ls
+          unzip -o openocd*.zip || true
+          ls
+          unzip -o riscv-toolchain*.zip || true
+          ls
+      - name: Extract build (tar.gz)
+        if: runner.os == 'Linux'
+        run: |
+          tar -xvf pico-sdk-tools*.tar.gz
+          ls
+          tar -xvf picotool*.tar.gz
+          ls
+          tar -xvf openocd*.tar.gz || true
+          ls
+          tar -xvf riscv-toolchain*.tar.gz || true
+          ls
+
+      - name: Clean runner (MacOS)
+        if: runner.os == 'macOS'
+        run: |
+          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/uninstall.sh)"
+      - name: Install prerequisites (Ubuntu)
+        if: runner.os == 'Linux' && runner.environment == 'github-hosted'
+        run: sudo apt install libftdi1-2 libhidapi-hidraw0
+
+      - name: Test default stuff runs
+        run: |
+          ./picotool/picotool version
+          ./pioasm/pioasm --version
+      - name: Test openocd runs
+        if: env.SKIP_OPENOCD != 1
+        run: |
+          ./openocd --version
+      - name: Test riscv-toolchain runs
+        if: env.SKIP_RISCV != 1
+        run: |
+          ./bin/riscv32-unknown-elf-gcc --version
+          ./bin/riscv32-unknown-elf-gdb --version
+

build_linux.sh

@@ -7,7 +7,7 @@ SKIP_RISCV=${SKIP_RISCV-0}
 SKIP_OPENOCD=${SKIP_OPENOCD-0}
 
 # Install prerequisites
-sudo apt install -y jq cmake libtool automake libusb-1.0-0-dev libhidapi-dev libftdi1-dev libjim-dev patchelf
+sudo apt install -y jq cmake libtool automake libusb-1.0-0-dev libhidapi-dev libftdi1-dev patchelf
 # RISC-V prerequisites
 sudo apt install -y autoconf automake autotools-dev curl python3 python3-pip libmpc-dev libmpfr-dev libgmp-dev gawk build-essential bison flex texinfo gperf libtool patchutils bc zlib1g-dev ninja-build git cmake libglib2.0-dev libslirp-dev
 # RPi Only prerequisites

build_macos.sh

@@ -7,10 +7,13 @@ SKIP_RISCV=${SKIP_RISCV-0}
 SKIP_OPENOCD=${SKIP_OPENOCD-0}
 SKIP_PICOTOOL=${SKIP_PICOTOOL-0}
 
+echo "Running on $(uname -m)"
+
 # Install prerequisites
-arch -x86_64 /usr/local/bin/brew install jq libtool libusb automake hidapi jimtcl --quiet
 if [[ $(uname -m) == 'arm64' ]]; then
-    arch -arm64 /opt/homebrew/bin/brew install jq libtool libusb automake hidapi jimtcl --quiet
+    arch -arm64 /opt/homebrew/bin/brew  install jq libtool libusb automake hidapi --quiet
+else
+    arch -x86_64 /usr/local/bin/brew    install jq libtool libusb automake hidapi --quiet
 fi
 # RISC-V prerequisites
 echo "Listing local"
@@ -19,9 +22,10 @@ rm /usr/local/bin/2to3* || true
 rm /usr/local/bin/idle3* || true
 rm /usr/local/bin/pip* || true
 rm /usr/local/bin/py* || true
-arch -x86_64 /usr/local/bin/brew install python3 gawk gnu-sed make gmp mpfr libmpc isl zlib expat texinfo flock libslirp --quiet
 if [[ $(uname -m) == 'arm64' ]]; then
-    arch -arm64 /opt/homebrew/bin/brew install python3 gawk gnu-sed make gmp mpfr libmpc isl zlib expat texinfo flock libslirp --quiet
+    arch -arm64 /opt/homebrew/bin/brew  install python3 gawk gnu-sed make gmp mpfr libmpc isl zlib expat texinfo flock libslirp --quiet
+else
+    arch -x86_64 /usr/local/bin/brew    install python3 gawk gnu-sed make gmp mpfr libmpc isl zlib expat texinfo flock libslirp --quiet
 fi
 
 repos=$(cat config/repositories.json | jq -c '.repositories.[]')
@@ -76,82 +80,10 @@ if [[ "$SKIP_RISCV" != 1 ]]; then
     echo "RISC-V dylibs copied"
 fi
 if [[ "$SKIP_PICOTOOL" != 1 ]]; then
-    arch -x86_64 ../packages/macos/picotool/build-picotool.sh
-    if [[ $(uname -m) == 'arm64' ]]; then
-        arch -arm64 ../packages/macos/picotool/build-picotool.sh
-    fi
+    ../packages/macos/picotool/build-picotool.sh
     echo "Picotool Build Complete"
 
-    ../packages/macos/get-dylibs.sh "picotool-install-x86_64"
-    if [[ $(uname -m) == 'arm64' ]]; then
-        ../packages/macos/get-dylibs.sh "picotool-install-arm64"
-    fi
+    ../packages/macos/get-dylibs.sh "picotool-install-$(uname -m)"
     echo "Picotool dylibs copied"
-
-    ../packages/macos/make-universal.sh "pico-sdk-tools" "pioasm" "pioasm"
-    echo "Pioasm Universal Merge Complete"
-    ../packages/macos/make-universal.sh "picotool-install" "picotool" "picotool"
-    echo "Picotool Universal Merge Complete"
 fi
 cd ..
-
-topd=$PWD
-
-if [[ "$SKIP_PICOTOOL" != 1 ]]; then
-    echo "Packaging picotool"
-    if [ ${version:0:1} -ge 2 ]; then
-        # Package pico-sdk-tools separately as well
-
-        filename="pico-sdk-tools-${version}-${suffix}.zip"
-
-        echo "Saving pico-sdk-tools package to $filename"
-        pushd "$builddir/pico-sdk-tools/"
-        tar -a -cf "$topd/bin/$filename" * .keep
-        popd
-    fi
-
-    # Package picotool separately as well
-    version=$("./$builddir/picotool-install/picotool/picotool" version -s)
-    echo "Picotool version $version"
-
-    filename="picotool-${version}-${suffix}.zip"
-
-    echo "Saving picotool package to $filename"
-    pushd "$builddir/picotool-install/"
-    tar -a -cf "$topd/bin/$filename" * .keep
-    popd
-fi
-
-if [[ "$SKIP_OPENOCD" != 1 ]]; then
-    echo "Packaging OpenOCD"
-    # Package OpenOCD separately as well
-
-    version=($("./$builddir/openocd-install-$(uname -m)/usr/local/bin/openocd" --version 2>&1))
-    version=${version[0]}
-    version=${version[3]}
-    version=$(echo $version | cut -d "-" -f 1)
-
-    echo "OpenOCD version $version"
-
-    filename="openocd-${version}-$(uname -m)-${suffix}.zip"
-
-    echo "Saving OpenOCD package to $filename"
-    pushd "$builddir/openocd-install-$(uname -m)/usr/local/bin"
-    tar -a -cf "$topd/bin/$filename" * -C "../share/openocd" "scripts"
-    popd
-fi
-
-if [[ "$SKIP_RISCV" != 1 ]]; then
-    echo "Packaging RISC-V Toolchain"
-    # Package riscv toolchain separately as well
-    version=$("./$builddir/riscv-install-$(uname -m)/bin/riscv32-unknown-elf-gcc" -dumpversion)
-    version=$(echo $version | cut -d "." -f 1)
-    echo "Risc-V Toolchain version $version"
-
-    filename="riscv-toolchain-${version}-$(uname -m)-${suffix}.zip"
-
-    echo "Saving RISC-V Toolchain package to $filename"
-    pushd "$builddir/riscv-install-$(uname -m)/"
-    tar -a -cf "$topd/bin/$filename" *
-    popd
-fi