diff --git a/.github/scripts/check_no_std.sh b/.github/scripts/check_no_std.sh index 0306df8724..14e08a3059 100755 --- a/.github/scripts/check_no_std.sh +++ b/.github/scripts/check_no_std.sh @@ -8,11 +8,16 @@ no_std_packages=( commonware-storage ) -target="thumbv7em-none-eabihf" +target="riscv32imac-unknown-none-elf" +image="ghcr.io/commonwarexyz/monorepo/rust-riscv32imac-cross@sha256:652f5ff21c943935bc1caf7cf0c65b38127381c66b423f70f86dc7785d93ce85" base_rustflags="${RUSTFLAGS:-}" for package in "${no_std_packages[@]}"; do - build_cmd=(cargo build -p "$package" --no-default-features --target "$target" --release) + build_cmd=(docker run \ + --rm \ + -v `pwd`:/workdir \ + -w="/workdir" \ + "$image" cargo +nightly build -p "$package" -Zbuild-std=core,alloc --no-default-features --target "$target" --release) pretty_cmd="${build_cmd[*]}" if [ -n "$CI" ]; then echo "::group::${pretty_cmd}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..4bc67faf84 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,165 @@ +name: Build and Publish Docker Image + +on: + workflow_dispatch: + inputs: + target: + type: choice + description: Which image to release + required: true + options: + - riscv-unknown-elf-toolchain + - rust-riscv32imac-cross + - rust-riscv64imac-cross + push: + tags: + # matches tags like `service/v1.0.0` + - "*/v*" + +env: + REGISTRY: ghcr.io + REGISTRY_IMAGE: ghcr.io/commonwarexyz/monorepo + GIT_REF_NAME: ${{ github.ref_name }} + +jobs: + prepare: + name: Prepare Bake + runs-on: ubuntu-latest + permissions: + packages: write + outputs: + matrix: ${{ steps.platforms.outputs.matrix }} + target: ${{ steps.target-spec.outputs.target }} + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Specify Target + id: target-spec + run: | + export TARGET="${{ inputs.target }}" + if [[ -z $TARGET ]]; then + export TARGET="${GIT_REF_NAME%/*}" + fi + echo "Target: $TARGET" + echo "target=$TARGET" >> $GITHUB_OUTPUT + - name: Create matrix + id: platforms + run: | + echo "matrix=$(docker buildx bake -f docker/docker-bake.hcl ${{ steps.target-spec.outputs.target }} --print | jq -cr '.target."${{ steps.target-spec.outputs.target }}".platforms')" >> ${GITHUB_OUTPUT} + - name: Show matrix + run: | + echo ${{ steps.platforms.outputs.matrix }} + - name: Docker meta + id: meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: ${{ env.REGISTRY_IMAGE }}/${{ steps.target-spec.outputs.target }} + tags: | + type=ref,event=branch + type=match,pattern=v(.*),group=1,event=tag + type=ref,event=pr + - name: Rename meta bake definition file + run: | + mv "${{ steps.meta.outputs.bake-file }}" "${{ runner.temp }}/bake-meta.json" + - name: Upload meta bake definition + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: bake-meta + path: ${{ runner.temp }}/bake-meta.json + if-no-files-found: error + retention-days: 1 + + build: + name: Build Image (${{ needs.prepare.outputs.target }} - ${{ matrix.platform }}) + runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-22.04-arm' }} + permissions: + packages: write + needs: + - prepare + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.prepare.outputs.matrix) }} + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + large-packages: false + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Download meta bake definition + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: bake-meta + path: ${{ runner.temp }} + - name: Authenticate with container registry + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + - name: Build + id: bake + uses: docker/bake-action@3acf805d94d93a86cce4ca44798a76464a75b88c # v6.9.0 + with: + files: | + ./docker/docker-bake.hcl + cwd://${{ runner.temp }}/bake-meta.json + targets: ${{ needs.prepare.outputs.target }} + set: | + *.tags= + *.platform=${{ matrix.platform }} + *.output=type=image,"name=${{ env.REGISTRY_IMAGE }}/${{ needs.prepare.outputs.target }}",push-by-digest=true,name-canonical=true,push=true + - name: Export digest + run: | + mkdir -p ${{ runner.temp }}/digests + digest="${{ fromJSON(steps.bake.outputs.metadata)[needs.prepare.outputs.target]['containerimage.digest'] }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + - name: Upload digest + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Publish Manifest (${{ needs.prepare.outputs.target }}) + runs-on: ubuntu-latest + permissions: + packages: write + needs: + - build + - prepare + steps: + - name: Download meta bake definition + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: bake-meta + path: ${{ runner.temp }} + - name: Download digests + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + - name: Authenticate with container registry + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + run: | + docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map(select(startswith("${{ env.REGISTRY_IMAGE }}/${{ needs.prepare.outputs.target }}")) | "-t " + .) | join(" ")' ${{ runner.temp }}/bake-meta.json) \ + $(printf '${{ env.REGISTRY_IMAGE }}/${{ needs.prepare.outputs.target }}@sha256:%s ' *) + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}/${{ needs.prepare.outputs.target }}:$(jq -r '.target."docker-metadata-action".args.DOCKER_META_VERSION' ${{ runner.temp }}/bake-meta.json) diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml index bbcb153ecf..b7bbeeea61 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/fast.yml @@ -176,12 +176,6 @@ jobs: uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 - name: Run setup uses: ./.github/actions/setup - - name: Add minimal no_std target - run: rustup target add thumbv7em-none-eabihf - - name: Install ARM bare-metal toolchain (required to compile C from x86 worker) - run: | - sudo apt-get update - sudo apt-get install -y gcc-arm-none-eabi - name: Check no_std compatibility run: ./.github/scripts/check_no_std.sh diff --git a/README.md b/README.md index 70293943ab..9d4d4683d9 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ _Examples may include insecure code (i.e. deriving keypairs from an integer argu _Sometimes, we opt to maintain software that is neither a primitive nor an example to make it easier to interact with the Commonware Library. Unless otherwise indicated, code in this section is intended to be used in production. Please refer to our [security policy](./SECURITY.md) before disclosing an exploit publicly._ * [docs](./docs): Access information about Commonware at https://commonware.xyz. +* [docker](./docker): Dockerfiles used for cross-compilation and CI. * [macros](./macros/README.md): Augment the development of primitives with procedural macros. * [pipeline](./pipeline): Mechanisms under development. * [utils](./utils/README.md): Leverage common functionality across multiple primitives. diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..148fcd6126 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,67 @@ +# `commonware-docker` + +This directory contains all of the repositories' dockerfiles as well as the [bake file](https://docs.docker.com/build/bake/) +used to define this repository's docker build configuration. + +## Install Dependencies + +* `docker`: https://www.docker.com/get-started/ +* `docker-buildx`: https://github.com/docker/buildx?tab=readme-ov-file#installing + +## Building Locally + +To build any image in the bake file locally, use `docker buildx bake`: + +```sh +export TARGET="" + +# Optional: adjust the tag for the image +# Defaults to `commonware:local` +export DEFAULT_TAG="my-image:local" + +# Optional: Override the platforms to build the image for. +# Defaults to `linux/amd64,linux/arm64` +export PLATFORMS="" + +docker buildx bake \ + --progress plain \ + -f docker/docker-bake.hcl \ + $TARGET +``` + +#### Troubleshooting + +If you receive an error like the following: + +``` +ERROR: Multi-platform build is not supported for the docker driver. +Switch to a different driver, or turn on the containerd image store, and try again. +Learn more at https://docs.docker.com/go/build-multi-platform/ +``` + +Create and activate a new builder and retry the bake command. + +```sh +docker buildx create --name commonware-builder --use +``` + +## Cutting a Release (for maintainers / forks) + +To cut a release of the docker image for any of the targets, cut a new annotated tag for the target like so: + +```sh +# Example formats: +# - `rust-riscv-cross/v0.1.0-beta.8` +# - `riscv-unknown-elf-toolchain/v1.2.0` +TAG="/" +git tag -a $TAG -m "" && git push origin tag $TAG +``` + +To run the workflow manually, navigate over to the ["Build and Publish Docker Image"](https://github.com/commonwarexyz/monorepo/actions/workflows/docker.yaml) +action. From there, run a `workflow_dispatch` trigger, select the tag you just pushed, and then finally select the image to release. + +Or, if you prefer to use the `gh` CLI, you can run: +```sh +gh workflow run "Build and Publish Docker Image" --ref -f target= +``` + diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl new file mode 100644 index 0000000000..a34e146ea6 --- /dev/null +++ b/docker/docker-bake.hcl @@ -0,0 +1,50 @@ +variable "REGISTRY" { + default = "ghcr.io" +} + +variable "REPOSITORY" { + default = "commonwarexyz/monorepo" +} + +variable "DEFAULT_TAG" { + default = "commonware:local" +} + +variable "PLATFORMS" { + // Only specify a single platform when `--load` ing into docker. + // Multi-platform is supported when outputting to disk or pushing to a registry. + // Multi-platform builds can be tested locally with: --set="*.output=type=image,push=false" + default = "linux/amd64,linux/arm64" +} + +// Special target: https://github.com/docker/metadata-action#bake-definition +target "docker-metadata-action" { + tags = ["${DEFAULT_TAG}"] +} + +target "riscv-unknown-elf-toolchain" { + inherits = ["docker-metadata-action"] + context = "." + dockerfile = "docker/riscv-unknown-elf-toolchain.dockerfile" + platforms = split(",", PLATFORMS) +} + +target "rust-riscv32imac-cross" { + inherits = ["docker-metadata-action"] + context = "." + dockerfile = "docker/rust-riscv-cross.dockerfile" + args = { + ARCH = "riscv32imac" + } + platforms = split(",", PLATFORMS) +} + +target "rust-riscv64imac-cross" { + inherits = ["docker-metadata-action"] + context = "." + dockerfile = "docker/rust-riscv-cross.dockerfile" + args = { + ARCH = "riscv64imac" + } + platforms = split(",", PLATFORMS) +} diff --git a/docker/riscv-unknown-elf-toolchain.dockerfile b/docker/riscv-unknown-elf-toolchain.dockerfile new file mode 100644 index 0000000000..4ccdf88530 --- /dev/null +++ b/docker/riscv-unknown-elf-toolchain.dockerfile @@ -0,0 +1,46 @@ +FROM ubuntu:22.04 + +# Install core build dependencies +RUN apt-get update && \ + apt-get install --assume-yes --no-install-recommends \ + ca-certificates \ + autoconf \ + automake \ + autotools-dev \ + curl \ + python3 \ + python3-pip \ + python3-tomli \ + libmpc-dev \ + libmpfr-dev \ + libgmp-dev \ + gawk \ + build-essential \ + bison \ + flex \ + texinfo \ + gperf \ + libtool \ + patchutils \ + bc \ + zlib1g-dev \ + libexpat-dev \ + ninja-build \ + git \ + cmake \ + libglib2.0-dev \ + libslirp-dev + +ENV RISCV=/opt/riscv +ENV RISCV_TAG=2025.09.28 +ENV PATH=$PATH:$RISCV/bin + +# https://github.com/riscv-collab/riscv-gnu-toolchain/issues/1669#issuecomment-2682013720 +RUN git clone https://github.com/riscv/riscv-gnu-toolchain --branch $RISCV_TAG && \ + cd riscv-gnu-toolchain && \ + sed -i '/shallow = true/d' .gitmodules && \ + sed -i 's/--depth 1//g' Makefile.in && \ + ./configure --prefix=$RISCV --enable-multilib && \ + make && \ + cd .. && \ + rm -rf riscv-gnu-toolchain diff --git a/docker/rust-riscv-cross.dockerfile b/docker/rust-riscv-cross.dockerfile new file mode 100644 index 0000000000..5007c2c4d0 --- /dev/null +++ b/docker/rust-riscv-cross.dockerfile @@ -0,0 +1,47 @@ +FROM ghcr.io/commonwarexyz/monorepo/riscv-unknown-elf-toolchain@sha256:09897c042df2487352e499a5ec121e4ad8ea4828d691aa2e6b9435e5ca59f25b AS rv-bare +FROM ubuntu:22.04 + +ARG ARCH + +ENV SHELL=/bin/bash +ENV DEBIAN_FRONTEND=noninteractive + +# Install core dependencies +RUN apt-get update && apt-get install --assume-yes --no-install-recommends \ + ca-certificates \ + build-essential \ + autoconf \ + automake \ + autotools-dev \ + git \ + curl \ + make \ + cmake \ + xxd \ + g++-riscv64-linux-gnu \ + libc6-dev-riscv64-cross \ + binutils-riscv64-linux-gnu \ + llvm \ + clang + +# Copy the RISC-V GNU toolchain from the previous stage +COPY --from=rv-bare /opt/riscv /opt/riscv + +# Install Rustup and Rust +ENV RUST_VERSION=nightly +ENV PATH="/root/.cargo/bin:${PATH}" +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y --default-toolchain ${RUST_VERSION} --component rust-src + +# Add the RV toolchain to the path +ENV PATH="/opt/riscv/bin:${PATH}" + +# Set up the env vars to instruct rustc to use the correct compiler and linker +ENV CC_riscv32_unknown_none_elf=riscv64-linux-gnu-gcc \ + CC_riscv64_unknown_none_elf=riscv64-linux-gnu-gcc \ + CXX_riscv32_unknown_none_elf=riscv64-linux-gnu-g++ \ + CXX_riscv64_unknown_none_elf=riscv64-linux-gnu-g++ \ + CARGO_TARGET_RISCV32_UNKNOWN_NONE_ELF_LINKER=riscv64-linux-gnu-gcc \ + CARGO_TARGET_RISCV64_UNKNOWN_NONE_ELF_LINKER=riscv64-linux-gnu-gcc \ + RUSTFLAGS="-Clink-arg=-e_start" \ + CARGO_BUILD_TARGET="$ARCH-unknown-none-elf" \ + RUSTUP_TOOLCHAIN=${RUST_VERSION}