From 163101be52ead7a7523402f79dc0081fe1ec9816 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Tue, 5 Mar 2024 17:02:23 -0700 Subject: [PATCH 1/4] feat(*): add node installer dir and workflow Signed-off-by: Vaughn Dice --- .github/workflows/ci.yaml | 5 ++ .github/workflows/docker-build-push.yaml | 2 +- .github/workflows/node-installer.yaml | 69 ++++++++++++++++++++++++ .github/workflows/release.yaml | 38 +++++++++---- node-installer/Dockerfile | 23 ++++++++ node-installer/Makefile | 18 +++++++ node-installer/README.md | 9 ++++ node-installer/script/installer.sh | 65 ++++++++++++++++++++++ 8 files changed, 217 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/node-installer.yaml create mode 100644 node-installer/Dockerfile create mode 100644 node-installer/Makefile create mode 100644 node-installer/README.md create mode 100644 node-installer/script/installer.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0ffe6b4..b1c74fb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,6 +26,11 @@ jobs: test: true build: uses: ./.github/workflows/build.yaml + publish-node-installer-image: + uses: ./.github/workflows/node-installer.yaml + needs: build + with: + release: false test: needs: build runs-on: ubuntu-latest diff --git a/.github/workflows/docker-build-push.yaml b/.github/workflows/docker-build-push.yaml index aa7b20b..a0382fe 100644 --- a/.github/workflows/docker-build-push.yaml +++ b/.github/workflows/docker-build-push.yaml @@ -26,7 +26,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set RELEASE_VERSION ENV var - run: echo "RELEASE_VERSION=${GITHUB_REF:10}" >> $GITHUB_ENV + run: echo "RELEASE_VERSION=$(echo -n ${GITHUB_REF} | cut -d '/' -f 3)" >> $GITHUB_ENV - name: lowercase the runner OS name shell: bash run: | diff --git a/.github/workflows/node-installer.yaml b/.github/workflows/node-installer.yaml new file mode 100644 index 0000000..6517619 --- /dev/null +++ b/.github/workflows/node-installer.yaml @@ -0,0 +1,69 @@ +name: Publish node-installer image + +on: + workflow_call: + inputs: + ref: + description: 'the git ref for the associated workflow' + type: string + required: true + +jobs: + # Note: assumes being called in a workflow where build has already run and + # required artifacts have been uploaded + publish: + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set RELEASE_VERSION env var + run: | + if [[ "${{ startsWith(inputs.ref, 'refs/tags/v')}}" == "true" ]]; then + echo "RELEASE_VERSION=$(echo -n ${{ inputs.ref }} | cut -d '/' -f 3)" >> $GITHUB_ENV + else + echo "RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD)" >> $GITHUB_ENV + fi + + - uses: actions/download-artifact@v3 + with: + path: _artifacts + + # Setup buildx to build multiarch image: https://github.com/docker/build-push-action/blob/master/docs/advanced/multi-platform.md + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: setup buildx + uses: docker/setup-buildx-action@v2 + + - name: login to GitHub container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Build and push node-installer image + # TODO: remove once https://github.com/spinkube/runtime-class-manager handles this + - name: untar musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir + run: | + mkdir -p ./node-installer/.tmp/linux/amd64 + mkdir -p ./node-installer/.tmp/linux/arm64 + for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done + for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done + + - name: build and push node-installer image + uses: docker/build-push-action@v5 + with: + push: true + tags: | + ghcr.io/${{ github.repository }}/node-installer:${{ env.RELEASE_VERSION }} + context: node-installer + platforms: linux/amd64,linux/arm64 + + - name: clear + if: always() + run: | + rm -f ${HOME}/.docker/config.json diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bd61368..a413893 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,6 +1,8 @@ name: release on: push: + branches: + - main tags: - "v[0-9]+.[0-9]+.*" jobs: @@ -12,31 +14,42 @@ jobs: with: test: false + publish-node-installer-image: + uses: ./.github/workflows/node-installer.yaml + needs: build + with: + ref: ${{ github.ref }} + release: permissions: contents: write packages: write needs: build - if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Set RELEASE_VERSION ENV var - run: echo "RELEASE_VERSION=${GITHUB_REF:10}" >> $GITHUB_ENV - - name: lowercase the runner OS name - shell: bash run: | - OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]') - echo "RUNNER_OS=$OS" >> $GITHUB_ENV + if [[ "${{ startsWith(github.ref, 'refs/tags/v') }}" == "true" ]]; then + echo "RELEASE_VERSION=${{ github.ref_name }}" >> $GITHUB_ENV + else + echo "RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD)" >> $GITHUB_ENV + fi + + - uses: actions/download-artifact@v3 + with: + path: _artifacts + - name: copy release workload assets into _dist + if: startsWith(github.ref, 'refs/tags/v') run: | mkdir -p _dist cp ./deployments/workloads/runtime.yaml _dist/runtime.yaml cp ./deployments/workloads/workload.yaml _dist/workload.yaml - - uses: actions/download-artifact@v3 - with: - path: _artifacts + - name: create release + if: startsWith(github.ref, 'refs/tags/v') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -51,8 +64,10 @@ jobs: # Setup buildx to build multiarch image: https://github.com/docker/build-push-action/blob/master/docs/advanced/multi-platform.md - name: Set up QEMU uses: docker/setup-qemu-action@v3 + - name: setup buildx uses: docker/setup-buildx-action@v2 + - name: login to GitHub container registry uses: docker/login-action@v2 with: @@ -67,17 +82,18 @@ jobs: mkdir -p ./deployments/k3d/.tmp/linux/arm64 for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./deployments/k3d/.tmp/linux/amd64; done for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./deployments/k3d/.tmp/linux/arm64; done + - name: build and push k3d shim image uses: docker/build-push-action@v5 with: push: true tags: | ghcr.io/${{ github.repository }}/k3d:${{ env.RELEASE_VERSION }} - ghcr.io/${{ github.repository }}/k3d:latest context: deployments/k3d platforms: linux/amd64,linux/arm64 build-args: | STAGE=release + - name: clear if: always() run: | diff --git a/node-installer/Dockerfile b/node-installer/Dockerfile new file mode 100644 index 0000000..2d3985b --- /dev/null +++ b/node-installer/Dockerfile @@ -0,0 +1,23 @@ +# Based on: https://github.com/KWasm/kwasm-node-installer/blob/main/images/installer/Dockerfile + +FROM scratch AS bin +ARG TARGETPLATFORM +COPY ./.tmp/${TARGETPLATFORM} / + +FROM ubuntu:22.04 AS download-containerd-runwasi +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y curl + +RUN mkdir -p /release/bin/ \ + && curl -L https://github.com/containerd/runwasi/releases/download/containerd-shim-wasmedge%2Fv0.3.0/containerd-shim-wasmedge-$(uname -m | sed s/arm64/aarch64/g | sed s/amd64/x86_64/g).tar.gz | tar -xzf - -C /release/bin/ \ + && curl -L https://github.com/containerd/runwasi/releases/download/containerd-shim-wasmtime%2Fv0.3.0/containerd-shim-wasmtime-$(uname -m | sed s/arm64/aarch64/g | sed s/amd64/x86_64/g).tar.gz | tar -xzf - -C /release/bin/ \ + && curl -L https://github.com/containerd/runwasi/releases/download/containerd-shim-wasmer%2Fv0.3.0/containerd-shim-wasmer-$(uname -m | sed s/arm64/aarch64/g | sed s/amd64/x86_64/g).tar.gz | tar -xzf - -C /release/bin/ + +FROM busybox + +COPY script/installer.sh /script/installer.sh +COPY --link --from=bin / /assets +COPY --link --from=download-containerd-runwasi /release/bin/containerd-shim-wasmedge-v1 /assets/ +COPY --link --from=download-containerd-runwasi /release/bin/containerd-shim-wasmer-v1 /assets/ +COPY --link --from=download-containerd-runwasi /release/bin/containerd-shim-wasmtime-v1 /assets/ +CMD sh /script/installer.sh wasmedge diff --git a/node-installer/Makefile b/node-installer/Makefile new file mode 100644 index 0000000..68d08fd --- /dev/null +++ b/node-installer/Makefile @@ -0,0 +1,18 @@ +SPIN_VERSION = v2 +IMAGE_NAME ?= ghcr.io/spinkube/containerd-shim-spin/node-installer +PLATFORM ?= linux/amd64 +ARCH ?= x86_64 +TARGET ?= $(ARCH)-unknown-linux-musl + +compile-musl: + make build-spin-cross-$(TARGET) -C ../ + +move-musl-to-tmp: compile-musl + mkdir -p ./.tmp + cp ../../containerd-shim-spin/target/$(TARGET)/release/containerd-shim-spin-$(SPIN_VERSION) ./.tmp/ + +build-multi-installer-image: move-musl-to-tmp + docker buildx build -t $(IMAGE_NAME) --platform linux/amd64,linux/arm64 . + +build-dev-installer-image: move-musl-to-tmp + docker buildx build -t $(IMAGE_NAME) --load --platform $(PLATFORM) . diff --git a/node-installer/README.md b/node-installer/README.md new file mode 100644 index 0000000..957fcb7 --- /dev/null +++ b/node-installer/README.md @@ -0,0 +1,9 @@ +This directory contains resources for a custom node-installer image +intended to be used in conjunction with the [Kwasm Operator](https://github.com/KWasm/kwasm-operator). + +This version of the image only contains the containerd-shim-spin-v2 shim, as +opposed to the default [kwasm-node-installer image](https://github.com/KWasm/kwasm-node-installer) +which also bundles other shims. + +The intention is for the [spinkube/runtime-class-manager](https://github.com/spinkube/runtime-class-manager) +project to handle this concern in the future. diff --git a/node-installer/script/installer.sh b/node-installer/script/installer.sh new file mode 100644 index 0000000..8a7cd88 --- /dev/null +++ b/node-installer/script/installer.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env sh +set -euo pipefail + +# Based on: https://github.com/KWasm/kwasm-node-installer/blob/main/script/installer.sh + +KWASM_DIR=/opt/kwasm + +CONTAINERD_CONF=/etc/containerd/config.toml +IS_MICROK8S=false +IS_K3S=false +IS_RKE2_AGENT=false +if ps aux | grep kubelet | grep -q snap/microk8s; then + CONTAINERD_CONF=/var/snap/microk8s/current/args/containerd-template.toml + IS_MICROK8S=true + if nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- ls /var/snap/microk8s/current/args/containerd-template.toml > /dev/null 2>&1 ;then + KWASM_DIR=/var/snap/microk8s/common/kwasm + else + echo "Installer seems to run on microk8s but 'containerd-template.toml' not found." + exit 1 + fi +elif ls $NODE_ROOT/var/lib/rancher/rke2/agent/etc/containerd/config.toml > /dev/null 2>&1 ; then + IS_RKE2_AGENT=true + cp $NODE_ROOT/var/lib/rancher/rke2/agent/etc/containerd/config.toml $NODE_ROOT/var/lib/rancher/rke2/agent/etc/containerd/config.toml.tmpl + CONTAINERD_CONF=/var/lib/rancher/rke2/agent/etc/containerd/config.toml.tmpl +elif ls $NODE_ROOT/var/lib/rancher/k3s/agent/etc/containerd/config.toml > /dev/null 2>&1 ; then + IS_K3S=true + cp $NODE_ROOT/var/lib/rancher/k3s/agent/etc/containerd/config.toml $NODE_ROOT/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl + CONTAINERD_CONF=/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl +fi + +mkdir -p $NODE_ROOT$KWASM_DIR/bin/ + +cp /assets/containerd-shim-* $NODE_ROOT$KWASM_DIR/bin/ + +# TODO check if runtime config is already present +if ! grep -q wasmtime $NODE_ROOT$CONTAINERD_CONF; then + echo ' +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin] + runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasmedge] + runtime_type = "'$KWASM_DIR'/bin/containerd-shim-wasmedge-v1" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasmer] + runtime_type = "'$KWASM_DIR'/bin/containerd-shim-wasmer-v1" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasmtime] + runtime_type = "'$KWASM_DIR'/bin/containerd-shim-wasmtime-v1" +' >> $NODE_ROOT$CONTAINERD_CONF + rm -Rf $NODE_ROOT$KWASM_DIR/active +fi + +if [ ! -f $NODE_ROOT$KWASM_DIR/active ]; then + touch $NODE_ROOT$KWASM_DIR/active + if $IS_MICROK8S; then + nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- systemctl restart snap.microk8s.daemon-containerd + elif ls $NODE_ROOT/etc/init.d/containerd > /dev/null 2>&1 ; then + nsenter --target 1 --mount --uts --ipc --net -- /etc/init.d/containerd restart + elif ls $NODE_ROOT/etc/init.d/k3s > /dev/null 2>&1 ; then + nsenter --target 1 --mount --uts --ipc --net -- /etc/init.d/k3s restart + elif $IS_RKE2_AGENT; then + nsenter --target 1 --mount --uts --ipc --net -- /bin/systemctl restart rke2-agent + else + nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- /bin/systemctl restart containerd + fi +else + echo "No change in containerd/config.toml" +fi From ca8508468e8665e0ef2ec0470380780266f8c2e4 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Wed, 6 Mar 2024 17:31:44 -0700 Subject: [PATCH 2/4] ref(node-installer): rm installation of runtimes Signed-off-by: Vaughn Dice --- node-installer/Dockerfile | 24 +++--------------------- node-installer/script/installer.sh | 14 ++++---------- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/node-installer/Dockerfile b/node-installer/Dockerfile index 2d3985b..ca7df49 100644 --- a/node-installer/Dockerfile +++ b/node-installer/Dockerfile @@ -1,23 +1,5 @@ -# Based on: https://github.com/KWasm/kwasm-node-installer/blob/main/images/installer/Dockerfile - -FROM scratch AS bin -ARG TARGETPLATFORM -COPY ./.tmp/${TARGETPLATFORM} / - -FROM ubuntu:22.04 AS download-containerd-runwasi -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y curl - -RUN mkdir -p /release/bin/ \ - && curl -L https://github.com/containerd/runwasi/releases/download/containerd-shim-wasmedge%2Fv0.3.0/containerd-shim-wasmedge-$(uname -m | sed s/arm64/aarch64/g | sed s/amd64/x86_64/g).tar.gz | tar -xzf - -C /release/bin/ \ - && curl -L https://github.com/containerd/runwasi/releases/download/containerd-shim-wasmtime%2Fv0.3.0/containerd-shim-wasmtime-$(uname -m | sed s/arm64/aarch64/g | sed s/amd64/x86_64/g).tar.gz | tar -xzf - -C /release/bin/ \ - && curl -L https://github.com/containerd/runwasi/releases/download/containerd-shim-wasmer%2Fv0.3.0/containerd-shim-wasmer-$(uname -m | sed s/arm64/aarch64/g | sed s/amd64/x86_64/g).tar.gz | tar -xzf - -C /release/bin/ - FROM busybox - +ARG TARGETPLATFORM COPY script/installer.sh /script/installer.sh -COPY --link --from=bin / /assets -COPY --link --from=download-containerd-runwasi /release/bin/containerd-shim-wasmedge-v1 /assets/ -COPY --link --from=download-containerd-runwasi /release/bin/containerd-shim-wasmer-v1 /assets/ -COPY --link --from=download-containerd-runwasi /release/bin/containerd-shim-wasmtime-v1 /assets/ -CMD sh /script/installer.sh wasmedge +COPY ./.tmp/${TARGETPLATFORM} /assets +CMD sh /script/installer.sh diff --git a/node-installer/script/installer.sh b/node-installer/script/installer.sh index 8a7cd88..43ef011 100644 --- a/node-installer/script/installer.sh +++ b/node-installer/script/installer.sh @@ -1,7 +1,8 @@ #!/usr/bin/env sh set -euo pipefail -# Based on: https://github.com/KWasm/kwasm-node-installer/blob/main/script/installer.sh +# Based on https://github.com/KWasm/kwasm-node-installer/blob/main/script/installer.sh +# Distilled to only configuring the Spin shim KWASM_DIR=/opt/kwasm @@ -30,19 +31,12 @@ fi mkdir -p $NODE_ROOT$KWASM_DIR/bin/ -cp /assets/containerd-shim-* $NODE_ROOT$KWASM_DIR/bin/ +cp /assets/containerd-shim-spin-v2 $NODE_ROOT$KWASM_DIR/bin/ -# TODO check if runtime config is already present -if ! grep -q wasmtime $NODE_ROOT$CONTAINERD_CONF; then +if ! grep -q spin $NODE_ROOT$CONTAINERD_CONF; then echo ' [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin] runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasmedge] - runtime_type = "'$KWASM_DIR'/bin/containerd-shim-wasmedge-v1" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasmer] - runtime_type = "'$KWASM_DIR'/bin/containerd-shim-wasmer-v1" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasmtime] - runtime_type = "'$KWASM_DIR'/bin/containerd-shim-wasmtime-v1" ' >> $NODE_ROOT$CONTAINERD_CONF rm -Rf $NODE_ROOT$KWASM_DIR/active fi From 132e4c23cdb18f3059b0dd20bfa8fafa4b2883f2 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Wed, 6 Mar 2024 19:45:08 -0700 Subject: [PATCH 3/4] fix(ci.yaml): update node-installer usage Signed-off-by: Vaughn Dice --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b1c74fb..7fa9167 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,7 +30,7 @@ jobs: uses: ./.github/workflows/node-installer.yaml needs: build with: - release: false + ref: ${{ github.ref }} test: needs: build runs-on: ubuntu-latest From 167ab94187ab0c1a575b2dd36a3f41bb4fc0ef6b Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Thu, 7 Mar 2024 13:09:15 -0700 Subject: [PATCH 4/4] ci(node-installer): gate job to only run if branch on origin Signed-off-by: Vaughn Dice --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7fa9167..571ad85 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,6 +29,9 @@ jobs: publish-node-installer-image: uses: ./.github/workflows/node-installer.yaml needs: build + # This action requires use of the GITHUB_TOKEN to publish the image + # By default, PRs from forks don't have access, so we only run when the PR branch is on origin. + if: ${{ ! github.event.pull_request.head.repo.fork }} with: ref: ${{ github.ref }} test: