Upgrade tool management (#1842)

* upgrade tool management Signed-off-by: Alex Goodman <[email protected]> * update version file on release Signed-off-by: Alex Goodman <[email protected]> --------- Signed-off-by: Alex Goodman <[email protected]>
anchore · May 9, 2024 · 24d5d4f · 24d5d4f
1 parent e0c2b90
commit 24d5d4f
Show file tree

Hide file tree

Showing 15 changed files with 723 additions and 471 deletions.
diff --git a/.binny.yaml b/.binny.yaml
@@ -0,0 +1,104 @@
+tools:
+  # we want to use a pinned version of binny to manage the toolchain (so binny manages itself!)
+  - name: binny
+    version:
+      want: v0.7.0
+    method: github-release
+    with:
+      repo: anchore/binny
+
+  # used to produce SBOMs during release
+  - name: syft
+    version:
+      want: latest
+    method: github-release
+    with:
+      repo: anchore/syft
+
+  # used to sign mac binaries at release
+  - name: quill
+    version:
+      want: v0.4.1
+    method: github-release
+    with:
+      repo: anchore/quill
+
+  # used for linting
+  - name: golangci-lint
+    version:
+      want: v1.57.2
+    method: github-release
+    with:
+      repo: golangci/golangci-lint
+
+  # used for showing the changelog at release
+  - name: glow
+    version:
+      want: v1.5.1
+    method: github-release
+    with:
+      repo: charmbracelet/glow
+
+  # used for signing the checksums file at release
+  - name: cosign
+    version:
+      want: v2.2.4
+    method: github-release
+    with:
+      repo: sigstore/cosign
+
+  # used in integration tests to verify JSON schemas
+  - name: yajsv
+    version:
+      want: v1.4.1
+    method: github-release
+    with:
+      repo: neilpa/yajsv
+
+  # used to release all artifacts
+  - name: goreleaser
+    version:
+      want: v1.25.1
+    method: github-release
+    with:
+      repo: goreleaser/goreleaser
+
+  # used for organizing imports during static analysis
+  - name: gosimports
+    version:
+      want: v0.3.8
+    method: github-release
+    with:
+      repo: rinchsan/gosimports
+
+  # used at release to generate the changelog
+  - name: chronicle
+    version:
+      want: v0.8.0
+    method: github-release
+    with:
+      repo: anchore/chronicle
+
+  # used during static analysis for license compliance
+  - name: bouncer
+    version:
+      want: v0.4.0
+    method: github-release
+    with:
+      repo: wagoodman/go-bouncer
+
+  # used for running all local and CI tasks
+  - name: task
+    version:
+      want: v3.36.0
+    method: github-release
+    with:
+      repo: go-task/task
+
+  # used for triggering a release
+  - name: gh
+    version:
+      want: v2.48.0
+    method: github-release
+    with:
+      repo: cli/cli
diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml
@@ -9,14 +9,18 @@ inputs:
     description: "Python version to install"
     required: true
     default: "3.10"
+  go-dependencies:
+    description: "Download go dependencies"
+    required: true
+    default: "true"
   cache-key-prefix:
     description: "Prefix all cache keys with this value"
     required: true
-    default: "831180ac26"
-  build-cache-key-prefix:
-    description: "Prefix build cache key with this value"
+    default: "1ac8281053"
+  compute-fingerprints:
+    description: "Compute test fixture fingerprints"
     required: true
-    default: "f8b6d31dea"
+    default: "true"
   bootstrap-apt-packages:
     description: "Space delimited list of tools to install via apt"
     default: "libxml2-utils"
@@ -26,37 +30,29 @@ runs:
   steps:
     # note: go mod and build is automatically cached on default with v4+
     - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0
+      if: inputs.go-version != ''
       with:
         go-version: ${{ inputs.go-version }}
 
     - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
         python-version: ${{ inputs.python-version }}
 
-    - name: Restore python cache
-      id: python-venv-cache
-      uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
-      with:
-        path: |
-          test/quality/venv
-          test/quality/vulnerability-match-labels/venv
-        key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-python-${{ inputs.python-version }}-${{ hashFiles('**/test/quality/**/requirements.txt') }}
-
     - name: Restore tool cache
       id: tool-cache
-      uses: actions/cache@v3
+      uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2
       with:
-        path: ${{ github.workspace }}/.tmp
-        key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('Makefile') }}
+        path: ${{ github.workspace }}/.tool
+        key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('.binny.yaml') }}
 
-    - name: (cache-miss) Bootstrap project tools
+    - name: Install project tools
       shell: bash
-      if: steps.tool-cache.outputs.cache-hit != 'true'
-      run: make bootstrap-tools
+      run: make tools
 
-    - name: Bootstrap go dependencies
+    - name: Install go dependencies
+      if: inputs.go-dependencies == 'true'
       shell: bash
-      run: make bootstrap-go
+      run: make ci-bootstrap-go
 
     - name: Install apt packages
       if: inputs.bootstrap-apt-packages != ''
@@ -65,5 +61,6 @@ runs:
         DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }}
 
     - name: Create all cache fingerprints
+      if: inputs.compute-fingerprints == 'true'
       shell: bash
       run: make fingerprints
diff --git a/.github/scripts/go-mod-tidy-check.sh b/.github/scripts/go-mod-tidy-check.sh
@@ -4,19 +4,18 @@ set -eu
 ORIGINAL_STATE_DIR=$(mktemp -d "TEMP-original-state-XXXXXXXXX")
 TIDY_STATE_DIR=$(mktemp -d "TEMP-tidy-state-XXXXXXXXX")
 
-trap "cp -v ${ORIGINAL_STATE_DIR}/* ./ && rm -fR ${ORIGINAL_STATE_DIR} ${TIDY_STATE_DIR}" EXIT
+trap "cp -p ${ORIGINAL_STATE_DIR}/* ./ && git update-index -q --refresh && rm -fR ${ORIGINAL_STATE_DIR} ${TIDY_STATE_DIR}" EXIT
 
-echo "Capturing original state of files..."
-cp -v go.mod go.sum "${ORIGINAL_STATE_DIR}"
+# capturing original state of files...
+cp go.mod go.sum "${ORIGINAL_STATE_DIR}"
 
-echo "Capturing state of go.mod and go.sum after running go mod tidy..."
+# capturing state of go.mod and go.sum after running go mod tidy...
 go mod tidy
-cp -v go.mod go.sum "${TIDY_STATE_DIR}"
-echo ""
+cp go.mod go.sum "${TIDY_STATE_DIR}"
 
 set +e
 
-# Detect difference between the git HEAD state and the go mod tidy state
+# detect difference between the git HEAD state and the go mod tidy state
 DIFF_MOD=$(diff -u "${ORIGINAL_STATE_DIR}/go.mod" "${TIDY_STATE_DIR}/go.mod")
 DIFF_SUM=$(diff -u "${ORIGINAL_STATE_DIR}/go.sum" "${TIDY_STATE_DIR}/go.sum")
 

diff --git a/.github/scripts/syft-released-version-check.sh b/.github/scripts/syft-released-version-check.sh
diff --git a/.github/scripts/update-version-file.sh b/.github/scripts/update-version-file.sh
@@ -2,8 +2,8 @@
 set -ue
 
 BIN="grype"
-DISTDIR=$1
-VERSION=$2
+VERSION_FILE="VERSION"
+VERSION=$1
 
 # the source of truth as to whether we want to notify users of an update is if the release just created is NOT
 # flagged as a pre-release on github
@@ -12,10 +12,9 @@ if [[ "$(curl -SsL https://api.github.com/repos/anchore/${BIN}/releases/tags/${V
    exit 0
 fi
 
-echo "creating and publishing version file"
+echo "creating and publishing version file (${VERSION})"
 
 # create a version file for version-update checks
-VERSION_FILE="${DISTDIR}/VERSION"
 echo "${VERSION}" | tee "${VERSION_FILE}"
 
 # upload the version file that supports the application version update check

diff --git a/.github/workflows/release-version-file.yaml b/.github/workflows/release-version-file.yaml
@@ -0,0 +1,32 @@
+name: "Release"
+
+on:
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: release version to update the version file with (prefixed with v)
+        required: true
+
+  workflow_call:
+    inputs:
+      version:
+        type: string
+        description: release version to update the version file with (prefixed with v)
+        required: true
+
+jobs:
+
+  release:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #v4.1.4
+
+      - name: Update version file
+        run: make ci-release-version-file
+        env:
+          RELEASE_VERSION: ${{ github.event.inputs.version }}
+          # for updating the VERSION file in S3...
+          AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}
+
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -90,6 +90,10 @@ jobs:
           echo "CLI Test (Linux) Status: ${{ steps.cli-linux.outputs.conclusion }}"
           false
 
+  # only release core assets within the "release" job. Any other assets not already under the purview of the
+  # goreleaser configuration should be added as separate jobs to allow for debugging separately from the release workflow
+  # as well as not accidentally be re-run as a step multiple times (as could be done within the release workflow) as
+  # not all actions are guaranteed to be idempotent.
   release:
     needs: [quality-gate]
     runs-on: ubuntu-20.04
@@ -121,9 +125,6 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Cosign install
-        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
-
       - name: Tag release
         run: |
           git config user.name "anchoreci"
@@ -146,10 +147,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # for updating brew formula in anchore/homebrew-syft
           GITHUB_BREW_TOKEN: ${{ secrets.ANCHOREOPS_GITHUB_OSS_WRITE_TOKEN }}
-          # for updating the VERSION file in S3...
-          AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}
-
 
       - uses: anchore/sbom-action@7ccf588e3cf3cc2611714c2eeae48550fbc17552 # v0.15.11
         continue-on-error: true
@@ -165,3 +162,10 @@ jobs:
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
         if: ${{ success() }}
+
+  release-version-file:
+    needs: [release]
+    uses: ./.github/workflows/release-version-file.yaml
+    with:
+      version: ${{ github.event.inputs.version }}
+    secrets: inherit