From 3c9a48d04ee48ef6c1f6dbd7b11c28c71c8ed5a2 Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Wed, 4 Dec 2024 09:56:42 +0100 Subject: [PATCH] ci: add renovate (#1262) We need a way to update the dependencies mentioned inside the Dockerfile definition. Dependabot does not support this but Renovate does, allowing to treat arbitrary strings as version identifiers. This also includes support for updating the kubectl, kustomize, and helm versions included in the Dockerfile. --- .github/renovate-config.json5 | 97 ++++++++++++++++++++++++++ .github/workflows/acceptance-tests.yml | 21 +----- .github/workflows/docker.yml | 30 -------- .github/workflows/renovate.yml | 79 +++++++++++++++++++++ Dockerfile | 20 +++--- Makefile | 2 +- dagger/dagger.gen.go | 48 +++---------- dagger/main.go | 12 +--- 8 files changed, 197 insertions(+), 112 deletions(-) create mode 100644 .github/renovate-config.json5 create mode 100644 .github/workflows/renovate.yml diff --git a/.github/renovate-config.json5 b/.github/renovate-config.json5 new file mode 100644 index 000000000..501b6a814 --- /dev/null +++ b/.github/renovate-config.json5 @@ -0,0 +1,97 @@ +{ + $schema: "https://docs.renovatebot.com/renovate-schema.json", + branchPrefix: "grafanarenovatebot/", + customDatasources: { + "kubectl": { + "defaultRegistryUrlTemplate": "https://cdn.dl.k8s.io/release/stable.txt", + "format": "plain", + "transformTemplates": [ + "{\"releases\": [releases . {\"version\": $substring(version, 1)}]}", + ], + }, + "helm": { + "defaultRegistryUrlTemplate": "https://api.github.com/repos/helm/helm/releases", + "format": "json", + "transformTemplates": [ + "{\"releases\": [$.tag_name . {\"version\": $substring($, 1)}]}", + ], + }, + "kustomize": { + "defaultRegistryUrlTemplate": "https://api.github.com/repos/kubernetes-sigs/kustomize/releases", + "format": "json", + "transformTemplates": [ + "{\"releases\": [$$ [$match(tag_name, /kustomize.*/) and $not(draft) and $not(prerelease) ] . {\"version\": $substringAfter(tag_name, \"/v\")}]}", + ], + }, + }, + + customManagers: [ + { + "customType": "regex", + "fileMatch": ["Dockerfile"], + "matchStrings": [ + "ARG KUBECTL_VERSION=(?\\S+)", + ], + "datasourceTemplate": "custom.kubectl", + "depNameTemplate": "kubectl", + }, + { + "customType": "regex", + "fileMatch": ["Dockerfile"], + "matchStrings": [ + "ARG HELM_VERSION=(?\\S+)", + ], + "datasourceTemplate": "custom.helm", + "depNameTemplate": "helm", + "versioningTemplate": "semver", + }, + { + "customType": "regex", + "fileMatch": ["Dockerfile"], + "matchStrings": [ + "ARG KUSTOMIZE_VERSION=(?\\S+)", + ], + "datasourceTemplate": "custom.kustomize", + "depNameTemplate": "kustomize", + "versioningTemplate": "semver", + }, + ], + dependencyDashboard: false, + enabledManagers: ["custom.regex"], + forkProcessing: "enabled", + globalExtends: [":pinDependencies", "config:best-practices"], + onboarding: false, + osvVulnerabilityAlerts: true, + packageRules: [ + { + labels: ["update-major"], + matchUpdateTypes: ["major"], + }, + { + labels: ["update-minor"], + matchUpdateTypes: ["minor"], + }, + { + automerge: true, + labels: ["automerge-patch"], + matchUpdateTypes: ["patch"], + }, + { + labels: ["update-digest"], + matchUpdateTypes: ["digest"], + }, + { + // Run the custom matcher on early Monday mornings (UTC) + schedule: "* 0-4 * * 1", + matchPackageNames: ["ghcr.io/renovatebot/renovate"], + }, + ], + platformCommit: "enabled", + rebaseWhen: "behind-base-branch", + requireConfig: "optional", + vulnerabilityAlerts: { + automerge: true, + enabled: true, + labels: ["automerge-security-update"], + }, +} diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 3e4573815..427e70c9f 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -21,25 +21,6 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: "Determine dependency versions" - id: "versions" - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - script: | - const helmRelease = await github.rest.repos.getLatestRelease({ - 'owner': 'helm', - 'repo': 'helm', - }); - core.setOutput('helm', helmRelease.data.tag_name); - console.log('Helm version', helmRelease.data.tag_name); - const kustomizeReleases = await github.rest.repos.listReleases({ - 'owner': 'kubernetes-sigs', - 'repo': 'kustomize', - }); - const kustomizeRelease = kustomizeReleases.data.filter(release => release.tag_name.startsWith('kustomize') && !release.draft && !release.prerelease).map(release => release.tag_name.split('/')[1])[0]; - console.log('Kustomize version', kustomizeRelease); - core.setOutput('kustomize', kustomizeRelease); - - name: Call Dagger Function id: dagger uses: dagger/dagger-for-github@e5153f5610d82ac9f3f848f3a25ad9d696641068 # v7.0.1 @@ -47,4 +28,4 @@ jobs: version: "0.14.0" verb: call dagger-flags: "--silent" - args: "acceptance-tests --root-dir .:source-files --acceptance-tests-dir ./acceptance-tests --kustomize-version ${{ steps.versions.outputs.kustomize }} --helm-version ${{ steps.versions.outputs.helm }}" + args: "acceptance-tests --root-dir .:source-files --acceptance-tests-dir ./acceptance-tests" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c3d54d74a..96366ec85 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -36,34 +36,7 @@ env: type=semver,pattern={{version}},value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} jobs: - determine-versions: - runs-on: ubuntu-latest - outputs: - helm: ${{ steps.versions.outputs.helm }} - kustomize: ${{ steps.versions.outputs.kustomize }} - steps: - - name: "Determine dependency versions" - id: "versions" - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - script: | - const helmRelease = await github.rest.repos.getLatestRelease({ - 'owner': 'helm', - 'repo': 'helm', - }); - core.setOutput('helm', helmRelease.data.tag_name); - console.log('Helm version', helmRelease.data.tag_name); - const kustomizeReleases = await github.rest.repos.listReleases({ - 'owner': 'kubernetes-sigs', - 'repo': 'kustomize', - }); - const kustomizeRelease = kustomizeReleases.data.filter(release => release.tag_name.startsWith('kustomize') && !release.draft && !release.prerelease).map(release => release.tag_name.split('/')[1])[0]; - console.log('Kustomize version', kustomizeRelease); - core.setOutput('kustomize', kustomizeRelease); - build: - needs: - - determine-versions strategy: fail-fast: false matrix: @@ -98,9 +71,6 @@ jobs: context: . labels: ${{ steps.meta.outputs.labels }} outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.event_name == 'push' }} - build-args: | - HELM_VERSION=${{ needs.determine-versions.outputs.helm }} - KUSTOMIZE_VERSION=${{ needs.determine-versions.outputs.kustomize }} - name: Export digest id: digest diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml new file mode 100644 index 000000000..299ae574a --- /dev/null +++ b/.github/workflows/renovate.yml @@ -0,0 +1,79 @@ +name: Renovate +on: + schedule: + # Offset by 12 minutes to avoid busy times on the hour + - cron: 12 */4 * * * + + pull_request: + paths: + - .github/renovate-config.json5 + - .github/workflows/renovate.yml + types: + - edited + - opened + - ready_for_review + - synchronize + + push: + branches: + - main + paths: + - .github/renovate-config.json5 + - .github/workflows/renovate.yml + + workflow_dispatch: + inputs: + dry-run: + description: "Run Renovate in dry-run mode" + required: false + default: false + type: boolean + +jobs: + renovate: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + sparse-checkout: | + .github/renovate-config.json5 + + - name: Retrieve renovate secrets + id: get-secrets + uses: grafana/shared-workflows/actions/get-vault-secrets@97c6f45f01d4bca8a3b1acfe397113ce88858a81 # get-vault-secrets-v1.0.1 + with: + common_secrets: | + GRAFANA_RENOVATE_APP_ID=grafana-renovate-app:app-id + GRAFANA_RENOVATE_PRIVATE_KEY=grafana-renovate-app:private-key + + - name: Generate token + id: generate-token + uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0 + with: + app-id: ${{ env.GRAFANA_RENOVATE_APP_ID }} + private-key: ${{ env.GRAFANA_RENOVATE_PRIVATE_KEY }} + + - name: Self-hosted Renovate + uses: renovatebot/github-action@936628dfbff213ab2eb95033c5e123cfcaf09ebb # v41.0.5 + with: + configurationFile: .github/renovate-config.json5 + # renovate: datasource=docker depName=ghcr.io/renovatebot/renovate + renovate-version: 39.42.4@sha256:c5d718e312cdacc0746e37f13c215ff498be28c51e50efd24c070ae29f5b636a + token: ${{ steps.generate-token.outputs.token }} + env: + LOG_LEVEL: ${{ github.event_name == 'pull_request' && 'debug' || 'info' }} + # For pull requests, this means we'll get the dependencies of the PR's + # branch, so you can fix/change things and see the results in the PR's + # run. By default, Renovate will clone the main/default branch. + RENOVATE_BASE_BRANCHES: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || null }} + # Dry run if the event is pull_request, or workflow_dispatch AND the dry-run input is true + RENOVATE_DRY_RUN: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry-run == 'true')) && 'full' || null }} + RENOVATE_PLATFORM: github + RENOVATE_REPOSITORIES: ${{ github.repository }} + RENOVATE_USERNAME: GrafanaRenovateBot diff --git a/Dockerfile b/Dockerfile index 5fa0db8a2..4dfba9676 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # download kubectl FROM golang:1.23.3-alpine AS kubectl +ARG KUBECTL_VERSION=1.31.3 RUN apk add --no-cache curl -RUN export VERSION=$(curl -s https://cdn.dl.k8s.io/release/stable.txt) &&\ - export OS=$(go env GOOS) && \ +RUN export OS=$(go env GOOS) && \ export ARCH=$(go env GOARCH) &&\ - curl -o /usr/local/bin/kubectl -L https://cdn.dl.k8s.io/release/${VERSION}/bin/${OS}/${ARCH}/kubectl &&\ + curl -o /usr/local/bin/kubectl -L https://cdn.dl.k8s.io/release/v${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl &&\ chmod +x /usr/local/bin/kubectl # build jsonnet-bundler @@ -19,25 +19,21 @@ RUN apk add --no-cache git make bash &&\ FROM golang:1.23.3-alpine AS helm WORKDIR /tmp/helm -ARG HELM_VERSION +ARG HELM_VERSION=3.16.3 RUN apk add --no-cache jq curl RUN export OS=$(go env GOOS) && \ export ARCH=$(go env GOARCH) &&\ - if [[ -z ${HELM_VERSION} ]]; then export HELM_VERSION=$(curl --silent "https://api.github.com/repos/helm/helm/releases" | jq -r '.[0].tag_name'); fi && \ - curl -SL "https://get.helm.sh/helm-${HELM_VERSION}-${OS}-${ARCH}.tar.gz" > helm.tgz && \ + curl -SL "https://get.helm.sh/helm-v${HELM_VERSION}-${OS}-${ARCH}.tar.gz" > helm.tgz && \ tar -xvf helm.tgz --strip-components=1 FROM golang:1.23.3-alpine AS kustomize WORKDIR /tmp/kustomize -ARG KUSTOMIZE_VERSION +ARG KUSTOMIZE_VERSION=5.5.0 RUN apk add --no-cache jq curl -# Get the latest version of kustomize -# Releases are filtered by their name since the kustomize repository exposes multiple products in the releases RUN export OS=$(go env GOOS) &&\ export ARCH=$(go env GOARCH) &&\ - if [[ -z ${KUSTOMIZE_VERSION} ]]; then export KUSTOMIZE_VERSION=$(curl --silent "https://api.github.com/repos/kubernetes-sigs/kustomize/releases" | jq -r '[ .[] | select(.name | startswith("kustomize")) ] | .[0].tag_name | split("/")[1]'); fi && \ - echo "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_${OS}_${ARCH}.tar.gz" && \ - curl -SL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_${OS}_${ARCH}.tar.gz" > kustomize.tgz && \ + echo "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_${OS}_${ARCH}.tar.gz" && \ + curl -SL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_${OS}_${ARCH}.tar.gz" > kustomize.tgz && \ tar -xvf kustomize.tgz FROM golang:1.23.3 AS build diff --git a/Makefile b/Makefile index 7039d0e23..ce933e53d 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ test: go test ./... -bench=. -benchmem acceptance-tests: - dagger call acceptance-tests --root-dir .:source-files --acceptance-tests-dir ./acceptance-tests --kustomize-version "" --helm-version "" + dagger call acceptance-tests --root-dir .:source-files --acceptance-tests-dir ./acceptance-tests # Compilation dev: diff --git a/dagger/dagger.gen.go b/dagger/dagger.gen.go index 49e2e2c7b..6065e2f5c 100644 --- a/dagger/dagger.gen.go +++ b/dagger/dagger.gen.go @@ -157,21 +157,7 @@ func invoke(ctx context.Context, parentJSON []byte, parentName string, fnName st panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg rootDir", err)) } } - var helmVersion string - if inputArgs["helmVersion"] != nil { - err = json.Unmarshal([]byte(inputArgs["helmVersion"]), &helmVersion) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg helmVersion", err)) - } - } - var kustomizeVersion string - if inputArgs["kustomizeVersion"] != nil { - err = json.Unmarshal([]byte(inputArgs["kustomizeVersion"]), &kustomizeVersion) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg kustomizeVersion", err)) - } - } - return (*Tanka).Build(&parent, ctx, rootDir, helmVersion, kustomizeVersion), nil + return (*Tanka).Build(&parent, ctx, rootDir), nil case "GetGoVersion": var parent Tanka err = json.Unmarshal(parentJSON, &parent) @@ -199,20 +185,6 @@ func invoke(ctx context.Context, parentJSON []byte, parentName string, fnName st panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg rootDir", err)) } } - var helmVersion string - if inputArgs["helmVersion"] != nil { - err = json.Unmarshal([]byte(inputArgs["helmVersion"]), &helmVersion) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg helmVersion", err)) - } - } - var kustomizeVersion string - if inputArgs["kustomizeVersion"] != nil { - err = json.Unmarshal([]byte(inputArgs["kustomizeVersion"]), &kustomizeVersion) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg kustomizeVersion", err)) - } - } var acceptanceTestsDir *dagger.Directory if inputArgs["acceptanceTestsDir"] != nil { err = json.Unmarshal([]byte(inputArgs["acceptanceTestsDir"]), &acceptanceTestsDir) @@ -220,7 +192,7 @@ func invoke(ctx context.Context, parentJSON []byte, parentName string, fnName st panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg acceptanceTestsDir", err)) } } - return (*Tanka).AcceptanceTests(&parent, ctx, rootDir, helmVersion, kustomizeVersion, acceptanceTestsDir) + return (*Tanka).AcceptanceTests(&parent, ctx, rootDir, acceptanceTestsDir) default: return nil, fmt.Errorf("unknown function %s", fnName) } @@ -232,22 +204,18 @@ func invoke(ctx context.Context, parentJSON []byte, parentName string, fnName st dag.Function("Build", dag.TypeDef().WithObject("Container")). WithSourceMap(dag.SourceMap("main.go", 15, 1)). - WithArg("rootDir", dag.TypeDef().WithObject("Directory"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 15, 44)}). - WithArg("helmVersion", dag.TypeDef().WithKind(dagger.TypeDefKindStringKind), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 15, 71)}). - WithArg("kustomizeVersion", dag.TypeDef().WithKind(dagger.TypeDefKindStringKind), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 15, 91)})). + WithArg("rootDir", dag.TypeDef().WithObject("Directory"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 15, 44)})). WithFunction( dag.Function("GetGoVersion", dag.TypeDef().WithKind(dagger.TypeDefKindStringKind)). - WithSourceMap(dag.SourceMap("main.go", 29, 1)). - WithArg("file", dag.TypeDef().WithObject("File"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 29, 51)})). + WithSourceMap(dag.SourceMap("main.go", 23, 1)). + WithArg("file", dag.TypeDef().WithObject("File"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 23, 51)})). WithFunction( dag.Function("AcceptanceTests", dag.TypeDef().WithKind(dagger.TypeDefKindStringKind)). - WithSourceMap(dag.SourceMap("main.go", 45, 1)). - WithArg("rootDir", dag.TypeDef().WithObject("Directory"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 45, 54)}). - WithArg("helmVersion", dag.TypeDef().WithKind(dagger.TypeDefKindStringKind), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 45, 81)}). - WithArg("kustomizeVersion", dag.TypeDef().WithKind(dagger.TypeDefKindStringKind), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 45, 101)}). - WithArg("acceptanceTestsDir", dag.TypeDef().WithObject("Directory"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 45, 126)}))), nil + WithSourceMap(dag.SourceMap("main.go", 39, 1)). + WithArg("rootDir", dag.TypeDef().WithObject("Directory"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 39, 54)}). + WithArg("acceptanceTestsDir", dag.TypeDef().WithObject("Directory"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 39, 81)}))), nil default: return nil, fmt.Errorf("unknown object %s", parentName) } diff --git a/dagger/main.go b/dagger/main.go index 7e5b412f9..79cd568ec 100644 --- a/dagger/main.go +++ b/dagger/main.go @@ -12,14 +12,8 @@ import ( type Tanka struct{} -func (m *Tanka) Build(ctx context.Context, rootDir *dagger.Directory, helmVersion string, kustomizeVersion string) *dagger.Container { +func (m *Tanka) Build(ctx context.Context, rootDir *dagger.Directory) *dagger.Container { buildArgs := make([]dagger.BuildArg, 0, 2) - if helmVersion != "" { - buildArgs = append(buildArgs, dagger.BuildArg{Name: "HELM_VERSION", Value: helmVersion}) - } - if kustomizeVersion != "" { - buildArgs = append(buildArgs, dagger.BuildArg{Name: "KUSTOMIZE_VERSION", Value: kustomizeVersion}) - } return dag.Container(). Build(rootDir, dagger.ContainerBuildOpts{ BuildArgs: buildArgs, @@ -42,12 +36,12 @@ func (m *Tanka) GetGoVersion(ctx context.Context, file *dagger.File) (string, er return "", fmt.Errorf("no Go version found") } -func (m *Tanka) AcceptanceTests(ctx context.Context, rootDir *dagger.Directory, helmVersion string, kustomizeVersion string, acceptanceTestsDir *dagger.Directory) (string, error) { +func (m *Tanka) AcceptanceTests(ctx context.Context, rootDir *dagger.Directory, acceptanceTestsDir *dagger.Directory) (string, error) { goVersion, err := m.GetGoVersion(ctx, rootDir.File("go.mod")) if err != nil { return "", err } - buildContainer := m.Build(ctx, rootDir, helmVersion, kustomizeVersion) + buildContainer := m.Build(ctx, rootDir) k3s := dag.K3S("k3sdemo") k3sSrv, err := k3s.Server().Start(ctx)