[ingester] Optionally delay circuit breaker activation until after tr… #36225
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - r[0-9]+ # Trigger builds after a push to weekly branches | |
| tags: | |
| # The following regex matches the Mimir release tag. Tag filters not as strict due to different regex system on Github Actions. | |
| - mimir-[0-9]+.[0-9]+.[0-9]+** | |
| pull_request: | |
| concurrency: | |
| # Cancel any running workflow for the same branch when new commits are pushed. | |
| # We group both by ref_name (available when CI is triggered by a push to a branch/tag) | |
| # and head_ref (available when CI is triggered by a PR). | |
| group: "${{ github.ref_name }}-${{ github.head_ref }}" | |
| cancel-in-progress: true | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Get build image from Makefile | |
| id: build_image_step | |
| run: echo "build_image=$(make print-build-image)" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| build_image: ${{ steps.build_image_step.outputs.build_image }} | |
| # Determine if we will deploy (aka push) the image to the registry. | |
| is_deploy: ${{ (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/r')) && github.event_name == 'push' && github.repository == 'grafana/mimir' }} | |
| goversion: | |
| runs-on: ubuntu-latest | |
| needs: prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Get Go Version | |
| id: go-version | |
| run: | | |
| echo "version=$(make BUILD_IN_CONTAINER=false print-go-version)" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| version: ${{ steps.go-version.outputs.version }} | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| # Commands in the Makefile are hardcoded with an assumed file structure of the CI container | |
| # Symlink ensures paths specified in previous commands don’t break | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Get golangci-lint cache path | |
| id: golangcilintcache | |
| run: | | |
| echo "path=$(golangci-lint cache status | grep 'Dir: ' | cut -d ' ' -f2)" >> "$GITHUB_OUTPUT" | |
| - name: Cache golangci-lint cache | |
| uses: actions/cache@v4 | |
| with: | |
| key: lint-golangci-lint-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile') }} | |
| path: ${{ steps.golangcilintcache.outputs.path }} | |
| - name: Get Go cache paths | |
| id: goenv | |
| run: | | |
| echo "gocache=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" | |
| echo "gomodcache=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" | |
| - name: Cache Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| key: lint-go-build-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }} | |
| path: ${{ steps.goenv.outputs.gocache }} | |
| # Although we use vendoring, this linting job downloads all modules to verify that what is vendored is correct, | |
| # so it'll use GOMODCACHE. Other jobs don't need this. | |
| - name: Cache Go module cache | |
| uses: actions/cache@v4 | |
| with: | |
| key: lint-go-mod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }} | |
| path: ${{ steps.goenv.outputs.gomodcache }} | |
| - name: Lint | |
| run: make BUILD_IN_CONTAINER=false lint | |
| - name: Check Vendor Directory | |
| run: make BUILD_IN_CONTAINER=false mod-check | |
| - name: Check Protos | |
| run: make BUILD_IN_CONTAINER=false check-protos | |
| - name: Check Generated Documentation | |
| run: make BUILD_IN_CONTAINER=false check-doc | |
| - name: Check White Noise | |
| run: make BUILD_IN_CONTAINER=false check-white-noise | |
| - name: Check License Header | |
| run: make BUILD_IN_CONTAINER=false check-license | |
| - name: Check Docker-Compose YAML | |
| run: make BUILD_IN_CONTAINER=false check-mimir-microservices-mode-docker-compose-yaml check-mimir-read-write-mode-docker-compose-yaml | |
| - name: Check Generated OTLP Code | |
| run: make BUILD_IN_CONTAINER=false check-generated-otlp-code | |
| doc-validator: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: grafana/doc-validator:v5.2.0 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Run doc-validator tool (mimir) | |
| run: > | |
| doc-validator | |
| '--skip-checks=^canonical-does-not-match-pretty-URL$' | |
| docs/sources/mimir | |
| /docs/mimir/latest | |
| | reviewdog | |
| -f=rdjsonl | |
| --fail-on-error | |
| --filter-mode=nofilter | |
| --name=doc-validator | |
| --reporter=github-pr-review | |
| env: | |
| REVIEWDOG_GITHUB_API_TOKEN: "${{ secrets.GITHUB_TOKEN }}" | |
| - name: Run doc-validator tool (helm-charts) | |
| run: > | |
| doc-validator | |
| docs/sources/helm-charts | |
| /docs/helm-charts/mimir-distributed/latest | |
| | reviewdog | |
| -f=rdjsonl | |
| --fail-on-error | |
| --filter-mode=nofilter | |
| --name=doc-validator | |
| --reporter=github-pr-review | |
| env: | |
| REVIEWDOG_GITHUB_API_TOKEN: "${{ secrets.GITHUB_TOKEN }}" | |
| lint-jsonnet: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| # Commands in the Makefile are hardcoded with an assumed file structure of the CI container | |
| # Symlink ensures paths specified in previous commands don’t break | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Check Mixin | |
| run: make BUILD_IN_CONTAINER=false check-mixin | |
| - name: Check Mixin Tests | |
| run: make BUILD_IN_CONTAINER=false check-mixin-tests | |
| - name: Check Mixin with Mimirtool rules check | |
| run: make BUILD_IN_CONTAINER=false check-mixin-mimirtool-rules | |
| - name: Check Jsonnet Manifests | |
| run: make BUILD_IN_CONTAINER=false check-jsonnet-manifests | |
| - name: Check Jsonnet Getting Started | |
| run: make BUILD_IN_CONTAINER=false check-jsonnet-getting-started | |
| - name: Check Jsonnet Tests | |
| run: make BUILD_IN_CONTAINER=false check-jsonnet-tests | |
| lint-helm: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| # Commands in the Makefile are hardcoded with an assumed file structure of the CI container | |
| # Symlink ensures paths specified in previous commands don’t break | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| version: v3.8.2 | |
| - name: Check Helm Tests | |
| run: make BUILD_IN_CONTAINER=false check-helm-tests | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # Do not abort other groups when one fails. | |
| fail-fast: false | |
| # Split tests into 4 groups. | |
| matrix: | |
| test_group_id: [0, 1, 2, 3] | |
| test_group_total: [4] | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Get Go build cache path | |
| id: gocache | |
| run: | | |
| echo "path=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" | |
| - name: Cache Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| # Cache is shared between test groups. | |
| key: test-go-build-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }} | |
| path: ${{ steps.gocache.outputs.path }} | |
| - name: Run Tests | |
| run: | | |
| echo "Running unit tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)" | |
| ./.github/workflows/scripts/run-unit-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }} | |
| test-docs: | |
| uses: ./.github/workflows/test-docs.yml | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Install Docker Client | |
| run: ./.github/workflows/scripts/install-docker.sh | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Get Go build cache path | |
| id: gocache | |
| run: | | |
| echo "path=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" | |
| - name: Cache Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| key: build-go-build-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }} | |
| path: ${{ steps.gocache.outputs.path }} | |
| - name: Build Multiarch Docker Images Locally | |
| # Ignore mimir-build-image and mimir-rules-action. | |
| run: | | |
| ./.github/workflows/scripts/build-images.sh /tmp/images $(make list-image-targets | grep -v -E '/mimir-build-image/|/mimir-rules-action/') | |
| - name: Build Archive With Docker Images | |
| run: | | |
| tar cvf images.tar /tmp/images | |
| - name: Upload Archive with Docker Images | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Docker Images | |
| path: ./images.tar | |
| - name: Build Mimir with race-detector | |
| run: | | |
| # When building uptodate_race target, we create two images since | |
| # a Dockerfile.alpine exists. The alpine image is built with | |
| # the `-alpine` suffix. | |
| # We build both until we have finished migrating to distroless. | |
| # We test the distroless race image in every integration test. | |
| # We test the (legacy) alpine race image when deploying (aka pushing). | |
| make BUILD_IN_CONTAINER=false cmd/mimir/.uptodate_race | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| export MIMIR_DISTROLESS_IMAGE="grafana/mimir:$IMAGE_TAG_RACE" | |
| export MIMIR_ALPINE_IMAGE="grafana/mimir-alpine:$IMAGE_TAG_RACE" | |
| docker save $MIMIR_DISTROLESS_IMAGE -o ./mimir_race_image_distroless | |
| docker save $MIMIR_ALPINE_IMAGE -o ./mimir_race_image_alpine | |
| - name: Upload archive with race-enabled Mimir | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Race-enabled Mimir | |
| path: | | |
| ./mimir_race_image_alpine | |
| ./mimir_race_image_distroless | |
| integration: | |
| needs: [goversion, build, prepare] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # Do not abort other groups when one fails. | |
| fail-fast: false | |
| # Split tests into 6 groups. | |
| matrix: | |
| test_group_id: [0, 1, 2, 3, 4, 5] | |
| test_group_total: [6] | |
| steps: | |
| - name: Upgrade golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ needs.goversion.outputs.version }} | |
| cache: false # We manage caching ourselves below to maintain consistency with the other jobs that don't use setup-go. | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Install Docker Client | |
| run: sudo ./.github/workflows/scripts/install-docker.sh | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| sudo mkdir -p /go/src/github.com/grafana/mimir | |
| sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Get Go build cache path | |
| id: gocache | |
| run: | | |
| echo "path=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" | |
| - name: Cache Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| # Cache is shared between test groups. | |
| key: integration-go-build-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }} | |
| path: ${{ steps.gocache.outputs.path }} | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images from Archive | |
| run: tar xvf images.tar -C / | |
| - name: Load Mimirtool Image into Docker | |
| run: | | |
| export IMAGE_TAG=$(make image-tag) | |
| # skopeo will by default load system-specific version of the image (linux/amd64). | |
| # note that this doesn't use skopeo version from our build-image, because we don't use build-image when running integration tests. | |
| # that's why we use docker run to run latest version. | |
| docker run -v /tmp/images:/tmp/images -v /var/run/docker.sock:/var/run/docker.sock quay.io/skopeo/stable:v1.15.1 copy oci-archive:/tmp/images/mimirtool.oci "docker-daemon:grafana/mimirtool:$IMAGE_TAG" | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Race-enabled Mimir | |
| - name: Load race-enabled mimir into Docker | |
| run: | | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| docker load -i ./mimir_race_image_distroless | |
| docker run "grafana/mimir:$IMAGE_TAG_RACE" --version | |
| - name: Preload Images | |
| # We download docker images used by integration tests so that all images are available | |
| # locally and the download time doesn't account in the test execution time, which is subject | |
| # to a timeout | |
| run: go run ./tools/pre-pull-images | xargs -n1 -P4 docker pull | |
| - name: Integration Tests | |
| run: | | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| export MIMIR_IMAGE="grafana/mimir:$IMAGE_TAG_RACE" | |
| export IMAGE_TAG=$(make image-tag) | |
| export MIMIRTOOL_IMAGE="grafana/mimirtool:$IMAGE_TAG" | |
| export MIMIR_CHECKOUT_DIR="/go/src/github.com/grafana/mimir" | |
| echo "Running integration tests with image: $MIMIR_IMAGE (Mimir), $MIMIRTOOL_IMAGE (Mimirtool)" | |
| echo "Running integration tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)" | |
| ./.github/workflows/scripts/run-integration-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }} | |
| integration-alpine: | |
| needs: [goversion, build, prepare] | |
| runs-on: ubuntu-latest | |
| if: needs.prepare.outputs.is_deploy == 'true' | |
| strategy: | |
| # Do not abort other groups when one fails. | |
| fail-fast: false | |
| # Split tests into 6 groups. | |
| matrix: | |
| test_group_id: [0, 1, 2, 3, 4, 5] | |
| test_group_total: [6] | |
| steps: | |
| - name: Upgrade golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ needs.goversion.outputs.version }} | |
| cache: false # We manage caching ourselves below to maintain consistency with the other jobs that don't use setup-go. | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Install Docker Client | |
| run: sudo ./.github/workflows/scripts/install-docker.sh | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| sudo mkdir -p /go/src/github.com/grafana/mimir | |
| sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Get Go build cache path | |
| id: gocache | |
| run: | | |
| echo "path=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" | |
| - name: Cache Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| # Cache is shared between test groups. | |
| key: integration-go-build-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }} | |
| path: ${{ steps.gocache.outputs.path }} | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images from Archive | |
| run: tar xvf images.tar -C / | |
| - name: Load Mimirtool Image into Docker | |
| run: | | |
| export IMAGE_TAG=$(make image-tag) | |
| # skopeo will by default load system-specific version of the image (linux/amd64). | |
| # note that this doesn't use skopeo version from our build-image, because we don't use build-image when running integration tests. | |
| # that's why we use docker run to run latest version. | |
| docker run -v /tmp/images:/tmp/images -v /var/run/docker.sock:/var/run/docker.sock quay.io/skopeo/stable:v1.15.1 copy oci-archive:/tmp/images/mimirtool.oci "docker-daemon:grafana/mimirtool:$IMAGE_TAG" | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Race-enabled Mimir | |
| - name: Load race-enabled mimir into Docker | |
| run: | | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| docker load -i ./mimir_race_image_alpine | |
| docker run "grafana/mimir-alpine:$IMAGE_TAG_RACE" --version | |
| - name: Preload Images | |
| # We download docker images used by integration tests so that all images are available | |
| # locally and the download time doesn't account in the test execution time, which is subject | |
| # to a timeout | |
| run: go run ./tools/pre-pull-images | xargs -n1 -P4 docker pull | |
| - name: Integration Tests | |
| run: | | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| export MIMIR_IMAGE="grafana/mimir-alpine:$IMAGE_TAG_RACE" | |
| export IMAGE_TAG=$(make image-tag) | |
| export MIMIRTOOL_IMAGE="grafana/mimirtool:$IMAGE_TAG" | |
| export MIMIR_CHECKOUT_DIR="/go/src/github.com/grafana/mimir" | |
| echo "Running integration tests with image: $MIMIR_IMAGE (Mimir), $MIMIRTOOL_IMAGE (Mimirtool)" | |
| echo "Running integration tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)" | |
| ./.github/workflows/scripts/run-integration-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }} | |
| deploy: | |
| needs: [prepare, build, test, lint, integration, integration-alpine] | |
| # Only deploy images on pushes to the grafana/mimir repo, which either are tag pushes or weekly release branch pushes. | |
| if: needs.prepare.outputs.is_deploy == 'true' | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Install Docker Client | |
| run: ./.github/workflows/scripts/install-docker.sh | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images from Archive | |
| run: tar xvf images.tar -C / | |
| - name: Deploy | |
| run: | | |
| if [ -n "$DOCKER_PASSWORD" ]; then | |
| printenv DOCKER_PASSWORD | skopeo login -u "$DOCKER_USERNAME" --password-stdin docker.io | |
| fi | |
| ./.github/workflows/scripts/push-images.sh /tmp/images grafana/ $(make image-tag) | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} |