diff --git a/.github/workflows/scripts/install-docker.sh b/.github/workflows/scripts/install-docker.sh index 519c7410818..080cf6bd59d 100755 --- a/.github/workflows/scripts/install-docker.sh +++ b/.github/workflows/scripts/install-docker.sh @@ -2,10 +2,30 @@ set -x VER="28.0.4" -curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz + +# Detect architecture +ARCH=$(uname -m) +case $ARCH in + x86_64) + DOCKER_ARCH="x86_64" + BUILDX_ARCH="amd64" + ;; + aarch64) + DOCKER_ARCH="aarch64" + BUILDX_ARCH="arm64" + ;; + *) + echo "Unsupported architecture: $ARCH" + exit 1 + ;; +esac + +echo "Installing Docker $VER for architecture: $ARCH (docker: $DOCKER_ARCH, buildx: $BUILDX_ARCH)" + +curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/$DOCKER_ARCH/docker-$VER.tgz tar -xz -C /tmp -f /tmp/docker-$VER.tgz mkdir -vp ~/.docker/cli-plugins/ -curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx +curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-$BUILDX_ARCH" > ~/.docker/cli-plugins/docker-buildx chmod a+x ~/.docker/cli-plugins/docker-buildx mv /tmp/docker/* /usr/bin docker run --privileged --rm tonistiigi/binfmt --install all diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 92653428b29..0b3372eea5f 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -152,19 +152,57 @@ jobs: integration: needs: build - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} + timeout-minutes: 50 strategy: fail-fast: false matrix: - tags: - - requires_docker - - integration_alertmanager - - integration_backward_compatibility - - integration_memberlist - - integration_querier - - integration_ruler - - integration_query_fuzz - - integration_remote_write_v2 + include: + - runner: ubuntu-24.04 + arch: amd64 + tags: requires_docker + - runner: ubuntu-24.04 + arch: amd64 + tags: integration_alertmanager + - runner: ubuntu-24.04 + arch: amd64 + tags: integration_backward_compatibility + - runner: ubuntu-24.04 + arch: amd64 + tags: integration_memberlist + - runner: ubuntu-24.04 + arch: amd64 + tags: integration_querier + - runner: ubuntu-24.04 + arch: amd64 + tags: integration_ruler + - runner: ubuntu-24.04 + arch: amd64 + tags: integration_query_fuzz + - runner: ubuntu-24.04 + arch: amd64 + tags: integration_remote_write_v2 + - runner: ubuntu-24.04-arm + arch: arm64 + tags: requires_docker + - runner: ubuntu-24.04-arm + arch: arm64 + tags: integration_alertmanager + - runner: ubuntu-24.04-arm + arch: arm64 + tags: integration_memberlist + - runner: ubuntu-24.04-arm + arch: arm64 + tags: integration_ruler + - runner: ubuntu-24.04-arm + arch: arm64 + tags: integration_remote_write_v2 + - runner: ubuntu-24.04-arm + arch: arm64 + tags: integration_backward_compatibility + - runner: ubuntu-24.04-arm + arch: arm64 + tags: integration_querier steps: - name: Upgrade golang uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 @@ -197,20 +235,13 @@ jobs: docker pull consul:1.8.4 docker pull gcr.io/etcd-development/etcd:v3.4.7 if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then - docker pull quay.io/cortexproject/cortex:v1.13.1 - docker pull quay.io/cortexproject/cortex:v1.13.2 - docker pull quay.io/cortexproject/cortex:v1.14.0 - docker pull quay.io/cortexproject/cortex:v1.14.1 - docker pull quay.io/cortexproject/cortex:v1.15.0 - docker pull quay.io/cortexproject/cortex:v1.15.1 - docker pull quay.io/cortexproject/cortex:v1.15.2 - docker pull quay.io/cortexproject/cortex:v1.15.3 docker pull quay.io/cortexproject/cortex:v1.16.0 docker pull quay.io/cortexproject/cortex:v1.16.1 docker pull quay.io/cortexproject/cortex:v1.17.0 docker pull quay.io/cortexproject/cortex:v1.17.1 docker pull quay.io/cortexproject/cortex:v1.18.0 docker pull quay.io/cortexproject/cortex:v1.18.1 + docker pull quay.io/cortexproject/cortex:v1.19.0 elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then docker pull quay.io/cortexproject/cortex:v1.18.1 docker pull quay.io/prometheus/prometheus:v3.6.0 @@ -220,19 +251,29 @@ jobs: env: TEST_TAGS: ${{ matrix.tags }} - name: Integration Tests + timeout-minutes: 45 run: | export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}" export IMAGE_TAG=$(make image-tag) - export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64" + export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:${IMAGE_TAG}-${{ matrix.arch }}" export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex" - echo "Running integration tests with image: $CORTEX_IMAGE" + echo "Running integration tests on ${{ matrix.arch }} with image: ${CORTEX_IMAGE}" go test -tags=slicelabels,integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/... env: IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }} integration-configs-db: needs: build - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 steps: - name: Checkout Repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -245,10 +286,12 @@ jobs: - name: Extract Docker Images Archive run: tar -xvf images.tar -C / - name: Run Integration Configs Tests + timeout-minutes: 15 # Github Actions does not support TTY in their default runners yet run: | touch build-image/.uptodate MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations + echo "Running configs integration tests on ${{ matrix.arch }}" make BUILD_IMAGE=quay.io/cortexproject/build-image:master-59491e9aae TTY='' configs-integration-test deploy_website: diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index f38442b8eb1..a230bb8a2b9 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -1,5 +1,4 @@ //go:build integration_alertmanager -// +build integration_alertmanager package integration diff --git a/integration/api_endpoints_test.go b/integration/api_endpoints_test.go index 5e5d683e060..ddcfada4f04 100644 --- a/integration/api_endpoints_test.go +++ b/integration/api_endpoints_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/asserts.go b/integration/asserts.go index b19de6dfb32..4ebcac02a92 100644 --- a/integration/asserts.go +++ b/integration/asserts.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package integration diff --git a/integration/backward_compatibility_test.go b/integration/backward_compatibility_test.go index 75ae91edb57..2d727094bcb 100644 --- a/integration/backward_compatibility_test.go +++ b/integration/backward_compatibility_test.go @@ -1,5 +1,4 @@ //go:build integration_backward_compatibility -// +build integration_backward_compatibility package integration @@ -30,49 +29,9 @@ type versionsImagesFlags struct { var ( // If you change the image tag, remember to update it in the preloading done // by GitHub Actions too (see .github/workflows/test-build-deploy.yml). + // Per https://cortexmetrics.io/docs/configuration/v1guarantees/#flags-config-and-minor-version-upgrades, + // we only need to support backward compatibility for the last 3 minor versions. previousVersionImages = map[string]*versionsImagesFlags{ - "quay.io/cortexproject/cortex:v1.13.1": { - flagsForOldImage: func(m map[string]string) map[string]string { - m["-ingester.stream-chunks-when-using-blocks"] = "true" - return m - }, - flagsForNewImage: func(m map[string]string) map[string]string { - m["-ingester.client.grpc-compression"] = "snappy" - return m - }, - }, - "quay.io/cortexproject/cortex:v1.13.2": { - flagsForOldImage: func(m map[string]string) map[string]string { - m["-ingester.stream-chunks-when-using-blocks"] = "true" - return m - }, - flagsForNewImage: func(m map[string]string) map[string]string { - m["-ingester.client.grpc-compression"] = "snappy" - return m - }, - }, - "quay.io/cortexproject/cortex:v1.14.0": { - flagsForOldImage: func(m map[string]string) map[string]string { - return m - }, - flagsForNewImage: func(m map[string]string) map[string]string { - m["-ingester.client.grpc-compression"] = "snappy" - return m - }, - }, - "quay.io/cortexproject/cortex:v1.14.1": { - flagsForOldImage: func(m map[string]string) map[string]string { - return m - }, - flagsForNewImage: func(m map[string]string) map[string]string { - m["-ingester.client.grpc-compression"] = "snappy" - return m - }, - }, - "quay.io/cortexproject/cortex:v1.15.0": nil, - "quay.io/cortexproject/cortex:v1.15.1": nil, - "quay.io/cortexproject/cortex:v1.15.2": nil, - "quay.io/cortexproject/cortex:v1.15.3": nil, "quay.io/cortexproject/cortex:v1.16.0": nil, "quay.io/cortexproject/cortex:v1.16.1": nil, "quay.io/cortexproject/cortex:v1.17.0": nil, diff --git a/integration/blocks_storage_backends_test.go b/integration/blocks_storage_backends_test.go index 8fc93fe58fd..862018815df 100644 --- a/integration/blocks_storage_backends_test.go +++ b/integration/blocks_storage_backends_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/configs.go b/integration/configs.go index c95a6bdc764..479c85574a2 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package integration diff --git a/integration/distributor_mixed_ha_samples_test.go b/integration/distributor_mixed_ha_samples_test.go index 6d303f3514a..7e66f6045f2 100644 --- a/integration/distributor_mixed_ha_samples_test.go +++ b/integration/distributor_mixed_ha_samples_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/e2e/scenario_test.go b/integration/e2e/scenario_test.go index cf1f6b22348..d1457e3943a 100644 --- a/integration/e2e/scenario_test.go +++ b/integration/e2e/scenario_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package e2e_test diff --git a/integration/e2e/service_test.go b/integration/e2e/service_test.go index 38ffa353a26..f322bd435bc 100644 --- a/integration/e2e/service_test.go +++ b/integration/e2e/service_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package e2e diff --git a/integration/format_query_api_test.go b/integration/format_query_api_test.go index cd07ae699aa..893a1db6c9b 100644 --- a/integration/format_query_api_test.go +++ b/integration/format_query_api_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/getting_started_single_process_config_local_test.go b/integration/getting_started_single_process_config_local_test.go index 0a7434c5a3f..92362fc8e47 100644 --- a/integration/getting_started_single_process_config_local_test.go +++ b/integration/getting_started_single_process_config_local_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/getting_started_single_process_config_test.go b/integration/getting_started_single_process_config_test.go index 624928abf0a..f65b2ae86c2 100644 --- a/integration/getting_started_single_process_config_test.go +++ b/integration/getting_started_single_process_config_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/getting_started_with_gossiped_ring_test.go b/integration/getting_started_with_gossiped_ring_test.go index 4bcd8a090b4..b11b566a29e 100644 --- a/integration/getting_started_with_gossiped_ring_test.go +++ b/integration/getting_started_with_gossiped_ring_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/grpc_server_test.go b/integration/grpc_server_test.go index 4eab7029d11..7987733d197 100644 --- a/integration/grpc_server_test.go +++ b/integration/grpc_server_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/ingester_limits_test.go b/integration/ingester_limits_test.go index e94c83d5bfc..ad71ebe5416 100644 --- a/integration/ingester_limits_test.go +++ b/integration/ingester_limits_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/ingester_metadata_test.go b/integration/ingester_metadata_test.go index 0d0893bf665..1a7de999747 100644 --- a/integration/ingester_metadata_test.go +++ b/integration/ingester_metadata_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/ingester_sharding_test.go b/integration/ingester_sharding_test.go index aeffc1f0cca..0a7571b94a6 100644 --- a/integration/ingester_sharding_test.go +++ b/integration/ingester_sharding_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/ingester_stream_push_test.go b/integration/ingester_stream_push_test.go index 87b95565b99..4d9677cf726 100644 --- a/integration/ingester_stream_push_test.go +++ b/integration/ingester_stream_push_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/integration_memberlist_single_binary_test.go b/integration/integration_memberlist_single_binary_test.go index 68bd343ee06..a79642e1dd5 100644 --- a/integration/integration_memberlist_single_binary_test.go +++ b/integration/integration_memberlist_single_binary_test.go @@ -1,5 +1,4 @@ //go:build integration_memberlist -// +build integration_memberlist package integration diff --git a/integration/kv_test.go b/integration/kv_test.go index 0252f6024f7..65b4e1daa56 100644 --- a/integration/kv_test.go +++ b/integration/kv_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/native_histogram_test.go b/integration/native_histogram_test.go index 68a7c2deb67..34cd31e8714 100644 --- a/integration/native_histogram_test.go +++ b/integration/native_histogram_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/otlp_test.go b/integration/otlp_test.go index fe83c1852fa..7ec911da96f 100644 --- a/integration/otlp_test.go +++ b/integration/otlp_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/parquet_querier_test.go b/integration/parquet_querier_test.go index e085cef99d1..f4bb5a6c60d 100644 --- a/integration/parquet_querier_test.go +++ b/integration/parquet_querier_test.go @@ -1,5 +1,4 @@ //go:build integration_query_fuzz -// +build integration_query_fuzz package integration diff --git a/integration/parse_query_api_test.go b/integration/parse_query_api_test.go index 06db800a922..833fecb6db3 100644 --- a/integration/parse_query_api_test.go +++ b/integration/parse_query_api_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/querier_remote_read_test.go b/integration/querier_remote_read_test.go index 99a62ebb631..7f19bbb38fa 100644 --- a/integration/querier_remote_read_test.go +++ b/integration/querier_remote_read_test.go @@ -1,5 +1,4 @@ //go:build integration_querier -// +build integration_querier package integration diff --git a/integration/querier_sharding_test.go b/integration/querier_sharding_test.go index f9d12f7f3bd..414c212affc 100644 --- a/integration/querier_sharding_test.go +++ b/integration/querier_sharding_test.go @@ -1,5 +1,4 @@ //go:build integration_querier -// +build integration_querier package integration diff --git a/integration/querier_tenant_federation_test.go b/integration/querier_tenant_federation_test.go index 5222b1c8e6e..9e5aff81081 100644 --- a/integration/querier_tenant_federation_test.go +++ b/integration/querier_tenant_federation_test.go @@ -1,5 +1,4 @@ //go:build integration_querier -// +build integration_querier package integration diff --git a/integration/querier_test.go b/integration/querier_test.go index 27929ba5d86..d5840d356aa 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -1,5 +1,4 @@ //go:build integration_querier -// +build integration_querier package integration @@ -8,6 +7,7 @@ import ( "encoding/json" "fmt" "net/http" + "runtime" "strconv" "strings" "testing" @@ -385,6 +385,12 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { for testName, testCfg := range tests { for _, thanosEngine := range []bool{false, true} { t.Run(fmt.Sprintf("%s,thanosEngine=%t", testName, thanosEngine), func(t *testing.T) { + // Skip Thanos engine tests on non-amd64 architectures due to timing-sensitive + // cache request count assertions that vary across architectures. + if runtime.GOARCH != "amd64" && thanosEngine { + t.Skip("Skipping test: Thanos engine tests only run on amd64 due to timing-sensitive assertions") + } + const blockRangePeriod = 5 * time.Second s, err := e2e.NewScenario(networkName) diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index 7a68b64a691..4b60eb0484a 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go index b12560be6f4..7ed30dec6d2 100644 --- a/integration/query_fuzz_test.go +++ b/integration/query_fuzz_test.go @@ -1,5 +1,4 @@ //go:build integration_query_fuzz -// +build integration_query_fuzz package integration diff --git a/integration/query_response_compression_test.go b/integration/query_response_compression_test.go index 8de7dafd1d6..cc28418a228 100644 --- a/integration/query_response_compression_test.go +++ b/integration/query_response_compression_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/remote_write_v2_test.go b/integration/remote_write_v2_test.go index c295b6a62ab..6a1f4996c64 100644 --- a/integration/remote_write_v2_test.go +++ b/integration/remote_write_v2_test.go @@ -1,5 +1,4 @@ //go:build integration_remote_write_v2 -// +build integration_remote_write_v2 package integration diff --git a/integration/resource_based_limiter_test.go b/integration/resource_based_limiter_test.go index 953bf7c7887..880e4747184 100644 --- a/integration/resource_based_limiter_test.go +++ b/integration/resource_based_limiter_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/ruler_test.go b/integration/ruler_test.go index 48bdaff5514..c36ee0f276a 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -1,5 +1,4 @@ //go:build integration_ruler -// +build integration_ruler package integration diff --git a/integration/runtime_config_test.go b/integration/runtime_config_test.go index 62f1ba18082..5762739c797 100644 --- a/integration/runtime_config_test.go +++ b/integration/runtime_config_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/utf8_test.go b/integration/utf8_test.go index 17b0348bfec..0863b05f030 100644 --- a/integration/utf8_test.go +++ b/integration/utf8_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/integration/util.go b/integration/util.go index 34355697d30..79df4958a38 100644 --- a/integration/util.go +++ b/integration/util.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package integration diff --git a/integration/zone_aware_test.go b/integration/zone_aware_test.go index 215d488ec3f..32ddd8ea423 100644 --- a/integration/zone_aware_test.go +++ b/integration/zone_aware_test.go @@ -1,5 +1,4 @@ //go:build requires_docker -// +build requires_docker package integration diff --git a/pkg/configs/db/dbtest/integration.go b/pkg/configs/db/dbtest/integration.go index 8c30c41bb5c..ae58d7987bd 100644 --- a/pkg/configs/db/dbtest/integration.go +++ b/pkg/configs/db/dbtest/integration.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package dbtest diff --git a/pkg/configs/db/dbtest/unit.go b/pkg/configs/db/dbtest/unit.go index f5d7635b086..f38748225a1 100644 --- a/pkg/configs/db/dbtest/unit.go +++ b/pkg/configs/db/dbtest/unit.go @@ -1,5 +1,4 @@ //go:build !integration -// +build !integration package dbtest