Merge branch '8.x' into mergify/bp/8.x/pr-15286

.buildkite/scripts/dra.sh

@@ -110,7 +110,14 @@ dra() {
 if [[ "${TYPE}" == "staging" ]]; then
   if [[ "${DRA_BRANCH}" != "8.x" ]]; then
     echo "${DRA_BRANCH} is not '8.x'"
-    dra "${TYPE}" "$dra_command"
+    if [[ "${DRA_BRANCH}" == "main" ]] ; then
+      # NOTE: qualifier is needed for main/staging at the moment. Skip builds if no ELASTIC_QUALIFIER
+      if [[ -n "${ELASTIC_QUALIFIER}" ]]; then
+        dra "${TYPE}" "$dra_command"
+      fi
+    else
+      dra "${TYPE}" "$dra_command"
+    fi
   fi
 else
   # NOTE: qualifier is not needed for snapshots, let's unset it.

.github/workflows/benchmarks.yml

@@ -104,12 +104,11 @@ jobs:
           username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
           password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}
 
-
       - uses: elastic/oblt-actions/google/auth@v1
 
       - uses: elastic/oblt-actions/aws/auth@v1
         with:
-          role-duration-seconds: 18000 # 5 hours
+          role-duration-seconds: 21600 # 6 hours
 
       - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2.2.2
         with:

.github/workflows/functional-tests.yml

@@ -0,0 +1,66 @@
+---
+name: functional-tests
+
+on:
+  workflow_dispatch: ~
+  schedule:
+    - cron: '0 3 * * 1-5'
+
+permissions:
+  contents: read
+  id-token: write
+
+env:
+  TF_VAR_BRANCH: ${{ github.ref_name }}
+  TF_VAR_BUILD_ID: ${{ github.run_id }}
+  TF_VAR_ENVIRONMENT: 'ci'
+  TF_VAR_REPO: ${{ github.repository }}
+  TERRAFORM_VERSION: 1.10.2
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        environment:
+          - 'qa'
+          - 'pro'
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: "${{ env.TERRAFORM_VERSION }}"
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'functionaltests/go.mod'
+
+      - uses: elastic/oblt-actions/google/auth@v1
+
+      - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2.2.2
+        with:
+          export_to_environment: true
+          secrets: |-
+            EC_API_KEY:elastic-observability/elastic-cloud-observability-team-${{ matrix.environment }}-api-key
+
+      - run: |
+          export TF_VAR_CREATED_DATE=$(date +%s)
+          cd functionaltests && go test -v -timeout=20m -target "${{ matrix.environment }}" ./
+
+  notify:
+    if: always()
+    runs-on: ubuntu-latest
+    needs:
+      - run
+    steps:
+      - id: check
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+      - uses: elastic/oblt-actions/slack/notify-result@v1
+        with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: "#apm-server"
+          status: ${{ steps.check.outputs.status }}

NOTICE.txt

@@ -5409,11 +5409,11 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/cons
 
 --------------------------------------------------------------------------------
 Dependency : go.opentelemetry.io/collector/pdata
-Version: v1.23.0
+Version: v1.24.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.23.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.24.0/LICENSE:
 
 
                                  Apache License

dev_docs/RELEASES.md

@@ -68,6 +68,14 @@ For patch releases, only the version on the existing major and minor version bra
 
 * Collaborate with the docs team on any release highlights or breaking changes that should be included in the APM Server guide.
 
+* Run DRA for a given qualifier. The Release Team will say what qualifier to use in the the #mission-control channel.
+  * Go to https://buildkite.com/elastic/apm-server-package
+  * Click on `New Build`.
+  * Choose the `Branch` where the release should come from (either `main`, `8.x` or `[0-9].[0-9]+)`_
+  * Click on `options`
+  * Add `ELASTIC_QUALIFIER=<qualifier>` (`<qualifier` should be replaced with the given qualifier)
+  * Click on `Create Build`.
+
 ## On release day
 
 * For **minor releases**: new branches need to be added to `conf.yml` in the `elastic/docs` repo. [Example](https://github.com/elastic/docs/pull/893/files#diff-4a701a5adb4359c6abf9b8e1cb38819fR925). **This is handled by the docs release manager.**

functionaltests/8_15_test.go

@@ -0,0 +1,152 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package functionaltests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest"
+
+	"github.com/elastic/apm-server/functionaltests/internal/esclient"
+	"github.com/elastic/apm-server/functionaltests/internal/gen"
+	"github.com/elastic/apm-server/functionaltests/internal/terraform"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
+	ecAPICheck(t)
+
+	start := time.Now()
+	ctx := context.Background()
+
+	t.Log("creating deploment with terraform")
+	tf, err := terraform.New(t, t.Name())
+	require.NoError(t, err)
+	ecTarget := terraform.Var("ec_target", *target)
+	ecRegion := terraform.Var("ec_region", regionFrom(*target))
+	version := terraform.Var("stack_version", "8.15.4")
+	name := terraform.Var("name", t.Name())
+	require.NoError(t, tf.Apply(ctx, ecTarget, ecRegion, version, name))
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Cleanup(func() {
+		if !t.Failed() || (t.Failed() && *cleanupOnFailure) {
+			t.Log("cleanup terraform resources")
+			require.NoError(t, tf.Destroy(ctx, ecTarget, ecRegion, name, version))
+		} else {
+			t.Log("test failed and cleanup-on-failure is false, skipping cleanup")
+		}
+	})
+
+	var escfg esclient.Config
+	tf.Output("apm_url", &escfg.APMServerURL)
+	tf.Output("es_url", &escfg.ElasticsearchURL)
+	tf.Output("username", &escfg.Username)
+	tf.Output("password", &escfg.Password)
+	tf.Output("kb_url", &escfg.KibanaURL)
+
+	t.Logf("created deployment %s", escfg.KibanaURL)
+
+	ecc, err := esclient.New(escfg)
+	require.NoError(t, err)
+
+	t.Log("creating APM API key")
+	apikey, err := ecc.CreateAPMAPIKey(ctx, t.Name())
+	require.NoError(t, err)
+
+	g := gen.New(escfg.APMServerURL, apikey)
+	g.Logger = zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
+
+	previous, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+
+	g.RunBlockingWait(ctx, ecc, expectedIngestForASingleRun(), previous, 1*time.Minute)
+
+	beforeUpgradeCount, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+	assertDocCount(t, beforeUpgradeCount, previous, expectedIngestForASingleRun())
+
+	t.Log("check data streams")
+	var dss []types.DataStream
+	dss, err = ecc.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     1,
+		IndicesManagedBy: []string{"Data stream lifecycle"},
+	}, dss)
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Log("upgrade to 8.16.0")
+	require.NoError(t, tf.Apply(ctx, ecTarget, ecRegion, name, terraform.Var("stack_version", "8.16.0")))
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Log("check number of documents after upgrade")
+	afterUpgradeCount, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+	// We assert that no changes happened in the number of documents after upgrade
+	// to ensure the state didn't change before running the next ingestion round
+	// and further assertions.
+	// We don't expect any change here unless something broke during the upgrade.
+	assertDocCount(t, afterUpgradeCount, esclient.APMDataStreamsDocCount{}, beforeUpgradeCount)
+
+	t.Log("check data streams after upgrade, no rollover expected")
+	dss, err = ecc.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     1,
+		IndicesManagedBy: []string{"Data stream lifecycle"},
+	}, dss)
+
+	g.RunBlockingWait(ctx, ecc, expectedIngestForASingleRun(), previous, 1*time.Minute)
+
+	t.Log("check number of documents")
+	afterUpgradeIngestionCount, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+	assertDocCount(t, afterUpgradeIngestionCount, afterUpgradeCount, expectedIngestForASingleRun())
+
+	// Confirm datastreams are
+	// v managed by DSL if created after 8.15.0
+	// x managed by ILM if created before 8.15.0
+	t.Log("check data streams and verify lazy rollover happened")
+	dss2, err := ecc.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     2,
+		IndicesManagedBy: []string{"Data stream lifecycle", "Data stream lifecycle"},
+	}, dss2)
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	res, err := ecc.GetESErrorLogs(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, res.Hits.Total.Value)
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/main.tf

@@ -0,0 +1,20 @@
+module "ec_deployment" {
+  source = "../../testing/infra/terraform/modules/ec_deployment"
+  region = var.ec_region
+
+  deployment_template    = "aws-storage-optimized"
+  deployment_name_prefix = var.name
+
+  // self monitoring is enabled so we can inspect Elasticsearch
+  // logs from tests.
+  observability_deployment = "self"
+
+  apm_server_size = "1g"
+
+  elasticsearch_size       = "4g"
+  elasticsearch_zone_count = 1
+
+  stack_version = var.stack_version
+
+  tags = merge(local.ci_tags, module.tags.tags)
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/outputs.tf

@@ -0,0 +1,19 @@
+output "apm_url" {
+  value = module.ec_deployment.apm_url
+}
+
+output "es_url" {
+  value = module.ec_deployment.elasticsearch_url
+}
+
+output "username" {
+  value = module.ec_deployment.elasticsearch_username
+}
+output "password" {
+  value     = module.ec_deployment.elasticsearch_password
+  sensitive = true
+}
+
+output "kb_url" {
+  value = module.ec_deployment.kibana_url
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/tags.tf

@@ -0,0 +1,20 @@
+resource "time_static" "created_date" {}
+
+locals {
+  ci_tags = {
+    environment  = var.ENVIRONMENT
+    repo         = var.REPO
+    branch       = var.BRANCH
+    build        = var.BUILD_ID
+    created_date = coalesce(var.CREATED_DATE, time_static.created_date.unix)
+  }
+  project = "apm-server-functionaltest"
+}
+
+module "tags" {
+  source = "../../testing/infra/terraform/modules/tags"
+  # use the convention for team/shared owned resources if we are running in CI. 
+  # assume this is an individually owned resource otherwise. 
+  project = local.project
+}
+

functionaltests/TestUpgrade_8_15_4_to_8_16_0/terraform.tf

@@ -0,0 +1,21 @@
+terraform {
+  required_version = ">= 0.12.29"
+
+  required_providers {
+    ec = {
+      source  = "elastic/ec"
+      version = "0.5.1"
+    }
+  }
+}
+
+locals {
+  api_endpoints = {
+    qa  = "https://public-api.qa.cld.elstc.co"
+    pro = "https://api.elastic-cloud.com"
+  }
+}
+
+provider "ec" {
+  endpoint = local.api_endpoints[var.ec_target]
+}