Merge branch 'main' into testing-benchmark-okta-aws-cli

CHANGELOG.asciidoc

@@ -1,4 +1,5 @@
 // tag::list[]
+* <<apm-release-notes-9.0>>
 * <<apm-release-notes-8.18>>
 * <<apm-release-notes-8.17>>
 * <<apm-release-notes-8.16>>
@@ -21,6 +22,8 @@
 // end::list[]
 
 // tag::includes[]
+include::./changelogs/head.asciidoc[]
+include::./changelogs/9.0.asciidoc[]
 include::./changelogs/8.18.asciidoc[]
 include::./changelogs/8.17.asciidoc[]
 include::./changelogs/8.16.asciidoc[]

changelogs/9.0.asciidoc

@@ -0,0 +1,27 @@
+[[apm-release-notes-9.0]]
+== APM version 9.0
+* <<apm-release-notes-9.0.0>>
+
+[float]
+[[apm-release-notes-9.0.0]]
+=== APM version 9.0.0
+
+https://github.com/elastic/apm-server/compare/v\...v9.0.0[View commits]
+
+[float]
+==== Bug fixes
+- Fix overflow in validation of `apm-server.agent.config.cache.expiration` on 32-bit architectures {pull}15216[15216]
+
+[float]
+==== Breaking Changes
+- Change `sampling.tail.storage_limit` default to `0`. While `0` means unlimited local tail-sampling database size, it now enforces a max 90% disk usage on the disk where the data directory is located. Any tail sampling writes after this threshold will be rejected, similar to what happens when tail-sampling database size exceeds a non-0 storage limit. Setting `sampling.tail.storage_limit` to non-0 maintains the existing behavior which limits the tail-sampling database size to `sampling.tail.storage_limit` and does not have the new disk usage threshold check. {pull}15467[15467]
+
+[float]
+==== Deprecations
+
+[float]
+==== Intake API Changes
+
+[float]
+==== Added
+- Tail-based sampling: Storage layer is rewritten to use Pebble database instead of BadgerDB. The new implementation offers a substantial throughput increase while consuming significantly less memory. Disk usage is lower and more stable. See PR for benchmark details. {pull}15235[15235]

changelogs/all-breaking-changes.asciidoc

@@ -7,6 +7,24 @@
 This section describes the breaking changes and deprecations introduced in this release
 and previous minor versions.
 
+// tag::90-bc[]
+[float]
+[[breaking-changes-9.0]]
+=== 9.0
+
+The following breaking changes are introduced in APM version 9.0.0:
+
+- Change `sampling.tail.storage_limit` default to `0`.
+While `0` means unlimited local tail-sampling database size,
+it now enforces a max 90% disk usage on the disk where the data directory is located.
+Any tail sampling writes after this threshold will be rejected,
+similar to what happens when tail-sampling database size exceeds a non-0 storage limit.
+Setting `sampling.tail.storage_limit` to non-0 maintains the existing behavior
+which limits the tail-sampling database size to `sampling.tail.storage_limit`
+and does not have the new disk usage threshold check.
+For more details, see https://github.com/elastic/apm-server/pull/15467[PR #15467]
+// end::90-bc[]
+
 // tag::811-bc[]
 [float]
 [[breaking-changes-8.11]]

changelogs/head.asciidoc

@@ -1,7 +1,7 @@
 [[release-notes-head]]
 == APM version HEAD
 
-https://github.com/elastic/apm-server/compare/8.18\...8.x[View commits]
+https://github.com/elastic/apm-server/compare/9.0\...main[View commits]
 
 [float]
 ==== Breaking Changes

cmd/intake-receiver/version.go

@@ -18,4 +18,4 @@
 package main
 
 // version matches the APM Server's version
-const version = "9.0.0"
+const version = "9.1.0"

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v1.2.0
 	github.com/elastic/apm-data v1.16.0
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241231140711-7806f1a2cb26
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250131124117-04eac62055e9
 	github.com/elastic/elastic-agent-client/v7 v7.17.0
 	github.com/elastic/elastic-agent-libs v0.18.3
 	github.com/elastic/elastic-agent-system-metrics v0.11.7
@@ -25,9 +25,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/hashicorp/golang-lru v1.0.2
-	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901
 	github.com/libp2p/go-reuseport v0.4.0
-	github.com/modern-go/reflect2 v1.0.2
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/ryanuber/go-glob v1.0.0
@@ -74,16 +72,15 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect
 	github.com/dlclark/regexp2 v1.8.1 // indirect
-	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/dop251/goja v0.0.0-20230427124612-428fc442ff5f // indirect
 	github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 // indirect
 	github.com/eapache/go-resiliency v1.7.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
-	github.com/ebitengine/purego v0.8.0 // indirect
+	github.com/ebitengine/purego v0.8.1 // indirect
 	github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
 	github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f // indirect
-	github.com/elastic/go-structform v0.0.10 // indirect
+	github.com/elastic/go-structform v0.0.12 // indirect
 	github.com/elastic/go-windows v1.0.2 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
 	github.com/elastic/opentelemetry-lib v0.14.0 // indirect
@@ -110,6 +107,7 @@ require (
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -119,6 +117,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
@@ -131,7 +130,7 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect
-	github.com/shirou/gopsutil/v4 v4.24.9 // indirect
+	github.com/shirou/gopsutil/v4 v4.24.10 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect

go.sum

@@ -113,14 +113,14 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE=
-github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE=
+github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/elastic/apm-aggregation v1.2.0 h1:UndqG3ccBTjyxTqHujBVjcbVLb9qG1clxRcrp9JRelI=
 github.com/elastic/apm-aggregation v1.2.0/go.mod h1:YllYwPYVV27pbuPfjRtQAKo6eSSrh13PZr38RKYd810=
 github.com/elastic/apm-data v1.16.0 h1:LkJFoNkadIyqXNo3EMm98J38j9HWDMWUe6F74GaXtJ4=
 github.com/elastic/apm-data v1.16.0/go.mod h1:ST2P1yshhN2U3IbBYyn+Ni3VOFTifavNzMfh7E9zLHY=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241231140711-7806f1a2cb26 h1:qyI4AIRfdnTwesqHN9KYdbGpcJ4m4lIKlR+f8zbKIO0=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241231140711-7806f1a2cb26/go.mod h1:CgYrkUfXIWj/lcPc5ipnO0/2OEx4GmstkLzyUHilme8=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250131124117-04eac62055e9 h1:yV4r4O/aVyPXvXhqxpeRqDyF5G+sy2CmTOKpBFOsXQM=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250131124117-04eac62055e9/go.mod h1:84zbls1RDKu/SHmRykLUkpaF0ZVme7P+TCDV6mZ7sKw=
 github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0JojNasPgYIA47gpuuns=
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.17.0 h1:TPLrEHF4kJ3RkmQzZPffrniY4WeW4vriHZbOAzM1hFo=
@@ -139,8 +139,8 @@ github.com/elastic/go-elasticsearch/v8 v8.17.0 h1:e9cWksE/Fr7urDRmGPGp47Nsp4/mvN
 github.com/elastic/go-elasticsearch/v8 v8.17.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64=
 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt7YZoupKlxZ093qTZYdGou3EhfTF1U0B4=
 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko=
-github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w=
-github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
+github.com/elastic/go-structform v0.0.12 h1:HXpzlAKyej8T7LobqKDThUw7BMhwV6Db24VwxNtgxCs=
+github.com/elastic/go-structform v0.0.12/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
 github.com/elastic/go-sysinfo v1.15.0 h1:54pRFlAYUlVNQ2HbXzLVZlV+fxS7Eax49stzg95M4Xw=
 github.com/elastic/go-sysinfo v1.15.0/go.mod h1:jPSuTgXG+dhhh0GKIyI2Cso+w5lPJ5PvVqKlL8LV/Hk=
 github.com/elastic/go-ucfg v0.8.8 h1:54KIF/2zFKfl0MzsSOCGOsZ3O2bnjFQJ0nDJcLhviyk=
@@ -355,8 +355,8 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/shirou/gopsutil/v4 v4.24.9 h1:KIV+/HaHD5ka5f570RZq+2SaeFsb/pq+fp2DGNWYoOI=
-github.com/shirou/gopsutil/v4 v4.24.9/go.mod h1:3fkaHNeYsUFCGZ8+9vZVWtbyM1k2eRnlL+bWO8Bxa/Q=
+github.com/shirou/gopsutil/v4 v4.24.10 h1:7VOzPtfw/5YDU+jLEoBwXwxJbQetULywoSV4RYY7HkM=
+github.com/shirou/gopsutil/v4 v4.24.10/go.mod h1:s4D/wg+ag4rG0WO7AiTj2BeYCRhym0vM7DHbZRxnIT8=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
@@ -426,8 +426,8 @@ go.opentelemetry.io/collector/pdata v1.24.0 h1:D6j92eAzmAbQgivNBUnt8r9juOl8ugb+i
 go.opentelemetry.io/collector/pdata v1.24.0/go.mod h1:cf3/W9E/uIvPS4MR26SnMFJhraUCattzzM6qusuONuc=
 go.opentelemetry.io/collector/semconv v0.116.0 h1:63xCZomsKJAWmKGWD3lnORiE3WKW6AO4LjnzcHzGx3Y=
 go.opentelemetry.io/collector/semconv v0.116.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM=
 go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
 go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
 go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=

internal/agentcfg/elasticsearch.go

@@ -125,22 +125,22 @@ func (f *ElasticsearchFetcher) Fetch(ctx context.Context, query Query) (Result,
 		// Happy path: serve fetch requests using an initialized cache.
 		f.mu.RLock()
 		defer f.mu.RUnlock()
-		f.esFetchCount.Add(ctx, 1)
+		f.esFetchCount.Add(context.Background(), 1)
 		return matchAgentConfig(query, f.cache), nil
 	}
 
 	if f.fallbackFetcher != nil {
-		f.esFetchFallbackCount.Add(ctx, 1)
+		f.esFetchFallbackCount.Add(context.Background(), 1)
 		return f.fallbackFetcher.Fetch(ctx, query)
 	}
 
 	if f.invalidESCfg.Load() {
-		f.esFetchInvalidCount.Add(ctx, 1)
+		f.esFetchInvalidCount.Add(context.Background(), 1)
 		f.rateLimitedLogger.Errorf("rejecting fetch request: no valid elasticsearch config")
 		return Result{}, errors.New(ErrNoValidElasticsearchConfig)
 	}
 
-	f.esFetchUnavailableCount.Add(ctx, 1)
+	f.esFetchUnavailableCount.Add(context.Background(), 1)
 	f.rateLimitedLogger.Warnf("rejecting fetch request: infrastructure is not ready")
 	return Result{}, errors.New(ErrInfrastructureNotReady)
 }
@@ -233,9 +233,9 @@ func (f *ElasticsearchFetcher) refreshCache(ctx context.Context) (err error) {
 
 	defer func() {
 		if err != nil {
-			f.esCacheRefreshFailures.Add(ctx, 1)
+			f.esCacheRefreshFailures.Add(context.Background(), 1)
 		} else {
-			f.esCacheRefreshSuccesses.Add(ctx, 1)
+			f.esCacheRefreshSuccesses.Add(context.Background(), 1)
 		}
 	}()
 
@@ -267,7 +267,7 @@ func (f *ElasticsearchFetcher) refreshCache(ctx context.Context) (err error) {
 	f.cache = buffer
 	f.mu.Unlock()
 	f.cacheInitialized.Store(true)
-	f.esCacheEntriesCount.Record(ctx, int64(len(f.cache)))
+	f.esCacheEntriesCount.Record(context.Background(), int64(len(f.cache)))
 	f.last = time.Now()
 	return nil
 }

internal/beatcmd/beat_test.go

@@ -563,7 +563,12 @@ func TestRunManager_Reloader_newRunnerError(t *testing.T) {
 	require.NoError(t, err)
 	defer manager.Stop()
 
-	assert.Equal(t, "failed to load input config: newRunner error", <-inputFailedMsg)
+	select {
+	case msg := <-inputFailedMsg:
+		assert.Equal(t, "failed to load input config: newRunner error", msg)
+	case <-time.After(10 * time.Second):
+		t.Fatal("timed out waiting for input failed msg")
+	}
 }
 
 func runBeat(t testing.TB, beat *Beat) (stop func() error) {

internal/beatcmd/reloader.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/joeshaw/multierror"
 	"go.elastic.co/apm/module/apmotel/v2"
 	"go.opentelemetry.io/otel/metric"
 	"golang.org/x/sync/errgroup"
@@ -138,15 +137,15 @@ func (r *Reloader) Run(ctx context.Context) error {
 // Note: reloadInputs may be called before the Reloader is running.
 func (r *Reloader) reloadInputs(configs []*reload.ConfigWithMeta) error {
 	if n := len(configs); n != 1 {
-		var errs multierror.Errors
+		var errs []error
 		for _, cfg := range configs {
 			unitErr := cfgfile.UnitError{
 				Err:    fmt.Errorf("only 1 input supported, got %d", n),
 				UnitID: cfg.InputUnitID,
 			}
 			errs = append(errs, unitErr)
 		}
-		return errs.Err()
+		return errors.Join(errs...)
 	}
 	r.mu.Lock()
 	defer r.mu.Unlock()
@@ -156,21 +155,21 @@ func (r *Reloader) reloadInputs(configs []*reload.ConfigWithMeta) error {
 	// increasing revision number.
 	revision, err := cfg.Int("revision", -1)
 	if err != nil {
-		return multierror.Errors{
+		return errors.Join(
 			cfgfile.UnitError{
 				Err:    fmt.Errorf("failed to extract input config revision: %w", err),
 				UnitID: configs[0].InputUnitID,
 			},
-		}.Err()
+		)
 	}
 
 	if err := r.reload(cfg, r.outputConfig, r.apmTracingConfig); err != nil {
-		return multierror.Errors{
+		return errors.Join(
 			cfgfile.UnitError{
 				Err:    fmt.Errorf("failed to load input config: %w", err),
 				UnitID: configs[0].InputUnitID,
 			},
-		}.Err()
+		)
 	}
 	r.inputConfig = cfg
 	r.logger.With(logp.Int64("revision", revision)).Info("loaded input config")

internal/beatcmd/reloader_test.go

@@ -87,7 +87,7 @@ func TestReloader(t *testing.T) {
 	err = registry.GetInputList().Reload([]*reload.ConfigWithMeta{{
 		Config: config.MustNewConfigFrom(`{}`),
 	}})
-	assert.EqualError(t, err, "1 error: failed to extract input config revision: missing field accessing 'revision'")
+	assert.EqualError(t, err, "failed to extract input config revision: missing field accessing 'revision'")
 	assertNoReload()
 
 	err = registry.GetInputList().Reload([]*reload.ConfigWithMeta{{
@@ -119,7 +119,7 @@ func TestReloader(t *testing.T) {
 	err = registry.GetInputList().Reload([]*reload.ConfigWithMeta{{
 		Config: config.MustNewConfigFrom(`{"revision": 2, "error": true}`),
 	}})
-	assert.EqualError(t, err, "1 error: failed to load input config: no runner for you")
+	assert.EqualError(t, err, "failed to load input config: no runner for you")
 	assertNoReload() // error occurred during reload, nothing changes
 	expectNoEvent(t, r1.stopped, "runner should not have been stopped")
 

internal/beater/config/config_test.go

@@ -362,8 +362,9 @@ func TestUnpackConfig(t *testing.T) {
 						ESConfig:              elasticsearch.DefaultConfig(),
 						Interval:              1 * time.Minute,
 						IngestRateDecayFactor: 0.25,
-						StorageLimit:          "3GB",
-						StorageLimitParsed:    3000000000,
+						StorageLimit:          "0",
+						StorageLimitParsed:    0,
+						DiskUsageThreshold:    0.9,
 						TTL:                   30 * time.Minute,
 					},
 				},
@@ -410,11 +411,12 @@ func TestUnpackConfig(t *testing.T) {
 					},
 				},
 				"sampling.tail": map[string]interface{}{
-					"enabled":           false,
-					"policies":          []map[string]interface{}{{"sample_rate": 0.5}},
-					"interval":          "2m",
-					"ingest_rate_decay": 1.0,
-					"storage_limit":     "1GB",
+					"enabled":              false,
+					"policies":             []map[string]interface{}{{"sample_rate": 0.5}},
+					"interval":             "2m",
+					"ingest_rate_decay":    1.0,
+					"storage_limit":        "1GB",
+					"disk_usage_threshold": 0.8,
 				},
 				"data_streams": map[string]interface{}{
 					"namespace": "foo",
@@ -495,6 +497,7 @@ func TestUnpackConfig(t *testing.T) {
 						IngestRateDecayFactor: 1.0,
 						StorageLimit:          "1GB",
 						StorageLimitParsed:    1000000000,
+						DiskUsageThreshold:    0.8,
 						TTL:                   30 * time.Minute,
 					},
 				},