TBS: change default disk usage threshold to 0.8 (#15524)

Change TBS default usage threshold to 0.8 for a lower risk. Change log to be slightly more readable. (cherry picked from commit 9ccfae6) # Conflicts: # changelogs/9.0.asciidoc
elastic · Feb 3, 2025 · 495d16a · 495d16a
1 parent a87f648
commit 495d16a
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 5 deletions.
diff --git a/changelogs/9.0.asciidoc b/changelogs/9.0.asciidoc
@@ -0,0 +1,27 @@
+[[apm-release-notes-9.0]]
+== APM version 9.0
+* <<apm-release-notes-9.0.0>>
+
+[float]
+[[apm-release-notes-9.0.0]]
+=== APM version 9.0.0
+
+https://github.com/elastic/apm-server/compare/v\...v9.0.0[View commits]
+
+[float]
+==== Bug fixes
+- Fix overflow in validation of `apm-server.agent.config.cache.expiration` on 32-bit architectures {pull}15216[15216]
+
+[float]
+==== Breaking Changes
+- Change `sampling.tail.storage_limit` default to `0`. While `0` means unlimited local tail-sampling database size, it now enforces a max 80% disk usage on the disk where the data directory is located. Any tail sampling writes after this threshold will be rejected, similar to what happens when tail-sampling database size exceeds a non-0 storage limit. Setting `sampling.tail.storage_limit` to non-0 maintains the existing behavior which limits the tail-sampling database size to `sampling.tail.storage_limit` and does not have the new disk usage threshold check. {pull}15467[15467] {pull}15524[15524]
+
+[float]
+==== Deprecations
+
+[float]
+==== Intake API Changes
+
+[float]
+==== Added
+- Tail-based sampling: Storage layer is rewritten to use Pebble database instead of BadgerDB. The new implementation offers a substantial throughput increase while consuming significantly less memory. Disk usage is lower and more stable. See PR for benchmark details. {pull}15235[15235]
diff --git a/changelogs/all-breaking-changes.asciidoc b/changelogs/all-breaking-changes.asciidoc
@@ -16,13 +16,14 @@ The following breaking changes are introduced in APM version 9.0.0:
 
 - Change `sampling.tail.storage_limit` default to `0`.
 While `0` means unlimited local tail-sampling database size,
-it now enforces a max 90% disk usage on the disk where the data directory is located.
+it now enforces a max 80% disk usage on the disk where the data directory is located.
 Any tail sampling writes after this threshold will be rejected,
 similar to what happens when tail-sampling database size exceeds a non-0 storage limit.
 Setting `sampling.tail.storage_limit` to non-0 maintains the existing behavior
 which limits the tail-sampling database size to `sampling.tail.storage_limit`
 and does not have the new disk usage threshold check.
-For more details, see https://github.com/elastic/apm-server/pull/15467[PR #15467]
+For more details, see https://github.com/elastic/apm-server/pull/15467[PR #15467] and
+https://github.com/elastic/apm-server/pull/15524[PR #15524]
 // end::90-bc[]
 
 // tag::811-bc[]

diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go
@@ -364,7 +364,7 @@ func TestUnpackConfig(t *testing.T) {
 						IngestRateDecayFactor: 0.25,
 						StorageLimit:          "0",
 						StorageLimitParsed:    0,
-						DiskUsageThreshold:    0.9,
+						DiskUsageThreshold:    0.8,
 						TTL:                   30 * time.Minute,
 					},
 				},

diff --git a/internal/beater/config/sampling.go b/internal/beater/config/sampling.go
@@ -161,7 +161,7 @@ func defaultTailSamplingConfig() TailSamplingConfig {
 		IngestRateDecayFactor: 0.25,
 		TTL:                   30 * time.Minute,
 		StorageLimit:          "0",
-		DiskUsageThreshold:    0.9,
+		DiskUsageThreshold:    0.8,
 		DiscardOnWriteFailure: false,
 	}
 	parsed, err := humanize.ParseBytes(cfg.StorageLimit)

diff --git a/x-pack/apm-server/sampling/eventstorage/storage_manager.go b/x-pack/apm-server/sampling/eventstorage/storage_manager.go
@@ -449,7 +449,7 @@ func (sm *StorageManager) NewReadWriter(storageLimit uint64, diskUsageThreshold
 		return uint64(float64(sm.cachedDiskStat.total.Load()) * diskUsageThreshold)
 	}
 	// the total disk space could change in runtime, but it is still useful to print it out in logs.
-	sm.logger.Infof("setting disk usage threshold to %.2f of total disk space of %0.1fgb", diskUsageThreshold, float64(sm.cachedDiskStat.total.Load())/gb)
+	sm.logger.Infof("setting disk usage threshold to %.0f%% of total disk space of %0.1fgb", diskUsageThreshold*100, float64(sm.cachedDiskStat.total.Load())/gb)
 	diskThresholdChecker := NewStorageLimitCheckerFunc(sm.diskUsed, diskThreshold)
 	rw = NewStorageLimitReadWriter(
 		fmt.Sprintf("disk usage threshold %.2f", diskUsageThreshold),