Implement span compression and dropping fast exit spans

docs/configuration.asciidoc

@@ -498,6 +498,101 @@ Any labels set by the application via the agent's public API will override globa
 
 NOTE: This option requires APM Server 7.2 or later. It will have no effect on older versions.
 
+
+[float]
+[[config-span-compression-enabled]]
+==== `SpanCompressionEnabled` (added[1.14])
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+Setting this option to true will enable span compression feature. Span compression reduces the collection, processing, and storage overhead, and removes clutter from the UI. 
+The tradeoff is that some information such as DB statements of all the compressed spans will not be collected.
+
+[options="header"]
+|============
+| Environment variable name    | IConfiguration key
+| `ELASTIC_APM_SPAN_COMPRESSION_ENABLED` | `ElasticApm:SpanCompressionEnabled`
+|============
+
+[options="header"]
+|============
+| Default  | Type
+| `true`   | Boolean
+|============
+
+[float]
+[[config-span-compression-exact-match-max-duration]]
+==== `SpanCompressionExactMatchMaxDuration` (added[1.14])
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+Consecutive spans that are exact match and that are under this threshold will be compressed into a single composite span.
+This option does not apply to composite spans. This reduces the collection, processing, and storage overhead, and removes clutter from the UI.
+The tradeoff is that the DB statements of all the compressed spans will not be collected.
+
+[options="header"]
+|============
+| Environment variable name                           | IConfiguration key
+| `ELASTIC_APM_SPAN_COMPRESSION_EXACT_MATCH_MAX_DURATION` | `ElasticApm:SpanCompressionExactMatchMaxDuration`
+|============
+
+[options="header"]
+|============
+| Default | Type
+| `50ms`  | TimeDuration
+|============
+
+
+[float]
+[[config-span-compression-same-kind-max-duration]]
+==== `SpanCompressionSameKindMaxDuration` (added[1.14])
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+Consecutive spans to the same destination that are under this threshold will be compressed into a single composite span.
+This option does not apply to composite spans.
+This reduces the collection, processing, and storage overhead, and removes clutter from the UI. The tradeoff is that the DB statements of all the compressed spans will not be collected.
+
+[options="header"]
+|============
+| Environment variable name                             | IConfiguration key
+| `ELASTIC_APM_SPAN_COMPRESSION_SAME_KIND_MAX_DURATION` | `ElasticApm:SpanCompressionSameKindMaxDuration`
+|============
+
+[options="header"]
+|============
+| Default | Type
+| `5ms`   | TimeDuration
+|============
+
+
+
+[float]
+[[config-exit-span-min-duration]]
+==== `ExitSpanMinDuration` (added[1.14])
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+Sets the minimum duration of exit spans. Exit spans with a duration lesser than this threshold are attempted to be discarded.
+If the exit span is equal or greater the threshold, it should be kept.
+In some cases exit spans cannot be discarded. For example, spans that propagate the trace context to downstream services,
+such as outgoing HTTP requests, can't be discarded.
+However, external calls that don't propagate context, such as calls to a database, can be discarded using this threshold.
+Additionally, spans that lead to an error can't be discarded.
+
+[options="header"]
+|============
+| Environment variable name            | IConfiguration key
+| `ELASTIC_APM_EXIT_SPAN_MIN_DURATION` | `ElasticApm:ExitSpanMinDuration`
+|============
+
+[options="header"]
+|============
+| Default | Type
+| `1ms`  | TimeDuration
+|============
+
+
 [[config-reporter]]
 === Reporter configuration options
 
@@ -1130,6 +1225,7 @@ If your application is called by an {dotnet5} application that does not have an
 | <<config-enabled,`Enabled`>> | No | Core
 | <<config-environment,`Environment`>> | No | Core
 | <<config-excluded-namespaces,`ExcludedNamespaces`>> | No | Stacktrace
+| <<config-exit-span-min-duration,`ExitSpanMinDuration`>> | Yes | Core, Performance
 | <<config-flush-interval,`FlushInterval`>> | No | Reporter
 | <<config-global-labels,`GlobalLabels`>> | No | Core
 | <<config-ignore-message-queues,`IgnoreMessageQueues`>> | Yes | Messaging, Performance
@@ -1146,6 +1242,9 @@ If your application is called by an {dotnet5} application that does not have an
 | <<config-service-name,`ServiceName`>> | No | Core
 | <<config-service-node-name, `ServiceNodeName`>> | No | Core
 | <<config-service-version,`ServiceVersion`>> | No | Core
+| <<config-span-compression-enabled,`SpanCompressionEnabled`>> | Yes | Core, Performance
+| <<config-span-compression-exact-match-max-duration,`SpanCompressionExactMatchMaxDuration`>> | Yes | Core, Performance
+| <<config-span-compression-same-kind-max-duration,`SpanCompressionSameKindMaxDuration`>> | Yes | Core, Performance
 | <<config-span-frames-min-duration,`SpanFramesMinDuration`>> | No | Stacktrace, Performance
 | <<config-stack-trace-limit,`StackTraceLimit`>> | No | Stacktrace, Performance
 | <<config-trace-context-ignore-sampled-false,`TraceContextIgnoreSampledFalse`>> | No | Core

sample/ApiSamples/Program.cs

@@ -19,6 +19,10 @@ internal class Program
 	{
 		private static void Main(string[] args)
 		{
+			CompressionSample();
+
+			Console.ReadKey();
+
 			if (args.Length == 1) //in case it's started with arguments, parse DistributedTracingData from them
 			{
 				var distributedTracingData = DistributedTracingData.TryDeserializeFromString(args[0]);
@@ -52,6 +56,37 @@ private static void Main(string[] args)
 			}
 		}
 
+		public static void CompressionSample()
+		{
+			Environment.SetEnvironmentVariable("ELASTIC_APM_SPAN_COMPRESSION_ENABLED", "true");
+			Environment.SetEnvironmentVariable("ELASTIC_APM_SPAN_COMPRESSION_EXACT_MATCH_MAX_DURATION", "1s");
+
+			Agent.Tracer.CaptureTransaction("Foo", "Bar", t =>
+			{
+				t.CaptureSpan("foo1", "bar", span1 =>
+				{
+					for (var i = 0; i < 10; i++)
+					{
+						span1.CaptureSpan("Select * From Table1", ApiConstants.TypeDb, (s) =>
+						{
+							s.Context.Db = new Database() { Type = "mssql", Instance = "01" };
+						}, ApiConstants.SubtypeMssql, isExitSpan: true);
+					}
+
+					span1.CaptureSpan("foo2", "randomSpan", () => { });
+
+
+					for (var i = 0; i < 10; i++)
+					{
+						span1.CaptureSpan("Select * From Table2", ApiConstants.TypeDb, (s2) =>
+						{
+							s2.Context.Db = new Database() { Type = "mssql", Instance = "01" };
+						}, ApiConstants.SubtypeMssql, isExitSpan: true);
+					}
+				});
+			});
+		}
+
 		public static void SampleCustomTransaction()
 		{
 			WriteLineToConsole($"{nameof(SampleCustomTransaction)} started");
@@ -264,9 +299,7 @@ public static void SampleSpanWithCustomContextFillAll()
 					{
 						span.Context.Db = new Database
 						{
-							Statement = "GET /_all/_search?q=tag:wow",
-							Type = Database.TypeElasticsearch,
-							Instance = "MyInstance"
+							Statement = "GET /_all/_search?q=tag:wow", Type = Database.TypeElasticsearch, Instance = "MyInstance"
 						};
 					});
 			});

src/Elastic.Apm/BackendComm/CentralConfig/CentralConfigurationFetcher.cs

@@ -33,7 +33,8 @@ internal class CentralConfigurationFetcher : BackendCommComponentBase, ICentralC
 		private long _dbgIterationsCount;
 		private EntityTagHeaderValue _eTag;
 
-		internal CentralConfigurationFetcher(IApmLogger logger, IConfigurationStore configurationStore, ICentralConfigurationResponseParser centralConfigurationResponseParser,
+		internal CentralConfigurationFetcher(IApmLogger logger, IConfigurationStore configurationStore,
+			ICentralConfigurationResponseParser centralConfigurationResponseParser,
 			Service service,
 			HttpMessageHandler httpMessageHandler = null, IAgentTimer agentTimer = null, string dbgName = null
 		) : this(logger, configurationStore, configurationStore.CurrentSnapshot, service, httpMessageHandler, agentTimer, dbgName) =>
@@ -49,7 +50,8 @@ internal CentralConfigurationFetcher(IApmLogger logger, IConfigurationStore conf
 		/// snapshots)
 		/// when passing isEnabled: initialConfigSnapshot.CentralConfig and config: initialConfigSnapshot to base
 		/// </summary>
-		private CentralConfigurationFetcher(IApmLogger logger, IConfigurationStore configurationStore, IConfiguration initialConfiguration, Service service
+		private CentralConfigurationFetcher(IApmLogger logger, IConfigurationStore configurationStore, IConfiguration initialConfiguration,
+			Service service
 			, HttpMessageHandler httpMessageHandler, IAgentTimer agentTimer, string dbgName
 		)
 			: base( /* isEnabled: */ initialConfiguration.CentralConfig, logger, ThisClassName, service, initialConfiguration, httpMessageHandler)
@@ -127,10 +129,11 @@ protected override async Task WorkLoopIteration()
 				else
 				{
 					waitInfo = new WaitInfoS(WaitTimeIfAnyError, "Default wait time is used because exception was thrown"
-							+ " while fetching configuration from APM Server and parsing it.");
+						+ " while fetching configuration from APM Server and parsing it.");
 				}
 
-				_logger.IfLevel(level)?.LogException(ex, "Exception was thrown while fetching configuration from APM Server and parsing it."
+				_logger.IfLevel(level)
+					?.LogException(ex, "Exception was thrown while fetching configuration from APM Server and parsing it."
 						+ " ETag: `{ETag}'. URL: `{Url}'. Apm Server base URL: `{ApmServerUrl}'. WaitInterval: {WaitInterval}."
 						+ " dbgIterationsCount: {dbgIterationsCount}."
 						+ Environment.NewLine + "+-> Request:{HttpRequest}"
@@ -141,7 +144,9 @@ protected override async Task WorkLoopIteration()
 						, HttpClient.BaseAddress.Sanitize().ToString()
 						, waitInfo.Interval.ToHms(),
 						_dbgIterationsCount
-						, httpRequest == null ? " N/A" : Environment.NewLine + httpRequest.Sanitize(_configurationStore.CurrentSnapshot.SanitizeFieldNames).ToString().Indent()
+						, httpRequest == null
+							? " N/A"
+							: Environment.NewLine + httpRequest.Sanitize(_configurationStore.CurrentSnapshot.SanitizeFieldNames).ToString().Indent()
 						, httpResponse == null ? " N/A" : Environment.NewLine + httpResponse.ToString().Indent()
 						, httpResponseBody == null ? "N/A" : httpResponseBody.Length.ToString()
 						, httpResponseBody == null ? " N/A" : Environment.NewLine + httpResponseBody.Indent());
@@ -152,7 +157,8 @@ protected override async Task WorkLoopIteration()
 				httpResponse?.Dispose();
 			}
 
-			_logger.Trace()?.Log("Waiting {WaitInterval}... {WaitReason}. dbgIterationsCount: {dbgIterationsCount}."
+			_logger.Trace()
+				?.Log("Waiting {WaitInterval}... {WaitReason}. dbgIterationsCount: {dbgIterationsCount}."
 					, waitInfo.Interval.ToHms(), waitInfo.Reason, _dbgIterationsCount);
 			await _agentTimer.Delay(_agentTimer.Now + waitInfo.Interval, CancellationTokenSource.Token).ConfigureAwait(false);
 		}
@@ -186,11 +192,13 @@ private HttpRequestMessage BuildHttpRequest(EntityTagHeaderValue eTag)
 
 		private async Task<(HttpResponseMessage, string)> FetchConfigHttpResponseAsync(HttpRequestMessage httpRequest) =>
 			await _agentTimer.AwaitOrTimeout(FetchConfigHttpResponseImplAsync(httpRequest)
-				, _agentTimer.Now + GetConfigHttpRequestTimeout, CancellationTokenSource.Token).ConfigureAwait(false);
+					, _agentTimer.Now + GetConfigHttpRequestTimeout, CancellationTokenSource.Token)
+				.ConfigureAwait(false);
 
 		private void UpdateConfigStore(CentralConfigurationReader centralConfigurationReader)
 		{
-			_logger.Info()?.Log("Updating " + nameof(ConfigurationStore) + ". New central configuration: {CentralConfiguration}", centralConfigurationReader);
+			_logger.Info()
+				?.Log("Updating " + nameof(ConfigurationStore) + ". New central configuration: {CentralConfiguration}", centralConfigurationReader);
 
 			_configurationStore.CurrentSnapshot = new WrappingConfiguration(_initialSnapshot, centralConfigurationReader
 				, $"{_initialSnapshot.Description()} + central (ETag: `{centralConfigurationReader.ETag}')");
@@ -252,6 +260,7 @@ internal WrappingConfiguration(IConfiguration wrapped, CentralConfigurationReade
 
 			public string Environment => _wrapped.Environment;
 			public IReadOnlyCollection<string> ExcludedNamespaces => _wrapped.ExcludedNamespaces;
+			public double ExitSpanMinDuration => _centralConfiguration.ExitSpanMinDuration ?? _wrapped.ExitSpanMinDuration;
 
 			public TimeSpan FlushInterval => _wrapped.FlushInterval;
 
@@ -285,12 +294,21 @@ internal WrappingConfiguration(IConfiguration wrapped, CentralConfigurationReade
 			public string ServiceNodeName => _wrapped.ServiceNodeName;
 
 			public string ServiceVersion => _wrapped.ServiceVersion;
+			public bool SpanCompressionEnabled => _centralConfiguration.SpanCompressionEnabled ?? _wrapped.SpanCompressionEnabled;
+
+			public double SpanCompressionExactMatchMaxDuration =>
+				_centralConfiguration.SpanCompressionExactMatchMaxDuration ?? _wrapped.SpanCompressionExactMatchMaxDuration;
+
+			public double SpanCompressionSameKindMaxDuration =>
+				_centralConfiguration.SpanCompressionSameKindMaxDuration ?? _wrapped.SpanCompressionSameKindMaxDuration;
 
 			public double SpanFramesMinDurationInMilliseconds =>
 				_centralConfiguration.SpanFramesMinDurationInMilliseconds ?? _wrapped.SpanFramesMinDurationInMilliseconds;
 
 			public int StackTraceLimit => _centralConfiguration.StackTraceLimit ?? _wrapped.StackTraceLimit;
-			public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls => _centralConfiguration.TransactionIgnoreUrls ?? _wrapped.TransactionIgnoreUrls;
+
+			public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls =>
+				_centralConfiguration.TransactionIgnoreUrls ?? _wrapped.TransactionIgnoreUrls;
 
 			public int TransactionMaxSpans => _centralConfiguration.TransactionMaxSpans ?? _wrapped.TransactionMaxSpans;
 

src/Elastic.Apm/BackendComm/CentralConfig/CentralConfigurationReader.cs

@@ -18,8 +18,9 @@ internal class CentralConfigurationReader : AbstractConfigurationReader
 
 		private readonly CentralConfigurationResponseParser.CentralConfigPayload _configPayload;
 
-		public CentralConfigurationReader(IApmLogger logger, CentralConfigurationResponseParser.CentralConfigPayload configPayload, string eTag) : base(logger,
-			ThisClassName)
+		public CentralConfigurationReader(IApmLogger logger, CentralConfigurationResponseParser.CentralConfigPayload configPayload, string eTag) :
+			base(logger,
+				ThisClassName)
 		{
 			_configPayload = configPayload;
 			ETag = eTag;
@@ -53,6 +54,14 @@ public CentralConfigurationReader(IApmLogger logger, CentralConfigurationRespons
 
 		internal double? TransactionSampleRate { get; private set; }
 
+		internal bool? SpanCompressionEnabled { get; private set; }
+
+		internal double? SpanCompressionExactMatchMaxDuration { get; private set; }
+
+		internal double? SpanCompressionSameKindMaxDuration { get; private set; }
+
+		internal double? ExitSpanMinDuration { get; private set; }
+
 		private void UpdateConfigurationValues()
 		{
 			CaptureBody = GetConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.CaptureBodyKey, ParseCaptureBody);
@@ -62,19 +71,32 @@ private void UpdateConfigurationValues()
 				ParseTransactionMaxSpans);
 			TransactionSampleRate = GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.TransactionSampleRateKey,
 				ParseTransactionSampleRate);
-			CaptureHeaders = GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.CaptureHeadersKey, ParseCaptureHeaders);
+			CaptureHeaders =
+				GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.CaptureHeadersKey, ParseCaptureHeaders);
 			LogLevel = GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.LogLevelKey, ParseLogLevel);
 			SpanFramesMinDurationInMilliseconds =
 				GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.SpanFramesMinDurationKey,
 					ParseSpanFramesMinDurationInMilliseconds);
-			StackTraceLimit = GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.StackTraceLimitKey, ParseStackTraceLimit);
+			StackTraceLimit = GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.StackTraceLimitKey,
+				ParseStackTraceLimit);
 			Recording = GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.Recording, ParseRecording);
 			SanitizeFieldNames =
 				GetConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.SanitizeFieldNames, ParseSanitizeFieldNamesImpl);
 			TransactionIgnoreUrls =
 				GetConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.TransactionIgnoreUrls, ParseTransactionIgnoreUrlsImpl);
 			IgnoreMessageQueues =
 				GetConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.IgnoreMessageQueues, ParseIgnoreMessageQueuesImpl);
+			SpanCompressionEnabled =
+				GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.SpanCompressionEnabled,
+					ParseSpanCompressionEnabled);
+			SpanCompressionExactMatchMaxDuration =
+				GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.SpanCompressionExactMatchMaxDuration,
+					ParseSpanCompressionExactMatchMaxDuration);
+			SpanCompressionSameKindMaxDuration =
+				GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.SpanCompressionSameKindMaxDuration,
+					ParseSpanCompressionSameKindMaxDuration);
+			ExitSpanMinDuration =
+				GetSimpleConfigurationValue(CentralConfigurationResponseParser.CentralConfigPayload.ExitSpanMinDuration, ParseExitSpanMinDuration);
 		}
 
 		private ConfigurationKeyValue BuildKv(string key, string value) =>

src/Elastic.Apm/BackendComm/CentralConfig/CentralConfigurationResponseParser.cs

@@ -147,6 +147,10 @@ internal class CentralConfigPayload
 			internal const string TransactionIgnoreUrls = "transaction_ignore_urls";
 			internal const string TransactionMaxSpansKey = "transaction_max_spans";
 			internal const string TransactionSampleRateKey = "transaction_sample_rate";
+			internal const string SpanCompressionEnabled = "span_compression_enabled";
+			internal const string SpanCompressionExactMatchMaxDuration = "span_compression_exact_match_max_duration";
+			internal const string SpanCompressionSameKindMaxDuration = "span_compression_same_kind_max_duration";
+			internal const string ExitSpanMinDuration = "exit_span_min_duration";
 
 			internal static readonly ISet<string> SupportedOptions = new HashSet<string>
 			{
@@ -162,6 +166,10 @@ internal class CentralConfigPayload
 				TransactionIgnoreUrls,
 				TransactionMaxSpansKey,
 				TransactionSampleRateKey,
+				SpanCompressionEnabled,
+				SpanCompressionExactMatchMaxDuration,
+				SpanCompressionSameKindMaxDuration,
+				ExitSpanMinDuration
 			};
 
 			private readonly IDictionary<string, string> _keyValues;