span (experimental): Compress short exit spans

CHANGELOG.asciidoc

@@ -25,6 +25,7 @@ https://github.com/elastic/apm-agent-go/compare/v1.14.0...master[View commits]
 
 - Deprecate `http.request.socket.encrypted` and stop recording it in `module/apmhttp`, `module/apmgrpc` and `module/apmfiber`. {pull}1129[#(1129)]
 - Collect and send span destination service timing statistics about the dropped spans to the apm-server. {pull}1132[#(1132)]
+- Experimental support to compress short exit spans into a composite span. Disabled by default. {pull}1134[#(1134)]
 
 [[release-notes-1.x]]
 === Go Agent version 1.x

apmtest/debug.go

@@ -0,0 +1,161 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package apmtest // import "go.elastic.co/apm/apmtest"
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+	"text/tabwriter"
+	"time"
+	"unicode/utf8"
+
+	"go.elastic.co/apm/internal/apmmath"
+	"go.elastic.co/apm/model"
+)
+
+// WriteTraceTable displays the trace as a table which can be used on tests to aid
+// debugging.
+func WriteTraceTable(writer io.Writer, tx model.Transaction, spans []model.Span) {
+	w := tabwriter.NewWriter(writer, 2, 4, 2, ' ', tabwriter.TabIndent)
+	fmt.Fprintln(w, "#\tNAME\tTYPE\tCOMP\tN\tDURATION\tOFFSET\tSPAN ID\tPARENT ID\tTRACE ID")
+
+	fmt.Fprintf(w, "TX\t%s\t%s\t-\t-\t%f\t%d\t%x\t%x\t%x\n", tx.Name,
+		tx.Type, tx.Duration,
+		0,
+		tx.ID, tx.ParentID, tx.TraceID,
+	)
+
+	sort.SliceStable(spans, func(i, j int) bool {
+		return time.Time(spans[i].Timestamp).Before(time.Time(spans[j].Timestamp))
+	})
+	for i, span := range spans {
+		count := 1
+		if span.Composite != nil {
+			count = span.Composite.Count
+		}
+
+		fmt.Fprintf(w, "%d\t%s\t%s\t%v\t%d\t%f\t+%d\t%x\t%x\t%x\n", i, span.Name,
+			span.Type, span.Composite != nil, count, span.Duration,
+			time.Time(span.Timestamp).Sub(time.Time(tx.Timestamp))/1e3,
+			span.ID, span.ParentID, span.TraceID,
+		)
+	}
+	w.Flush()
+}
+
+// WriteTraceWaterfall the trace waterfall "console output" to the specified
+// writer sorted by timestamp.
+func WriteTraceWaterfall(w io.Writer, tx model.Transaction, spans []model.Span) {
+	maxDuration := time.Duration(tx.Duration * float64(time.Millisecond))
+	if maxDuration == 0 {
+		for _, span := range spans {
+			maxDuration += time.Duration(span.Duration * float64(time.Millisecond))
+		}
+	}
+
+	maxWidth := int64(72)
+	buf := new(bytes.Buffer)
+	if tx.Duration > 0.0 {
+		writeSpan(buf, int(maxWidth), 0, fmt.Sprintf("transaction (%x) - %s", tx.ID, maxDuration.String()))
+	}
+
+	sort.SliceStable(spans, func(i, j int) bool {
+		return time.Time(spans[i].Timestamp).Before(time.Time(spans[j].Timestamp))
+	})
+
+	for _, span := range spans {
+		pos := int(apmmath.Round(
+			float64(time.Time(span.Timestamp).Sub(time.Time(tx.Timestamp))) /
+				float64(maxDuration) * float64(maxWidth),
+		))
+		tDur := time.Duration(span.Duration * float64(time.Millisecond))
+		dur := float64(tDur) / float64(maxDuration)
+		width := int(apmmath.Round(dur * float64(maxWidth)))
+		if width == int(maxWidth) {
+			width = int(maxWidth) - 1
+		}
+
+		spancontent := fmt.Sprintf("%s %s - %s",
+			span.Type, span.Name,
+			time.Duration(span.Duration*float64(time.Millisecond)).String(),
+		)
+		if span.Composite != nil {
+			spancontent = fmt.Sprintf("%d %s - %s",
+				span.Composite.Count, span.Name,
+				time.Duration(span.Duration*float64(time.Millisecond)).String(),
+			)
+		}
+		writeSpan(buf, width, pos, spancontent)
+	}
+
+	io.Copy(w, buf)
+}
+
+func writeSpan(buf *bytes.Buffer, width, pos int, content string) {
+	spaceRune := ' '
+	fillRune := '_'
+	startRune := '|'
+	endRune := '|'
+
+	// Prevent the spans from going out of bounds.
+	if pos == width {
+		pos = pos - 2
+	} else if pos >= width {
+		pos = pos - 1
+	}
+
+	for i := 0; i < int(pos); i++ {
+		buf.WriteRune(spaceRune)
+	}
+
+	if width <= 1 {
+		width = 1
+		// Write the first letter of the span type when the width is too small.
+		startRune, _ = utf8.DecodeRuneInString(content)
+	}
+
+	var written int
+	written, _ = buf.WriteRune(startRune)
+	if len(content) >= int(width)-1 {
+		content = content[:int(width)-1]
+	}
+
+	spacing := (width - len(content) - 2) / 2
+	for i := 0; i < spacing; i++ {
+		n, _ := buf.WriteRune(fillRune)
+		written += n
+	}
+
+	n, _ := buf.WriteString(content)
+	written += n
+	for i := 0; i < spacing; i++ {
+		n, _ := buf.WriteRune(fillRune)
+		written += n
+	}
+
+	if written < width {
+		buf.WriteRune(fillRune)
+	}
+	if width > 1 {
+		buf.WriteRune(endRune)
+	}
+
+	buf.WriteString("\n")
+}

config.go

@@ -63,6 +63,14 @@ const (
 	envUseElasticTraceparentHeader = "ELASTIC_APM_USE_ELASTIC_TRACEPARENT_HEADER"
 	envCloudProvider               = "ELASTIC_APM_CLOUD_PROVIDER"
 
+	// NOTE(marclop) Experimental settings
+	// span_compression (default `false`)
+	envSpanCompressionEnabled = "ELASTIC_APM_SPAN_COMPRESSION_ENABLED"
+	// span_compression_exact_match_max_duration (default `50ms`)
+	envSpanCompressionExactMatchMaxDuration = "ELASTIC_APM_SPAN_COMPRESSION_EXACT_MATCH_MAX_DURATION"
+	// span_compression_same_kind_max_duration (default `5ms`)
+	envSpanCompressionSameKindMaxDuration = "ELASTIC_APM_SPAN_COMPRESSION_SAME_KIND_MAX_DURATION"
+
 	// NOTE(axw) profiling environment variables are experimental.
 	// They may be removed in a future minor version without being
 	// considered a breaking change.
@@ -87,6 +95,11 @@ const (
 	maxAPIRequestSize    = 5 * configutil.MByte
 	minMetricsBufferSize = 10 * configutil.KByte
 	maxMetricsBufferSize = 100 * configutil.MByte
+
+	// Experimental Span Compressions default setting values
+	defaultSpanCompressionEnabled               = false
+	defaultSpanCompressionExactMatchMaxDuration = 50 * time.Millisecond
+	defaultSpanCompressionSameKindMaxDuration   = 5 * time.Millisecond
 )
 
 var (
@@ -298,6 +311,26 @@ func initialUseElasticTraceparentHeader() (bool, error) {
 	return configutil.ParseBoolEnv(envUseElasticTraceparentHeader, true)
 }
 
+func initialSpanCompressionEnabled() (bool, error) {
+	return configutil.ParseBoolEnv(envSpanCompressionEnabled,
+		defaultSpanCompressionEnabled,
+	)
+}
+
+func initialSpanCompressionExactMatchMaxDuration() (time.Duration, error) {
+	return configutil.ParseDurationEnv(
+		envSpanCompressionExactMatchMaxDuration,
+		defaultSpanCompressionExactMatchMaxDuration,
+	)
+}
+
+func initialSpanCompressionSameKindMaxDuration() (time.Duration, error) {
+	return configutil.ParseDurationEnv(
+		envSpanCompressionSameKindMaxDuration,
+		defaultSpanCompressionSameKindMaxDuration,
+	)
+}
+
 func initialCPUProfileIntervalDuration() (time.Duration, time.Duration, error) {
 	interval, err := configutil.ParseDurationEnv(envCPUProfileInterval, 0)
 	if err != nil || interval <= 0 {
@@ -532,4 +565,9 @@ type instrumentationConfigValues struct {
 	propagateLegacyHeader bool
 	sanitizedFieldNames   wildcard.Matchers
 	ignoreTransactionURLs wildcard.Matchers
+
+	// compressed spans.
+	spanCompressionEnabled               bool
+	spanCompressionExactMatchMaxDuration time.Duration
+	spanCompressionSameKindMaxDuration   time.Duration
 }

docs/configuration.asciidoc

@@ -606,3 +606,68 @@ automatically collect the cloud metadata.
 
 Valid options are `"none"`, `"auto"`, `"aws"`, `"gcp"`, and `"azure"`
 If this config value is set to `"none"`, then no cloud metadata will be collected.
+
+[float]
+[[config-span-compression-enabled]]
+=== `ELASTIC_APM_SPAN_COMPRESSION_ENABLED`
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+[options="header"]
+|============
+| Environment                            | Default
+| `ELASTIC_APM_SPAN_COMPRESSION_ENABLED` | `false`
+|============
+
+When enabled, the agent will attempt to compress _short_ exit spans that share the
+same parent into a composite span. The exact duration for what is considered
+_short_, depends on the compression strategy used (`same_kind` or `exact_match`).
+
+In order for a span to be compressible, these conditions need to be met:
+
+* Spans are exit spans.
+* Spans are siblings (share the same parent).
+* Spans have not propagated their context downstream.
+* Each span duration is equal or lower to the compression strategy maximum duration.
+* Spans are compressed with `same_kind` strategy when these attributes are equal:
+** `span.type`.
+** `span.subtype`.
+** `span.context.destination.service.resource`
+* Spans are compressed with `exact_match` strategy when all the previous conditions
+are met and the `span.name` is equal.
+
+Compressing short exit spans should provide some storage savings for services that
+create a lot of consecutive short exit spans to for example databases or cache
+services which are generally uninteresting when viewing a trace.
+
+NOTE: This feature is experimental and requires APM Server v7.15 or later.
+
+[float]
+[[config-span-compression-exact-match-duration]]
+=== `ELASTIC_APM_SPAN_COMPRESSION_EXACT_MATCH_MAX_DURATION`
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+[options="header"]
+|============
+| Environment                                             | Default
+| `ELASTIC_APM_SPAN_COMPRESSION_EXACT_MATCH_MAX_DURATION` | `50ms`
+|============
+
+The maximum duration to consider for compressing sibling exit spans that are an
+exact match for compression.
+
+[float]
+[[config-span-compression-same-kind-duration]]
+=== `ELASTIC_APM_SPAN_COMPRESSION_SAME_KIND_MAX_DURATION`
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+[options="header"]
+|============
+| Environment                                            | Default
+| `ELASTIC_APM_SPAN_COMPRESSION_SAME_KIND_MAX_DURATION`  | `5ms`
+|============
+
+The maximum duration to consider for compressing sibling exit spans that are of
+the same kind for compression.

utils_go10.go → internal/apmmath/round_go10.go

@@ -18,10 +18,11 @@
 //go:build go1.10
 // +build go1.10
 
-package apm // import "go.elastic.co/apm"
+package apmmath
 
 import "math"
 
-func round(x float64) float64 {
+// Round is the current math.Round implementation for >= Go1.10.
+func Round(x float64) float64 {
 	return math.Round(x)
 }

utils_go9.go → internal/apmmath/round_go9.go

@@ -18,14 +18,14 @@
 //go:build !go1.10
 // +build !go1.10
 
-package apm // import "go.elastic.co/apm"
+package apmmath
 
 import "math"
 
-// Implementation of math.Round for Go < 1.10.
+// Round Implementation of math.Round for Go < 1.10.
 //
 // Code shamelessly copied from pkg/math.
-func round(x float64) float64 {
+func Round(x float64) float64 {
 	t := math.Trunc(x)
 	if math.Abs(x-t) >= 0.5 {
 		return t + math.Copysign(1, x)

internal/apmstrings/concat10.go

@@ -0,0 +1,38 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build go1.10
+// +build go1.10
+
+package apmstrings
+
+import "strings"
+
+// Concat concatenates all the string arguments efficiently.
+func Concat(elements ...string) string {
+	var builder strings.Builder
+	var length int
+	for i := range elements {
+		length += len(elements[i])
+	}
+	builder.Grow(length)
+
+	for i := range elements {
+		builder.WriteString(elements[i])
+	}
+	return builder.String()
+}