Introduce dedicated fields for internal metrics

Add dedicated fields to model.Transaction and model.Span
for well-defined, internal transaction and span metrics.
Drop unknown metrics received in metricsets with any
transaction or span fields defined.

Internal metrics will be indexed with strict mapping.
We already have explicitly defined fields for these
metrics. This PR ensures that unknown metrics do not
slip through. If an agent is upgraded before the server
and some new internal metric starts being produced, that
must not be indexed as it would cause other internal
metrics to be rejected by Elasticsearch.

Drop the unused `metricset.period` field that was being
added to service_destination metrics.

Author: axw

beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json

@@ -650,16 +650,12 @@
                 "name": "java",
                 "version": "1.10.0"
             },
-            "byte_counter": 1,
             "container": {
                 "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4"
             },
-            "dotted.float.gauge": 6.12,
-            "double_gauge": 3.141592653589793,
             "ecs": {
                 "version": "1.10.0"
             },
-            "float_gauge": 9.16,
             "host": {
                 "architecture": "amd64",
                 "hostname": "node-name",
@@ -669,7 +665,6 @@
                     "platform": "Linux"
                 }
             },
-            "integer_gauge": 42767,
             "kubernetes": {
                 "namespace": "default",
                 "node": {
@@ -687,9 +682,7 @@
                 "segment": 5,
                 "success": true
             },
-            "long_gauge": 3147483648,
             "metricset.name": "span_breakdown",
-            "negative.d.o.t.t.e.d": -1022,
             "observer": {
                 "ephemeral_id": "00000000-0000-0000-0000-000000000000",
                 "id": "fbba762a-14dd-412c-b7e9-b79f903eb492",
@@ -729,22 +722,23 @@
                 },
                 "version": "4.3.0"
             },
-            "short_counter": 227,
             "span": {
+                "self_time": {
+                    "count": 1,
+                    "sum.us": 633
+                },
                 "subtype": "mysql",
                 "type": "db"
             },
-            "span.self_time.count": 1,
-            "span.self_time.sum.us": 633.288,
             "transaction": {
+                "breakdown.count": 12,
+                "duration": {
+                    "count": 2,
+                    "sum.us": 12
+                },
                 "name": "GET/",
                 "type": "request"
-            },
-            "transaction.breakdown.count": 12,
-            "transaction.duration.count": 2,
-            "transaction.duration.sum.us": 12,
-            "transaction.self_time.count": 2,
-            "transaction.self_time.sum.us": 10
+            }
         }
     ]
 }

beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json

@@ -6,27 +6,20 @@
                 "name": "elastic-node",
                 "version": "3.14.0"
             },
-            "byte_counter": 1,
-            "dotted.float.gauge": 6.12,
-            "double_gauge": 3.141592653589793,
             "ecs": {
                 "version": "1.10.0"
             },
-            "float_gauge": 9.16,
             "host": {
                 "ip": "127.0.0.1"
             },
-            "integer_gauge": 42767,
             "labels": {
                 "code": 200,
                 "some": "abc",
                 "success": true,
                 "tag1": "one",
                 "tag2": 2
             },
-            "long_gauge": 3147483648,
             "metricset.name": "span_breakdown",
-            "negative.d.o.t.t.e.d": -1022,
             "observer": {
                 "ephemeral_id": "00000000-0000-0000-0000-000000000000",
                 "id": "fbba762a-14dd-412c-b7e9-b79f903eb492",
@@ -50,22 +43,23 @@
                     "name": "node-1"
                 }
             },
-            "short_counter": 227,
             "span": {
+                "self_time": {
+                    "count": 1,
+                    "sum.us": 633
+                },
                 "subtype": "mysql",
                 "type": "db"
             },
-            "span.self_time.count": 1,
-            "span.self_time.sum.us": 633.288,
             "transaction": {
+                "breakdown.count": 12,
+                "duration": {
+                    "count": 2,
+                    "sum.us": 12
+                },
                 "name": "GET /",
                 "type": "request"
             },
-            "transaction.breakdown.count": 12,
-            "transaction.duration.count": 2,
-            "transaction.duration.sum.us": 12,
-            "transaction.self_time.count": 2,
-            "transaction.self_time.sum.us": 10,
             "user": {
                 "email": "user@mail.com",
                 "id": "axb123hg",

changelogs/head.asciidoc

@@ -13,6 +13,8 @@ https://github.com/elastic/apm-server/compare/7.13\...master[View commits]
 - Removed unused support for top-level metricsets and metricset tags for RUMv3 {pull}6065[6065]
 - Removed `apm-server.mode` configuration, and "experimental" fields {pull}6086[6086]
 - `transaction.sampled` is now only set for sampled transactions {pull}6066[6066]
+- Unknown metrics are dropped when `transaction.*` or `span.*` are present in a metricset {pull}6111[6111]
+- Removed `metricset.period` from service_destination metrics {pull}6111[6111]
 
 [float]
 ==== Bug fixes

model/metricset.go

@@ -18,6 +18,8 @@
 package model
 
 import (
+	"time"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 )
 
@@ -81,6 +83,12 @@ type MetricsetSample struct {
 	// If Counts and Values are specified, then Value will be ignored.
 	Value float64
 
+	// Histogram holds bucket values and counts for histogram metrics.
+	Histogram
+}
+
+// Histogram holds bucket values and counts for a histogram metric.
+type Histogram struct {
 	// Values holds the bucket values for histogram metrics.
 	//
 	// These values must be provided in ascending order.
@@ -96,6 +104,35 @@ type MetricsetSample struct {
 	Counts []int64
 }
 
+func (h *Histogram) fields() common.MapStr {
+	if len(h.Counts) == 0 {
+		return nil
+	}
+	var fields mapStr
+	fields.set("counts", h.Counts)
+	fields.set("values", h.Values)
+	return common.MapStr(fields)
+}
+
+// AggregatedDuration holds a count and sum of aggregated durations.
+type AggregatedDuration struct {
+	// Count holds the number of durations aggregated.
+	Count int
+
+	// Sum holds the sum of aggregated durations.
+	Sum time.Duration
+}
+
+func (a *AggregatedDuration) fields() common.MapStr {
+	if a.Count == 0 {
+		return nil
+	}
+	var fields mapStr
+	fields.set("count", a.Count)
+	fields.set("sum.us", a.Sum.Microseconds())
+	return common.MapStr(fields)
+}
+
 func (me *Metricset) setFields(fields *mapStr) {
 	if me.TimeseriesInstanceID != "" {
 		fields.set("timeseries", common.MapStr{"instance": me.TimeseriesInstanceID})
@@ -119,10 +156,7 @@ func (me *Metricset) setFields(fields *mapStr) {
 
 func (s *MetricsetSample) set(name string, fields *mapStr) {
 	if s.Type == MetricTypeHistogram {
-		fields.set(name, common.MapStr{
-			"counts": s.Counts,
-			"values": s.Values,
-		})
+		fields.set(name, s.Histogram.fields())
 	} else {
 		fields.set(name, s.Value)
 	}