From 25aec1f79274ab500f24f685fd469cbcb57f20ae Mon Sep 17 00:00:00 2001 From: Marc Lopez Rubio Date: Mon, 6 Dec 2021 16:00:06 +0800 Subject: [PATCH] Add systemtest, changelog entry Signed-off-by: Marc Lopez Rubio --- changelogs/head.asciidoc | 1 + docs/open-telemetry.asciidoc | 2 +- .../approvals/TestOTLPGRPCLogs.approved.json | 55 ++++++++++++++++++ systemtest/go.mod | 5 +- systemtest/go.sum | 16 +++++ systemtest/otlp_test.go | 58 +++++++++++++++++++ 6 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 systemtest/approvals/TestOTLPGRPCLogs.approved.json diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index af96436d0f..d86a1ddd16 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -79,6 +79,7 @@ https://github.com/elastic/apm-server/compare/7.15\...master[View commits] - The `labels` indexed field is now ECS compliant (string only) and added a new `numeric_labels` object that holds labels with numeric values {pull}6633[6633] - Modify default standalone apm-server config values to be more in line with the default managed apm-server values {pull}6675[6675] - APM Server is now using a new Elasticsearch output implementation {pull}6656[6656] +- APM Server now supports receiving OpenTelemetry Logs on the OTLP/GRPC receiver {pull}6768[6768] [float] ==== Deprecated diff --git a/docs/open-telemetry.asciidoc b/docs/open-telemetry.asciidoc index ec453be426..ccfe3a2cdc 100644 --- a/docs/open-telemetry.asciidoc +++ b/docs/open-telemetry.asciidoc @@ -401,4 +401,4 @@ Here is an example of AWS Lambda Node.js function managed with Terraform and the [[open-telemetry-logs-limitations]] ===== OpenTelemetry logs -* OpenTelemetry logs are not yet supported https://github.com/elastic/apm-server/issues/5491[#5491] +* OpenTelemetry logs are supported with **beta support** from 8.0 https://github.com/elastic/apm-server/pull/6768[#6768]. diff --git a/systemtest/approvals/TestOTLPGRPCLogs.approved.json b/systemtest/approvals/TestOTLPGRPCLogs.approved.json new file mode 100644 index 0000000000..8cb4fd4459 --- /dev/null +++ b/systemtest/approvals/TestOTLPGRPCLogs.approved.json @@ -0,0 +1,55 @@ +{ + "events": [ + { + "@timestamp": "1970-01-01T00:00:01.000Z", + "agent": { + "name": "otlp/go", + "version": "unknown" + }, + "data_stream.dataset": "apm.app", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "ecs": { + "version": "dynamic" + }, + "event": { + "action": "doOperation()", + "severity": 9 + }, + "labels": { + "key": "value" + }, + "log": { + "level": "Info" + }, + "message": "a log message", + "numeric_labels": { + "numeric_key": 1234 + }, + "observer": { + "ephemeral_id": "dynamic", + "hostname": "dynamic", + "id": "dynamic", + "type": "apm-server", + "version": "dynamic", + "version_major": "dynamic" + }, + "processor": { + "event": "log", + "name": "log" + }, + "service": { + "language": { + "name": "go" + }, + "name": "unknown" + }, + "span": { + "id": "0200000000000000" + }, + "trace": { + "id": "01000000000000000000000000000000" + } + } + ] +} diff --git a/systemtest/go.mod b/systemtest/go.mod index 4bfb4fea58..51f1d72048 100644 --- a/systemtest/go.mod +++ b/systemtest/go.mod @@ -29,7 +29,7 @@ require ( go.uber.org/zap v1.15.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 - google.golang.org/grpc v1.40.0 + google.golang.org/grpc v1.42.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) @@ -75,6 +75,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.1.1 // indirect go.opencensus.io v0.22.3 // indirect + go.opentelemetry.io/collector/model v0.34.0 go.opentelemetry.io/otel/internal/metric v0.23.0 // indirect go.opentelemetry.io/proto/otlp v0.9.0 // indirect go.uber.org/atomic v1.6.0 // indirect @@ -85,7 +86,7 @@ require ( golang.org/x/text v0.3.6 // indirect golang.org/x/tools v0.1.5 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect + google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect howett.net/plist v0.0.0-20201203080718-1454fab16a06 // indirect diff --git a/systemtest/go.sum b/systemtest/go.sum index 2e5a114212..96420e2818 100644 --- a/systemtest/go.sum +++ b/systemtest/go.sum @@ -104,7 +104,11 @@ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -205,6 +209,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -715,6 +720,10 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/collector/model v0.34.0 h1:JmuBhBvX0l0bYDBAG9GtJVQKcIQRspPLgHfZqgHLpJc= +go.opentelemetry.io/collector/model v0.34.0/go.mod h1:+7YCSjJG+MqiIFjauzt7oM2qkqBsaJWh5hcsO4fwsAc= +go.opentelemetry.io/collector/model v0.40.0 h1:UgdWfBnJBQL4atFv6LhlYq67Ts/vFDbwGknvsIsm/g8= +go.opentelemetry.io/collector/model v0.40.0/go.mod h1:dXqjAeml+cB+YzJ3kUnd3v5/JvGAKl3MqHXfgSWRIo8= go.opentelemetry.io/otel v1.0.0-RC3/go.mod h1:Ka5j3ua8tZs4Rkq4Ex3hwgBgOchyPVq5S6P2lz//nKQ= go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI= go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= @@ -940,6 +949,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -997,6 +1007,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1045,6 +1056,8 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08 h1:pc16UedxnxXXtGxHCSUhafAoVHQZ0yXl8ZelMH4EETc= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1060,8 +1073,11 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/systemtest/otlp_test.go b/systemtest/otlp_test.go index f7efc9bf46..0f6787becb 100644 --- a/systemtest/otlp_test.go +++ b/systemtest/otlp_test.go @@ -25,6 +25,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/model/otlpgrpc" + "go.opentelemetry.io/collector/model/pdata" + semconv "go.opentelemetry.io/collector/model/semconv/v1.5.0" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric" @@ -42,6 +45,7 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -135,6 +139,28 @@ func TestOTLPGRPCMetrics(t *testing.T) { assert.True(t, gjson.GetBytes(doc.RawSource, "beats_stats.metrics.apm-server.otlp.grpc.metrics.consumer").Exists()) } +func TestOTLPGRPCLogs(t *testing.T) { + systemtest.CleanupElasticsearch(t) + srv := apmservertest.NewServer(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + conn, err := grpc.Dial(serverAddr(srv), grpc.WithInsecure(), grpc.WithBlock()) + require.NoError(t, err) + defer conn.Close() + + logsClient := otlpgrpc.NewLogsClient(conn) + + logs := newLogs("a log message") + _, err = logsClient.Export(ctx, logs) + require.NoError(t, err) + + result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm*", estest.TermQuery{ + Field: "processor.event", Value: "log", + }) + systemtest.ApproveEvents(t, t.Name(), result.Hits.Hits) +} + func TestOTLPGRPCAuth(t *testing.T) { systemtest.CleanupElasticsearch(t) srv := apmservertest.NewUnstartedServer(t) @@ -429,3 +455,35 @@ func (m *idGeneratorFuncs) NewIDs(ctx context.Context) (trace.TraceID, trace.Spa func (m *idGeneratorFuncs) NewSpanID(ctx context.Context, traceID trace.TraceID) trace.SpanID { return m.newSpanID(ctx, traceID) } + +func newLogs(body interface{}) pdata.Logs { + logs := pdata.NewLogs() + resourceLogs := logs.ResourceLogs().AppendEmpty() + logs.ResourceLogs().At(0).Resource().Attributes().InitFromMap(map[string]pdata.AttributeValue{ + semconv.AttributeTelemetrySDKLanguage: pdata.NewAttributeValueString("go"), + }) + instrumentationLogs := resourceLogs.InstrumentationLibraryLogs().AppendEmpty() + otelLog := instrumentationLogs.Logs().AppendEmpty() + otelLog.SetTraceID(pdata.NewTraceID([16]byte{1})) + otelLog.SetSpanID(pdata.NewSpanID([8]byte{2})) + otelLog.SetName("doOperation()") + otelLog.SetSeverityNumber(pdata.SeverityNumberINFO) + otelLog.SetSeverityText("Info") + otelLog.SetTimestamp(pdata.NewTimestampFromTime(time.Unix(1, 0))) + otelLog.Attributes().InitFromMap(map[string]pdata.AttributeValue{ + "key": pdata.NewAttributeValueString("value"), + "numeric_key": pdata.NewAttributeValueDouble(1234), + }) + + switch b := body.(type) { + case string: + otelLog.Body().SetStringVal(b) + case int: + otelLog.Body().SetIntVal(int64(b)) + case float64: + otelLog.Body().SetDoubleVal(float64(b)) + case bool: + otelLog.Body().SetBoolVal(b) + } + return logs +}