Added support for collecting statistics about dropped exit spans

CHANGELOG.asciidoc

@@ -34,6 +34,7 @@ endif::[]
 * Added support for compressing spans - {pull}2477[#2477]
 * Added microsecond durations with `us` as unit - {pull}2496[#2496]
 * Added support for dropping fast exit spans - {pull}2491[#2491]
+* Added support for collecting statistics about dropped exit spans - {pull}2505[#2505]
 
 [float]
 ===== Performance improvements

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/ElasticApmTracer.java

@@ -383,6 +383,10 @@ public void endTransaction(Transaction transaction) {
 
     public void endSpan(Span span) {
         if (!span.isSampled()) {
+            Transaction transaction = span.getTransaction();
+            if (transaction != null) {
+                transaction.captureDroppedSpan(span);
+            }
             span.decrementReferences();
             return;
         }
@@ -401,7 +405,7 @@ public void endSpan(Span span) {
             logger.debug("Discarding span {}", span);
             Transaction transaction = span.getTransaction();
             if (transaction != null) {
-                transaction.getSpanCount().getDropped().incrementAndGet();
+                transaction.captureDroppedSpan(span);
             }
             span.decrementReferences();
             return;

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/transaction/DroppedSpanStats.java

@@ -0,0 +1,171 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.impl.transaction;
+
+import co.elastic.apm.agent.objectpool.Allocator;
+import co.elastic.apm.agent.objectpool.ObjectPool;
+import co.elastic.apm.agent.objectpool.Recyclable;
+import co.elastic.apm.agent.objectpool.impl.QueueBasedObjectPool;
+import org.jctools.queues.atomic.MpmcAtomicArrayQueue;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class DroppedSpanStats implements Iterable<Map.Entry<DroppedSpanStats.StatsKey, DroppedSpanStats.Stats>>, Recyclable {
+
+    public static class StatsKey implements Recyclable {
+        private String destinationServiceResource;
+        private Outcome outcome;
+
+        public StatsKey() {
+
+        }
+
+        public StatsKey init(CharSequence destinationServiceResource, Outcome outcome) {
+            this.destinationServiceResource = destinationServiceResource.toString();
+            this.outcome = outcome;
+            return this;
+        }
+
+        public String getDestinationServiceResource() {
+            return destinationServiceResource;
+        }
+
+        public Outcome getOutcome() {
+            return outcome;
+        }
+
+        @Override
+        public void resetState() {
+            destinationServiceResource = null;
+            outcome = null;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            StatsKey statsKey = (StatsKey) o;
+            return destinationServiceResource.equals(statsKey.destinationServiceResource) && outcome == statsKey.outcome;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(destinationServiceResource, outcome);
+        }
+    }
+
+    public static class Stats implements Recyclable {
+        private final AtomicInteger count = new AtomicInteger(0);
+        private final AtomicLong sum = new AtomicLong(0L);
+
+        public int getCount() {
+            return count.get();
+        }
+
+        public long getSum() {
+            return sum.get();
+        }
+
+        @Override
+        public void resetState() {
+            count.set(0);
+            sum.set(0L);
+        }
+    }
+
+    private static final ObjectPool<StatsKey> statsKeyObjectPool = QueueBasedObjectPool.<StatsKey>ofRecyclable(new MpmcAtomicArrayQueue<StatsKey>(512), false, new Allocator<StatsKey>() {
+        @Override
+        public StatsKey createInstance() {
+            return new StatsKey();
+        }
+    });
+
+    private static ObjectPool<Stats> statsObjectPool = QueueBasedObjectPool.<Stats>ofRecyclable(new MpmcAtomicArrayQueue<Stats>(512), false, new Allocator<Stats>() {
+        @Override
+        public Stats createInstance() {
+            return new Stats();
+        }
+    });
+
+    private final ConcurrentMap<StatsKey, Stats> statsMap = new ConcurrentHashMap<>();
+
+    //only used during testing
+    Stats getStats(String destinationServiceResource, Outcome outcome) {
+        return statsMap.get(new StatsKey().init(destinationServiceResource, outcome));
+    }
+
+    public void captureDroppedSpan(Span span) {
+        StringBuilder resource = span.getContext().getDestination().getService().getResource();
+        if (!span.isExit() || resource.length() == 0) {
+            return;
+        }
+
+        Stats stats = getOrCreateStats(resource.toString(), span.getOutcome());
+        if (stats == null) {
+            return;
+        }
+
+        if (span.isComposite()) {
+            stats.count.addAndGet(span.getComposite().getCount());
+        } else {
+            stats.count.incrementAndGet();
+        }
+        stats.sum.addAndGet(span.getDuration());
+    }
+
+    private Stats getOrCreateStats(String resource, Outcome oucome) {
+        StatsKey statsKey = statsKeyObjectPool.createInstance().init(resource, oucome);
+        Stats stats = statsMap.get(statsKey);
+        if (stats != null || statsMap.size() > 127) {
+            statsKeyObjectPool.recycle(statsKey);
+            return stats;
+        }
+
+        stats = statsObjectPool.createInstance();
+
+        Stats oldStats = statsMap.putIfAbsent(statsKey, stats);
+        if (oldStats != null) {
+            statsKeyObjectPool.recycle(statsKey);
+            statsObjectPool.recycle(stats);
+            return oldStats;
+        }
+
+        return stats;
+    }
+
+    @Override
+    public Iterator<Map.Entry<StatsKey, Stats>> iterator() {
+        return statsMap.entrySet().iterator();
+    }
+
+    @Override
+    public void resetState() {
+        for (Map.Entry<StatsKey, Stats> e : statsMap.entrySet()) {
+            statsKeyObjectPool.recycle(e.getKey());
+            statsObjectPool.recycle(e.getValue());
+        }
+        statsMap.clear();
+    }
+}

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/transaction/Transaction.java

@@ -55,6 +55,7 @@ protected Labels.Mutable initialValue() {
      */
     private final TransactionContext context = new TransactionContext();
     private final SpanCount spanCount = new SpanCount();
+    private final DroppedSpanStats droppedSpanStats = new DroppedSpanStats();
     /**
      * type: subtype: timer
      * <p>
@@ -263,6 +264,17 @@ public SpanCount getSpanCount() {
         return spanCount;
     }
 
+    public void captureDroppedSpan(Span span) {
+        if (span.isSampled()) {
+            spanCount.getDropped().incrementAndGet();
+        }
+        droppedSpanStats.captureDroppedSpan(span);
+    }
+
+    public DroppedSpanStats getDroppedSpanStats() {
+        return droppedSpanStats;
+    }
+
     boolean isSpanLimitReached() {
         return getSpanCount().isSpanLimitReached(maxSpans);
     }
@@ -277,6 +289,7 @@ public void resetState() {
         context.resetState();
         result = null;
         spanCount.resetState();
+        droppedSpanStats.resetState();
         type = null;
         noop = false;
         maxSpans = 0;

apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java

@@ -48,6 +48,7 @@
 import co.elastic.apm.agent.impl.metadata.SystemInfo;
 import co.elastic.apm.agent.impl.stacktrace.StacktraceConfiguration;
 import co.elastic.apm.agent.impl.transaction.Composite;
+import co.elastic.apm.agent.impl.transaction.DroppedSpanStats;
 import co.elastic.apm.agent.impl.transaction.Faas;
 import co.elastic.apm.agent.impl.transaction.FaasTrigger;
 import co.elastic.apm.agent.impl.transaction.Id;
@@ -666,6 +667,9 @@ private void serializeTransaction(final Transaction transaction) {
         serializeFaas(transaction.getFaas());
         serializeContext(transaction, transaction.getContext(), traceContext);
         serializeSpanCount(transaction.getSpanCount());
+        if (transaction.isSampled()) {
+            serializeDroppedSpanStats(transaction.getDroppedSpanStats());
+        }
         double sampleRate = traceContext.getSampleRate();
         if (!Double.isNaN(sampleRate)) {
             writeField("sample_rate", sampleRate);
@@ -1116,6 +1120,32 @@ private void serializeSpanCount(final SpanCount spanCount) {
         jw.writeByte(COMMA);
     }
 
+    private void serializeDroppedSpanStats(final DroppedSpanStats droppedSpanStats) {
+        writeFieldName("dropped_spans_stats");
+        jw.writeByte(ARRAY_START);
+
+        int i = 0;
+        for (Map.Entry<DroppedSpanStats.StatsKey, DroppedSpanStats.Stats> stats : droppedSpanStats) {
+            if (i++ >= 128) {
+                break;
+            }
+            jw.writeByte(OBJECT_START);
+            writeField("destination_service_resource", stats.getKey().getDestinationServiceResource());
+            writeField("outcome", stats.getKey().getOutcome().toString());
+            writeFieldName("duration");
+            jw.writeByte(OBJECT_START);
+            writeField("count", stats.getValue().getCount());
+            writeFieldName("sum");
+            jw.writeByte(OBJECT_START);
+            writeLastField("us", stats.getValue().getSum());
+            jw.writeByte(OBJECT_END);
+            jw.writeByte(OBJECT_END);
+            jw.writeByte(OBJECT_END);
+        }
+        jw.writeByte(ARRAY_END);
+        jw.writeByte(COMMA);
+    }
+
     private void serializeContext(@Nullable final Transaction transaction, final TransactionContext context, TraceContext traceContext) {
         writeFieldName("context");
         jw.writeByte(OBJECT_START);
@@ -1529,6 +1559,11 @@ private void writeLastField(final String fieldName, final int value) {
         NumberConverter.serialize(value, jw);
     }
 
+    private void writeLastField(final String fieldName, final long value) {
+        writeFieldName(fieldName);
+        NumberConverter.serialize(value, jw);
+    }
+
     private void writeField(final String fieldName, final boolean value) {
         writeFieldName(fieldName);
         BoolConverter.serialize(value, jw);

apm-agent-core/src/test/java/co/elastic/apm/agent/impl/transaction/FastExitSpanTest.java

@@ -68,6 +68,10 @@ void testExitSpanBelowDuration() {
         SpanCount spanCount = reporter.getFirstTransaction().getSpanCount();
         assertThat(spanCount.getReported().get()).isEqualTo(0);
         assertThat(spanCount.getDropped().get()).isEqualTo(1);
+
+        DroppedSpanStats droppedSpanStats = reporter.getFirstTransaction().getDroppedSpanStats();
+        assertThat(droppedSpanStats.getStats("postgresql", Outcome.SUCCESS).getCount()).isEqualTo(1);
+        assertThat(droppedSpanStats.getStats("postgresql", Outcome.SUCCESS).getSum()).isEqualTo(49_999L);
     }
 
     @Test
@@ -88,6 +92,10 @@ void testCompositeExitSpanBelowDuration() {
         SpanCount spanCount = reporter.getFirstTransaction().getSpanCount();
         assertThat(spanCount.getReported().get()).isEqualTo(0);
         assertThat(spanCount.getDropped().get()).isEqualTo(3);
+
+        DroppedSpanStats droppedSpanStats = reporter.getFirstTransaction().getDroppedSpanStats();
+        assertThat(droppedSpanStats.getStats("postgresql", Outcome.SUCCESS).getCount()).isEqualTo(3);
+        assertThat(droppedSpanStats.getStats("postgresql", Outcome.SUCCESS).getSum()).isEqualTo(30_000L);
     }
 
     @Test
@@ -104,6 +112,9 @@ void testExitSpanAboveDuration() {
         SpanCount spanCount = reporter.getFirstTransaction().getSpanCount();
         assertThat(spanCount.getReported().get()).isEqualTo(1);
         assertThat(spanCount.getDropped().get()).isEqualTo(0);
+
+        DroppedSpanStats droppedSpanStats = reporter.getFirstTransaction().getDroppedSpanStats();
+        assertThat(droppedSpanStats.getStats("postgresql", Outcome.SUCCESS)).isNull();
     }
 
     @Test
@@ -124,6 +135,9 @@ void testCompositeExitSpanAboveDuration() {
         SpanCount spanCount = reporter.getFirstTransaction().getSpanCount();
         assertThat(spanCount.getReported().get()).isEqualTo(1);
         assertThat(spanCount.getDropped().get()).isEqualTo(2);
+
+        DroppedSpanStats droppedSpanStats = reporter.getFirstTransaction().getDroppedSpanStats();
+        assertThat(droppedSpanStats.getStats("postgresql", Outcome.SUCCESS)).isNull();
     }
 
     private Transaction startTransaction() {