Skip to content

Commit 68a90b4

Browse files
JacekLachJacekLach
committed
Introduce requestId MDC property
Logs from concurrent incoming requests with the same traceid are currently difficult to tell apart (one can look at thread names etc, but not across executors). This introduces a new MDC property managed by Tracers, which stores a unique identifier when the top span of the current trace is a SERVER_INCOMING span.
1 parent 4702bd0 commit 68a90b4

File tree

8 files changed

+187
-49
lines changed

8 files changed

+187
-49
lines changed

‎changelog/@unreleased/pr-364.v2.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
type: feature
2+
feature:
3+
description: Introduce a new MDC property, _requestId, that disambiguates span stacks
4+
that belong to the same traceid but are different requests.
5+
links:
6+
- https://github.com/palantir/tracing-java/pull/364

‎tracing-jersey/src/main/java/com/palantir/tracing/jersey/TraceEnrichingFilter.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public final class TraceEnrichingFilter implements ContainerRequestFilter, Conta
4646
* This is the name of the trace id property we set on {@link ContainerRequestContext}.
4747
*/
4848
public static final String TRACE_ID_PROPERTY_NAME = "com.palantir.tracing.traceId";
49+
public static final String REQUEST_ID_PROPERTY_NAME = "com.palantir.tracing.requestId";
4950
public static final String SAMPLED_PROPERTY_NAME = "com.palantir.tracing.sampled";
5051

5152
@Context
@@ -65,24 +66,31 @@ public void filter(ContainerRequestContext requestContext) throws IOException {
6566
String traceId = requestContext.getHeaderString(TraceHttpHeaders.TRACE_ID);
6667
String spanId = requestContext.getHeaderString(TraceHttpHeaders.SPAN_ID);
6768

69+
// XXX: leaks abstraction and ties us to requestId === spanId of initial span
70+
// however the leak is contained without this repository, so maybe OK?
71+
// alternative is to expose requestId as a property on the current trace to all clients.
72+
// or read the property from the MDC at this point
73+
String requestId;
74+
6875
// Set up thread-local span that inherits state from HTTP headers
6976
if (Strings.isNullOrEmpty(traceId)) {
7077
// HTTP request did not indicate a trace; initialize trace state and create a span.
7178
Tracer.initTrace(getObservabilityFromHeader(requestContext), Tracers.randomId());
72-
Tracer.fastStartSpan(operation, SpanType.SERVER_INCOMING);
79+
requestId = Tracer.startSpan(operation, SpanType.SERVER_INCOMING).getSpanId();
7380
} else {
7481
Tracer.initTrace(getObservabilityFromHeader(requestContext), traceId);
7582
if (spanId == null) {
76-
Tracer.fastStartSpan(operation, SpanType.SERVER_INCOMING);
83+
requestId = Tracer.startSpan(operation, SpanType.SERVER_INCOMING).getSpanId();
7784
} else {
7885
// caller's span is this span's parent.
79-
Tracer.fastStartSpan(operation, spanId, SpanType.SERVER_INCOMING);
86+
requestId = Tracer.startSpan(operation, spanId, SpanType.SERVER_INCOMING).getSpanId();
8087
}
8188
}
8289

8390
// Give asynchronous downstream handlers access to the trace id
8491
requestContext.setProperty(TRACE_ID_PROPERTY_NAME, Tracer.getTraceId());
8592
requestContext.setProperty(SAMPLED_PROPERTY_NAME, Tracer.isTraceObservable());
93+
requestContext.setProperty(REQUEST_ID_PROPERTY_NAME, requestId);
8694
}
8795

8896
// Handles outgoing response

‎tracing-jersey/src/test/java/com/palantir/tracing/jersey/TraceEnrichingFilterTest.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,36 @@ public void testTraceState_withHeaderUsesTraceId() {
117117
assertThat(spanCaptor.getValue().getOperation()).isEqualTo("Jersey: GET /trace");
118118
}
119119

120+
@Test
121+
public void testTraceState_withHeaderUsesTraceIdWithDifferentLocalIds() {
122+
Response response = target.path("/top-span").request()
123+
.header(TraceHttpHeaders.TRACE_ID, "traceId")
124+
.header(TraceHttpHeaders.PARENT_SPAN_ID, "parentSpanId")
125+
.header(TraceHttpHeaders.SPAN_ID, "spanId")
126+
.get();
127+
assertThat(response.getHeaderString(TraceHttpHeaders.TRACE_ID)).isEqualTo("traceId");
128+
assertThat(response.getHeaderString(TraceHttpHeaders.PARENT_SPAN_ID)).isNull();
129+
assertThat(response.getHeaderString(TraceHttpHeaders.SPAN_ID)).isNull();
130+
verify(observer).consume(spanCaptor.capture());
131+
assertThat(spanCaptor.getValue().getOperation()).isEqualTo("Jersey: GET /top-span");
132+
133+
String firstLocalTrace = response.readEntity(String.class);
134+
assertThat(firstLocalTrace).isNotEmpty();
135+
136+
// make the query again
137+
Response otherResponse = target.path("/top-span").request()
138+
.header(TraceHttpHeaders.TRACE_ID, "traceId")
139+
.header(TraceHttpHeaders.PARENT_SPAN_ID, "parentSpanId")
140+
.header(TraceHttpHeaders.SPAN_ID, "spanId")
141+
.get();
142+
assertThat(otherResponse.getHeaderString(TraceHttpHeaders.TRACE_ID)).isEqualTo("traceId");
143+
assertThat(otherResponse.getHeaderString(TraceHttpHeaders.PARENT_SPAN_ID)).isNull();
144+
assertThat(otherResponse.getHeaderString(TraceHttpHeaders.SPAN_ID)).isNull();
145+
String otherLocalTrace = otherResponse.readEntity(String.class);
146+
assertThat(otherLocalTrace).isNotEmpty();
147+
assertThat(otherLocalTrace).isNotEqualTo(firstLocalTrace);
148+
}
149+
120150
@Test
121151
public void testTraceState_respectsMethod() {
122152
Response response = target.path("/trace").request()
@@ -218,6 +248,8 @@ public void testFilter_setsMdcIfTraceIdHeaderIsPresent() throws Exception {
218248
TraceEnrichingFilter.INSTANCE.filter(request);
219249
assertThat(MDC.get(Tracers.TRACE_ID_KEY)).isEqualTo("traceId");
220250
verify(request).setProperty(TraceEnrichingFilter.TRACE_ID_PROPERTY_NAME, "traceId");
251+
// the top span id is randomly generated
252+
verify(request).setProperty(eq(TraceEnrichingFilter.REQUEST_ID_PROPERTY_NAME), anyString());
221253
// Note: this will be set to a random value; we want to check whether the value is being set
222254
verify(request).setProperty(eq(TraceEnrichingFilter.SAMPLED_PROPERTY_NAME), anyBoolean());
223255
}
@@ -262,6 +294,11 @@ public void getFailingTraceOperation() {
262294
throw new RuntimeException();
263295
}
264296

297+
@Override
298+
public String getRequestId() {
299+
return MDC.get(Tracers.REQUEST_ID_KEY);
300+
}
301+
265302
@Override
266303
public StreamingOutput getFailingStreamingTraceOperation() {
267304
return os -> {
@@ -296,6 +333,10 @@ public interface TracingTestService {
296333
@Path("/failing-trace")
297334
void getFailingTraceOperation();
298335

336+
@GET
337+
@Path("/top-span")
338+
String getRequestId();
339+
299340
@GET
300341
@Path("/failing-streaming-trace")
301342
@Produces(MediaType.APPLICATION_OCTET_STREAM)

‎tracing-undertow/src/main/java/com/palantir/tracing/undertow/TracedOperationHandler.java‎

Lines changed: 23 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public final class TracedOperationHandler implements HttpHandler {
4545
* Attachment to check whether the current request is being traced.
4646
*/
4747
public static final AttachmentKey<Boolean> IS_SAMPLED_ATTACHMENT = AttachmentKey.create(Boolean.class);
48+
public static final AttachmentKey<String> REQUEST_ID_ATTACHMENT = AttachmentKey.create(String.class);
4849

4950
private static final HttpString TRACE_ID = HttpString.tryFromString(TraceHttpHeaders.TRACE_ID);
5051
private static final HttpString SPAN_ID = HttpString.tryFromString(TraceHttpHeaders.SPAN_ID);
@@ -60,11 +61,32 @@ public TracedOperationHandler(HttpHandler delegate, String operation) {
6061

6162
@Override
6263
public void handleRequest(HttpServerExchange exchange) throws Exception {
63-
String traceId = initializeTrace(exchange);
64+
HeaderMap headers = exchange.getRequestHeaders();
65+
// TODO(rfink): Log/warn if we find multiple headers?
66+
String traceIdHeader = headers.getFirst(TRACE_ID); // nullable
67+
String traceId;
68+
// XXX: again, nasty abstraction leak
69+
String requestId;
70+
71+
// Set up thread-local span that inherits state from HTTP headers
72+
if (Strings.isNullOrEmpty(traceIdHeader)) {
73+
traceId = initializeNewTrace(headers);
74+
requestId = Tracer.startSpan(operation, SpanType.SERVER_INCOMING).getSpanId();
75+
} else {
76+
traceId = traceIdHeader;
77+
Tracer.initTrace(getObservabilityFromHeader(headers), traceIdHeader);
78+
String spanId = headers.getFirst(SPAN_ID); // nullable
79+
if (spanId == null) {
80+
requestId = Tracer.startSpan(operation, SpanType.SERVER_INCOMING).getSpanId();
81+
} else {
82+
requestId = Tracer.startSpan(operation, spanId, SpanType.SERVER_INCOMING).getSpanId();
83+
}
84+
}
6485

6586
// Populate response before calling delegate since delegate might commit the response.
6687
exchange.getResponseHeaders().put(TRACE_ID, traceId);
6788
exchange.putAttachment(IS_SAMPLED_ATTACHMENT, Tracer.isTraceObservable());
89+
exchange.putAttachment(REQUEST_ID_ATTACHMENT, requestId);
6890
try {
6991
delegate.handleRequest(exchange);
7092
} finally {
@@ -83,39 +105,11 @@ private static Observability getObservabilityFromHeader(HeaderMap headers) {
83105
}
84106
}
85107

86-
/** Initializes trace state and a root span for this request, returning the traceId. */
87-
private String initializeTrace(HttpServerExchange exchange) {
88-
HeaderMap headers = exchange.getRequestHeaders();
89-
// TODO(rfink): Log/warn if we find multiple headers?
90-
String traceId = headers.getFirst(TRACE_ID); // nullable
91-
92-
// Set up thread-local span that inherits state from HTTP headers
93-
if (Strings.isNullOrEmpty(traceId)) {
94-
return initializeNewTrace(headers);
95-
} else {
96-
initializeTraceFromExisting(headers, traceId);
97-
}
98-
return traceId;
99-
}
100-
101-
/** Initializes trace state given a trace-id header from the client. */
102-
private void initializeTraceFromExisting(HeaderMap headers, String traceId) {
103-
Tracer.initTrace(getObservabilityFromHeader(headers), traceId);
104-
String spanId = headers.getFirst(SPAN_ID); // nullable
105-
if (spanId == null) {
106-
Tracer.fastStartSpan(operation, SpanType.SERVER_INCOMING);
107-
} else {
108-
// caller's span is this span's parent.
109-
Tracer.fastStartSpan(operation, spanId, SpanType.SERVER_INCOMING);
110-
}
111-
}
112-
113108
/** Initializes trace state for a request without tracing headers. */
114109
private String initializeNewTrace(HeaderMap headers) {
115110
// HTTP request did not indicate a trace; initialize trace state and create a span.
116111
String newTraceId = Tracers.randomId();
117112
Tracer.initTrace(getObservabilityFromHeader(headers), newTraceId);
118-
Tracer.fastStartSpan(operation, SpanType.SERVER_INCOMING);
119113
return newTraceId;
120114
}
121115

‎tracing-undertow/src/test/java/com/palantir/tracing/undertow/TracedOperationHandlerTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,21 @@ public void whenTraceIsInHeader_usesGivenTraceId() throws Exception {
102102
assertThat(exchange.getResponseHeaders().getFirst(TraceHttpHeaders.TRACE_ID)).isEqualTo(traceId);
103103
}
104104

105+
@Test
106+
public void whenTraceIsInHeader_usesGivenTraceIdWithDifferentRequestIds() throws Exception {
107+
setRequestTraceId(traceId);
108+
exchange.getRequestHeaders().put(HttpString.tryFromString(TraceHttpHeaders.IS_SAMPLED), "1");
109+
110+
handler.handleRequest(exchange);
111+
String firstRequestId = exchange.getAttachment(TracedOperationHandler.REQUEST_ID_ATTACHMENT);
112+
assertThat(firstRequestId).isNotEmpty();
113+
114+
handler.handleRequest(exchange);
115+
String secondRequestId = exchange.getAttachment(TracedOperationHandler.REQUEST_ID_ATTACHMENT);
116+
assertThat(secondRequestId).isNotEmpty();
117+
assertThat(firstRequestId).isNotEqualTo(secondRequestId);
118+
}
119+
105120
@Test
106121
public void whenParentSpanIsGiven_usesParentSpan() throws Exception {
107122
setRequestTraceId(traceId);

‎tracing/src/main/java/com/palantir/tracing/Trace.java‎

Lines changed: 49 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,8 @@ final String getTraceId() {
136136
return traceId;
137137
}
138138

139+
abstract Optional<String> getRequestId();
140+
139141
abstract Optional<String> getOriginatingSpanId();
140142

141143
/** Returns a copy of this Trace which can be independently mutated. */
@@ -195,6 +197,18 @@ boolean isObservable() {
195197
return true;
196198
}
197199

200+
@Override
201+
Optional<String> getRequestId() {
202+
if (stack.isEmpty()) {
203+
return Optional.empty();
204+
}
205+
OpenSpan top = stack.peekFirst();
206+
if (top.type() != SpanType.SERVER_INCOMING) {
207+
return Optional.empty();
208+
}
209+
return Optional.of(top.getSpanId());
210+
}
211+
198212
@Override
199213
Optional<String> getOriginatingSpanId() {
200214
if (stack.isEmpty()) {
@@ -220,36 +234,52 @@ private static final class Unsampled extends Trace {
220234
* This allows thread trace state to be cleared when all "started" spans have been "removed".
221235
*/
222236
private int numberOfSpans;
223-
private Optional<String> originatingSpanId = Optional.empty();
224-
225-
private Unsampled(int numberOfSpans, String traceId) {
237+
private Optional<String> originatingSpanId;
238+
private Optional<String> requestId;
239+
240+
private Unsampled(
241+
int numberOfSpans,
242+
String traceId,
243+
Optional<String> originatingSpanId,
244+
Optional<String> requestId) {
226245
super(traceId);
227246
this.numberOfSpans = numberOfSpans;
247+
this.originatingSpanId = originatingSpanId;
248+
this.requestId = requestId;
228249
validateNumberOfSpans();
229250
}
230251

231252
private Unsampled(String traceId) {
232-
this(0, traceId);
253+
this(0, traceId, Optional.empty(), Optional.empty());
233254
}
234255

235256
@Override
236-
void fastStartSpan(String _operation, String parentSpanId, SpanType _type) {
237-
startSpan(Optional.of(parentSpanId));
257+
void fastStartSpan(String _operation, String parentSpanId, SpanType type) {
258+
if (numberOfSpans == 0) {
259+
originatingSpanId = Optional.of(parentSpanId);
260+
if (type == SpanType.SERVER_INCOMING) {
261+
requestId = Optional.of(Tracers.randomId());
262+
}
263+
}
264+
numberOfSpans++;
238265
}
239266

240267
@Override
241-
void fastStartSpan(String _operation, SpanType _type) {
268+
void fastStartSpan(String _operation, SpanType type) {
269+
if (numberOfSpans == 0 && type == SpanType.SERVER_INCOMING) {
270+
requestId = Optional.of(Tracers.randomId());
271+
}
242272
numberOfSpans++;
243273
}
244274

245275
@Override
246276
protected void push(OpenSpan span) {
247-
startSpan(span.getParentSpanId());
248-
}
249-
250-
private void startSpan(Optional<String> parentSpanId) {
251277
if (numberOfSpans == 0) {
252-
originatingSpanId = parentSpanId;
278+
originatingSpanId = span.getParentSpanId();
279+
280+
if (span.type() == SpanType.SERVER_INCOMING) {
281+
requestId = Optional.of(span.getSpanId());
282+
}
253283
}
254284
numberOfSpans++;
255285
}
@@ -267,6 +297,7 @@ Optional<OpenSpan> pop() {
267297
}
268298
if (numberOfSpans == 0) {
269299
originatingSpanId = Optional.empty();
300+
requestId = Optional.empty();
270301
}
271302
return Optional.empty();
272303
}
@@ -282,14 +313,19 @@ boolean isObservable() {
282313
return false;
283314
}
284315

316+
@Override
317+
Optional<String> getRequestId() {
318+
return requestId;
319+
}
320+
285321
@Override
286322
Optional<String> getOriginatingSpanId() {
287323
return originatingSpanId;
288324
}
289325

290326
@Override
291327
Trace deepCopy() {
292-
return new Unsampled(numberOfSpans, getTraceId());
328+
return new Unsampled(numberOfSpans, getTraceId(), getOriginatingSpanId(), getRequestId());
293329
}
294330

295331
/** Internal validation, this should never fail because {@link #pop()} only decrements positive values. */

0 commit comments

Comments
 (0)