diff --git a/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java b/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java index be835c171..af9515a9f 100644 --- a/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java @@ -328,7 +328,8 @@ public SearchResponse handle(IndexState indexState, SearchRequest searchRequest) InnerHitFetchTask::getDiagnostic))); } searchContext.getResponseBuilder().setDiagnostics(diagnostics); - + // TODO: These are the diagnostics I want to publish to prometheus, I'll try to figure out + // where other metrics are being published and follow that pattern if (profileResultBuilder != null) { searchContext.getResponseBuilder().setProfileResult(profileResultBuilder); } diff --git a/src/main/java/com/yelp/nrtsearch/server/monitoring/SearchResponseCollector.java b/src/main/java/com/yelp/nrtsearch/server/monitoring/SearchResponseCollector.java index 164d82dc9..0622d44a8 100644 --- a/src/main/java/com/yelp/nrtsearch/server/monitoring/SearchResponseCollector.java +++ b/src/main/java/com/yelp/nrtsearch/server/monitoring/SearchResponseCollector.java @@ -120,6 +120,7 @@ public static void updateSearchResponseMetrics( .labelValues(index, "facet:" + entry.getKey()) .observe(entry.getValue()); } + searchStageLatencyMs.labelValues(index, "rescore").observe(diagnostics.getRescoreTimeMs());//adding extra rescore metric to avoid calculating average of all rescorers for (Map.Entry entry : diagnostics.getRescorersTimeMsMap().entrySet()) { searchStageLatencyMs .labelValues(index, "rescorer:" + entry.getKey()) @@ -139,6 +140,9 @@ public MetricSnapshots collect() { try { metrics.add(searchTimeoutCount.collect()); metrics.add(searchTerminatedEarlyCount.collect()); + metrics.add(searchStageLatencyMs.collect());// Just adding this here should mean it gets published to prometheus, is that what we want? + // when is publishVerboseMetrics set to true, I couldn't find this metric in the grafana shard without any filters? + // maybe it makes sense to add an extra parameter like publishSearchStageLatencyMs with default value true (?) boolean publishVerboseMetrics = false; Set indexNames = globalState.getIndexNames(); @@ -151,7 +155,6 @@ public MetricSnapshots collect() { if (publishVerboseMetrics) { metrics.add(searchResponseSizeBytes.collect()); metrics.add(searchResponseTotalHits.collect()); - metrics.add(searchStageLatencyMs.collect()); } } catch (Exception e) { logger.warn("Error getting search response metrics: ", e);