Service monitor scrap tempo metrics for monolithic

.chloggen/fix_monolithic_metrics.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempomonolithic
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Scrap tempo metrics for monolithic.
+
+# One or more tracking issues related to the change
+issues: [1275]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

internal/manifests/monolithic/configmap.go

@@ -177,8 +177,9 @@ func buildTempoConfig(opts Options) ([]byte, error) {
 	config.Server.HttpServerReadTimeout = opts.Tempo.Spec.Timeout.Duration
 	config.Server.HttpServerWriteTimeout = opts.Tempo.Spec.Timeout.Duration
 	if tempo.Spec.Multitenancy.IsGatewayEnabled() {
+		// We need this to scrap metrics.
+		config.Server.HTTPListenAddress = "0.0.0.0"
 		// all connections to tempo must go via gateway
-		config.Server.HTTPListenAddress = "localhost"
 		config.Server.GRPCListenAddress = "localhost"
 	}
 

internal/manifests/monolithic/servicemonitor.go

@@ -12,9 +12,14 @@ func BuildServiceMonitor(opts Options) *monitoringv1.ServiceMonitor {
 	tempo := opts.Tempo
 	if tempo.Spec.Multitenancy.IsGatewayEnabled() {
 		labels := ComponentLabels(manifestutils.GatewayComponentName, tempo.Name)
-		return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.GatewayComponentName, manifestutils.GatewayInternalHttpPortName)
+		return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.GatewayComponentName,
+			[]string{
+				manifestutils.GatewayInternalHttpPortName,
+				manifestutils.HttpPortName,
+			})
 	} else {
 		labels := ComponentLabels(manifestutils.TempoMonolithComponentName, tempo.Name)
-		return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.TempoMonolithComponentName, manifestutils.HttpPortName)
+		return servicemonitor.NewServiceMonitor(
+			tempo.Namespace, tempo.Name, labels, false, manifestutils.TempoMonolithComponentName, []string{manifestutils.HttpPortName})
 	}
 }

internal/manifests/monolithic/servicemonitor_test.go

@@ -120,7 +120,23 @@ func TestBuildServiceMonitorGateway(t *testing.T) {
 						TargetLabel:  "job",
 					},
 				},
-			}},
+			}, {
+				Scheme: "http",
+				Port:   "http",
+				Path:   "/metrics",
+				RelabelConfigs: []*monitoringv1.RelabelConfig{
+					{
+						SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_service_label_app_kubernetes_io_instance"},
+						TargetLabel:  "cluster",
+					},
+					{
+						SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_namespace", "__meta_kubernetes_service_label_app_kubernetes_io_component"},
+						Separator:    ptr.To("/"),
+						TargetLabel:  "job",
+					},
+				},
+			},
+			},
 			NamespaceSelector: monitoringv1.NamespaceSelector{
 				MatchNames: []string{"default"},
 			},

internal/manifests/monolithic/services.go

@@ -146,6 +146,12 @@ func buildGatewayService(opts Options) *corev1.Service {
 			Port:       manifestutils.GatewayPortInternalHTTPServer,
 			TargetPort: intstr.FromString(manifestutils.GatewayInternalHttpPortName),
 		},
+		{
+			Name:       manifestutils.HttpPortName,
+			Protocol:   corev1.ProtocolTCP,
+			Port:       manifestutils.PortHTTPServer,
+			TargetPort: intstr.FromString(manifestutils.HttpPortName),
+		},
 	}
 
 	if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil &&

internal/manifests/monolithic/services_test.go

@@ -269,6 +269,12 @@ func TestBuildServices(t *testing.T) {
 								Port:       8081,
 								TargetPort: intstr.FromString("internal"),
 							},
+							{
+								Name:       "http",
+								Protocol:   corev1.ProtocolTCP,
+								Port:       3200,
+								TargetPort: intstr.FromString("http"),
+							},
 							{
 								Name:       "otlp-grpc",
 								Protocol:   corev1.ProtocolTCP,

internal/manifests/servicemonitor/servicemonitor.go

@@ -35,14 +35,14 @@ func BuildServiceMonitors(params manifestutils.Params) []client.Object {
 
 func buildServiceMonitor(params manifestutils.Params, component string, port string) *monitoringv1.ServiceMonitor {
 	labels := manifestutils.ComponentLabels(component, params.Tempo.Name)
-	return NewServiceMonitor(params.Tempo.Namespace, params.Tempo.Name, labels, params.CtrlConfig.Gates.HTTPEncryption, component, port)
+	return NewServiceMonitor(params.Tempo.Namespace, params.Tempo.Name, labels, params.CtrlConfig.Gates.HTTPEncryption, component, []string{port})
 }
 
 func buildFrontEndServiceMonitor(params manifestutils.Params, port string) *monitoringv1.ServiceMonitor {
 	labels := manifestutils.ComponentLabels(manifestutils.QueryFrontendComponentName, params.Tempo.Name)
 	tls := params.CtrlConfig.Gates.HTTPEncryption && params.Tempo.Spec.Template.Gateway.Enabled
 	return NewServiceMonitor(params.Tempo.Namespace, params.Tempo.Name, labels, tls,
-		manifestutils.QueryFrontendComponentName, port)
+		manifestutils.QueryFrontendComponentName, []string{port})
 }
 
 // NewServiceMonitor creates a ServiceMonitor.
@@ -52,7 +52,7 @@ func NewServiceMonitor(
 	labels labels.Set,
 	tls bool,
 	component string,
-	port string,
+	ports []string,
 ) *monitoringv1.ServiceMonitor {
 	scheme := "http"
 	var tlsConfig *monitoringv1.TLSConfig
@@ -91,6 +91,30 @@ func NewServiceMonitor(
 		}
 	}
 
+	var endpoints []monitoringv1.Endpoint
+
+	for _, port := range ports {
+		endpoints = append(endpoints, monitoringv1.Endpoint{
+			Scheme:    scheme,
+			Port:      port,
+			Path:      "/metrics",
+			TLSConfig: tlsConfig,
+			// Custom relabel configs to be compatible with predefined Tempo dashboards:
+			// https://grafana.com/docs/tempo/latest/operations/monitoring/#dashboards
+			RelabelConfigs: []*monitoringv1.RelabelConfig{
+				{
+					SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_service_label_app_kubernetes_io_instance"},
+					TargetLabel:  "cluster",
+				},
+				{
+					SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_namespace", "__meta_kubernetes_service_label_app_kubernetes_io_component"},
+					Separator:    ptr.To("/"),
+					TargetLabel:  "job",
+				},
+			},
+		})
+	}
+
 	return &monitoringv1.ServiceMonitor{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: monitoringv1.SchemeGroupVersion.String(),
@@ -102,25 +126,7 @@ func NewServiceMonitor(
 			Labels:    labels,
 		},
 		Spec: monitoringv1.ServiceMonitorSpec{
-			Endpoints: []monitoringv1.Endpoint{{
-				Scheme:    scheme,
-				Port:      port,
-				Path:      "/metrics",
-				TLSConfig: tlsConfig,
-				// Custom relabel configs to be compatible with predefined Tempo dashboards:
-				// https://grafana.com/docs/tempo/latest/operations/monitoring/#dashboards
-				RelabelConfigs: []*monitoringv1.RelabelConfig{
-					{
-						SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_service_label_app_kubernetes_io_instance"},
-						TargetLabel:  "cluster",
-					},
-					{
-						SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_namespace", "__meta_kubernetes_service_label_app_kubernetes_io_component"},
-						Separator:    ptr.To("/"),
-						TargetLabel:  "job",
-					},
-				},
-			}},
+			Endpoints: endpoints,
 			NamespaceSelector: monitoringv1.NamespaceSelector{
 				MatchNames: []string{namespace},
 			},

tests/e2e-openshift/monolithic-multitenancy-openshift/01-assert.yaml

@@ -29,26 +29,6 @@ status:
     started: true
   phase: Running
 
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: tempo-mmo-gateway
-spec:
-  ports:
-  - name: public
-    port: 8080
-    protocol: TCP
-    targetPort: public
-  - name: internal
-    port: 8081
-    protocol: TCP
-    targetPort: internal
-  - name: otlp-grpc
-    port: 4317
-    protocol: TCP
-    targetPort: grpc-public
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -88,6 +68,30 @@ subjects:
   namespace: chainsaw-monolithic-multitenancy
 
 ---
+apiVersion: v1
+kind: Service
+metadata:
+    name: tempo-mmo-gateway
+spec:
+    ports:
+    - name: public
+      protocol: TCP
+      port: 8080
+      targetPort: public
+    - name: internal
+      protocol: TCP
+      port: 8081
+      targetPort: internal
+    - name: http
+      protocol: TCP
+      port: 3200
+      targetPort: http
+    - name: otlp-grpc
+      protocol: TCP
+      port: 4317
+      targetPort: grpc-public
+---
+
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -114,6 +118,20 @@ spec:
       - __meta_kubernetes_service_label_app_kubernetes_io_component
       targetLabel: job
     scheme: http
+  - path: /metrics
+    port: http
+    relabelings:
+      - action: replace
+        sourceLabels:
+          - __meta_kubernetes_service_label_app_kubernetes_io_instance
+        targetLabel: cluster
+      - action: replace
+        separator: /
+        sourceLabels:
+          - __meta_kubernetes_namespace
+          - __meta_kubernetes_service_label_app_kubernetes_io_component
+        targetLabel: job
+    scheme: http
   namespaceSelector:
     matchNames:
     - chainsaw-monolithic-multitenancy

tests/e2e-openshift/monolithic-multitenancy-openshift/check_metrics.sh

@@ -7,9 +7,7 @@ TOKEN=$(oc create token e2e-test-metrics-reader -n $NAMESPACE)
 THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host')
 
 #Check TempoMonolithc metircs
-# Tempo component metrics not exposed due to bug. https://issues.redhat.com/browse/TRACING-5472
-#metrics="tempo_query_frontend_queries_total tempo_distributor_bytes_received_total tempo_distributor_spans_received_total tempo_ingester_bytes_received_total tempo_distributor_traces_per_batch_count tempo_build_info"
-metrics="http_request_duration_seconds_bucket"
+metrics="tempo_query_frontend_queries_total tempo_distributor_bytes_received_total tempo_distributor_spans_received_total tempo_ingester_bytes_received_total tempo_distributor_traces_per_batch_count tempo_build_info"
 
 for metric in $metrics; do
 query="$metric"