Add bitnami signals source to kafka (#1315)

* Add bitnami signals source to kafka * Revert groupLabels * fmt * fmt
grafana · Aug 22, 2024 · d2347d0 · d2347d0
1 parent 740488f
commit d2347d0
Show file tree

Hide file tree

Showing 12 changed files with 176 additions and 3 deletions.
diff --git a/kafka-observ-lib/README.md b/kafka-observ-lib/README.md
@@ -6,6 +6,7 @@ The library supports two metrics sources (`metricsSource`):
 
 - `prometheus`: JMX configs from [prometheus/jmx_exporter](https://github.com/prometheus/jmx_exporter/blob/main/example_configs/kafka-2_0_0.yml) and [kafka_exporter](https://github.com/danielqsj/kafka_exporter)
 - `grafanacloud`: JMX configs from [kafka-mixin](../kafka-mixin/jmx) and [kafka_exporter fork](https://github.com/grafana/kafka_exporter) (used in grafana-agent/alloy).
+- `bitnami`: JMX configs from [bitnami](https://github.com/bitnami/charts/blob/main/bitnami/kafka/templates/metrics/jmx-configmap.yaml) helm chart
 
 You can also pick on the three JVM metrics sources (`jvmMetricsSource`):
   - `jmx_exporter` if you use jmx_exporter in http mode or javaagent mode with the additional config snippet (see README)

diff --git a/kafka-observ-lib/config.libsonnet b/kafka-observ-lib/config.libsonnet
@@ -9,7 +9,7 @@
   dashboardTags: ['kafka'],
   alertKafkaLagTooHighThreshold: '100',
   alertKafkaLagTooHighSeverity: 'critical',
-  metricsSource: 'prometheus',  //or grafanacloud. See README
+  metricsSource: 'prometheus',  //or grafanacloud, bitnami. See README
 
   // 'jmx_exporter' if you use jmx_exporter in http mode or javaagent mode with the additional config snippet (see README)
   // 'prometheus_old' if you use jmx_exporter in javaagent mode and version prior to 1.0.1

diff --git a/kafka-observ-lib/main.libsonnet b/kafka-observ-lib/main.libsonnet
@@ -30,7 +30,10 @@ local zookeeperlib = import 'zookeeper-observ-lib/main.libsonnet';
           uid: this.config.uid,
           dashboardNamePrefix: this.config.dashboardNamePrefix,
           dashboardTags: this.config.dashboardTags,
-          metricsSource: this.config.metricsSource,
+          metricsSource:
+            if this.config.metricsSource == 'prometheus' then 'prometheus'
+            else if this.config.metricsSource == 'grafanacloud' then 'grafanacloud'
+            else if this.config.metricsSource == 'bitnami' then 'prometheus',
         }
       ),
 

diff --git a/kafka-observ-lib/rows.libsonnet b/kafka-observ-lib/rows.libsonnet
@@ -53,7 +53,7 @@ local commonlib = import 'common-lib/common/main.libsonnet';
 
 
         +
-        if type == 'prometheus' then
+        if type == 'prometheus' || type == 'bitnami' then
           [
             panels.consumerGroup.consumerGroupConsumeRate { gridPos+: { w: 12, h: 8 } },
             panels.consumerGroup.consumerGroupLag { gridPos+: { w: 12, h: 8 } },

diff --git a/kafka-observ-lib/signals/broker.libsonnet b/kafka-observ-lib/signals/broker.libsonnet
@@ -10,6 +10,7 @@ function(this)
     discoveryMetric: {
       prometheus: 'kafka_controller_kafkacontroller_activecontrollercount',
       grafanacloud: 'kafka_controller_kafkacontroller_activecontrollercount',
+      bitnami: 'kafka_controller_kafkacontroller_activecontrollercount_value',
     },
     signals: {
       brokerMessagesInPerSec: {
@@ -24,6 +25,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_messagesinpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_messagesinpersec_count{%(queriesSelector)s}',
+          },
         },
       },
       brokerBytesInPerSec: {
@@ -38,6 +42,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_bytesinpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_bytesinpersec_count{%(queriesSelector)s}',
+          },
         },
       },
       brokerBytesOutPerSec: {
@@ -52,6 +59,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_bytesoutpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_bytesoutpersec_count{%(queriesSelector)s}',
+          },
         },
       },
 

diff --git a/kafka-observ-lib/signals/cluster.libsonnet b/kafka-observ-lib/signals/cluster.libsonnet
@@ -11,6 +11,7 @@ function(this)
     discoveryMetric: {
       prometheus: 'kafka_controller_kafkacontroller_activecontrollercount',
       grafanacloud: 'kafka_controller_kafkacontroller_activecontrollercount',
+      bitnami: 'kafka_controller_kafkacontroller_activecontrollercount_value',
     },
     signals: {
       activeControllers: {
@@ -30,6 +31,10 @@ function(this)
             {
               expr: 'kafka_controller_kafkacontroller_activecontrollercount{%(queriesSelector)s}',
             },
+          bitnami:
+            {
+              expr: 'kafka_controller_kafkacontroller_activecontrollercount_value{%(queriesSelector)s}',
+            },
         },
       },
       // used in status map
@@ -86,6 +91,29 @@ function(this)
                 },
               ],
             },
+          bitnami:
+            {
+              expr: 'kafka_controller_kafkacontroller_activecontrollercount_value{%(queriesSelector)s}',
+              aggKeepLabels: this.instanceLabels,
+              legendCustomTemplate: '{{ %s }}' % xtd.array.slice(this.instanceLabels, -1),
+              valueMappings: [
+                {
+                  type: 'value',
+                  options: {
+                    '0': {
+                      text: 'follower',
+                      color: 'light-purple',
+                      index: 0,
+                    },
+                    '1': {
+                      text: 'controller',
+                      color: 'light-blue',
+                      index: 1,
+                    },
+                  },
+                },
+              ],
+            },
         },
       },
       brokersCount: {
@@ -105,6 +133,10 @@ function(this)
             {
               expr: 'kafka_server_kafkaserver_brokerstate{%(queriesSelector)s}',
             },
+          bitnami:
+            {
+              expr: 'kafka_server_kafkaserver_total_brokerstate_value{%(queriesSelector)s}',
+            },
         },
       },
 
@@ -120,6 +152,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_messagesinpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_messagesinpersec_count{%(queriesSelector)s}',
+          },
         },
       },
       clusterBytesInPerSec: {
@@ -134,6 +169,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_bytesinpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_bytesinpersec_count{%(queriesSelector)s}',
+          },
         },
       },
       clusterBytesOutPerSec: {
@@ -148,6 +186,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_bytesoutpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_bytesoutpersec_count{%(queriesSelector)s}',
+          },
         },
       },
     },

diff --git a/kafka-observ-lib/signals/consumerGroup.libsonnet b/kafka-observ-lib/signals/consumerGroup.libsonnet
@@ -11,6 +11,7 @@ function(this)
     discoveryMetric: {
       prometheus: 'kafka_consumergroup_lag',  // https://github.com/danielqsj/kafka_exporter?tab=readme-ov-file#metrics
       grafanacloud: 'kafka_consumergroup_uncomitted_offsets',  // https://github.com/grafana/kafka_exporter/blob/master/exporter/exporter.go#L887
+      bitnami: self.prometheus,
     },
     signals: {
       consumerGroupLag: {
@@ -28,6 +29,7 @@ function(this)
             aggKeepLabels: ['consumergroup', 'topic'],
             expr: 'kafka_consumergroup_uncomitted_offsets{%(queriesSelector)s}',
           },
+          bitnami: self.prometheus,
         },
       },
 
@@ -57,7 +59,9 @@ function(this)
             expr: 'kafka_consumergroup_current_offset{%(queriesSelector)s}',
           },
           grafanacloud: self.prometheus,
+          bitnami: self.prometheus,
         },
+
       },
     },
   }
diff --git a/kafka-observ-lib/signals/conversion.libsonnet b/kafka-observ-lib/signals/conversion.libsonnet
@@ -10,6 +10,7 @@ function(this)
     discoveryMetric: {
       prometheus: 'kafka_server_brokertopicmetrics_producemessageconversions_total',
       grafanacloud: 'kafka_server_brokertopicmetrics_producemessageconversionspersec',
+      bitnami: 'kafka_server_brokertopicmetrics_total_producemessageconversionspersec_count',
     },
     signals: {
       producerConversion: {
@@ -24,6 +25,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_producemessageconversionspersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_producemessageconversionspersec_count{%(queriesSelector)s}',
+          },
         },
       },
       consumerConversion: {
@@ -38,6 +42,9 @@ function(this)
           grafanacloud: {
             expr: 'kafka_server_brokertopicmetrics_fetchmessageconversionspersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            expr: 'kafka_server_brokertopicmetrics_fetchmessageconversionspersec_count{%(queriesSelector)s}',
+          },
         },
       },
     },

diff --git a/kafka-observ-lib/signals/replicaManager.libsonnet b/kafka-observ-lib/signals/replicaManager.libsonnet
@@ -10,6 +10,7 @@ function(this)
     discoveryMetric: {
       prometheus: 'kafka_server_replicamanager_isrshrinks_total',
       grafanacloud: 'kafka_server_replicamanager_isrshrinkspersec',
+      bitnami: 'kafka_server_replicamanager_total_isrshrinkspersec_count',
     },
     signals: {
       local s = this,
@@ -39,6 +40,10 @@ function(this)
             {
               expr: '%(aggFunction)s by (%(agg)s)  (rate(kafka_server_replicamanager_isrshrinks_total{%(queriesSelector)s}[%(interval)s]))',
             },
+          bitnami:
+            {
+              expr: '%(aggFunction)s by (%(agg)s)  (rate(kafka_server_replicamanager_total_isrshrinkspersec_count{%(queriesSelector)s}[%(interval)s]))',
+            },
         },
       },
       isrExpands: {
@@ -67,6 +72,9 @@ function(this)
             {
               expr: '%(aggFunction)s by (%(agg)s) (rate(kafka_server_replicamanager_isrexpands_total{%(queriesSelector)s}[%(interval)s]))',
             },
+          bitnami: {
+            expr: '%(aggFunction)s by (%(agg)s) (rate(kafka_server_replicamanager_total_isrexpandspersec_count{%(queriesSelector)s}[%(interval)s]))',
+          },
         },
       },
       onlinePartitions: {
@@ -86,6 +94,10 @@ function(this)
             {
               expr: 'kafka_server_replicamanager_partitioncount{%(queriesSelector)s}',
             },
+          bitnami:
+            {
+              expr: 'kafka_server_replicamanager_total_partitioncount_value{%(queriesSelector)s}',
+            },
         },
       },
       offlinePartitions: {
@@ -105,6 +117,9 @@ function(this)
             {
               expr: 'kafka_controller_kafkacontroller_offlinepartitionscount{%(queriesSelector)s}',
             },
+          bitnami: {
+            expr: 'kafka_controller_kafkacontroller_offlinepartitionscount_value{%(queriesSelector)s}',
+          },
         },
       },
       underReplicatedPartitions: {
@@ -115,6 +130,7 @@ function(this)
         type: 'gauge',
         unit: 'short',
         aggFunction: 'sum',
+        optional: true,
         sources: {
           grafanacloud:
             {
@@ -134,6 +150,7 @@ function(this)
         type: 'gauge',
         unit: 'short',
         aggFunction: 'sum',
+        optional: true,
         sources: {
           grafanacloud:
             {
@@ -153,6 +170,7 @@ function(this)
         type: 'raw',
         unit: 'short',
         aggFunction: 'sum',
+        optional: true,
         sources: {
           grafanacloud:
             {
@@ -181,6 +199,9 @@ function(this)
             {
               expr: 'kafka_controller_kafkacontroller_preferredreplicaimbalancecount{%(queriesSelector)s}',
             },
+          bitnami: {
+            expr: 'kafka_controller_kafkacontroller_preferredreplicaimbalancecount_value{%(queriesSelector)s}',
+          },
         },
       },
 

diff --git a/kafka-observ-lib/signals/topic.libsonnet b/kafka-observ-lib/signals/topic.libsonnet
@@ -11,6 +11,7 @@ function(this)
     discoveryMetric: {
       prometheus: 'kafka_log_log_logstartoffset',  //https://github.com/danielqsj/kafka_exporter?tab=readme-ov-file#metrics
       grafanacloud: self.prometheus,
+      bitnami: 'kafka_log_log_logstartoffset',
     },
     signals: {
       topicMessagesPerSec: {
@@ -24,6 +25,7 @@ function(this)
             expr: 'kafka_topic_partition_current_offset{%(queriesSelector)s}',
           },
           grafanacloud: self.prometheus,
+          bitnami: self.prometheus,
         },
       },
       // used in table:
@@ -39,6 +41,7 @@ function(this)
             expr: 'kafka_topic_partition_current_offset{%(queriesSelector)s}',
           },
           grafanacloud: self.prometheus,
+          bitnami: self.prometheus,
         },
       },
       // JMX exporter extras
@@ -56,6 +59,10 @@ function(this)
             aggKeepLabels: ['topic'],
             expr: 'kafka_server_brokertopicmetrics_bytesinpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            aggKeepLabels: ['topic'],
+            expr: 'kafka_server_brokertopicmetrics_bytesinpersec_count{%(queriesSelector)s}',
+          },
         },
       },
       topicBytesOutPerSec: {
@@ -72,6 +79,10 @@ function(this)
             aggKeepLabels: ['topic'],
             expr: 'kafka_server_brokertopicmetrics_bytesoutpersec{%(queriesSelector)s}',
           },
+          bitnami: {
+            aggKeepLabels: ['topic'],
+            expr: 'kafka_server_brokertopicmetrics_bytesoutpersec_count{%(queriesSelector)s}',
+          },
         },
       },
       topicLogStartOffset: {
@@ -90,6 +101,10 @@ function(this)
             aggKeepLabels: ['topic', 'partition'],
             expr: 'kafka_log_log_logstartoffset{%(queriesSelector)s}',
           },
+          bitnami: {
+            aggKeepLabels: ['topic', 'partition'],
+            expr: 'kafka_log_log_logstartoffset{%(queriesSelector)s}',
+          },
         },
       },
       topicLogEndOffset: {
@@ -106,6 +121,7 @@ function(this)
             expr: 'kafka_log_log_logendoffset{%(queriesSelector)s}',
           },
           grafanacloud: self.prometheus,
+          bitnami: self.prometheus,
         },
       },
       topicLogSize: {
@@ -121,6 +137,7 @@ function(this)
             expr: 'kafka_log_log_size{%(queriesSelector)s}',
           },
           grafanacloud: self.prometheus,
+          bitnami: self.prometheus,
         },
       },