TT-6799 Prometheus pump: Support to disable certain metric families from exposition. (#492)

* support disabling prometheus metric families

* update readme and changelog

* fix language

* disabling of metrics should only work for the base metrics

* do not collect and log out disabled metrics

* exclude base metrics entirely if they are disabled

* checkpoint

* update docs

Author: hsyed-dojo

Diff for: CHANGELOG.md

@@ -4,8 +4,9 @@
 
 - Initial connector which outputs all analytics into a single elasticsearch index
 - Enabled the ES connecter to save all output to a rolling index rather than a static index
+- Add support to the Prometheus pump to exclude metric families from exposition 
 
 ## v0.1
 
 - Initial connector to replace MongoDB connector in Tyk
-- Added support for uptime purging (redis/mongo only)
+- Added support for uptime purging (redis/mongo only)

Diff for: README.md

@@ -483,6 +483,7 @@ Tyk expose the following counters:
 And the following Histogram for latencies:
 - tyk_latency{type, api}
 
+Note: base metric families can be removed by configuring the `disabled_metrics` property.
 
 #### Custom Prometheus metrics
 From Pump 1.6+ it's possible to add custom prometheus metrics using the `custom_metrics` configuration.
@@ -536,6 +537,7 @@ TYK_PMP_PUMPS_PROMETHEUS_TYPE=prometheus
 TYK_PMP_PUMPS_PROMETHEUS_META_ADDR=localhost:9090
 TYK_PMP_PUMPS_PROMETHEUS_META_PATH=/metrics
 TYK_PMP_PUMPS_PROMETHEUS_META_CUSTOMMETRICS=[]
+TYK_PMP_PUMPS_PROMETHEUS_META_DISABLED_METRICS=[]
 ```
 
 ## DogStatsD

Diff for: go.sum

@@ -324,7 +324,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=

Diff for: pumps/prometheus.go

@@ -39,6 +39,8 @@ type PrometheusConf struct {
 	// This will enable an experimental feature that will aggregate the histogram metrics request time values before exposing them to prometheus.
 	// Enabling this will reduce the CPU usage of your prometheus pump but you will loose histogram precision. Experimental.
 	AggregateObservations bool `json:"aggregate_observations" mapstructure:"aggregate_observations"`
+	// Metrics to exclude from exposition. Currently, excludes only the base metrics.
+	DisabledMetrics []string `json:"disabled_metrics" mapstructure:"disabled_metrics"`
 	// Custom Prometheus metrics.
 	CustomMetrics []PrometheusMetric `json:"custom_metrics" mapstructure:"custom_metrics"`
 }
@@ -69,7 +71,7 @@ type PrometheusMetric struct {
 	aggregatedObservations bool
 }
 
-//histogramCounter is a helper struct to mantain the totalRequestTime and hits in memory
+// histogramCounter is a helper struct to mantain the totalRequestTime and hits in memory
 type histogramCounter struct {
 	totalRequestTime uint64
 	hits             uint64
@@ -91,7 +93,7 @@ func (p *PrometheusPump) New() Pump {
 	return &newPump
 }
 
-//CreateBasicMetrics stores all the predefined pump metrics in allMetrics slice
+// CreateBasicMetrics stores all the predefined pump metrics in allMetrics slice
 func (p *PrometheusPump) CreateBasicMetrics() {
 
 	//counter metrics
@@ -160,13 +162,7 @@ func (p *PrometheusPump) Init(conf interface{}) error {
 	}
 
 	//first we init the base metrics
-	for _, metric := range p.allMetrics {
-		metric.aggregatedObservations = p.conf.AggregateObservations
-		errInit := metric.InitVec()
-		if errInit != nil {
-			p.log.Error(errInit)
-		}
-	}
+	p.initBaseMetrics()
 
 	//then we check the custom ones
 	p.InitCustomMetrics()
@@ -183,7 +179,27 @@ func (p *PrometheusPump) Init(conf interface{}) error {
 	return nil
 }
 
-//InitCustomMetrics initialise custom prometheus metrics based on p.conf.CustomMetrics and add them into p.allMetrics
+func (p *PrometheusPump) initBaseMetrics() {
+	toDisableSet := map[string]struct{}{}
+	for _, metric := range p.conf.DisabledMetrics {
+		toDisableSet[metric] = struct{}{}
+	}
+	// exclude disabled base metrics if needed. This disables exposition entirely during scrapes.
+	trimmedAllMetrics := make([]*PrometheusMetric, 0, len(p.allMetrics))
+	for _, metric := range p.allMetrics {
+		if _, isDisabled := toDisableSet[metric.Name]; isDisabled {
+			continue
+		}
+		metric.aggregatedObservations = p.conf.AggregateObservations
+		if errInit := metric.InitVec(); errInit != nil {
+			p.log.Error(errInit)
+		}
+		trimmedAllMetrics = append(trimmedAllMetrics, metric)
+	}
+	p.allMetrics = trimmedAllMetrics
+}
+
+// InitCustomMetrics initialise custom prometheus metrics based on p.conf.CustomMetrics and add them into p.allMetrics
 func (p *PrometheusPump) InitCustomMetrics() {
 	if len(p.conf.CustomMetrics) > 0 {
 		customMetrics := []*PrometheusMetric{}
@@ -358,7 +374,7 @@ func (pm *PrometheusMetric) GetLabelsValues(decoded analytics.AnalyticsRecord) [
 	return values
 }
 
-//Inc is going to fill counterMap and histogramMap with the data from record.
+// Inc is going to fill counterMap and histogramMap with the data from record.
 func (pm *PrometheusMetric) Inc(values ...string) error {
 	switch pm.MetricType {
 	case COUNTER_TYPE:
@@ -370,7 +386,7 @@ func (pm *PrometheusMetric) Inc(values ...string) error {
 	return nil
 }
 
-//Observe will fill hitogramMap with the sum of totalRequest and hits per label value if aggregate_observations is true. If aggregate_observations is set to false (default) it will execute prometheus Observe directly.
+// Observe will fill hitogramMap with the sum of totalRequest and hits per label value if aggregate_observations is true. If aggregate_observations is set to false (default) it will execute prometheus Observe directly.
 func (pm *PrometheusMetric) Observe(requestTime int64, values ...string) error {
 	switch pm.MetricType {
 	case HISTOGRAM_TYPE:
@@ -399,10 +415,10 @@ func (pm *PrometheusMetric) Observe(requestTime int64, values ...string) error {
 	return nil
 }
 
-//Expose executes prometheus library functions using the counter/histogram vector from the PrometheusMetric struct.
-//If the PrometheusMetric is COUNTER_TYPE, it will execute prometheus client Add function to add the counters from counterMap to the labels value metric
-//If the PrometheusMetric is HISTOGRAM_TYPE and aggregate_observations config is true, it will calculate the average value of the metrics in the histogramMap and execute prometheus Observe.
-//If aggregate_observations is false, it won't do anything since it means that we already exposed the metric.
+// Expose executes prometheus library functions using the counter/histogram vector from the PrometheusMetric struct.
+// If the PrometheusMetric is COUNTER_TYPE, it will execute prometheus client Add function to add the counters from counterMap to the labels value metric
+// If the PrometheusMetric is HISTOGRAM_TYPE and aggregate_observations config is true, it will calculate the average value of the metrics in the histogramMap and execute prometheus Observe.
+// If aggregate_observations is false, it won't do anything since it means that we already exposed the metric.
 func (pm *PrometheusMetric) Expose() error {
 	switch pm.MetricType {
 	case COUNTER_TYPE:
@@ -426,7 +442,7 @@ func (pm *PrometheusMetric) Expose() error {
 	return nil
 }
 
-//getAverageRequestTime returns the average request time of an histogramCounter dividing the sum of all the RequestTimes by the hits.
+// getAverageRequestTime returns the average request time of an histogramCounter dividing the sum of all the RequestTimes by the hits.
 func (c histogramCounter) getAverageRequestTime() float64 {
 	return float64(c.totalRequestTime / c.hits)
 }

Diff for: pumps/prometheus_test.go

@@ -2,14 +2,16 @@ package pumps
 
 import (
 	"errors"
+	"io"
 	"testing"
 
 	"github.com/TykTechnologies/tyk-pump/analytics"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestInitVec(t *testing.T) {
+func TestPrometheusInitVec(t *testing.T) {
 	tcs := []struct {
 		testName     string
 		customMetric PrometheusMetric
@@ -82,7 +84,7 @@ func TestInitVec(t *testing.T) {
 	}
 }
 
-func TestInitCustomMetrics(t *testing.T) {
+func TestPrometheusInitCustomMetrics(t *testing.T) {
 	tcs := []struct {
 		testName              string
 		metrics               []PrometheusMetric
@@ -183,7 +185,7 @@ func TestInitCustomMetrics(t *testing.T) {
 	}
 }
 
-func TestGetLabelsValues(t *testing.T) {
+func TestPrometheusGetLabelsValues(t *testing.T) {
 	tcs := []struct {
 		testName       string
 		customMetric   PrometheusMetric
@@ -539,7 +541,7 @@ func TestPrometheusCreateBasicMetrics(t *testing.T) {
 
 }
 
-func TestEnsureLabels(t *testing.T) {
+func TestPrometheusEnsureLabels(t *testing.T) {
 	testCases := []struct {
 		name                 string
 		metricType           string
@@ -601,3 +603,34 @@ func TestEnsureLabels(t *testing.T) {
 		})
 	}
 }
+
+func TestPrometheusDisablingMetrics(t *testing.T) {
+	p := &PrometheusPump{}
+	newPump := p.New().(*PrometheusPump)
+
+	log := logrus.New()
+	log.Out = io.Discard
+	newPump.log = logrus.NewEntry(log)
+
+	newPump.conf = &PrometheusConf{DisabledMetrics: []string{"tyk_http_status_per_path"}}
+
+	newPump.initBaseMetrics()
+
+	defer func() {
+		for i := range newPump.allMetrics {
+			if newPump.allMetrics[i].MetricType == COUNTER_TYPE {
+				prometheus.Unregister(newPump.allMetrics[i].counterVec)
+			} else if newPump.allMetrics[i].MetricType == HISTOGRAM_TYPE {
+				prometheus.Unregister(newPump.allMetrics[i].histogramVec)
+			}
+		}
+	}()
+
+	metricMap := map[string]*PrometheusMetric{}
+	for _, metric := range newPump.allMetrics {
+		metricMap[metric.Name] = metric
+	}
+
+	assert.Contains(t, metricMap, "tyk_http_status")
+	assert.NotContains(t, metricMap, "tyk_http_status_per_path")
+}