From 7633ac551e53dac56f3f80e23ea2d2bbff1eb9eb Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Tue, 2 May 2023 18:31:30 +0200 Subject: [PATCH 01/16] Extract fake controller to use it in other tests Signed-off-by: Lucas Thiesen --- pkg/collector/fake_collector_test.go | 57 +++++++++++++++++++++++++ pkg/collector/skipper_collector_test.go | 35 --------------- 2 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 pkg/collector/fake_collector_test.go diff --git a/pkg/collector/fake_collector_test.go b/pkg/collector/fake_collector_test.go new file mode 100644 index 00000000..1077c6ca --- /dev/null +++ b/pkg/collector/fake_collector_test.go @@ -0,0 +1,57 @@ +package collector + +import ( + "time" + + autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/metrics/pkg/apis/custom_metrics" +) + +type FakeCollectorPlugin struct { + metrics []CollectedMetric + config map[string]string +} + +type FakeCollector struct { + metrics []CollectedMetric + interval time.Duration + stub func() ([]CollectedMetric, error) +} + +func (c *FakeCollector) GetMetrics() ([]CollectedMetric, error) { + if c.stub != nil { + v, err := c.stub() + return v, err + } + + return c.metrics, nil +} + +func (FakeCollector) Interval() time.Duration { + return time.Minute +} + +func (p *FakeCollectorPlugin) NewCollector( + hpa *autoscalingv2.HorizontalPodAutoscaler, + config *MetricConfig, + interval time.Duration, +) (Collector, error) { + + p.config = config.Config + return &FakeCollector{metrics: p.metrics, interval: interval}, nil +} + +func makePlugin(metric int) *FakeCollectorPlugin { + return &FakeCollectorPlugin{ + metrics: []CollectedMetric{ + { + Custom: custom_metrics.MetricValue{Value: *resource.NewQuantity(int64(metric), resource.DecimalSI)}, + }, + }, + } +} + +func makeCollectorWithStub(f func() ([]CollectedMetric, error)) *FakeCollector { + return &FakeCollector{stub: f} +} diff --git a/pkg/collector/skipper_collector_test.go b/pkg/collector/skipper_collector_test.go index 63bcbd15..62e14392 100644 --- a/pkg/collector/skipper_collector_test.go +++ b/pkg/collector/skipper_collector_test.go @@ -658,38 +658,3 @@ func makeConfig(resourceName, namespace, kind, backend string, fakedAverage bool } return config } - -type FakeCollectorPlugin struct { - metrics []CollectedMetric - config map[string]string -} - -type FakeCollector struct { - metrics []CollectedMetric -} - -func (c *FakeCollector) GetMetrics() ([]CollectedMetric, error) { - return c.metrics, nil -} - -func (FakeCollector) Interval() time.Duration { - return time.Minute -} - -func (p *FakeCollectorPlugin) NewCollector(hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration) (Collector, error) { - if p.config != nil { - return nil, fmt.Errorf("config already assigned once: %v", p.config) - } - p.config = config.Config - return &FakeCollector{metrics: p.metrics}, nil -} - -func makePlugin(metric int) *FakeCollectorPlugin { - return &FakeCollectorPlugin{ - metrics: []CollectedMetric{ - { - Custom: custom_metrics.MetricValue{Value: *resource.NewQuantity(int64(metric), resource.DecimalSI)}, - }, - }, - } -} From aa7b64e63755e46176c3b0ac345d1577ab1c9b08 Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Tue, 25 Apr 2023 20:05:47 +0200 Subject: [PATCH 02/16] Add hostname RPS metric collector Signed-off-by: Lucas Thiesen --- pkg/collector/hostname_collector.go | 89 ++++++++ pkg/collector/hostname_collector_test.go | 261 +++++++++++++++++++++++ pkg/server/start.go | 21 ++ 3 files changed, 371 insertions(+) create mode 100644 pkg/collector/hostname_collector.go create mode 100644 pkg/collector/hostname_collector_test.go diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go new file mode 100644 index 00000000..d5a2c2c5 --- /dev/null +++ b/pkg/collector/hostname_collector.go @@ -0,0 +1,89 @@ +package collector + +import ( + "fmt" + "time" + + autoscalingv2 "k8s.io/api/autoscaling/v2beta2" +) + +const ( + HostnameMetricType = "hostname-rps" + HostnameRPSQuery = `scalar(sum(rate(%s{host=~"%s"}[1m])))` +) + +type HostnameCollectorPlugin struct { + metricName string + promPlugin CollectorPlugin +} + +type HostnameCollector struct { + interval time.Duration + promCollector Collector +} + +func NewHostnameCollectorPlugin( + promPlugin CollectorPlugin, + metricName string, +) (*HostnameCollectorPlugin, error) { + if metricName == "" { + return nil, fmt.Errorf("failed to initialize hostname collector plugin, metric name was not defined") + } + + return &HostnameCollectorPlugin{ + metricName: metricName, + promPlugin: promPlugin, + }, nil +} + +// NewCollector initializes a new skipper collector from the specified HPA. +func (p *HostnameCollectorPlugin) NewCollector( + hpa *autoscalingv2.HorizontalPodAutoscaler, + config *MetricConfig, + interval time.Duration, +) (Collector, error) { + if config == nil { + return nil, fmt.Errorf("Metric config not present, it is not possible to initialize the collector.") + } + // Need to copy config and add a promQL query in order to get + // RPS data from a specific hostname from prometheus. The idea + // of the copy is to not modify the original config struct. + confCopy := *config + hostname := config.Config["hostname"] + + if hostname == "" { + return nil, fmt.Errorf("hostname not specified, unable to create collector") + } + + confCopy.Config = map[string]string{ + "query": fmt.Sprintf(HostnameRPSQuery, p.metricName, hostname), + } + + c, err := p.promPlugin.NewCollector(hpa, &confCopy, interval) + if err != nil { + return nil, err + } + + return &HostnameCollector{ + interval: interval, + promCollector: c, + }, nil +} + +// GetMetrics gets hostname metrics from Prometheus +func (c *HostnameCollector) GetMetrics() ([]CollectedMetric, error) { + v, err := c.promCollector.GetMetrics() + if err != nil { + return nil, err + } + + if len(v) != 1 { + return nil, fmt.Errorf("expected to only get one metric value, got %d", len(v)) + } + return v, nil +} + +// Interval returns the interval at which the collector should run. +func (c *HostnameCollector) Interval() time.Duration { + return c.interval +} diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/hostname_collector_test.go new file mode 100644 index 00000000..1239ca08 --- /dev/null +++ b/pkg/collector/hostname_collector_test.go @@ -0,0 +1,261 @@ +package collector + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/metrics/pkg/apis/external_metrics" +) + +func TestHostnameCollectorPluginConstructor(tt *testing.T) { + for _, testcase := range []struct { + msg string + name string + isValid bool + }{ + {"No metric name", "", false}, + {"Valid metric name", "a_valid_metric_name", true}, + } { + tt.Run(testcase.msg, func(t *testing.T) { + + fakePlugin := &FakeCollectorPlugin{} + plugin, err := NewHostnameCollectorPlugin(fakePlugin, testcase.name) + + if testcase.isValid { + require.NoError(t, err) + require.NotNil(t, plugin) + require.Equal(t, testcase.name, plugin.metricName) + require.Equal(t, fakePlugin, plugin.promPlugin) + } else { + require.NotNil(t, err) + require.Nil(t, plugin) + } + }) + } +} + +func TestHostnamePluginNewCollector(tt *testing.T) { + fakePlugin := &FakeCollectorPlugin{} + + plugin := &HostnameCollectorPlugin{ + metricName: "a_valid_one", + promPlugin: fakePlugin, + } + interval := time.Duration(42) + expectedQuery := `scalar(sum(rate(a_valid_one{host=~"foo.bar.baz"}[1m])))` + + for _, testcase := range []struct { + msg string + config *MetricConfig + shouldWork bool + }{ + {"No hostname config", &MetricConfig{Config: make(map[string]string)}, false}, + {"Nil metric config", nil, false}, + {"Valid hostname no prom query config", &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz"}}, true}, + {"Valid hostname with prom query config", &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz", "query": "some_other_query"}}, true}, + } { + tt.Run(testcase.msg, func(t *testing.T) { + c, err := plugin.NewCollector( + &autoscalingv2.HorizontalPodAutoscaler{}, + testcase.config, + interval, + ) + + if testcase.shouldWork { + require.NotNil(t, c) + require.Nil(t, err) + require.Equal(t, fakePlugin.config["query"], expectedQuery) + } else { + require.Nil(t, c) + require.NotNil(t, err) + } + }) + } +} + +func TestHostnameCollectorGetMetrics(tt *testing.T) { + genericErr := fmt.Errorf("This is an error") + expectedMetric := *resource.NewQuantity(int64(42), resource.DecimalSI) + + for _, testcase := range []struct { + msg string + stub func() ([]CollectedMetric, error) + shouldWork bool + }{ + { + "Internal collector error", + func() ([]CollectedMetric, error) { + return nil, genericErr + }, + false, + }, + { + "Invalid metric collection from internal collector", + func() ([]CollectedMetric, error) { + return []CollectedMetric{ + {External: external_metrics.ExternalMetricValue{Value: *resource.NewQuantity(int64(24), resource.DecimalSI)}}, + {External: external_metrics.ExternalMetricValue{Value: *resource.NewQuantity(int64(42), resource.DecimalSI)}}, + }, nil + }, + false, + }, + { + "Internal collector return single metric", + func() ([]CollectedMetric, error) { + return []CollectedMetric{ + {External: external_metrics.ExternalMetricValue{Value: *resource.NewQuantity(int64(42), resource.DecimalSI)}}, + }, nil + }, + true, + }, + } { + tt.Run(testcase.msg, func(t *testing.T) { + fake := makeCollectorWithStub(testcase.stub) + c := &HostnameCollector{promCollector: fake} + m, err := c.GetMetrics() + + if testcase.shouldWork { + require.Nil(t, err) + require.NotNil(t, m) + require.Len(t, m, 1) + require.Equal(t, m[0].External.Value, expectedMetric) + } else { + require.NotNil(t, err) + require.Nil(t, m) + } + }) + } +} + +func TestHostnameCollectorInterval(t *testing.T) { + interval := time.Duration(42) + fakePlugin := &FakeCollectorPlugin{} + plugin := &HostnameCollectorPlugin{ + metricName: "a_valid_one", + promPlugin: fakePlugin, + } + c, err := plugin.NewCollector( + &autoscalingv2.HorizontalPodAutoscaler{}, + &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz"}}, + interval, + ) + + require.NotNil(t, c) + require.Nil(t, err) + require.Equal(t, interval, c.Interval()) +} + +func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { + expectedQuery := `scalar(sum(rate(a_metric{host=~"just.testing.com"}[1m])))` + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "metric-config.external.foo.hostname-rps/hostname": "just.testing.com", + }, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: "foo", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"type": "hostname-rps"}, + }, + }, + }, + }, + }, + }, + } + + factory := NewCollectorFactory() + fakePlugin := makePlugin(42) + hostnamePlugin, err := NewHostnameCollectorPlugin(fakePlugin, "a_metric") + require.NoError(t, err) + factory.RegisterExternalCollector([]string{HostnameMetricType}, hostnamePlugin) + conf, err := ParseHPAMetrics(hpa) + require.NoError(t, err) + require.Len(t, conf, 1) + + c, err := factory.NewCollector(hpa, conf[0], 0) + + require.NoError(t, err) + _, ok := c.(*HostnameCollector) + require.True(t, ok) + require.Equal(t, fakePlugin.config["query"], expectedQuery) + +} + +func TestHostnamePrometheusCollectorInteraction(t *testing.T) { + hostnameQuery := `scalar(sum(rate(a_metric{host=~"just.testing.com"}[1m])))` + promQuery := "sum(rate(rps[1m]))" + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "metric-config.external.foo.hostname-rps/hostname": "just.testing.com", + "metric-config.external.bar.prometheus/query": promQuery, + }, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: "foo", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"type": "hostname-rps"}, + }, + }, + }, + }, + { + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: "bar", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"type": "prometheus"}, + }, + }, + }, + }, + }, + }, + } + + factory := NewCollectorFactory() + promPlugin, err := NewPrometheusCollectorPlugin(nil, "http://prometheus") + require.NoError(t, err) + factory.RegisterExternalCollector([]string{PrometheusMetricType, PrometheusMetricNameLegacy}, promPlugin) + hostnamePlugin, err := NewHostnameCollectorPlugin(promPlugin, "a_metric") + require.NoError(t, err) + factory.RegisterExternalCollector([]string{HostnameMetricType}, hostnamePlugin) + + conf, err := ParseHPAMetrics(hpa) + require.NoError(t, err) + require.Len(t, conf, 2) + + collectors := make(map[string]Collector) + collectors["hostname"], err = factory.NewCollector(hpa, conf[0], 0) + require.NoError(t, err) + collectors["prom"], err = factory.NewCollector(hpa, conf[1], 0) + require.NoError(t, err) + + prom, ok := collectors["prom"].(*PrometheusCollector) + require.True(t, ok) + hostname, ok := collectors["hostname"].(*HostnameCollector) + require.True(t, ok) + hostnameProm, ok := hostname.promCollector.(*PrometheusCollector) + require.True(t, ok) + + require.Equal(t, prom.query, promQuery) + require.Equal(t, hostnameProm.query, hostnameQuery) +} diff --git a/pkg/server/start.go b/pkg/server/start.go index 86388c51..90da3a29 100644 --- a/pkg/server/start.go +++ b/pkg/server/start.go @@ -65,6 +65,7 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command { MetricsAddress: ":7979", ZMONTokenName: "zmon", CredentialsDir: "/meta/credentials", + HostnameRPSMetricName: "skipper_serve_host_duration_seconds_count", } cmd := &cobra.Command{ @@ -132,6 +133,10 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command { flags.DurationVar(&o.DefaultScheduledScalingWindow, "scaling-schedule-default-scaling-window", 10*time.Minute, "Default rampup and rampdown window duration for ScalingSchedules") flags.IntVar(&o.RampSteps, "scaling-schedule-ramp-steps", 10, "Number of steps used to rampup and rampdown ScalingSchedules. It's used to guarantee won't avoid reaching the max scaling due to the 10% minimum change rule.") flags.StringVar(&o.DefaultTimeZone, "scaling-schedule-default-time-zone", "Europe/Berlin", "Default time zone to use for ScalingSchedules.") + flags.StringVar(&o.HostnameRPSMetricName, "hostname-rps-metric-name", o.HostnameRPSMetricName, ""+ + "The name of the metric that should be used to query prometheus for RPS per hostname.") + flags.BoolVar(&o.HostnameRPSMetrics, "hostname-rps-metrics", o.HostnameRPSMetrics, ""+ + "whether to enable hostname RPS metric collector or not") return cmd } @@ -218,6 +223,18 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct } } } + + // Hostname collector, like skipper's, depends on prometheus being enabled. + // Also, to enable hostname metric its necessary to pass the metric name that + // will be used. This was built this way so we can support hostname metrics to + // any ingress provider, e.g. Skipper, Nginx, envoy etc, in a simple way. + if o.HostnameRPSMetrics && o.HostnameRPSMetricName != "" { + hostnamePlugin, err := collector.NewHostnameCollectorPlugin(promPlugin, o.HostnameRPSMetricName) + collectorFactory.RegisterExternalCollector([]string{collector.HostnameMetricType}, hostnamePlugin) + if err != nil { + return fmt.Errorf("failed to register hostname collector plugin: %v", err) + } + } } if o.InfluxDBAddress != "" { @@ -445,4 +462,8 @@ type AdapterServerOptions struct { RampSteps int // Default time zone to use for ScalingSchedules. DefaultTimeZone string + // Feature flag to enable hostname rps metric collector + HostnameRPSMetrics bool + // Name of the Prometheus metric that stores RPS by hostname for Hostname RPS metrics. + HostnameRPSMetricName string } From 65dd585813fdcd1cda531fed28c66a5be6dc0b2f Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Tue, 2 May 2023 20:06:49 +0200 Subject: [PATCH 03/16] Add documentation regarding hostname-rps collector to readme Signed-off-by: Lucas Thiesen --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/README.md b/README.md index 89048d12..c30e13dd 100644 --- a/README.md +++ b/README.md @@ -402,6 +402,51 @@ the `backend` label under `matchLabels` for the metric. The ingress annotation where the backend weights can be obtained can be specified through the flag `--skipper-backends-annotation`. +## Hostname RPS collector + +The Hostname collector, like Skipper collector, is a simple wrapper around the Prometheus collector to +make it easy to define an HPA for scaling based on the RPS measured for a given hostname. When +[skipper](https://github.com/zalando/skipper) is used as the ingress +implementation in your cluster everything should work automatically, in case another reverse proxy is used as ingress, like [Nginx](https://github.com/kubernetes/ingress-nginx) for example, its necessary to configure which prometheus metric should be used through `--hostname-rps-metric-name ` flag. Assuming `skipper-ingress` is being used or the appropriate metric name is passed using the flag mentioned previously this collector provides the correct Prometheus queries out of the +box so users don't have to define those manually. + +### Supported metrics + +| Metric | Description | Type | Kind | K8s Versions | +| ------------ | -------------- | ------- | -- | -- | +| `hostname-rps` | Scale based on requests per second for a certain hostname. | External | | `>=1.12` | + +### Example: External Metric + +This is an example of an HPA that will scale based on `hostname-rps` for the RPS measured in the hostname called `www.example.com`. + +```yaml +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: myapp-hpa + annotations: + metric-config.external.example-rps.hostname-rps/hostname: www.example.com +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: custom-metrics-consumer + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: External + external: + metric: + name: example-rps + selector: + matchLabels: + type: hostname-rps + target: + type: AverageValue + averageValue: "42" +``` + ## InfluxDB collector The InfluxDB collector maps [Flux](https://github.com/influxdata/flux) queries to metrics that can be used for scaling. From 02ec2282ab0ecbc5fd00542002dc30e033f01236 Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Wed, 3 May 2023 18:13:02 +0200 Subject: [PATCH 04/16] Add multi hostname per metric support Signed-off-by: Lucas Thiesen --- pkg/collector/hostname_collector.go | 16 +++++-- pkg/collector/hostname_collector_test.go | 58 ++++++++++++++++++------ 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index d5a2c2c5..d2d5ae24 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -2,6 +2,8 @@ package collector import ( "fmt" + "regexp" + "strings" "time" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" @@ -49,14 +51,20 @@ func (p *HostnameCollectorPlugin) NewCollector( // RPS data from a specific hostname from prometheus. The idea // of the copy is to not modify the original config struct. confCopy := *config - hostname := config.Config["hostname"] + hostnames := config.Config["hostname"] - if hostname == "" { - return nil, fmt.Errorf("hostname not specified, unable to create collector") + if ok, err := regexp.MatchString("^[a-zA-Z0-9.,-]+$", hostnames); !ok || err != nil { + return nil, fmt.Errorf( + "hostname is not specified or invalid format, unable to create collector", + ) } confCopy.Config = map[string]string{ - "query": fmt.Sprintf(HostnameRPSQuery, p.metricName, hostname), + "query": fmt.Sprintf( + HostnameRPSQuery, + p.metricName, + strings.Replace(strings.Replace(hostnames, ",", "|", -1), ".", "_", -1), + ), } c, err := p.promPlugin.NewCollector(hpa, &confCopy, interval) diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/hostname_collector_test.go index 1239ca08..91a9b6a5 100644 --- a/pkg/collector/hostname_collector_test.go +++ b/pkg/collector/hostname_collector_test.go @@ -47,17 +47,45 @@ func TestHostnamePluginNewCollector(tt *testing.T) { promPlugin: fakePlugin, } interval := time.Duration(42) - expectedQuery := `scalar(sum(rate(a_valid_one{host=~"foo.bar.baz"}[1m])))` for _, testcase := range []struct { - msg string - config *MetricConfig - shouldWork bool + msg string + config *MetricConfig + expectedQuery string + shouldWork bool }{ - {"No hostname config", &MetricConfig{Config: make(map[string]string)}, false}, - {"Nil metric config", nil, false}, - {"Valid hostname no prom query config", &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz"}}, true}, - {"Valid hostname with prom query config", &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz", "query": "some_other_query"}}, true}, + { + "No hostname config", + &MetricConfig{Config: make(map[string]string)}, + "", + false, + }, + { + "Nil metric config", + nil, + "", + false, + }, + { + "Valid hostname no prom query config", + &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz"}}, + `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz"}[1m])))`, + true, + }, + { + "Multiple valid hostnames no prom query config", + &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz,foz.bax.bas"}}, + `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz|foz_bax_bas"}[1m])))`, + true, + }, + { + "Valid hostname with prom query config", + &MetricConfig{ + Config: map[string]string{"hostname": "foo.bar.baz", "query": "some_other_query"}, + }, + `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz"}[1m])))`, + true, + }, } { tt.Run(testcase.msg, func(t *testing.T) { c, err := plugin.NewCollector( @@ -69,7 +97,7 @@ func TestHostnamePluginNewCollector(tt *testing.T) { if testcase.shouldWork { require.NotNil(t, c) require.Nil(t, err) - require.Equal(t, fakePlugin.config["query"], expectedQuery) + require.Equal(t, testcase.expectedQuery, fakePlugin.config["query"]) } else { require.Nil(t, c) require.NotNil(t, err) @@ -123,7 +151,7 @@ func TestHostnameCollectorGetMetrics(tt *testing.T) { require.Nil(t, err) require.NotNil(t, m) require.Len(t, m, 1) - require.Equal(t, m[0].External.Value, expectedMetric) + require.Equal(t, expectedMetric, m[0].External.Value) } else { require.NotNil(t, err) require.Nil(t, m) @@ -151,7 +179,7 @@ func TestHostnameCollectorInterval(t *testing.T) { } func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { - expectedQuery := `scalar(sum(rate(a_metric{host=~"just.testing.com"}[1m])))` + expectedQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])))` hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -189,12 +217,12 @@ func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { require.NoError(t, err) _, ok := c.(*HostnameCollector) require.True(t, ok) - require.Equal(t, fakePlugin.config["query"], expectedQuery) + require.Equal(t, expectedQuery, fakePlugin.config["query"]) } func TestHostnamePrometheusCollectorInteraction(t *testing.T) { - hostnameQuery := `scalar(sum(rate(a_metric{host=~"just.testing.com"}[1m])))` + hostnameQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])))` promQuery := "sum(rate(rps[1m]))" hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ @@ -256,6 +284,6 @@ func TestHostnamePrometheusCollectorInteraction(t *testing.T) { hostnameProm, ok := hostname.promCollector.(*PrometheusCollector) require.True(t, ok) - require.Equal(t, prom.query, promQuery) - require.Equal(t, hostnameProm.query, hostnameQuery) + require.Equal(t, promQuery, prom.query) + require.Equal(t, hostnameQuery, hostnameProm.query) } From 153d75435356fa2cc867159b4681cf66da57e99e Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Wed, 3 May 2023 20:49:14 +0200 Subject: [PATCH 05/16] Add support for traffic weight specification Signed-off-by: Lucas Thiesen --- pkg/collector/hostname_collector.go | 33 +++++++++++++++++++----- pkg/collector/hostname_collector_test.go | 32 ++++++++++++++--------- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index d2d5ae24..ab4b1823 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -3,6 +3,7 @@ package collector import ( "fmt" "regexp" + "strconv" "strings" "time" @@ -11,7 +12,7 @@ import ( const ( HostnameMetricType = "hostname-rps" - HostnameRPSQuery = `scalar(sum(rate(%s{host=~"%s"}[1m])))` + HostnameRPSQuery = `scalar(sum(rate(%s{host=~"%s"}[1m])) * %.4f)` ) type HostnameCollectorPlugin struct { @@ -51,19 +52,37 @@ func (p *HostnameCollectorPlugin) NewCollector( // RPS data from a specific hostname from prometheus. The idea // of the copy is to not modify the original config struct. confCopy := *config - hostnames := config.Config["hostname"] + //hostnames := config.Config["hostnames"] + //weights := config.Config["weights"] - if ok, err := regexp.MatchString("^[a-zA-Z0-9.,-]+$", hostnames); !ok || err != nil { - return nil, fmt.Errorf( - "hostname is not specified or invalid format, unable to create collector", - ) + if _, ok := config.Config["hostnames"]; !ok { + return nil, fmt.Errorf("hostname is not specified, unable to create collector") + } + hostnames := strings.Split(config.Config["hostnames"], ",") + for _, h := range hostnames { + if ok, err := regexp.MatchString("^[a-zA-Z0-9.-]+$", h); !ok || err != nil { + return nil, fmt.Errorf( + "Invalid hostname format, unable to create collector: %s", + h, + ) + } + } + + weight := 1.0 + if w, ok := config.Config["weight"]; ok { + num, err := strconv.ParseFloat(w, 64) + if err != nil { + return nil, fmt.Errorf("Could not parse weight annotation, unable to create collector: %s", w) + } + weight = num / 100.0 } confCopy.Config = map[string]string{ "query": fmt.Sprintf( HostnameRPSQuery, p.metricName, - strings.Replace(strings.Replace(hostnames, ",", "|", -1), ".", "_", -1), + strings.Replace(strings.Join(hostnames, "|"), ".", "_", -1), + weight, ), } diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/hostname_collector_test.go index 91a9b6a5..6f7bf7be 100644 --- a/pkg/collector/hostname_collector_test.go +++ b/pkg/collector/hostname_collector_test.go @@ -68,22 +68,28 @@ func TestHostnamePluginNewCollector(tt *testing.T) { }, { "Valid hostname no prom query config", - &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz"}}, - `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz"}[1m])))`, + &MetricConfig{Config: map[string]string{"hostnames": "foo.bar.baz"}}, + `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz"}[1m])) * 1.0000)`, + true, + }, + { + "Valid hostname no prom query config", + &MetricConfig{Config: map[string]string{"hostnames": "foo.bar.baz", "weight": "42"}}, + `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz"}[1m])) * 0.4200)`, true, }, { "Multiple valid hostnames no prom query config", - &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz,foz.bax.bas"}}, - `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz|foz_bax_bas"}[1m])))`, + &MetricConfig{Config: map[string]string{"hostnames": "foo.bar.baz,foz.bax.bas"}}, + `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz|foz_bax_bas"}[1m])) * 1.0000)`, true, }, { "Valid hostname with prom query config", &MetricConfig{ - Config: map[string]string{"hostname": "foo.bar.baz", "query": "some_other_query"}, + Config: map[string]string{"hostnames": "foo.bar.baz", "query": "some_other_query"}, }, - `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz"}[1m])))`, + `scalar(sum(rate(a_valid_one{host=~"foo_bar_baz"}[1m])) * 1.0000)`, true, }, } { @@ -169,7 +175,7 @@ func TestHostnameCollectorInterval(t *testing.T) { } c, err := plugin.NewCollector( &autoscalingv2.HorizontalPodAutoscaler{}, - &MetricConfig{Config: map[string]string{"hostname": "foo.bar.baz"}}, + &MetricConfig{Config: map[string]string{"hostnames": "foo.bar.baz"}}, interval, ) @@ -179,11 +185,12 @@ func TestHostnameCollectorInterval(t *testing.T) { } func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { - expectedQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])))` + expectedQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])) * 0.4200)` hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "metric-config.external.foo.hostname-rps/hostname": "just.testing.com", + "metric-config.external.foo.hostname-rps/hostnames": "just.testing.com", + "metric-config.external.foo.hostname-rps/weight": "42", }, }, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ @@ -222,13 +229,14 @@ func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { } func TestHostnamePrometheusCollectorInteraction(t *testing.T) { - hostnameQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])))` + hostnameQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])) * 0.4200)` promQuery := "sum(rate(rps[1m]))" hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "metric-config.external.foo.hostname-rps/hostname": "just.testing.com", - "metric-config.external.bar.prometheus/query": promQuery, + "metric-config.external.foo.hostname-rps/hostnames": "just.testing.com", + "metric-config.external.foo.hostname-rps/weight": "42", + "metric-config.external.bar.prometheus/query": promQuery, }, }, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ From 2f5d3f5a4203cc72a2fae19fabc0a3389ada5ec3 Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Wed, 3 May 2023 20:54:08 +0200 Subject: [PATCH 06/16] Change hostname collector type to requests-per-second Signed-off-by: Lucas Thiesen --- pkg/collector/hostname_collector.go | 2 +- pkg/collector/hostname_collector_test.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index ab4b1823..b2d5937f 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -11,7 +11,7 @@ import ( ) const ( - HostnameMetricType = "hostname-rps" + HostnameMetricType = "requests-per-second" HostnameRPSQuery = `scalar(sum(rate(%s{host=~"%s"}[1m])) * %.4f)` ) diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/hostname_collector_test.go index 6f7bf7be..06e76141 100644 --- a/pkg/collector/hostname_collector_test.go +++ b/pkg/collector/hostname_collector_test.go @@ -189,8 +189,8 @@ func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "metric-config.external.foo.hostname-rps/hostnames": "just.testing.com", - "metric-config.external.foo.hostname-rps/weight": "42", + "metric-config.external.foo.requests-per-second/hostnames": "just.testing.com", + "metric-config.external.foo.requests-per-second/weight": "42", }, }, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ @@ -201,7 +201,7 @@ func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { Metric: autoscalingv2.MetricIdentifier{ Name: "foo", Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"type": "hostname-rps"}, + MatchLabels: map[string]string{"type": "requests-per-second"}, }, }, }, @@ -234,9 +234,9 @@ func TestHostnamePrometheusCollectorInteraction(t *testing.T) { hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "metric-config.external.foo.hostname-rps/hostnames": "just.testing.com", - "metric-config.external.foo.hostname-rps/weight": "42", - "metric-config.external.bar.prometheus/query": promQuery, + "metric-config.external.foo.requests-per-second/hostnames": "just.testing.com", + "metric-config.external.foo.requests-per-second/weight": "42", + "metric-config.external.bar.prometheus/query": promQuery, }, }, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ @@ -247,7 +247,7 @@ func TestHostnamePrometheusCollectorInteraction(t *testing.T) { Metric: autoscalingv2.MetricIdentifier{ Name: "foo", Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"type": "hostname-rps"}, + MatchLabels: map[string]string{"type": "requests-per-second"}, }, }, }, From f28653de7441f94e787ad7caa673e43c148c78ab Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Thu, 4 May 2023 16:59:00 +0200 Subject: [PATCH 07/16] Use regexp.Compile to avoid performance problems Signed-off-by: Lucas Thiesen --- pkg/collector/hostname_collector.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index b2d5937f..86a62ec2 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -30,7 +30,7 @@ func NewHostnameCollectorPlugin( metricName string, ) (*HostnameCollectorPlugin, error) { if metricName == "" { - return nil, fmt.Errorf("failed to initialize hostname collector plugin, metric name was not defined") + return nil, fmt.Errorf("Failed to initialize hostname collector plugin, metric name was not defined") } return &HostnameCollectorPlugin{ @@ -52,15 +52,17 @@ func (p *HostnameCollectorPlugin) NewCollector( // RPS data from a specific hostname from prometheus. The idea // of the copy is to not modify the original config struct. confCopy := *config - //hostnames := config.Config["hostnames"] - //weights := config.Config["weights"] if _, ok := config.Config["hostnames"]; !ok { - return nil, fmt.Errorf("hostname is not specified, unable to create collector") + return nil, fmt.Errorf("Hostname is not specified, unable to create collector") + } + regex, err := regexp.Compile("^[a-zA-Z0-9.-]+$") + if err != nil { + return nil, fmt.Errorf("Failed to create regular expression to match hostname format") } hostnames := strings.Split(config.Config["hostnames"], ",") for _, h := range hostnames { - if ok, err := regexp.MatchString("^[a-zA-Z0-9.-]+$", h); !ok || err != nil { + if ok := regex.MatchString(h); !ok { return nil, fmt.Errorf( "Invalid hostname format, unable to create collector: %s", h, @@ -105,7 +107,7 @@ func (c *HostnameCollector) GetMetrics() ([]CollectedMetric, error) { } if len(v) != 1 { - return nil, fmt.Errorf("expected to only get one metric value, got %d", len(v)) + return nil, fmt.Errorf("Expected to only get one metric value, got %d", len(v)) } return v, nil } From b8532b756b45e42a19e0659a026dee76a8ffc95f Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Fri, 5 May 2023 12:05:09 +0200 Subject: [PATCH 08/16] Update HPA to autoscaling/v2 in new collectors Reference links: https://github.com/zalando-incubator/kube-metrics-adapter/pull/551 Signed-off-by: Lucas Thiesen --- pkg/collector/fake_collector_test.go | 3 ++- pkg/collector/hostname_collector.go | 3 ++- pkg/collector/hostname_collector_test.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/collector/fake_collector_test.go b/pkg/collector/fake_collector_test.go index 1077c6ca..4d6d91f3 100644 --- a/pkg/collector/fake_collector_test.go +++ b/pkg/collector/fake_collector_test.go @@ -3,7 +3,8 @@ package collector import ( "time" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + //autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/metrics/pkg/apis/custom_metrics" ) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index 86a62ec2..a1aa7628 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -7,7 +7,8 @@ import ( "strings" "time" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + //autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" ) const ( diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/hostname_collector_test.go index 06e76141..6aa62163 100644 --- a/pkg/collector/hostname_collector_test.go +++ b/pkg/collector/hostname_collector_test.go @@ -6,7 +6,8 @@ import ( "time" "github.com/stretchr/testify/require" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + //autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/metrics/pkg/apis/external_metrics" From d6a33fed63d1352a5f52637c8dc2f5e683d80aaf Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Fri, 5 May 2023 15:19:21 +0200 Subject: [PATCH 09/16] Update documentation for multi hostnames per metric and weight param Signed-off-by: Lucas Thiesen --- README.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c30e13dd..3cbd4b17 100644 --- a/README.md +++ b/README.md @@ -414,11 +414,11 @@ box so users don't have to define those manually. | Metric | Description | Type | Kind | K8s Versions | | ------------ | -------------- | ------- | -- | -- | -| `hostname-rps` | Scale based on requests per second for a certain hostname. | External | | `>=1.12` | +| `requests-per-second` | Scale based on requests per second for a certain hostname. | External | | `>=1.12` | ### Example: External Metric -This is an example of an HPA that will scale based on `hostname-rps` for the RPS measured in the hostname called `www.example.com`. +This is an example of an HPA that will scale based on `requests-per-second` for the RPS measured in the hostnames called: `www.example1.com` and `www.example2.com`; and weighted by 42%. ```yaml apiVersion: autoscaling/v2beta2 @@ -426,7 +426,8 @@ kind: HorizontalPodAutoscaler metadata: name: myapp-hpa annotations: - metric-config.external.example-rps.hostname-rps/hostname: www.example.com + metric-config.external.example-rps.requests-per-second/hostname: www.example1.com,www.example2.com + metric-config.external.example-rps.requests-per-second/weight: 42 spec: scaleTargetRef: apiVersion: apps/v1 @@ -441,11 +442,23 @@ spec: name: example-rps selector: matchLabels: - type: hostname-rps + type: requests-per-second target: type: AverageValue averageValue: "42" ``` +### Multiple hostnames per metric + +This metric supports a relation of n:1 with metrics. The way it works is the measured RPS is the sum of the RPS rate of each of the specified hostnames. This value is further modified by the weight parameter explained bellow. + +### Metric weighting based on backend + +There are ingress-controllers, like skipper-ingress, that supports sending traffic to different backends based on some kind of configuration, in case of skipper annotations +present on the `Ingress` object, or weights on the RouteGroup backends. By +default the number of replicas will be calculated based on the full traffic +served by these components. If however only the traffic being routed to +a specific hostname should be used then the weight for the configured hostname(s) might be specified via the `weight` annotation `metric-config.external..request-per-second/weight` for the metric being configured. + ## InfluxDB collector From cd986058e496961d6050168c54203aa246292013 Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Mon, 8 May 2023 14:35:32 +0200 Subject: [PATCH 10/16] Delete unwanted comments Signed-off-by: Lucas Thiesen --- pkg/collector/fake_collector_test.go | 1 - pkg/collector/hostname_collector.go | 1 - pkg/collector/hostname_collector_test.go | 1 - 3 files changed, 3 deletions(-) diff --git a/pkg/collector/fake_collector_test.go b/pkg/collector/fake_collector_test.go index 4d6d91f3..7b22719d 100644 --- a/pkg/collector/fake_collector_test.go +++ b/pkg/collector/fake_collector_test.go @@ -3,7 +3,6 @@ package collector import ( "time" - //autoscalingv2 "k8s.io/api/autoscaling/v2beta2" autoscalingv2 "k8s.io/api/autoscaling/v2" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/metrics/pkg/apis/custom_metrics" diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index a1aa7628..20ace385 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -7,7 +7,6 @@ import ( "strings" "time" - //autoscalingv2 "k8s.io/api/autoscaling/v2beta2" autoscalingv2 "k8s.io/api/autoscaling/v2" ) diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/hostname_collector_test.go index 6aa62163..c9a4d5e9 100644 --- a/pkg/collector/hostname_collector_test.go +++ b/pkg/collector/hostname_collector_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/stretchr/testify/require" - //autoscalingv2 "k8s.io/api/autoscaling/v2beta2" autoscalingv2 "k8s.io/api/autoscaling/v2" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" From ff6d479f1a7291a38e9f7b17deccd993dbcaaec8 Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Mon, 8 May 2023 16:16:23 +0200 Subject: [PATCH 11/16] Compile regex at plugin level for time optimization Signed-off-by: Lucas Thiesen --- pkg/collector/hostname_collector.go | 17 ++++++++++++----- pkg/collector/hostname_collector_test.go | 10 +++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index 20ace385..d52f8152 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -18,6 +18,7 @@ const ( type HostnameCollectorPlugin struct { metricName string promPlugin CollectorPlugin + pattern *regexp.Regexp } type HostnameCollector struct { @@ -33,9 +34,15 @@ func NewHostnameCollectorPlugin( return nil, fmt.Errorf("Failed to initialize hostname collector plugin, metric name was not defined") } + p, err := regexp.Compile("^[a-zA-Z0-9.-]+$") + if err != nil { + return nil, fmt.Errorf("Failed to create regular expression to match hostname format") + } + return &HostnameCollectorPlugin{ metricName: metricName, promPlugin: promPlugin, + pattern: p, }, nil } @@ -56,13 +63,13 @@ func (p *HostnameCollectorPlugin) NewCollector( if _, ok := config.Config["hostnames"]; !ok { return nil, fmt.Errorf("Hostname is not specified, unable to create collector") } - regex, err := regexp.Compile("^[a-zA-Z0-9.-]+$") - if err != nil { - return nil, fmt.Errorf("Failed to create regular expression to match hostname format") - } + hostnames := strings.Split(config.Config["hostnames"], ",") + if p.pattern == nil { + return nil, fmt.Errorf("Plugin did not specify hostname regex pattern, unable to create collector") + } for _, h := range hostnames { - if ok := regex.MatchString(h); !ok { + if ok := p.pattern.MatchString(h); !ok { return nil, fmt.Errorf( "Invalid hostname format, unable to create collector: %s", h, diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/hostname_collector_test.go index c9a4d5e9..c26ace7a 100644 --- a/pkg/collector/hostname_collector_test.go +++ b/pkg/collector/hostname_collector_test.go @@ -2,6 +2,7 @@ package collector import ( "fmt" + "regexp" "testing" "time" @@ -42,9 +43,13 @@ func TestHostnameCollectorPluginConstructor(tt *testing.T) { func TestHostnamePluginNewCollector(tt *testing.T) { fakePlugin := &FakeCollectorPlugin{} + pattern, err := regexp.Compile("^[a-zA-Z0-9.-]+$") + require.Nil(tt, err, "Something is up, regex compiling failed.") + plugin := &HostnameCollectorPlugin{ metricName: "a_valid_one", promPlugin: fakePlugin, + pattern: pattern, } interval := time.Duration(42) @@ -169,9 +174,12 @@ func TestHostnameCollectorGetMetrics(tt *testing.T) { func TestHostnameCollectorInterval(t *testing.T) { interval := time.Duration(42) fakePlugin := &FakeCollectorPlugin{} + pattern, err := regexp.Compile("^[a-zA-Z0-9.-]+$") + require.Nil(t, err, "Something is up, regex compiling failed.") plugin := &HostnameCollectorPlugin{ metricName: "a_valid_one", promPlugin: fakePlugin, + pattern: pattern, } c, err := plugin.NewCollector( &autoscalingv2.HorizontalPodAutoscaler{}, @@ -179,8 +187,8 @@ func TestHostnameCollectorInterval(t *testing.T) { interval, ) - require.NotNil(t, c) require.Nil(t, err) + require.NotNil(t, c) require.Equal(t, interval, c.Interval()) } From a276b64576eeef5a0c67714235db7bc57055bde9 Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Mon, 15 May 2023 22:39:33 +0200 Subject: [PATCH 12/16] Fix error string warnings Signed-off-by: Lucas Thiesen --- pkg/collector/hostname_collector.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/hostname_collector.go index d52f8152..263a7fc8 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/hostname_collector.go @@ -31,12 +31,12 @@ func NewHostnameCollectorPlugin( metricName string, ) (*HostnameCollectorPlugin, error) { if metricName == "" { - return nil, fmt.Errorf("Failed to initialize hostname collector plugin, metric name was not defined") + return nil, fmt.Errorf("failed to initialize hostname collector plugin, metric name was not defined") } p, err := regexp.Compile("^[a-zA-Z0-9.-]+$") if err != nil { - return nil, fmt.Errorf("Failed to create regular expression to match hostname format") + return nil, fmt.Errorf("failed to create regular expression to match hostname format") } return &HostnameCollectorPlugin{ @@ -53,7 +53,7 @@ func (p *HostnameCollectorPlugin) NewCollector( interval time.Duration, ) (Collector, error) { if config == nil { - return nil, fmt.Errorf("Metric config not present, it is not possible to initialize the collector.") + return nil, fmt.Errorf("metric config not present, it is not possible to initialize the collector") } // Need to copy config and add a promQL query in order to get // RPS data from a specific hostname from prometheus. The idea @@ -66,12 +66,12 @@ func (p *HostnameCollectorPlugin) NewCollector( hostnames := strings.Split(config.Config["hostnames"], ",") if p.pattern == nil { - return nil, fmt.Errorf("Plugin did not specify hostname regex pattern, unable to create collector") + return nil, fmt.Errorf("plugin did not specify hostname regex pattern, unable to create collector") } for _, h := range hostnames { if ok := p.pattern.MatchString(h); !ok { return nil, fmt.Errorf( - "Invalid hostname format, unable to create collector: %s", + "invalid hostname format, unable to create collector: %s", h, ) } @@ -81,16 +81,18 @@ func (p *HostnameCollectorPlugin) NewCollector( if w, ok := config.Config["weight"]; ok { num, err := strconv.ParseFloat(w, 64) if err != nil { - return nil, fmt.Errorf("Could not parse weight annotation, unable to create collector: %s", w) + return nil, fmt.Errorf("could not parse weight annotation, unable to create collector: %s", w) } weight = num / 100.0 } + + confCopy.Config = map[string]string{ "query": fmt.Sprintf( HostnameRPSQuery, p.metricName, - strings.Replace(strings.Join(hostnames, "|"), ".", "_", -1), + strings.ReplaceAll(strings.Join(hostnames, "|"), ".", "_"), weight, ), } @@ -114,7 +116,7 @@ func (c *HostnameCollector) GetMetrics() ([]CollectedMetric, error) { } if len(v) != 1 { - return nil, fmt.Errorf("Expected to only get one metric value, got %d", len(v)) + return nil, fmt.Errorf("expected to only get one metric value, got %d", len(v)) } return v, nil } @@ -123,3 +125,4 @@ func (c *HostnameCollector) GetMetrics() ([]CollectedMetric, error) { func (c *HostnameCollector) Interval() time.Duration { return c.interval } + From b89ca19e6a812d31de2dda366faaa8946d2491ce Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Tue, 23 May 2023 15:40:56 +0200 Subject: [PATCH 13/16] Rename collector to external RPS Signed-off-by: Lucas Thiesen --- ...collector.go => external_rps_collector.go} | 24 ++++++------- ...test.go => external_rps_collector_test.go} | 36 +++++++++---------- pkg/server/start.go | 22 ++++++------ 3 files changed, 41 insertions(+), 41 deletions(-) rename pkg/collector/{hostname_collector.go => external_rps_collector.go} (83%) rename pkg/collector/{hostname_collector_test.go => external_rps_collector_test.go} (86%) diff --git a/pkg/collector/hostname_collector.go b/pkg/collector/external_rps_collector.go similarity index 83% rename from pkg/collector/hostname_collector.go rename to pkg/collector/external_rps_collector.go index 263a7fc8..4d2d17ed 100644 --- a/pkg/collector/hostname_collector.go +++ b/pkg/collector/external_rps_collector.go @@ -11,25 +11,25 @@ import ( ) const ( - HostnameMetricType = "requests-per-second" - HostnameRPSQuery = `scalar(sum(rate(%s{host=~"%s"}[1m])) * %.4f)` + ExternalRPSMetricType = "requests-per-second" + ExternalRPSQuery = `scalar(sum(rate(%s{host=~"%s"}[1m])) * %.4f)` ) -type HostnameCollectorPlugin struct { +type ExternalRPSCollectorPlugin struct { metricName string promPlugin CollectorPlugin pattern *regexp.Regexp } -type HostnameCollector struct { +type ExternalRPSCollector struct { interval time.Duration promCollector Collector } -func NewHostnameCollectorPlugin( +func NewExternalRPSCollectorPlugin( promPlugin CollectorPlugin, metricName string, -) (*HostnameCollectorPlugin, error) { +) (*ExternalRPSCollectorPlugin, error) { if metricName == "" { return nil, fmt.Errorf("failed to initialize hostname collector plugin, metric name was not defined") } @@ -39,7 +39,7 @@ func NewHostnameCollectorPlugin( return nil, fmt.Errorf("failed to create regular expression to match hostname format") } - return &HostnameCollectorPlugin{ + return &ExternalRPSCollectorPlugin{ metricName: metricName, promPlugin: promPlugin, pattern: p, @@ -47,7 +47,7 @@ func NewHostnameCollectorPlugin( } // NewCollector initializes a new skipper collector from the specified HPA. -func (p *HostnameCollectorPlugin) NewCollector( +func (p *ExternalRPSCollectorPlugin) NewCollector( hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration, @@ -90,7 +90,7 @@ func (p *HostnameCollectorPlugin) NewCollector( confCopy.Config = map[string]string{ "query": fmt.Sprintf( - HostnameRPSQuery, + ExternalRPSQuery, p.metricName, strings.ReplaceAll(strings.Join(hostnames, "|"), ".", "_"), weight, @@ -102,14 +102,14 @@ func (p *HostnameCollectorPlugin) NewCollector( return nil, err } - return &HostnameCollector{ + return &ExternalRPSCollector{ interval: interval, promCollector: c, }, nil } // GetMetrics gets hostname metrics from Prometheus -func (c *HostnameCollector) GetMetrics() ([]CollectedMetric, error) { +func (c *ExternalRPSCollector) GetMetrics() ([]CollectedMetric, error) { v, err := c.promCollector.GetMetrics() if err != nil { return nil, err @@ -122,7 +122,7 @@ func (c *HostnameCollector) GetMetrics() ([]CollectedMetric, error) { } // Interval returns the interval at which the collector should run. -func (c *HostnameCollector) Interval() time.Duration { +func (c *ExternalRPSCollector) Interval() time.Duration { return c.interval } diff --git a/pkg/collector/hostname_collector_test.go b/pkg/collector/external_rps_collector_test.go similarity index 86% rename from pkg/collector/hostname_collector_test.go rename to pkg/collector/external_rps_collector_test.go index c26ace7a..afaf3540 100644 --- a/pkg/collector/hostname_collector_test.go +++ b/pkg/collector/external_rps_collector_test.go @@ -13,7 +13,7 @@ import ( "k8s.io/metrics/pkg/apis/external_metrics" ) -func TestHostnameCollectorPluginConstructor(tt *testing.T) { +func TestExternalRPSCollectorPluginConstructor(tt *testing.T) { for _, testcase := range []struct { msg string name string @@ -25,7 +25,7 @@ func TestHostnameCollectorPluginConstructor(tt *testing.T) { tt.Run(testcase.msg, func(t *testing.T) { fakePlugin := &FakeCollectorPlugin{} - plugin, err := NewHostnameCollectorPlugin(fakePlugin, testcase.name) + plugin, err := NewExternalRPSCollectorPlugin(fakePlugin, testcase.name) if testcase.isValid { require.NoError(t, err) @@ -40,13 +40,13 @@ func TestHostnameCollectorPluginConstructor(tt *testing.T) { } } -func TestHostnamePluginNewCollector(tt *testing.T) { +func TestExternalRPSPluginNewCollector(tt *testing.T) { fakePlugin := &FakeCollectorPlugin{} pattern, err := regexp.Compile("^[a-zA-Z0-9.-]+$") require.Nil(tt, err, "Something is up, regex compiling failed.") - plugin := &HostnameCollectorPlugin{ + plugin := &ExternalRPSCollectorPlugin{ metricName: "a_valid_one", promPlugin: fakePlugin, pattern: pattern, @@ -117,7 +117,7 @@ func TestHostnamePluginNewCollector(tt *testing.T) { } } -func TestHostnameCollectorGetMetrics(tt *testing.T) { +func TestExternalRPSCollectorGetMetrics(tt *testing.T) { genericErr := fmt.Errorf("This is an error") expectedMetric := *resource.NewQuantity(int64(42), resource.DecimalSI) @@ -155,7 +155,7 @@ func TestHostnameCollectorGetMetrics(tt *testing.T) { } { tt.Run(testcase.msg, func(t *testing.T) { fake := makeCollectorWithStub(testcase.stub) - c := &HostnameCollector{promCollector: fake} + c := &ExternalRPSCollector{promCollector: fake} m, err := c.GetMetrics() if testcase.shouldWork { @@ -171,12 +171,12 @@ func TestHostnameCollectorGetMetrics(tt *testing.T) { } } -func TestHostnameCollectorInterval(t *testing.T) { +func TestExternalRPSCollectorInterval(t *testing.T) { interval := time.Duration(42) fakePlugin := &FakeCollectorPlugin{} pattern, err := regexp.Compile("^[a-zA-Z0-9.-]+$") require.Nil(t, err, "Something is up, regex compiling failed.") - plugin := &HostnameCollectorPlugin{ + plugin := &ExternalRPSCollectorPlugin{ metricName: "a_valid_one", promPlugin: fakePlugin, pattern: pattern, @@ -192,7 +192,7 @@ func TestHostnameCollectorInterval(t *testing.T) { require.Equal(t, interval, c.Interval()) } -func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { +func TestExternalRPSCollectorAndCollectorFabricInteraction(t *testing.T) { expectedQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])) * 0.4200)` hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ @@ -220,9 +220,9 @@ func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { factory := NewCollectorFactory() fakePlugin := makePlugin(42) - hostnamePlugin, err := NewHostnameCollectorPlugin(fakePlugin, "a_metric") + hostnamePlugin, err := NewExternalRPSCollectorPlugin(fakePlugin, "a_metric") require.NoError(t, err) - factory.RegisterExternalCollector([]string{HostnameMetricType}, hostnamePlugin) + factory.RegisterExternalCollector([]string{ExternalRPSMetricType}, hostnamePlugin) conf, err := ParseHPAMetrics(hpa) require.NoError(t, err) require.Len(t, conf, 1) @@ -230,14 +230,14 @@ func TestHostnameCollectorAndCollectorFabricInteraction(t *testing.T) { c, err := factory.NewCollector(hpa, conf[0], 0) require.NoError(t, err) - _, ok := c.(*HostnameCollector) + _, ok := c.(*ExternalRPSCollector) require.True(t, ok) require.Equal(t, expectedQuery, fakePlugin.config["query"]) } -func TestHostnamePrometheusCollectorInteraction(t *testing.T) { - hostnameQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])) * 0.4200)` +func TestExternalRPSPrometheusCollectorInteraction(t *testing.T) { + externalRPSQuery := `scalar(sum(rate(a_metric{host=~"just_testing_com"}[1m])) * 0.4200)` promQuery := "sum(rate(rps[1m]))" hpa := &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ @@ -279,9 +279,9 @@ func TestHostnamePrometheusCollectorInteraction(t *testing.T) { promPlugin, err := NewPrometheusCollectorPlugin(nil, "http://prometheus") require.NoError(t, err) factory.RegisterExternalCollector([]string{PrometheusMetricType, PrometheusMetricNameLegacy}, promPlugin) - hostnamePlugin, err := NewHostnameCollectorPlugin(promPlugin, "a_metric") + hostnamePlugin, err := NewExternalRPSCollectorPlugin(promPlugin, "a_metric") require.NoError(t, err) - factory.RegisterExternalCollector([]string{HostnameMetricType}, hostnamePlugin) + factory.RegisterExternalCollector([]string{ExternalRPSMetricType}, hostnamePlugin) conf, err := ParseHPAMetrics(hpa) require.NoError(t, err) @@ -295,11 +295,11 @@ func TestHostnamePrometheusCollectorInteraction(t *testing.T) { prom, ok := collectors["prom"].(*PrometheusCollector) require.True(t, ok) - hostname, ok := collectors["hostname"].(*HostnameCollector) + hostname, ok := collectors["hostname"].(*ExternalRPSCollector) require.True(t, ok) hostnameProm, ok := hostname.promCollector.(*PrometheusCollector) require.True(t, ok) require.Equal(t, promQuery, prom.query) - require.Equal(t, hostnameQuery, hostnameProm.query) + require.Equal(t, externalRPSQuery, hostnameProm.query) } diff --git a/pkg/server/start.go b/pkg/server/start.go index 90da3a29..4ab86afb 100644 --- a/pkg/server/start.go +++ b/pkg/server/start.go @@ -65,7 +65,7 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command { MetricsAddress: ":7979", ZMONTokenName: "zmon", CredentialsDir: "/meta/credentials", - HostnameRPSMetricName: "skipper_serve_host_duration_seconds_count", + ExternalRPSMetricName: "skipper_serve_host_duration_seconds_count", } cmd := &cobra.Command{ @@ -133,9 +133,9 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command { flags.DurationVar(&o.DefaultScheduledScalingWindow, "scaling-schedule-default-scaling-window", 10*time.Minute, "Default rampup and rampdown window duration for ScalingSchedules") flags.IntVar(&o.RampSteps, "scaling-schedule-ramp-steps", 10, "Number of steps used to rampup and rampdown ScalingSchedules. It's used to guarantee won't avoid reaching the max scaling due to the 10% minimum change rule.") flags.StringVar(&o.DefaultTimeZone, "scaling-schedule-default-time-zone", "Europe/Berlin", "Default time zone to use for ScalingSchedules.") - flags.StringVar(&o.HostnameRPSMetricName, "hostname-rps-metric-name", o.HostnameRPSMetricName, ""+ + flags.StringVar(&o.ExternalRPSMetricName, "hostname-rps-metric-name", o.ExternalRPSMetricName, ""+ "The name of the metric that should be used to query prometheus for RPS per hostname.") - flags.BoolVar(&o.HostnameRPSMetrics, "hostname-rps-metrics", o.HostnameRPSMetrics, ""+ + flags.BoolVar(&o.ExternalRPSMetrics, "hostname-rps-metrics", o.ExternalRPSMetrics, ""+ "whether to enable hostname RPS metric collector or not") return cmd } @@ -224,13 +224,13 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct } } - // Hostname collector, like skipper's, depends on prometheus being enabled. + // External RPS collector, like skipper's, depends on prometheus being enabled. // Also, to enable hostname metric its necessary to pass the metric name that // will be used. This was built this way so we can support hostname metrics to // any ingress provider, e.g. Skipper, Nginx, envoy etc, in a simple way. - if o.HostnameRPSMetrics && o.HostnameRPSMetricName != "" { - hostnamePlugin, err := collector.NewHostnameCollectorPlugin(promPlugin, o.HostnameRPSMetricName) - collectorFactory.RegisterExternalCollector([]string{collector.HostnameMetricType}, hostnamePlugin) + if o.ExternalRPSMetrics && o.ExternalRPSMetricName != "" { + externalRPSPlugin, err := collector.NewExternalRPSCollectorPlugin(promPlugin, o.ExternalRPSMetricName) + collectorFactory.RegisterExternalCollector([]string{collector.ExternalRPSMetricType}, externalRPSPlugin) if err != nil { return fmt.Errorf("failed to register hostname collector plugin: %v", err) } @@ -462,8 +462,8 @@ type AdapterServerOptions struct { RampSteps int // Default time zone to use for ScalingSchedules. DefaultTimeZone string - // Feature flag to enable hostname rps metric collector - HostnameRPSMetrics bool - // Name of the Prometheus metric that stores RPS by hostname for Hostname RPS metrics. - HostnameRPSMetricName string + // Feature flag to enable external rps metric collector + ExternalRPSMetrics bool + // Name of the Prometheus metric that stores RPS by hostname for external RPS metrics. + ExternalRPSMetricName string } From d171e049bfbeb917022deff115bd4dc8dc80d385 Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Tue, 23 May 2023 15:55:20 +0200 Subject: [PATCH 14/16] Update documentation with new metric name Signed-off-by: Lucas Thiesen --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3cbd4b17..741ff33c 100644 --- a/README.md +++ b/README.md @@ -402,12 +402,12 @@ the `backend` label under `matchLabels` for the metric. The ingress annotation where the backend weights can be obtained can be specified through the flag `--skipper-backends-annotation`. -## Hostname RPS collector +## External RPS collector -The Hostname collector, like Skipper collector, is a simple wrapper around the Prometheus collector to +The External RPS collector, like Skipper collector, is a simple wrapper around the Prometheus collector to make it easy to define an HPA for scaling based on the RPS measured for a given hostname. When [skipper](https://github.com/zalando/skipper) is used as the ingress -implementation in your cluster everything should work automatically, in case another reverse proxy is used as ingress, like [Nginx](https://github.com/kubernetes/ingress-nginx) for example, its necessary to configure which prometheus metric should be used through `--hostname-rps-metric-name ` flag. Assuming `skipper-ingress` is being used or the appropriate metric name is passed using the flag mentioned previously this collector provides the correct Prometheus queries out of the +implementation in your cluster everything should work automatically, in case another reverse proxy is used as ingress, like [Nginx](https://github.com/kubernetes/ingress-nginx) for example, its necessary to configure which prometheus metric should be used through `--external-rps-metric-name ` flag. Assuming `skipper-ingress` is being used or the appropriate metric name is passed using the flag mentioned previously this collector provides the correct Prometheus queries out of the box so users don't have to define those manually. ### Supported metrics @@ -421,7 +421,7 @@ box so users don't have to define those manually. This is an example of an HPA that will scale based on `requests-per-second` for the RPS measured in the hostnames called: `www.example1.com` and `www.example2.com`; and weighted by 42%. ```yaml -apiVersion: autoscaling/v2beta2 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: myapp-hpa @@ -449,7 +449,7 @@ spec: ``` ### Multiple hostnames per metric -This metric supports a relation of n:1 with metrics. The way it works is the measured RPS is the sum of the RPS rate of each of the specified hostnames. This value is further modified by the weight parameter explained bellow. +This metric supports a relation of n:1 between hostnames and metrics. The way it works is the measured RPS is the sum of the RPS rate of each of the specified hostnames. This value is further modified by the weight parameter explained bellow. ### Metric weighting based on backend From 69df60e724447e3aba4905a692eafef1d36d97ea Mon Sep 17 00:00:00 2001 From: Lucas Thiesen Date: Tue, 23 May 2023 15:58:18 +0200 Subject: [PATCH 15/16] Update flags with new metric collector name Signed-off-by: Lucas Thiesen --- pkg/server/start.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/server/start.go b/pkg/server/start.go index 4ab86afb..f49bf384 100644 --- a/pkg/server/start.go +++ b/pkg/server/start.go @@ -133,10 +133,10 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command { flags.DurationVar(&o.DefaultScheduledScalingWindow, "scaling-schedule-default-scaling-window", 10*time.Minute, "Default rampup and rampdown window duration for ScalingSchedules") flags.IntVar(&o.RampSteps, "scaling-schedule-ramp-steps", 10, "Number of steps used to rampup and rampdown ScalingSchedules. It's used to guarantee won't avoid reaching the max scaling due to the 10% minimum change rule.") flags.StringVar(&o.DefaultTimeZone, "scaling-schedule-default-time-zone", "Europe/Berlin", "Default time zone to use for ScalingSchedules.") - flags.StringVar(&o.ExternalRPSMetricName, "hostname-rps-metric-name", o.ExternalRPSMetricName, ""+ + flags.StringVar(&o.ExternalRPSMetricName, "external-rps-metric-name", o.ExternalRPSMetricName, ""+ "The name of the metric that should be used to query prometheus for RPS per hostname.") - flags.BoolVar(&o.ExternalRPSMetrics, "hostname-rps-metrics", o.ExternalRPSMetrics, ""+ - "whether to enable hostname RPS metric collector or not") + flags.BoolVar(&o.ExternalRPSMetrics, "external-rps-metrics", o.ExternalRPSMetrics, ""+ + "whether to enable external RPS metric collector or not") return cmd } From 35e3fe83e845b3d3902b7a40688958507c0b5f35 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Thu, 25 May 2023 10:14:20 +0200 Subject: [PATCH 16/16] Update README.md Signed-off-by: Mikkel Oscar Lyderik Larsen --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 741ff33c..7808ec54 100644 --- a/README.md +++ b/README.md @@ -427,7 +427,7 @@ metadata: name: myapp-hpa annotations: metric-config.external.example-rps.requests-per-second/hostname: www.example1.com,www.example2.com - metric-config.external.example-rps.requests-per-second/weight: 42 + metric-config.external.example-rps.requests-per-second/weight: "42" spec: scaleTargetRef: apiVersion: apps/v1