From 8a07b11d3aa5eaae87aff6f9ed4e48eca434433d Mon Sep 17 00:00:00 2001 From: Chirag Bhatia Date: Wed, 8 Jan 2025 21:26:56 +0530 Subject: [PATCH 01/97] feat(scaler): Add TLS support for Artemis scaler Signed-off-by: Chirag Bhatia --- pkg/scalers/artemis_scaler.go | 49 +++++++++++++++++++++++++++--- pkg/scalers/artemis_scaler_test.go | 27 ++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/pkg/scalers/artemis_scaler.go b/pkg/scalers/artemis_scaler.go index c845beae037..ec27b34a12f 100644 --- a/pkg/scalers/artemis_scaler.go +++ b/pkg/scalers/artemis_scaler.go @@ -37,6 +37,12 @@ type artemisMetadata struct { QueueLength int64 `keda:"name=queueLength, order=triggerMetadata, optional, default=10"` ActivationQueueLength int64 `keda:"name=activationQueueLength, order=triggerMetadata, optional, default=10"` CorsHeader string `keda:"name=corsHeader, order=triggerMetadata, optional"` + UnsafeSsl bool `keda:"name=unsafeSsl, order=triggerMetadata, optional, default=false"` + TLS bool `keda:"name=tls, order=triggerMetadata, optional, default=false"` + CA string `keda:"name=ca, order=triggerMetadata, optional"` + Cert string `keda:"name=cert, order=triggerMetadata, optional"` + Key string `keda:"name=key, order=triggerMetadata, optional"` + KeyPassword string `keda:"name=keyPassword, order=triggerMetadata, optional"` } //revive:enable:var-naming @@ -77,15 +83,24 @@ func (a *artemisMetadata) Validate() error { if a.CorsHeader == "" { a.CorsHeader = fmt.Sprintf(defaultCorsHeader, a.ManagementEndpoint) } + + if (a.Cert == "") != (a.Key == "") { + return fmt.Errorf("both cert and key must be provided when using TLS") + } + + if a.TLS && a.CA == "" { + return fmt.Errorf("CA certificate must be provided when using TLS") + } + + if a.TLS && a.UnsafeSsl { + return fmt.Errorf("'tls' and 'unsafeSsl' cannot both be specified") + } + return nil } // NewArtemisQueueScaler creates a new artemis queue Scaler func NewArtemisQueueScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { - // do we need to guarantee this timeout for a specific - // reason? if not, we can have buildScaler pass in - // the global client - httpClient := kedautil.CreateHTTPClient(config.GlobalHTTPTimeout, false) metricType, err := GetMetricTargetType(config) if err != nil { @@ -96,6 +111,24 @@ func NewArtemisQueueScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { if err != nil { return nil, fmt.Errorf("error parsing artemis metadata: %w", err) } + // do we need to guarantee this timeout for a specific + // reason? if not, we can have buildScaler pass in + // the global client + httpClient := kedautil.CreateHTTPClient(config.GlobalHTTPTimeout, artemisMetadata.UnsafeSsl) + + if artemisMetadata.TLS { + tlsConfig, err := kedautil.NewTLSConfigWithPassword( + artemisMetadata.Cert, + artemisMetadata.Key, + artemisMetadata.KeyPassword, + artemisMetadata.CA, + artemisMetadata.UnsafeSsl, + ) + if err != nil { + return nil, fmt.Errorf("failed to configure TLS: %w", err) + } + httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(tlsConfig) + } return &artemisScaler{ metricType: metricType, @@ -149,7 +182,13 @@ func getAPIParameters(meta artemisMetadata) (artemisMetadata, error) { } func (s *artemisScaler) getMonitoringEndpoint() string { - replacer := strings.NewReplacer("<>", s.metadata.ManagementEndpoint, + scheme := "http" + + if s.metadata.TLS { + scheme = "https" + } + replacer := strings.NewReplacer( + "<>", fmt.Sprintf("%s://%s", scheme, s.metadata.ManagementEndpoint), "<>", s.metadata.QueueName, "<>", s.metadata.BrokerName, "<>", s.metadata.BrokerAddress) diff --git a/pkg/scalers/artemis_scaler_test.go b/pkg/scalers/artemis_scaler_test.go index 4140d8d3f63..af90f9d759b 100644 --- a/pkg/scalers/artemis_scaler_test.go +++ b/pkg/scalers/artemis_scaler_test.go @@ -163,3 +163,30 @@ func TestArtemisGetMetricSpecForScaling(t *testing.T) { } } } + +func TestArtemisTLSConfiguration(t *testing.T) { + metadata := map[string]string{ + "managementEndpoint": "localhost:8161", + "queueName": "queue1", + "brokerName": "broker-activemq", + "brokerAddress": "test", + "ca": "/path/to/ca.pem", + "cert": "/path/to/cert.pem", + "key": "/path/to/key.pem", + } + + resolvedEnv := map[string]string{ + "username": "admin", + "password": "admin", + } + + _, err := parseArtemisMetadata(&scalersconfig.ScalerConfig{ + ResolvedEnv: resolvedEnv, + TriggerMetadata: metadata, + AuthParams: artemisAuthParams, // Ensure valid AuthParams are provided + }) + + if err != nil { + t.Errorf("Expected success but got error: %v", err) + } +} From 7ac7c5a96add8da0189f129777aaa5e6cfe31648 Mon Sep 17 00:00:00 2001 From: Michael Yoo Date: Wed, 4 Sep 2024 22:30:16 +0900 Subject: [PATCH 02/97] Pulsar: remove deprecated trigger name (#6092) * remove deprecated trigger name in Pulsar scaler Signed-off-by: Kun Woo Yoo * Update CHANGELOG.md Signed-off-by: Jorge Turrado Ferrero Signed-off-by: Kun Woo Yoo * remove deprecated Pulsar trigger name from test Signed-off-by: Kun Woo Yoo --------- Signed-off-by: Kun Woo Yoo Signed-off-by: Jorge Turrado Ferrero Co-authored-by: Jorge Turrado Ferrero Signed-off-by: Chirag Bhatia --- CHANGELOG.md | 2 +- pkg/scalers/pulsar_scaler.go | 19 ++++--------------- pkg/scalers/pulsar_scaler_test.go | 9 +-------- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f86988e080..2692e6e4181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,7 +91,7 @@ New deprecation(s): ### Breaking Changes -- TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX)) +- **Pulsar Scaler**: remove `msgBacklog` trigger name ([#6059](https://github.com/kedacore/keda/issues/6059)) ### Other diff --git a/pkg/scalers/pulsar_scaler.go b/pkg/scalers/pulsar_scaler.go index c9732083113..66015982bd5 100644 --- a/pkg/scalers/pulsar_scaler.go +++ b/pkg/scalers/pulsar_scaler.go @@ -43,7 +43,6 @@ type pulsarMetadata struct { } const ( - msgBacklogMetricName = "msgBacklog" pulsarMetricType = "External" defaultMsgBacklogThreshold = 10 enable = "enable" @@ -133,7 +132,7 @@ func NewPulsarScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { }, nil } -func parsePulsarMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger) (pulsarMetadata, error) { +func parsePulsarMetadata(config *scalersconfig.ScalerConfig, _ logr.Logger) (pulsarMetadata, error) { meta := pulsarMetadata{} switch { case config.TriggerMetadata["adminURLFromEnv"] != "": @@ -182,23 +181,13 @@ func parsePulsarMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger) meta.msgBacklogThreshold = defaultMsgBacklogThreshold - // FIXME: msgBacklog support DEPRECATED to be removed in v2.14 - fmt.Println(config.TriggerMetadata) - if val, ok := config.TriggerMetadata[msgBacklogMetricName]; ok { - logger.V(1).Info("\"msgBacklog\" is deprecated and will be removed in v2.14, please use \"msgBacklogThreshold\" instead") + if val, ok := config.TriggerMetadata["msgBacklogThreshold"]; ok { t, err := strconv.ParseInt(val, 10, 64) if err != nil { - return meta, fmt.Errorf("error parsing %s: %w", msgBacklogMetricName, err) - } - meta.msgBacklogThreshold = t - } else if val, ok := config.TriggerMetadata["msgBacklogThreshold"]; ok { - t, err := strconv.ParseInt(val, 10, 64) - if err != nil { - return meta, fmt.Errorf("error parsing %s: %w", msgBacklogMetricName, err) + return meta, fmt.Errorf("error parsing %s: %w", "msgBacklogThreshold", err) } meta.msgBacklogThreshold = t } - // END FIXME // For backwards compatibility, we need to map "tls: enable" to if tls, ok := config.TriggerMetadata["tls"]; ok { @@ -212,7 +201,7 @@ func parsePulsarMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger) } auth, err := authentication.GetAuthConfigs(config.TriggerMetadata, config.AuthParams) if err != nil { - return meta, fmt.Errorf("error parsing %s: %w", msgBacklogMetricName, err) + return meta, fmt.Errorf("error parsing %s: %w", "msgBacklogThreshold", err) } if auth != nil && auth.EnableOAuth { diff --git a/pkg/scalers/pulsar_scaler_test.go b/pkg/scalers/pulsar_scaler_test.go index 5c7148cf5b6..68b183488eb 100644 --- a/pkg/scalers/pulsar_scaler_test.go +++ b/pkg/scalers/pulsar_scaler_test.go @@ -156,14 +156,7 @@ func TestParsePulsarMetadata(t *testing.T) { } var testDataMsgBacklogThreshold int64 - // FIXME: msgBacklog support DEPRECATED to be removed in v2.14 - if val, ok := testData.metadata["msgBacklog"]; ok { - testDataMsgBacklogThreshold, err = strconv.ParseInt(val, 10, 64) - if err != nil { - t.Errorf("error parseing msgBacklog: %v", err) - } - // END FiXME - } else if val, ok := testData.metadata["msgBacklogThreshold"]; ok { + if val, ok := testData.metadata["msgBacklogThreshold"]; ok { testDataMsgBacklogThreshold, err = strconv.ParseInt(val, 10, 64) if err != nil { t.Errorf("error parseing msgBacklogThreshold: %v", err) From 04dd321d3eab3e1f2956ecd3b028e0993d90f567 Mon Sep 17 00:00:00 2001 From: SpiritZhou Date: Thu, 5 Sep 2024 17:58:31 +0800 Subject: [PATCH 03/97] Provide ClusterCloudEventSource around the management of TriggerAuthentication/ClusterTriggerAuthentication resources (#6131) Signed-off-by: Chirag Bhatia --- CHANGELOG.md | 3 +- apis/eventing/v1alpha1/cloudevent_types.go | 21 +++- cmd/operator/main.go | 8 +- .../eventing.keda.sh_cloudeventsources.yaml | 16 ++- ...ting.keda.sh_clustercloudeventsources.yaml | 16 ++- ...clustertriggerauthentication_controller.go | 13 +- .../keda/triggerauthentication_controller.go | 12 +- controllers/keda/util/finalizer.go | 10 +- pkg/common/message/message.go | 8 ++ pkg/eventreason/eventreason.go | 6 + tests/internals/events/events_test.go | 118 ++++++++++++++++-- 11 files changed, 195 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2692e6e4181..ae977f01872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,8 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### New - **CloudEventSource**: Introduce ClusterCloudEventSource ([#3533](https://github.com/kedacore/keda/issues/3533)) -- **CloudEventSource**: Provide CloudEvents around the management of ScaledJobs resources ([#3523](https://github.com/kedacore/keda/issues/3523)) +- **CloudEventSource**: Provide ClusterCloudEventSource around the management of ScaledJobs resources ([#3523](https://github.com/kedacore/keda/issues/3523)) +- **CloudEventSource**: Provide ClusterCloudEventSource around the management of TriggerAuthentication/ClusterTriggerAuthentication resources ([#3524](https://github.com/kedacore/keda/issues/3524)) #### Experimental diff --git a/apis/eventing/v1alpha1/cloudevent_types.go b/apis/eventing/v1alpha1/cloudevent_types.go index c3cfaecad89..c0abd97dd15 100644 --- a/apis/eventing/v1alpha1/cloudevent_types.go +++ b/apis/eventing/v1alpha1/cloudevent_types.go @@ -17,7 +17,8 @@ limitations under the License. package v1alpha1 // CloudEventType contains the list of cloudevent types -// +kubebuilder:validation:Enum=keda.scaledobject.ready.v1;keda.scaledobject.failed.v1;keda.scaledobject.removed.v1;keda.scaledjob.ready.v1;keda.scaledjob.failed.v1;keda.scaledjob.removed.v1 +// +kubebuilder:validation:Enum=keda.scaledobject.ready.v1;keda.scaledobject.failed.v1;keda.scaledobject.removed.v1;keda.scaledjob.ready.v1;keda.scaledjob.failed.v1;keda.scaledjob.removed.v1;keda.authentication.triggerauthentication.created.v1;keda.authentication.triggerauthentication.updated.v1;keda.authentication.triggerauthentication.removed.v1;keda.authentication.clustertriggerauthentication.created.v1;keda.authentication.clustertriggerauthentication.updated.v1;keda.authentication.clustertriggerauthentication.removed.v1 + type CloudEventType string const ( @@ -38,6 +39,24 @@ const ( // ScaledJobRemovedType is for event when removed ScaledJob ScaledJobRemovedType CloudEventType = "keda.scaledjob.removed.v1" + + // TriggerAuthenticationCreatedType is for event when a new TriggerAuthentication is created + TriggerAuthenticationCreatedType CloudEventType = "keda.authentication.triggerauthentication.created.v1" + + // TriggerAuthenticationUpdatedType is for event when a TriggerAuthentication is updated + TriggerAuthenticationUpdatedType CloudEventType = "keda.authentication.triggerauthentication.updated.v1" + + // TriggerAuthenticationRemovedType is for event when a TriggerAuthentication is deleted + TriggerAuthenticationRemovedType CloudEventType = "keda.authentication.triggerauthentication.removed.v1" + + // ClusterTriggerAuthenticationCreatedType is for event when a new ClusterTriggerAuthentication is created + ClusterTriggerAuthenticationCreatedType CloudEventType = "keda.authentication.clustertriggerauthentication.created.v1" + + // ClusterTriggerAuthenticationCreatedType is for event when a ClusterTriggerAuthentication is updated + ClusterTriggerAuthenticationUpdatedType CloudEventType = "keda.authentication.clustertriggerauthentication.updated.v1" + + // ClusterTriggerAuthenticationRemovedType is for event when a ClusterTriggerAuthentication is deleted + ClusterTriggerAuthenticationRemovedType CloudEventType = "keda.authentication.clustertriggerauthentication.removed.v1" ) var AllEventTypes = []CloudEventType{ diff --git a/cmd/operator/main.go b/cmd/operator/main.go index aa81dc79fce..16f6899d230 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -246,15 +246,15 @@ func main() { os.Exit(1) } if err = (&kedacontrollers.TriggerAuthenticationReconciler{ - Client: mgr.GetClient(), - EventRecorder: eventRecorder, + Client: mgr.GetClient(), + EventHandler: eventEmitter, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TriggerAuthentication") os.Exit(1) } if err = (&kedacontrollers.ClusterTriggerAuthenticationReconciler{ - Client: mgr.GetClient(), - EventRecorder: eventRecorder, + Client: mgr.GetClient(), + EventHandler: eventEmitter, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterTriggerAuthentication") os.Exit(1) diff --git a/config/crd/bases/eventing.keda.sh_cloudeventsources.yaml b/config/crd/bases/eventing.keda.sh_cloudeventsources.yaml index d121df22074..53e8bcfe504 100644 --- a/config/crd/bases/eventing.keda.sh_cloudeventsources.yaml +++ b/config/crd/bases/eventing.keda.sh_cloudeventsources.yaml @@ -83,8 +83,6 @@ spec: properties: excludedEventTypes: items: - description: CloudEventType contains the list of cloudevent - types enum: - keda.scaledobject.ready.v1 - keda.scaledobject.failed.v1 @@ -92,12 +90,16 @@ spec: - keda.scaledjob.ready.v1 - keda.scaledjob.failed.v1 - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 type: string type: array includedEventTypes: items: - description: CloudEventType contains the list of cloudevent - types enum: - keda.scaledobject.ready.v1 - keda.scaledobject.failed.v1 @@ -105,6 +107,12 @@ spec: - keda.scaledjob.ready.v1 - keda.scaledjob.failed.v1 - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 type: string type: array type: object diff --git a/config/crd/bases/eventing.keda.sh_clustercloudeventsources.yaml b/config/crd/bases/eventing.keda.sh_clustercloudeventsources.yaml index 8c81de1f594..b5d07384483 100644 --- a/config/crd/bases/eventing.keda.sh_clustercloudeventsources.yaml +++ b/config/crd/bases/eventing.keda.sh_clustercloudeventsources.yaml @@ -81,8 +81,6 @@ spec: properties: excludedEventTypes: items: - description: CloudEventType contains the list of cloudevent - types enum: - keda.scaledobject.ready.v1 - keda.scaledobject.failed.v1 @@ -90,12 +88,16 @@ spec: - keda.scaledjob.ready.v1 - keda.scaledjob.failed.v1 - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 type: string type: array includedEventTypes: items: - description: CloudEventType contains the list of cloudevent - types enum: - keda.scaledobject.ready.v1 - keda.scaledobject.failed.v1 @@ -103,6 +105,12 @@ spec: - keda.scaledjob.ready.v1 - keda.scaledjob.failed.v1 - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 type: string type: array type: object diff --git a/controllers/keda/clustertriggerauthentication_controller.go b/controllers/keda/clustertriggerauthentication_controller.go index ceb7ae6185c..aabab91c4c3 100644 --- a/controllers/keda/clustertriggerauthentication_controller.go +++ b/controllers/keda/clustertriggerauthentication_controller.go @@ -18,18 +18,21 @@ package keda import ( "context" + "fmt" "sync" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + eventingv1alpha1 "github.com/kedacore/keda/v2/apis/eventing/v1alpha1" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/common/message" + "github.com/kedacore/keda/v2/pkg/eventemitter" "github.com/kedacore/keda/v2/pkg/eventreason" "github.com/kedacore/keda/v2/pkg/metricscollector" ) @@ -37,7 +40,7 @@ import ( // ClusterTriggerAuthenticationReconciler reconciles a ClusterTriggerAuthentication object type ClusterTriggerAuthenticationReconciler struct { client.Client - record.EventRecorder + eventemitter.EventHandler } type clusterTriggerAuthMetricsData struct { @@ -80,8 +83,12 @@ func (r *ClusterTriggerAuthenticationReconciler) Reconcile(ctx context.Context, r.updatePromMetrics(clusterTriggerAuthentication, req.NamespacedName.String()) if clusterTriggerAuthentication.ObjectMeta.Generation == 1 { - r.EventRecorder.Event(clusterTriggerAuthentication, corev1.EventTypeNormal, eventreason.ClusterTriggerAuthenticationAdded, "New ClusterTriggerAuthentication configured") + r.Emit(clusterTriggerAuthentication, req.NamespacedName.Namespace, corev1.EventTypeNormal, eventingv1alpha1.ClusterTriggerAuthenticationCreatedType, eventreason.ClusterTriggerAuthenticationAdded, message.ClusterTriggerAuthenticationCreatedMsg) + } else { + msg := fmt.Sprintf(message.ClusterTriggerAuthenticationUpdatedMsg, clusterTriggerAuthentication.Name) + r.Emit(clusterTriggerAuthentication, req.NamespacedName.Namespace, corev1.EventTypeNormal, eventingv1alpha1.ClusterTriggerAuthenticationUpdatedType, eventreason.ClusterTriggerAuthenticationUpdated, msg) } + return ctrl.Result{}, nil } diff --git a/controllers/keda/triggerauthentication_controller.go b/controllers/keda/triggerauthentication_controller.go index edca7ea8efc..b5ab9e1bd82 100755 --- a/controllers/keda/triggerauthentication_controller.go +++ b/controllers/keda/triggerauthentication_controller.go @@ -18,18 +18,21 @@ package keda import ( "context" + "fmt" "sync" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + eventingv1alpha1 "github.com/kedacore/keda/v2/apis/eventing/v1alpha1" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/common/message" + "github.com/kedacore/keda/v2/pkg/eventemitter" "github.com/kedacore/keda/v2/pkg/eventreason" "github.com/kedacore/keda/v2/pkg/metricscollector" "github.com/kedacore/keda/v2/pkg/util" @@ -38,7 +41,7 @@ import ( // TriggerAuthenticationReconciler reconciles a TriggerAuthentication object type TriggerAuthenticationReconciler struct { client.Client - record.EventRecorder + eventemitter.EventHandler } type triggerAuthMetricsData struct { @@ -81,7 +84,10 @@ func (r *TriggerAuthenticationReconciler) Reconcile(ctx context.Context, req ctr r.updatePromMetrics(triggerAuthentication, req.NamespacedName.String()) if triggerAuthentication.ObjectMeta.Generation == 1 { - r.EventRecorder.Event(triggerAuthentication, corev1.EventTypeNormal, eventreason.TriggerAuthenticationAdded, "New TriggerAuthentication configured") + r.Emit(triggerAuthentication, req.NamespacedName.Namespace, corev1.EventTypeNormal, eventingv1alpha1.TriggerAuthenticationCreatedType, eventreason.TriggerAuthenticationAdded, message.TriggerAuthenticationCreatedMsg) + } else { + msg := fmt.Sprintf(message.TriggerAuthenticationUpdatedMsg, triggerAuthentication.Name) + r.Emit(triggerAuthentication, req.NamespacedName.Namespace, corev1.EventTypeNormal, eventingv1alpha1.TriggerAuthenticationUpdatedType, eventreason.TriggerAuthenticationUpdated, msg) } return ctrl.Result{}, nil diff --git a/controllers/keda/util/finalizer.go b/controllers/keda/util/finalizer.go index c47343822e0..eb9d31db0a9 100644 --- a/controllers/keda/util/finalizer.go +++ b/controllers/keda/util/finalizer.go @@ -6,10 +6,11 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" + eventingv1alpha1 "github.com/kedacore/keda/v2/apis/eventing/v1alpha1" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/eventemitter" "github.com/kedacore/keda/v2/pkg/eventreason" ) @@ -19,7 +20,7 @@ const ( type authenticationReconciler interface { client.Client - record.EventRecorder + eventemitter.EventHandler UpdatePromMetricsOnDelete(string) } @@ -48,13 +49,16 @@ func EnsureAuthenticationResourceFinalizer(ctx context.Context, logger logr.Logg func FinalizeAuthenticationResource(ctx context.Context, logger logr.Logger, reconciler authenticationReconciler, authResource client.Object, namespacedName string) error { var authResourceType, reason string + var cloudEventType eventingv1alpha1.CloudEventType switch authResource.(type) { case *kedav1alpha1.TriggerAuthentication: authResourceType = "TriggerAuthentication" reason = eventreason.TriggerAuthenticationDeleted + cloudEventType = eventingv1alpha1.TriggerAuthenticationRemovedType case *kedav1alpha1.ClusterTriggerAuthentication: authResourceType = "ClusterTriggerAuthentication" reason = eventreason.ClusterTriggerAuthenticationDeleted + cloudEventType = eventingv1alpha1.ClusterTriggerAuthenticationRemovedType } if Contains(authResource.GetFinalizers(), authenticationFinalizer) { @@ -68,6 +72,6 @@ func FinalizeAuthenticationResource(ctx context.Context, logger logr.Logger, rec } logger.Info(fmt.Sprintf("Successfully finalized %s", authResourceType)) - reconciler.Event(authResource, corev1.EventTypeNormal, reason, fmt.Sprintf("%s was deleted", authResourceType)) + reconciler.Emit(authResource, namespacedName, corev1.EventTypeNormal, cloudEventType, reason, fmt.Sprintf("%s was deleted", authResourceType)) return nil } diff --git a/pkg/common/message/message.go b/pkg/common/message/message.go index b4a6458f3e2..4abcc0402ff 100644 --- a/pkg/common/message/message.go +++ b/pkg/common/message/message.go @@ -34,4 +34,12 @@ const ( ScaledJobReadyMsg = "ScaledJob is ready for scaling" ScaledJobRemoved = "ScaledJob was deleted" + + TriggerAuthenticationCreatedMsg = "New TriggerAuthentication configured" + + TriggerAuthenticationUpdatedMsg = "ClusterTriggerAuthentication %s is updated" + + ClusterTriggerAuthenticationCreatedMsg = "New ClusterTriggerAuthentication configured" + + ClusterTriggerAuthenticationUpdatedMsg = "ClusterTriggerAuthentication %s is updated" ) diff --git a/pkg/eventreason/eventreason.go b/pkg/eventreason/eventreason.go index 6fbc854ddc6..dd41cb6639a 100644 --- a/pkg/eventreason/eventreason.go +++ b/pkg/eventreason/eventreason.go @@ -77,6 +77,9 @@ const ( // TriggerAuthenticationFailed is for event when a TriggerAuthentication occurs error TriggerAuthenticationFailed = "TriggerAuthenticationFailed" + // TriggerAuthenticationUpdated is for event when a TriggerAuthentication is updated + TriggerAuthenticationUpdated = "ClusterTriggerAuthenticationUpdated" + // ClusterTriggerAuthenticationDeleted is for event when a ClusterTriggerAuthentication is deleted ClusterTriggerAuthenticationDeleted = "ClusterTriggerAuthenticationDeleted" @@ -85,4 +88,7 @@ const ( // ClusterTriggerAuthenticationFailed is for event when a ClusterTriggerAuthentication occurs error ClusterTriggerAuthenticationFailed = "ClusterTriggerAuthenticationFailed" + + // ClusterTriggerAuthenticationUpdated is for event when a ClusterTriggerAuthentication is updated + ClusterTriggerAuthenticationUpdated = "ClusterTriggerAuthenticationUpdated" ) diff --git a/tests/internals/events/events_test.go b/tests/internals/events/events_test.go index 5f3c4983696..3f9122d81fe 100644 --- a/tests/internals/events/events_test.go +++ b/tests/internals/events/events_test.go @@ -28,6 +28,10 @@ var ( scaledObjectName = fmt.Sprintf("%s-so", testName) scaledObjectTargetNotFoundName = fmt.Sprintf("%s-so-target-error", testName) scaledObjectTargetNoSubresourceName = fmt.Sprintf("%s-so-target-no-subresource", testName) + secretName = fmt.Sprintf("%s-secret", testName) + secretName2 = fmt.Sprintf("%s-secret-2", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + clusterTriggerAuthName = fmt.Sprintf("%s-cta", testName) scaledJobName = fmt.Sprintf("%s-sj", testName) scaledJobErrName = fmt.Sprintf("%s-sj-target-error", testName) @@ -43,6 +47,11 @@ type templateData struct { DaemonsetName string ScaledJobName string ScaledJobErrName string + SecretName string + SecretName2 string + SecretTargetName string + TriggerAuthName string + ClusterTriggerAuthName string } const ( @@ -223,6 +232,54 @@ spec: typex: Utilization value: "50" ` + + secretTemplate = `apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + AUTH_PASSWORD: U0VDUkVUCg== + AUTH_USERNAME: VVNFUgo= +` + secret2Template = `apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName2}} + namespace: {{.TestNamespace}} +data: + AUTH_PASSWORD: U0VDUkVUCg== + AUTH_USERNAME: VVNFUgo= +` + + triggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: username + name: {{.SecretTargetName}} + key: AUTH_USERNAME + - parameter: password + name: {{.SecretTargetName}} + key: AUTH_PASSWORD +` + + clusterTriggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: ClusterTriggerAuthentication +metadata: + name: {{.ClusterTriggerAuthName}} +spec: + secretTargetRef: + - parameter: username + name: {{.SecretTargetName}} + key: AUTH_USERNAME + - parameter: password + name: {{.SecretTargetName}} + key: AUTH_PASSWORD +` ) func TestEvents(t *testing.T) { @@ -242,6 +299,9 @@ func TestEvents(t *testing.T) { testScaledJobNormalEvent(t, kc, data) testScaledJobTargetNotSupportEventErr(t, kc, data) + + testTriggerAuthenticationEvent(t, kc, data) + // cleanup DeleteKubernetesResources(t, testNamespace, data, templates) } @@ -257,11 +317,15 @@ func getTemplateData() (templateData, []Template) { ScaledObjectTargetNoSubresourceName: scaledObjectTargetNoSubresourceName, ScaledJobName: scaledJobName, ScaledJobErrName: scaledJobErrName, + SecretName: secretName, + SecretName2: secretName2, + TriggerAuthName: triggerAuthName, + ClusterTriggerAuthName: clusterTriggerAuthName, }, []Template{} } -func checkingEvent(t *testing.T, scaledObject string, index int, eventreason string, message string) { - result, err := ExecuteCommand(fmt.Sprintf("kubectl get events -n %s --field-selector involvedObject.name=%s --sort-by=.metadata.creationTimestamp -o jsonpath=\"{.items[%d].reason}:{.items[%d].message}\"", testNamespace, scaledObject, index, index)) +func checkingEvent(t *testing.T, namespace string, scaledObject string, index int, eventreason string, message string) { + result, err := ExecuteCommand(fmt.Sprintf("kubectl get events -n %s --field-selector involvedObject.name=%s --sort-by=.metadata.creationTimestamp -o jsonpath=\"{.items[%d].reason}:{.items[%d].message}\"", namespace, scaledObject, index, index)) assert.NoError(t, err) lastEventMessage := strings.Trim(string(result), "\"") @@ -279,9 +343,9 @@ func testNormalEvent(t *testing.T, kc *kubernetes.Clientset, data templateData) KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 2, testNamespace) assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 2, 60, 1), "replica count should be 2 after 1 minute") - checkingEvent(t, scaledObjectName, 0, eventreason.KEDAScalersStarted, fmt.Sprintf(message.ScalerIsBuiltMsg, "kubernetes-workload")) - checkingEvent(t, scaledObjectName, 1, eventreason.KEDAScalersStarted, message.ScalerStartMsg) - checkingEvent(t, scaledObjectName, 2, eventreason.ScaledObjectReady, message.ScalerReadyMsg) + checkingEvent(t, testNamespace, scaledObjectName, 0, eventreason.KEDAScalersStarted, fmt.Sprintf(message.ScalerIsBuiltMsg, "kubernetes-workload")) + checkingEvent(t, testNamespace, scaledObjectName, 1, eventreason.KEDAScalersStarted, message.ScalerStartMsg) + checkingEvent(t, testNamespace, scaledObjectName, 2, eventreason.ScaledObjectReady, message.ScalerReadyMsg) KubectlDeleteWithTemplate(t, data, "deploymentTemplate", deploymentTemplate) KubectlDeleteWithTemplate(t, data, "monitoredDeploymentName", monitoredDeploymentTemplate) @@ -292,8 +356,8 @@ func testTargetNotFoundErr(t *testing.T, _ *kubernetes.Clientset, data templateD t.Log("--- testing target not found error event ---") KubectlApplyWithTemplate(t, data, "scaledObjectTargetErrTemplate", scaledObjectTargetErrTemplate) - checkingEvent(t, scaledObjectTargetNotFoundName, -2, eventreason.ScaledObjectCheckFailed, message.ScaleTargetNotFoundMsg) - checkingEvent(t, scaledObjectTargetNotFoundName, -1, eventreason.ScaledObjectCheckFailed, message.ScaleTargetErrMsg) + checkingEvent(t, testNamespace, scaledObjectTargetNotFoundName, -2, eventreason.ScaledObjectCheckFailed, message.ScaleTargetNotFoundMsg) + checkingEvent(t, testNamespace, scaledObjectTargetNotFoundName, -1, eventreason.ScaledObjectCheckFailed, message.ScaleTargetErrMsg) } func testTargetNotSupportEventErr(t *testing.T, _ *kubernetes.Clientset, data templateData) { @@ -301,8 +365,8 @@ func testTargetNotSupportEventErr(t *testing.T, _ *kubernetes.Clientset, data te KubectlApplyWithTemplate(t, data, "daemonSetTemplate", daemonSetTemplate) KubectlApplyWithTemplate(t, data, "scaledObjectTargetNotSupportTemplate", scaledObjectTargetNotSupportTemplate) - checkingEvent(t, scaledObjectTargetNoSubresourceName, -2, eventreason.ScaledObjectCheckFailed, message.ScaleTargetNoSubresourceMsg) - checkingEvent(t, scaledObjectTargetNoSubresourceName, -1, eventreason.ScaledObjectCheckFailed, message.ScaleTargetErrMsg) + checkingEvent(t, testNamespace, scaledObjectTargetNoSubresourceName, -2, eventreason.ScaledObjectCheckFailed, message.ScaleTargetNoSubresourceMsg) + checkingEvent(t, testNamespace, scaledObjectTargetNoSubresourceName, -1, eventreason.ScaledObjectCheckFailed, message.ScaleTargetErrMsg) } func testScaledJobNormalEvent(t *testing.T, kc *kubernetes.Clientset, data templateData) { @@ -315,9 +379,9 @@ func testScaledJobNormalEvent(t *testing.T, kc *kubernetes.Clientset, data templ KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 2, testNamespace) assert.True(t, WaitForJobCount(t, kc, testNamespace, 2, 60, 1), "replica count should be 2 after 1 minute") - checkingEvent(t, scaledJobName, 0, eventreason.KEDAScalersStarted, fmt.Sprintf(message.ScalerIsBuiltMsg, "kubernetes-workload")) - checkingEvent(t, scaledJobName, 1, eventreason.KEDAScalersStarted, message.ScalerStartMsg) - checkingEvent(t, scaledJobName, 2, eventreason.ScaledJobReady, message.ScaledJobReadyMsg) + checkingEvent(t, testNamespace, scaledJobName, 0, eventreason.KEDAScalersStarted, fmt.Sprintf(message.ScalerIsBuiltMsg, "kubernetes-workload")) + checkingEvent(t, testNamespace, scaledJobName, 1, eventreason.KEDAScalersStarted, message.ScalerStartMsg) + checkingEvent(t, testNamespace, scaledJobName, 2, eventreason.ScaledJobReady, message.ScaledJobReadyMsg) KubectlDeleteWithTemplate(t, data, "deploymentTemplate", deploymentTemplate) KubectlDeleteWithTemplate(t, data, "monitoredDeploymentName", monitoredDeploymentTemplate) @@ -328,5 +392,33 @@ func testScaledJobTargetNotSupportEventErr(t *testing.T, _ *kubernetes.Clientset t.Log("--- testing target not support error event ---") KubectlApplyWithTemplate(t, data, "scaledJobErrTemplate", scaledJobErrTemplate) - checkingEvent(t, scaledJobErrName, -1, eventreason.ScaledJobCheckFailed, "Failed to ensure ScaledJob is correctly created") + checkingEvent(t, testNamespace, scaledJobErrName, -1, eventreason.ScaledJobCheckFailed, "Failed to ensure ScaledJob is correctly created") +} + +func testTriggerAuthenticationEvent(t *testing.T, _ *kubernetes.Clientset, data templateData) { + t.Log("--- testing ScaledJob normal event ---") + + KubectlApplyWithTemplate(t, data, "secretTemplate", secretTemplate) + KubectlApplyWithTemplate(t, data, "secret2Template", secret2Template) + + data.SecretTargetName = secretName + KubectlApplyWithTemplate(t, data, "triggerAuthenticationTemplate", triggerAuthenticationTemplate) + + checkingEvent(t, testNamespace, triggerAuthName, 0, eventreason.TriggerAuthenticationAdded, message.TriggerAuthenticationCreatedMsg) + + KubectlApplyWithTemplate(t, data, "clusterTriggerAuthenticationTemplate", clusterTriggerAuthenticationTemplate) + + checkingEvent(t, "default", clusterTriggerAuthName, 0, eventreason.ClusterTriggerAuthenticationAdded, message.ClusterTriggerAuthenticationCreatedMsg) + + data.SecretTargetName = secretName2 + KubectlApplyWithTemplate(t, data, "triggerAuthenticationTemplate", triggerAuthenticationTemplate) + + checkingEvent(t, testNamespace, triggerAuthName, -1, eventreason.TriggerAuthenticationUpdated, fmt.Sprintf(message.TriggerAuthenticationUpdatedMsg, triggerAuthName)) + KubectlApplyWithTemplate(t, data, "clusterTriggerAuthenticationTemplate", clusterTriggerAuthenticationTemplate) + + checkingEvent(t, "default", clusterTriggerAuthName, -1, eventreason.ClusterTriggerAuthenticationUpdated, fmt.Sprintf(message.ClusterTriggerAuthenticationUpdatedMsg, clusterTriggerAuthName)) + KubectlDeleteWithTemplate(t, data, "secretTemplate", secretTemplate) + KubectlDeleteWithTemplate(t, data, "secret2Template", secret2Template) + KubectlDeleteWithTemplate(t, data, "triggerAuthenticationTemplate", triggerAuthenticationTemplate) + KubectlDeleteWithTemplate(t, data, "clusterTriggerAuthenticationTemplate", clusterTriggerAuthenticationTemplate) } From 71f2dd794b86cbec2ba010315a0d8d88b62cefec Mon Sep 17 00:00:00 2001 From: Dao Thanh Tung Date: Fri, 6 Sep 2024 15:23:11 +0100 Subject: [PATCH 04/97] Refactor aws dynamodb scaler (#5961) Signed-off-by: dttung2905 Signed-off-by: Chirag Bhatia --- pkg/scalers/aws_dynamodb_scaler.go | 130 ++++++---------------- pkg/scalers/aws_dynamodb_scaler_test.go | 102 +++++++++-------- pkg/scalers/scalersconfig/typed_config.go | 3 +- 3 files changed, 92 insertions(+), 143 deletions(-) diff --git a/pkg/scalers/aws_dynamodb_scaler.go b/pkg/scalers/aws_dynamodb_scaler.go index 72aff9beb46..f3c9bcac1f6 100644 --- a/pkg/scalers/aws_dynamodb_scaler.go +++ b/pkg/scalers/aws_dynamodb_scaler.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" @@ -28,18 +27,18 @@ type awsDynamoDBScaler struct { } type awsDynamoDBMetadata struct { - tableName string - awsRegion string - awsEndpoint string - keyConditionExpression string - expressionAttributeNames map[string]string - expressionAttributeValues map[string]types.AttributeValue - indexName string - targetValue int64 - activationTargetValue int64 awsAuthorization awsutils.AuthorizationMetadata + expressionAttributeValues map[string]types.AttributeValue + expressionAttributeNames map[string]string triggerIndex int metricName string + TableName string `keda:"name=tableName, order=triggerMetadata"` + AwsRegion string `keda:"name=awsRegion, order=triggerMetadata"` + AwsEndpoint string `keda:"name=awsEndpoint, order=triggerMetadata, optional"` + KeyConditionExpression string `keda:"name=keyConditionExpression, order=triggerMetadata"` + IndexName string `keda:"name=indexName, order=triggerMetadata, optional"` + TargetValue int64 `keda:"name=targetValue, order=triggerMetadata, optional, default=-1"` + ActivationTargetValue int64 `keda:"name=activationTargetValue, order=triggerMetadata, default=0"` } func NewAwsDynamoDBScaler(ctx context.Context, config *scalersconfig.ScalerConfig) (Scaler, error) { @@ -65,63 +64,24 @@ func NewAwsDynamoDBScaler(ctx context.Context, config *scalersconfig.ScalerConfi } var ( - // ErrAwsDynamoNoTableName is returned when "tableName" is missing from the config. - ErrAwsDynamoNoTableName = errors.New("no tableName given") - - // ErrAwsDynamoNoAwsRegion is returned when "awsRegion" is missing from the config. - ErrAwsDynamoNoAwsRegion = errors.New("no awsRegion given") - - // ErrAwsDynamoNoKeyConditionExpression is returned when "keyConditionExpression" is missing from the config. - ErrAwsDynamoNoKeyConditionExpression = errors.New("no keyConditionExpression given") - - // ErrAwsDynamoEmptyExpressionAttributeNames is returned when "expressionAttributeNames" is empty. - ErrAwsDynamoEmptyExpressionAttributeNames = errors.New("empty map") - - // ErrAwsDynamoInvalidExpressionAttributeNames is returned when "expressionAttributeNames" is an invalid JSON. - ErrAwsDynamoInvalidExpressionAttributeNames = errors.New("invalid expressionAttributeNames") - - // ErrAwsDynamoNoExpressionAttributeNames is returned when "expressionAttributeNames" is missing from the config. - ErrAwsDynamoNoExpressionAttributeNames = errors.New("no expressionAttributeNames given") - + ErrAwsDynamoNoTargetValue = errors.New("no targetValue given") // ErrAwsDynamoInvalidExpressionAttributeValues is returned when "expressionAttributeNames" is missing an invalid JSON. ErrAwsDynamoInvalidExpressionAttributeValues = errors.New("invalid expressionAttributeValues") - // ErrAwsDynamoNoExpressionAttributeValues is returned when "expressionAttributeValues" is missing from the config. ErrAwsDynamoNoExpressionAttributeValues = errors.New("no expressionAttributeValues given") - - // ErrAwsDynamoNoTargetValue is returned when "targetValue" is missing from the config. - ErrAwsDynamoNoTargetValue = errors.New("no targetValue given") + // ErrAwsDynamoInvalidExpressionAttributeNames is returned when "expressionAttributeNames" is an invalid JSON. + ErrAwsDynamoInvalidExpressionAttributeNames = errors.New("invalid expressionAttributeNames") + // ErrAwsDynamoEmptyExpressionAttributeNames is returned when "expressionAttributeNames" is empty. + ErrAwsDynamoEmptyExpressionAttributeNames = errors.New("empty map") + // ErrAwsDynamoNoExpressionAttributeNames is returned when "expressionAttributeNames" is missing from the config. + ErrAwsDynamoNoExpressionAttributeNames = errors.New("no expressionAttributeNames given") ) func parseAwsDynamoDBMetadata(config *scalersconfig.ScalerConfig) (*awsDynamoDBMetadata, error) { - meta := awsDynamoDBMetadata{} - - if val, ok := config.TriggerMetadata["tableName"]; ok && val != "" { - meta.tableName = val - } else { - return nil, ErrAwsDynamoNoTableName - } - - if val, ok := config.TriggerMetadata["awsRegion"]; ok && val != "" { - meta.awsRegion = val - } else { - return nil, ErrAwsDynamoNoAwsRegion - } - - if val, ok := config.TriggerMetadata["awsEndpoint"]; ok { - meta.awsEndpoint = val - } - - if val, ok := config.TriggerMetadata["indexName"]; ok { - meta.indexName = val - } - - if val, ok := config.TriggerMetadata["keyConditionExpression"]; ok && val != "" { - meta.keyConditionExpression = val - } else { - return nil, ErrAwsDynamoNoKeyConditionExpression + meta := &awsDynamoDBMetadata{} + if err := config.TypedConfig(meta); err != nil { + return nil, fmt.Errorf("error parsing DynamoDb metadata: %w", err) } - if val, ok := config.TriggerMetadata["expressionAttributeNames"]; ok && val != "" { names, err := json2Map(val) @@ -133,7 +93,6 @@ func parseAwsDynamoDBMetadata(config *scalersconfig.ScalerConfig) (*awsDynamoDBM } else { return nil, ErrAwsDynamoNoExpressionAttributeNames } - if val, ok := config.TriggerMetadata["expressionAttributeValues"]; ok && val != "" { values, err := json2DynamoMap(val) @@ -145,31 +104,10 @@ func parseAwsDynamoDBMetadata(config *scalersconfig.ScalerConfig) (*awsDynamoDBM } else { return nil, ErrAwsDynamoNoExpressionAttributeValues } - - if val, ok := config.TriggerMetadata["targetValue"]; ok && val != "" { - n, err := strconv.ParseInt(val, 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing metadata targetValue: %w", err) - } - - meta.targetValue = n - } else { - if config.AsMetricSource { - meta.targetValue = 0 - } else { - return nil, ErrAwsDynamoNoTargetValue - } - } - - if val, ok := config.TriggerMetadata["activationTargetValue"]; ok && val != "" { - n, err := strconv.ParseInt(val, 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing metadata activationTargetValue: %w", err) - } - - meta.activationTargetValue = n - } else { - meta.activationTargetValue = 0 + if meta.TargetValue == -1 && config.AsMetricSource { + meta.TargetValue = 0 + } else if meta.TargetValue == -1 && !config.AsMetricSource { + return nil, ErrAwsDynamoNoTargetValue } auth, err := awsutils.GetAwsAuthorization(config.TriggerUniqueKey, config.PodIdentity, config.TriggerMetadata, config.AuthParams, config.ResolvedEnv) @@ -181,20 +119,20 @@ func parseAwsDynamoDBMetadata(config *scalersconfig.ScalerConfig) (*awsDynamoDBM meta.triggerIndex = config.TriggerIndex meta.metricName = GenerateMetricNameWithIndex(config.TriggerIndex, - kedautil.NormalizeString(fmt.Sprintf("aws-dynamodb-%s", meta.tableName))) + kedautil.NormalizeString(fmt.Sprintf("aws-dynamodb-%s", meta.TableName))) - return &meta, nil + return meta, nil } func createDynamoDBClient(ctx context.Context, metadata *awsDynamoDBMetadata) (*dynamodb.Client, error) { - cfg, err := awsutils.GetAwsConfig(ctx, metadata.awsRegion, metadata.awsAuthorization) + cfg, err := awsutils.GetAwsConfig(ctx, metadata.AwsRegion, metadata.awsAuthorization) if err != nil { return nil, err } return dynamodb.NewFromConfig(*cfg, func(options *dynamodb.Options) { - if metadata.awsEndpoint != "" { - options.BaseEndpoint = aws.String(metadata.awsEndpoint) + if metadata.AwsEndpoint != "" { + options.BaseEndpoint = aws.String(metadata.AwsEndpoint) } }), nil } @@ -208,7 +146,7 @@ func (s *awsDynamoDBScaler) GetMetricsAndActivity(ctx context.Context, metricNam metric := GenerateMetricInMili(metricName, metricValue) - return []external_metrics.ExternalMetricValue{metric}, metricValue > float64(s.metadata.activationTargetValue), nil + return []external_metrics.ExternalMetricValue{metric}, metricValue > float64(s.metadata.ActivationTargetValue), nil } func (s *awsDynamoDBScaler) GetMetricSpecForScaling(context.Context) []v2.MetricSpec { @@ -216,7 +154,7 @@ func (s *awsDynamoDBScaler) GetMetricSpecForScaling(context.Context) []v2.Metric Metric: v2.MetricIdentifier{ Name: s.metadata.metricName, }, - Target: GetMetricTarget(s.metricType, s.metadata.targetValue), + Target: GetMetricTarget(s.metricType, s.metadata.TargetValue), } metricSpec := v2.MetricSpec{External: externalMetric, Type: externalMetricType} @@ -232,14 +170,14 @@ func (s *awsDynamoDBScaler) Close(context.Context) error { func (s *awsDynamoDBScaler) GetQueryMetrics(ctx context.Context) (float64, error) { dimensions := dynamodb.QueryInput{ - TableName: aws.String(s.metadata.tableName), - KeyConditionExpression: aws.String(s.metadata.keyConditionExpression), + TableName: aws.String(s.metadata.TableName), + KeyConditionExpression: aws.String(s.metadata.KeyConditionExpression), ExpressionAttributeNames: s.metadata.expressionAttributeNames, ExpressionAttributeValues: s.metadata.expressionAttributeValues, } - if s.metadata.indexName != "" { - dimensions.IndexName = aws.String(s.metadata.indexName) + if s.metadata.IndexName != "" { + dimensions.IndexName = aws.String(s.metadata.IndexName) } res, err := s.dbClient.Query(ctx, &dimensions) diff --git a/pkg/scalers/aws_dynamodb_scaler_test.go b/pkg/scalers/aws_dynamodb_scaler_test.go index a577f561e21..f6ec68314e8 100644 --- a/pkg/scalers/aws_dynamodb_scaler_test.go +++ b/pkg/scalers/aws_dynamodb_scaler_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strconv" "testing" "github.com/aws/aws-sdk-go-v2/service/dynamodb" @@ -38,6 +37,17 @@ type parseDynamoDBMetadataTestData struct { expectedError error } +var ( + // ErrAwsDynamoNoTableName is returned when "tableName" is missing from the config. + ErrAwsDynamoNoTableName = errors.New("missing required parameter \"tableName\"") + + // ErrAwsDynamoNoAwsRegion is returned when "awsRegion" is missing from the config. + ErrAwsDynamoNoAwsRegion = errors.New("missing required parameter \"awsRegion\"") + + // ErrAwsDynamoNoKeyConditionExpression is returned when "keyConditionExpression" is missing from the config. + ErrAwsDynamoNoKeyConditionExpression = errors.New("missing required parameter \"keyConditionExpression\"") +) + var dynamoTestCases = []parseDynamoDBMetadataTestData{ { name: "no tableName given", @@ -104,7 +114,7 @@ var dynamoTestCases = []parseDynamoDBMetadataTestData{ "targetValue": "no-valid", }, authParams: map[string]string{}, - expectedError: strconv.ErrSyntax, + expectedError: errors.New("error parsing DynamoDb metadata: unable to set param \"targetValue\" value"), }, { name: "invalid activationTargetValue given", @@ -118,7 +128,7 @@ var dynamoTestCases = []parseDynamoDBMetadataTestData{ "activationTargetValue": "no-valid", }, authParams: map[string]string{}, - expectedError: strconv.ErrSyntax, + expectedError: errors.New("unable to set param \"activationTargetValue\""), }, { name: "malformed expressionAttributeNames", @@ -185,12 +195,12 @@ var dynamoTestCases = []parseDynamoDBMetadataTestData{ authParams: testAWSDynamoAuthentication, expectedError: nil, expectedMetadata: &awsDynamoDBMetadata{ - tableName: "test", - awsRegion: "eu-west-1", - keyConditionExpression: "#yr = :yyyy", + TableName: "test", + AwsRegion: "eu-west-1", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - targetValue: 3, + TargetValue: 3, triggerIndex: 1, metricName: "s1-aws-dynamodb-test", awsAuthorization: awsutils.AuthorizationMetadata{ @@ -214,13 +224,13 @@ var dynamoTestCases = []parseDynamoDBMetadataTestData{ authParams: testAWSDynamoAuthentication, expectedError: nil, expectedMetadata: &awsDynamoDBMetadata{ - tableName: "test", - awsRegion: "eu-west-1", - awsEndpoint: "http://localhost:4566", - keyConditionExpression: "#yr = :yyyy", + TableName: "test", + AwsRegion: "eu-west-1", + AwsEndpoint: "http://localhost:4566", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - targetValue: 3, + TargetValue: 3, triggerIndex: 1, metricName: "s1-aws-dynamodb-test", awsAuthorization: awsutils.AuthorizationMetadata{ @@ -244,13 +254,13 @@ var dynamoTestCases = []parseDynamoDBMetadataTestData{ authParams: testAWSDynamoAuthentication, expectedError: nil, expectedMetadata: &awsDynamoDBMetadata{ - tableName: "test", - awsRegion: "eu-west-1", - keyConditionExpression: "#yr = :yyyy", + TableName: "test", + AwsRegion: "eu-west-1", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - activationTargetValue: 1, - targetValue: 3, + ActivationTargetValue: 1, + TargetValue: 3, triggerIndex: 1, metricName: "s1-aws-dynamodb-test", awsAuthorization: awsutils.AuthorizationMetadata{ @@ -274,13 +284,13 @@ var dynamoTestCases = []parseDynamoDBMetadataTestData{ authParams: testAWSDynamoAuthentication, expectedError: nil, expectedMetadata: &awsDynamoDBMetadata{ - tableName: "test", - awsRegion: "eu-west-1", - indexName: "test-index", - keyConditionExpression: "#yr = :yyyy", + TableName: "test", + AwsRegion: "eu-west-1", + IndexName: "test-index", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - targetValue: 3, + TargetValue: 3, triggerIndex: 1, metricName: "s1-aws-dynamodb-test", awsAuthorization: awsutils.AuthorizationMetadata{ @@ -346,48 +356,48 @@ var yearAttr = &types.AttributeValueMemberN{Value: target} var awsDynamoDBGetMetricTestData = []awsDynamoDBMetadata{ { - tableName: "ValidTable", - awsRegion: "eu-west-1", - keyConditionExpression: "#yr = :yyyy", + TableName: "ValidTable", + AwsRegion: "eu-west-1", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - targetValue: 3, + TargetValue: 3, }, { - tableName: testAWSDynamoErrorTable, - awsRegion: "eu-west-1", - keyConditionExpression: "#yr = :yyyy", + TableName: testAWSDynamoErrorTable, + AwsRegion: "eu-west-1", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - targetValue: 3, + TargetValue: 3, }, { - tableName: testAWSDynamoNoValueTable, - awsRegion: "eu-west-1", - keyConditionExpression: "#yr = :yyyy", + TableName: testAWSDynamoNoValueTable, + AwsRegion: "eu-west-1", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - targetValue: 3, + TargetValue: 3, }, { - tableName: testAWSDynamoIndexTable, - awsRegion: "eu-west-1", - indexName: "test-index", - keyConditionExpression: "#yr = :yyyy", + TableName: testAWSDynamoIndexTable, + AwsRegion: "eu-west-1", + IndexName: "test-index", + KeyConditionExpression: "#yr = :yyyy", expressionAttributeNames: map[string]string{"#yr": year}, expressionAttributeValues: map[string]types.AttributeValue{":yyyy": yearAttr}, - activationTargetValue: 3, - targetValue: 3, + ActivationTargetValue: 3, + TargetValue: 3, }, } func TestDynamoGetMetrics(t *testing.T) { for _, meta := range awsDynamoDBGetMetricTestData { - t.Run(meta.tableName, func(t *testing.T) { + t.Run(meta.TableName, func(t *testing.T) { scaler := awsDynamoDBScaler{"", &meta, &mockDynamoDB{}, logr.Discard()} value, _, err := scaler.GetMetricsAndActivity(context.Background(), "aws-dynamodb") - switch meta.tableName { + switch meta.TableName { case testAWSDynamoErrorTable: assert.EqualError(t, err, "error", "expect error because of dynamodb api error") case testAWSDynamoNoValueTable: @@ -403,11 +413,11 @@ func TestDynamoGetMetrics(t *testing.T) { func TestDynamoGetQueryMetrics(t *testing.T) { for _, meta := range awsDynamoDBGetMetricTestData { - t.Run(meta.tableName, func(t *testing.T) { + t.Run(meta.TableName, func(t *testing.T) { scaler := awsDynamoDBScaler{"", &meta, &mockDynamoDB{}, logr.Discard()} value, err := scaler.GetQueryMetrics(context.Background()) - switch meta.tableName { + switch meta.TableName { case testAWSDynamoErrorTable: assert.EqualError(t, err, "error", "expect error because of dynamodb api error") case testAWSDynamoNoValueTable: @@ -423,11 +433,11 @@ func TestDynamoGetQueryMetrics(t *testing.T) { func TestDynamoIsActive(t *testing.T) { for _, meta := range awsDynamoDBGetMetricTestData { - t.Run(meta.tableName, func(t *testing.T) { + t.Run(meta.TableName, func(t *testing.T) { scaler := awsDynamoDBScaler{"", &meta, &mockDynamoDB{}, logr.Discard()} _, value, err := scaler.GetMetricsAndActivity(context.Background(), "aws-dynamodb") - switch meta.tableName { + switch meta.TableName { case testAWSDynamoErrorTable: assert.EqualError(t, err, "error", "expect error because of dynamodb api error") case testAWSDynamoNoValueTable: diff --git a/pkg/scalers/scalersconfig/typed_config.go b/pkg/scalers/scalersconfig/typed_config.go index b06e1478b88..1833c49d628 100644 --- a/pkg/scalers/scalersconfig/typed_config.go +++ b/pkg/scalers/scalersconfig/typed_config.go @@ -62,7 +62,8 @@ const ( // separators for map and slice elements const ( - elemSeparator = "," + elemSeparator = "," + // TODO: support custom separator https://github.com/kedacore/keda/pull/5961/files#r1694991497 elemKeyValSeparator = "=" ) From c5171dd44e7b49c542214fce02349cd7832826be Mon Sep 17 00:00:00 2001 From: Dishant Kapadiya Date: Wed, 18 Sep 2024 04:36:13 -0700 Subject: [PATCH 05/97] Update Remote Container references to Dev Containers (#6140) Signed-off-by: Dishant Kapadiya Signed-off-by: Chirag Bhatia --- BUILD.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BUILD.md b/BUILD.md index fc80fe51ec1..560083ef37f 100644 --- a/BUILD.md +++ b/BUILD.md @@ -5,7 +5,7 @@ **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Building](#building) - - [Quick start with Visual Studio Code Remote - Containers](#quick-start-with-visual-studio-code-remote---containers) + - [Quick start with Visual Studio Code Dev Containers](#quick-start-with-visual-studio-code-dev-containers) - [Locally directly](#locally-directly) - [Deploying](#deploying) - [Custom KEDA locally outside cluster](#custom-keda-locally-outside-cluster) @@ -25,14 +25,14 @@ ## Building -### Quick start with [Visual Studio Code Remote - Containers](https://code.visualstudio.com/docs/remote/containers) +### Quick start with [Visual Studio Code Dev Containers](https://code.visualstudio.com/docs/remote/containers) This helps you pull and build quickly - dev containers launch the project inside a container with all the tooling required for a consistent and seamless developer experience. This means you don't have to install and configure your dev environment as the container handles this for you. -To get started install [VSCode](https://code.visualstudio.com/) and the [Remote Containers extensions]( +To get started install [VSCode](https://code.visualstudio.com/) and the [Dev Containers extensions]( https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) Clone the repo and launch code: @@ -43,7 +43,7 @@ cd keda code . ``` -Once VSCode launches run `CTRL+SHIFT+P -> Remote-Containers: Reopen in container` and then use the integrated +Once VSCode launches run `CTRL+SHIFT+P -> Dev Containers: Reopen in container` and then use the integrated terminal to run: ```bash From e8fdbbf3294358d87a53f6a73632583160998cc5 Mon Sep 17 00:00:00 2001 From: Jorge Turrado Ferrero Date: Wed, 18 Sep 2024 14:57:22 +0200 Subject: [PATCH 06/97] fix: Matrics API Scaler now can read any prometheus metric (#6078) Signed-off-by: Jorge Turrado Signed-off-by: Chirag Bhatia --- CHANGELOG.md | 2 +- go.mod | 5 + go.sum | 17 + pkg/scalers/metrics_api_scaler.go | 76 +- pkg/scalers/metrics_api_scaler_test.go | 16 + vendor/github.com/dennwc/varint/.gitignore | 2 + vendor/github.com/dennwc/varint/.travis.yml | 7 + vendor/github.com/dennwc/varint/LICENSE | 21 + vendor/github.com/dennwc/varint/README.md | 47 + vendor/github.com/dennwc/varint/proto.go | 244 + vendor/github.com/dennwc/varint/varint.go | 270 ++ vendor/github.com/go-kit/log/.gitignore | 15 + vendor/github.com/go-kit/log/LICENSE | 21 + vendor/github.com/go-kit/log/README.md | 156 + vendor/github.com/go-kit/log/doc.go | 116 + vendor/github.com/go-kit/log/json_logger.go | 91 + vendor/github.com/go-kit/log/level/doc.go | 33 + vendor/github.com/go-kit/log/level/level.go | 256 + vendor/github.com/go-kit/log/log.go | 179 + vendor/github.com/go-kit/log/logfmt_logger.go | 62 + vendor/github.com/go-kit/log/nop_logger.go | 8 + vendor/github.com/go-kit/log/staticcheck.conf | 1 + vendor/github.com/go-kit/log/stdlib.go | 151 + vendor/github.com/go-kit/log/sync.go | 113 + vendor/github.com/go-kit/log/value.go | 110 + vendor/github.com/go-logfmt/logfmt/.gitignore | 1 + .../github.com/go-logfmt/logfmt/CHANGELOG.md | 82 + vendor/github.com/go-logfmt/logfmt/LICENSE | 22 + vendor/github.com/go-logfmt/logfmt/README.md | 41 + vendor/github.com/go-logfmt/logfmt/decode.go | 254 + vendor/github.com/go-logfmt/logfmt/doc.go | 6 + vendor/github.com/go-logfmt/logfmt/encode.go | 322 ++ .../github.com/go-logfmt/logfmt/jsonstring.go | 277 ++ vendor/github.com/gogo/protobuf/types/any.go | 140 + .../github.com/gogo/protobuf/types/any.pb.go | 694 +++ .../github.com/gogo/protobuf/types/api.pb.go | 2134 +++++++++ vendor/github.com/gogo/protobuf/types/doc.go | 35 + .../gogo/protobuf/types/duration.go | 100 + .../gogo/protobuf/types/duration.pb.go | 517 ++ .../gogo/protobuf/types/duration_gogo.go | 100 + .../gogo/protobuf/types/empty.pb.go | 462 ++ .../gogo/protobuf/types/field_mask.pb.go | 738 +++ .../gogo/protobuf/types/protosize.go | 34 + .../gogo/protobuf/types/source_context.pb.go | 524 ++ .../gogo/protobuf/types/struct.pb.go | 2271 +++++++++ .../gogo/protobuf/types/timestamp.go | 130 + .../gogo/protobuf/types/timestamp.pb.go | 539 +++ .../gogo/protobuf/types/timestamp_gogo.go | 94 + .../github.com/gogo/protobuf/types/type.pb.go | 3355 +++++++++++++ .../gogo/protobuf/types/wrappers.pb.go | 2703 +++++++++++ .../gogo/protobuf/types/wrappers_gogo.go | 300 ++ vendor/github.com/grafana/regexp/.gitignore | 15 + vendor/github.com/grafana/regexp/LICENSE | 27 + vendor/github.com/grafana/regexp/README.md | 12 + vendor/github.com/grafana/regexp/backtrack.go | 367 ++ vendor/github.com/grafana/regexp/exec.go | 554 +++ vendor/github.com/grafana/regexp/onepass.go | 507 ++ vendor/github.com/grafana/regexp/regexp.go | 1285 +++++ .../grafana/regexp/syntax/compile.go | 296 ++ .../github.com/grafana/regexp/syntax/doc.go | 141 + .../grafana/regexp/syntax/make_perl_groups.pl | 113 + .../grafana/regexp/syntax/op_string.go | 26 + .../github.com/grafana/regexp/syntax/parse.go | 2114 ++++++++ .../grafana/regexp/syntax/perl_groups.go | 134 + .../github.com/grafana/regexp/syntax/prog.go | 347 ++ .../grafana/regexp/syntax/regexp.go | 320 ++ .../grafana/regexp/syntax/simplify.go | 151 + .../github.com/prometheus/prometheus/LICENSE | 201 + .../github.com/prometheus/prometheus/NOTICE | 108 + .../prometheus/model/exemplar/exemplar.go | 65 + .../model/histogram/float_histogram.go | 1175 +++++ .../prometheus/model/histogram/generic.go | 705 +++ .../prometheus/model/histogram/histogram.go | 512 ++ .../prometheus/model/histogram/test_utils.go | 52 + .../prometheus/model/labels/labels.go | 434 ++ .../prometheus/model/labels/labels_common.go | 235 + .../model/labels/labels_stringlabels.go | 645 +++ .../prometheus/model/labels/matcher.go | 120 + .../prometheus/model/labels/regexp.go | 125 + .../prometheus/model/labels/test_utils.go | 87 + .../prometheus/model/metadata/metadata.go | 23 + .../prometheus/model/textparse/README.md | 6 + .../prometheus/model/textparse/interface.go | 127 + .../model/textparse/openmetricslex.l | 80 + .../model/textparse/openmetricslex.l.go | 792 +++ .../model/textparse/openmetricsparse.go | 478 ++ .../prometheus/model/textparse/promlex.l | 103 + .../prometheus/model/textparse/promlex.l.go | 578 +++ .../prometheus/model/textparse/promparse.go | 441 ++ .../model/textparse/promtestdata.nometa.txt | 411 ++ .../model/textparse/promtestdata.txt | 529 ++ .../model/textparse/protobufparse.go | 611 +++ .../prometheus/model/timestamp/timestamp.go | 34 + .../prometheus/model/value/value.go | 34 + .../prompb/io/prometheus/client/metrics.pb.go | 4234 +++++++++++++++++ .../prompb/io/prometheus/client/metrics.proto | 157 + .../prometheus/promql/parser/ast.go | 492 ++ .../prometheus/promql/parser/functions.go | 427 ++ .../promql/parser/generated_parser.y | 919 ++++ .../promql/parser/generated_parser.y.go | 1976 ++++++++ .../prometheus/promql/parser/lex.go | 996 ++++ .../prometheus/promql/parser/parse.go | 984 ++++ .../promql/parser/posrange/posrange.go | 54 + .../prometheus/promql/parser/prettier.go | 166 + .../promql/parser/prettier_rules.md | 16 + .../prometheus/promql/parser/printer.go | 235 + .../prometheus/promql/parser/value.go | 45 + .../prometheus/prometheus/storage/buffer.go | 835 ++++ .../prometheus/prometheus/storage/fanout.go | 233 + .../prometheus/prometheus/storage/generic.go | 143 + .../prometheus/storage/interface.go | 432 ++ .../prometheus/prometheus/storage/lazy.go | 71 + .../prometheus/storage/memoized_iterator.go | 142 + .../prometheus/prometheus/storage/merge.go | 876 ++++ .../prometheus/prometheus/storage/noop.go | 97 + .../prometheus/storage/secondary.go | 109 + .../prometheus/prometheus/storage/series.go | 457 ++ .../prometheus/tsdb/chunkenc/bstream.go | 259 + .../prometheus/tsdb/chunkenc/chunk.go | 382 ++ .../tsdb/chunkenc/float_histogram.go | 940 ++++ .../prometheus/tsdb/chunkenc/histogram.go | 1058 ++++ .../tsdb/chunkenc/histogram_meta.go | 501 ++ .../prometheus/tsdb/chunkenc/varbit.go | 231 + .../prometheus/tsdb/chunkenc/xor.go | 507 ++ .../tsdb/chunks/chunk_write_queue.go | 264 + .../prometheus/tsdb/chunks/chunks.go | 754 +++ .../prometheus/tsdb/chunks/head_chunks.go | 1124 +++++ .../tsdb/chunks/head_chunks_other.go | 22 + .../tsdb/chunks/head_chunks_windows.go | 18 + .../prometheus/tsdb/chunks/queue.go | 141 + .../prometheus/tsdb/chunks/samples.go | 89 + .../prometheus/tsdb/errors/errors.go | 104 + .../prometheus/tsdb/fileutil/dir.go | 33 + .../prometheus/tsdb/fileutil/dir_unix.go | 23 + .../prometheus/tsdb/fileutil/dir_windows.go | 47 + .../prometheus/tsdb/fileutil/fileutil.go | 128 + .../prometheus/tsdb/fileutil/flock.go | 41 + .../prometheus/tsdb/fileutil/flock_js.go | 33 + .../prometheus/tsdb/fileutil/flock_plan9.go | 32 + .../prometheus/tsdb/fileutil/flock_solaris.go | 60 + .../prometheus/tsdb/fileutil/flock_unix.go | 55 + .../prometheus/tsdb/fileutil/flock_windows.go | 36 + .../prometheus/tsdb/fileutil/mmap.go | 72 + .../prometheus/tsdb/fileutil/mmap_386.go | 19 + .../prometheus/tsdb/fileutil/mmap_amd64.go | 19 + .../prometheus/tsdb/fileutil/mmap_arm64.go | 19 + .../prometheus/tsdb/fileutil/mmap_js.go | 30 + .../prometheus/tsdb/fileutil/mmap_unix.go | 31 + .../prometheus/tsdb/fileutil/mmap_windows.go | 46 + .../prometheus/tsdb/fileutil/preallocate.go | 54 + .../tsdb/fileutil/preallocate_darwin.go | 41 + .../tsdb/fileutil/preallocate_linux.go | 48 + .../tsdb/fileutil/preallocate_other.go | 26 + .../prometheus/tsdb/fileutil/sync.go | 25 + .../prometheus/tsdb/fileutil/sync_darwin.go | 28 + .../prometheus/tsdb/fileutil/sync_linux.go | 30 + .../util/annotations/annotations.go | 185 + .../prometheus/util/strutil/quote.go | 255 + .../prometheus/util/strutil/strconv.go | 64 + vendor/modules.txt | 34 + 160 files changed, 55000 insertions(+), 18 deletions(-) create mode 100644 vendor/github.com/dennwc/varint/.gitignore create mode 100644 vendor/github.com/dennwc/varint/.travis.yml create mode 100644 vendor/github.com/dennwc/varint/LICENSE create mode 100644 vendor/github.com/dennwc/varint/README.md create mode 100644 vendor/github.com/dennwc/varint/proto.go create mode 100644 vendor/github.com/dennwc/varint/varint.go create mode 100644 vendor/github.com/go-kit/log/.gitignore create mode 100644 vendor/github.com/go-kit/log/LICENSE create mode 100644 vendor/github.com/go-kit/log/README.md create mode 100644 vendor/github.com/go-kit/log/doc.go create mode 100644 vendor/github.com/go-kit/log/json_logger.go create mode 100644 vendor/github.com/go-kit/log/level/doc.go create mode 100644 vendor/github.com/go-kit/log/level/level.go create mode 100644 vendor/github.com/go-kit/log/log.go create mode 100644 vendor/github.com/go-kit/log/logfmt_logger.go create mode 100644 vendor/github.com/go-kit/log/nop_logger.go create mode 100644 vendor/github.com/go-kit/log/staticcheck.conf create mode 100644 vendor/github.com/go-kit/log/stdlib.go create mode 100644 vendor/github.com/go-kit/log/sync.go create mode 100644 vendor/github.com/go-kit/log/value.go create mode 100644 vendor/github.com/go-logfmt/logfmt/.gitignore create mode 100644 vendor/github.com/go-logfmt/logfmt/CHANGELOG.md create mode 100644 vendor/github.com/go-logfmt/logfmt/LICENSE create mode 100644 vendor/github.com/go-logfmt/logfmt/README.md create mode 100644 vendor/github.com/go-logfmt/logfmt/decode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/doc.go create mode 100644 vendor/github.com/go-logfmt/logfmt/encode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/jsonstring.go create mode 100644 vendor/github.com/gogo/protobuf/types/any.go create mode 100644 vendor/github.com/gogo/protobuf/types/any.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/api.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/doc.go create mode 100644 vendor/github.com/gogo/protobuf/types/duration.go create mode 100644 vendor/github.com/gogo/protobuf/types/duration.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/duration_gogo.go create mode 100644 vendor/github.com/gogo/protobuf/types/empty.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/field_mask.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/protosize.go create mode 100644 vendor/github.com/gogo/protobuf/types/source_context.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/struct.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/timestamp.go create mode 100644 vendor/github.com/gogo/protobuf/types/timestamp.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/timestamp_gogo.go create mode 100644 vendor/github.com/gogo/protobuf/types/type.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/wrappers.pb.go create mode 100644 vendor/github.com/gogo/protobuf/types/wrappers_gogo.go create mode 100644 vendor/github.com/grafana/regexp/.gitignore create mode 100644 vendor/github.com/grafana/regexp/LICENSE create mode 100644 vendor/github.com/grafana/regexp/README.md create mode 100644 vendor/github.com/grafana/regexp/backtrack.go create mode 100644 vendor/github.com/grafana/regexp/exec.go create mode 100644 vendor/github.com/grafana/regexp/onepass.go create mode 100644 vendor/github.com/grafana/regexp/regexp.go create mode 100644 vendor/github.com/grafana/regexp/syntax/compile.go create mode 100644 vendor/github.com/grafana/regexp/syntax/doc.go create mode 100644 vendor/github.com/grafana/regexp/syntax/make_perl_groups.pl create mode 100644 vendor/github.com/grafana/regexp/syntax/op_string.go create mode 100644 vendor/github.com/grafana/regexp/syntax/parse.go create mode 100644 vendor/github.com/grafana/regexp/syntax/perl_groups.go create mode 100644 vendor/github.com/grafana/regexp/syntax/prog.go create mode 100644 vendor/github.com/grafana/regexp/syntax/regexp.go create mode 100644 vendor/github.com/grafana/regexp/syntax/simplify.go create mode 100644 vendor/github.com/prometheus/prometheus/LICENSE create mode 100644 vendor/github.com/prometheus/prometheus/NOTICE create mode 100644 vendor/github.com/prometheus/prometheus/model/exemplar/exemplar.go create mode 100644 vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go create mode 100644 vendor/github.com/prometheus/prometheus/model/histogram/generic.go create mode 100644 vendor/github.com/prometheus/prometheus/model/histogram/histogram.go create mode 100644 vendor/github.com/prometheus/prometheus/model/histogram/test_utils.go create mode 100644 vendor/github.com/prometheus/prometheus/model/labels/labels.go create mode 100644 vendor/github.com/prometheus/prometheus/model/labels/labels_common.go create mode 100644 vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go create mode 100644 vendor/github.com/prometheus/prometheus/model/labels/matcher.go create mode 100644 vendor/github.com/prometheus/prometheus/model/labels/regexp.go create mode 100644 vendor/github.com/prometheus/prometheus/model/labels/test_utils.go create mode 100644 vendor/github.com/prometheus/prometheus/model/metadata/metadata.go create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/README.md create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/interface.go create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/openmetricslex.l create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/openmetricslex.l.go create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/promlex.l create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/promlex.l.go create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/promparse.go create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/promtestdata.nometa.txt create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/promtestdata.txt create mode 100644 vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go create mode 100644 vendor/github.com/prometheus/prometheus/model/timestamp/timestamp.go create mode 100644 vendor/github.com/prometheus/prometheus/model/value/value.go create mode 100644 vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.pb.go create mode 100644 vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.proto create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/ast.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/functions.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/lex.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/parse.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/posrange/posrange.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/prettier.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/prettier_rules.md create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/printer.go create mode 100644 vendor/github.com/prometheus/prometheus/promql/parser/value.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/buffer.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/fanout.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/generic.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/interface.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/lazy.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/memoized_iterator.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/merge.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/noop.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/secondary.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/series.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunkenc/bstream.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunkenc/float_histogram.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram_meta.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunkenc/varbit.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/chunk_write_queue.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_other.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_windows.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/queue.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/samples.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/dir.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/dir_unix.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/dir_windows.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/fileutil.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/flock.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/flock_js.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/flock_plan9.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/flock_solaris.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/flock_unix.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/flock_windows.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap_386.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap_amd64.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap_arm64.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap_js.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap_unix.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap_windows.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate_darwin.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate_linux.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate_other.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/sync.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/sync_darwin.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/fileutil/sync_linux.go create mode 100644 vendor/github.com/prometheus/prometheus/util/annotations/annotations.go create mode 100644 vendor/github.com/prometheus/prometheus/util/strutil/quote.go create mode 100644 vendor/github.com/prometheus/prometheus/util/strutil/strconv.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ae977f01872..71cef1b99cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,7 +78,7 @@ Here is an overview of all new **experimental** features: ### Fixes -- TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX)) +- **Metrics API**: Prometheus metrics can have multiple labels ([#6077](https://github.com/kedacore/keda/issues/6077)) ### Deprecations diff --git a/go.mod b/go.mod index b6eff2b66cc..4322a6cafee 100644 --- a/go.mod +++ b/go.mod @@ -210,6 +210,7 @@ require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dennwc/varint v1.0.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/eapache/go-resiliency v1.6.0 // indirect @@ -224,6 +225,8 @@ require ( github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-jose/go-jose/v4 v4.0.1 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -254,6 +257,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -308,6 +312,7 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/prometheus v0.49.0 github.com/rivo/uniseg v0.4.4 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/samber/lo v1.39.0 // indirect diff --git a/go.sum b/go.sum index 8ecdbc874fd..e7cefad3489 100644 --- a/go.sum +++ b/go.sum @@ -866,6 +866,8 @@ github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= @@ -891,6 +893,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0 h1:UyjtGmO0Uwl/K+zpzPwLoXzMhcN9xmnR2nrqJoBrg3c= github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0/go.mod h1:TJAXuFs2HcMib3sN5L0gUC+Q01Qvy3DemvA55WuC+iA= +github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s= +github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= @@ -1015,6 +1019,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= +github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= +github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -1088,6 +1094,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kivik/couchdb/v3 v3.4.1 h1:TlGYEFOmG5a0pN6MpDkIDdd+sn75+w5aSDTcEou02kk= github.com/go-kivik/couchdb/v3 v3.4.1/go.mod h1:scodbTTSS6vOAacJXaCx6XZ57qw8YH1JOvhMwvP0vuw= @@ -1099,6 +1106,8 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -1309,6 +1318,8 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= @@ -1539,6 +1550,8 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1590,10 +1603,14 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= +github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/prometheus v0.49.0 h1:i0CEhreJo3ZcZNeK7ulISinCac0MgL0krVOGgNmfFRY= +github.com/prometheus/prometheus v0.49.0/go.mod h1:aDogiyqmv3aBIWDb5z5Sdcxuuf2BOfiJwOIm9JGpMnI= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= diff --git a/pkg/scalers/metrics_api_scaler.go b/pkg/scalers/metrics_api_scaler.go index b3a678b3cc5..77b658217db 100644 --- a/pkg/scalers/metrics_api_scaler.go +++ b/pkg/scalers/metrics_api_scaler.go @@ -1,8 +1,6 @@ package scalers import ( - "bufio" - "bytes" "context" "encoding/xml" "errors" @@ -14,6 +12,8 @@ import ( "strings" "github.com/go-logr/logr" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/prometheus/promql/parser" "github.com/tidwall/gjson" "gopkg.in/yaml.v3" v2 "k8s.io/api/autoscaling/v2" @@ -264,25 +264,67 @@ func GetValueFromResponse(body []byte, valueLocation string, format APIFormat) ( // getValueFromPrometheusResponse uses provided valueLocation to access the numeric value in provided body func getValueFromPrometheusResponse(body []byte, valueLocation string) (float64, error) { - scanner := bufio.NewScanner(bytes.NewReader(body)) - for scanner.Scan() { - line := scanner.Text() - fields := strings.Fields(line) - if len(fields) == 0 || strings.HasPrefix(fields[0], "#") { - continue - } - if len(fields) == 2 && strings.HasPrefix(fields[0], valueLocation) { - value, err := strconv.ParseFloat(fields[1], 64) - if err != nil { - return 0, err - } - return value, nil + matchers, err := parser.ParseMetricSelector(valueLocation) + if err != nil { + return 0, err + } + metricName := "" + for _, v := range matchers { + if v.Name == "__name__" { + metricName = v.Value } } - - if err := scanner.Err(); err != nil { + // Ensure EOL + reader := strings.NewReader(strings.ReplaceAll(string(body), "\r\n", "\n")) + familiesParser := expfmt.TextParser{} + families, err := familiesParser.TextToMetricFamilies(reader) + if err != nil { return 0, err } + family, ok := families[metricName] + if !ok { + return 0, fmt.Errorf("metric '%s' not found", metricName) + } + + metrics := family.GetMetric() + for _, metric := range metrics { + labels := metric.GetLabel() + match := true + for _, matcher := range matchers { + matcherFound := false + if matcher == nil { + continue + } + // The name has been already validated, + // so we can skip it and check the other labels + if matcher.Name == "__name__" { + continue + } + for _, label := range labels { + if *label.Name == matcher.Name && + *label.Value == matcher.Value { + matcherFound = true + } + } + if !matcherFound { + match = false + } + } + if match { + untyped := metric.GetUntyped() + if untyped != nil && untyped.Value != nil { + return *untyped.Value, nil + } + counter := metric.GetCounter() + if counter != nil && counter.Value != nil { + return *counter.Value, nil + } + gauge := metric.GetGauge() + if gauge != nil && gauge.Value != nil { + return *gauge.Value, nil + } + } + } return 0, fmt.Errorf("value %s not found", valueLocation) } diff --git a/pkg/scalers/metrics_api_scaler_test.go b/pkg/scalers/metrics_api_scaler_test.go index cef44a7bfef..a580d8e1af1 100644 --- a/pkg/scalers/metrics_api_scaler_test.go +++ b/pkg/scalers/metrics_api_scaler_test.go @@ -125,6 +125,16 @@ func TestMetricsAPIGetMetricSpecForScaling(t *testing.T) { func TestGetValueFromResponse(t *testing.T) { inputJSON := []byte(`{"components":[{"id": "82328e93e", "tasks": 32, "str": "64", "k":"1k","wrong":"NaN"}],"count":2.43}`) inputYAML := []byte(`{components: [{id: 82328e93e, tasks: 32, str: '64', k: 1k, wrong: NaN}], count: 2.43}`) + inputPrometheus := []byte(`# HELP backend_queue_size Total number of items + # TYPE backend_queue_size counter + backend_queue_size{queueName="zero"} 0 + backend_queue_size{queueName="one"} 1 + backend_queue_size{queueName="two", instance="random"} 2 + backend_queue_size{queueName="two", instance="zero"} 20 + # HELP random_metric Random metric generate to include noise + # TYPE random_metric counter + random_metric 10 + `) testCases := []struct { name string @@ -143,6 +153,12 @@ func TestGetValueFromResponse(t *testing.T) { {name: "string", input: inputYAML, key: "components.0.str", format: YAMLFormat, expectVal: 64}, {name: "{}.[].{}", input: inputYAML, key: "components.0.tasks", format: YAMLFormat, expectVal: 32}, {name: "invalid data", input: inputYAML, key: "components.0.wrong", format: YAMLFormat, expectErr: true}, + + {name: "no labels", input: inputPrometheus, key: "random_metric", format: PrometheusFormat, expectVal: 10}, + {name: "one label", input: inputPrometheus, key: "backend_queue_size{queueName=\"one\"}", format: PrometheusFormat, expectVal: 1}, + {name: "multiple labels not queried", input: inputPrometheus, key: "backend_queue_size{queueName=\"two\"}", format: PrometheusFormat, expectVal: 2}, + {name: "multiple labels queried", input: inputPrometheus, key: "backend_queue_size{queueName=\"two\", instance=\"zero\"}", format: PrometheusFormat, expectVal: 20}, + {name: "invalid data", input: inputPrometheus, key: "backend_queue_size{invalid=test}", format: PrometheusFormat, expectErr: true}, } for _, tc := range testCases { diff --git a/vendor/github.com/dennwc/varint/.gitignore b/vendor/github.com/dennwc/varint/.gitignore new file mode 100644 index 00000000000..9385b6db18b --- /dev/null +++ b/vendor/github.com/dennwc/varint/.gitignore @@ -0,0 +1,2 @@ +*.o +*.txt \ No newline at end of file diff --git a/vendor/github.com/dennwc/varint/.travis.yml b/vendor/github.com/dennwc/varint/.travis.yml new file mode 100644 index 00000000000..b3da258f520 --- /dev/null +++ b/vendor/github.com/dennwc/varint/.travis.yml @@ -0,0 +1,7 @@ +language: go + +go: + - 1.12.x + +env: + - GO111MODULE=on \ No newline at end of file diff --git a/vendor/github.com/dennwc/varint/LICENSE b/vendor/github.com/dennwc/varint/LICENSE new file mode 100644 index 00000000000..8b3f68715c1 --- /dev/null +++ b/vendor/github.com/dennwc/varint/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Denys Smirnov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/dennwc/varint/README.md b/vendor/github.com/dennwc/varint/README.md new file mode 100644 index 00000000000..fe15b3b5003 --- /dev/null +++ b/vendor/github.com/dennwc/varint/README.md @@ -0,0 +1,47 @@ +# varint + +This package provides an optimized implementation of protobuf's varint encoding/decoding. +It has no dependencies. + +Benchmarks comparing to a `binary.Uvarint`: + +``` +benchmark old ns/op new ns/op delta +BenchmarkUvarint/1-8 4.13 2.85 -30.99% +BenchmarkUvarint/1_large-8 4.01 2.28 -43.14% +BenchmarkUvarint/2-8 6.23 2.87 -53.93% +BenchmarkUvarint/2_large-8 5.60 2.86 -48.93% +BenchmarkUvarint/3-8 6.55 3.44 -47.48% +BenchmarkUvarint/3_large-8 6.54 2.86 -56.27% +BenchmarkUvarint/4-8 7.30 3.71 -49.18% +BenchmarkUvarint/4_large-8 7.46 3.10 -58.45% +BenchmarkUvarint/5-8 8.31 4.12 -50.42% +BenchmarkUvarint/5_large-8 8.56 3.48 -59.35% +BenchmarkUvarint/6-8 9.42 4.66 -50.53% +BenchmarkUvarint/6_large-8 9.91 4.07 -58.93% +BenchmarkUvarint/7-8 10.6 5.28 -50.19% +BenchmarkUvarint/7_large-8 11.0 4.70 -57.27% +BenchmarkUvarint/8-8 11.7 6.02 -48.55% +BenchmarkUvarint/8_large-8 12.1 5.19 -57.11% +BenchmarkUvarint/9-8 12.9 6.83 -47.05% +BenchmarkUvarint/9_large-8 13.1 5.71 -56.41% +``` + +It also provides additional functionality like `UvarintSize` (similar to `sov*` in `gogo/protobuf`): + +``` +benchmark old ns/op new ns/op delta +BenchmarkUvarintSize/1-8 1.71 0.43 -74.85% +BenchmarkUvarintSize/2-8 2.56 0.57 -77.73% +BenchmarkUvarintSize/3-8 3.22 0.72 -77.64% +BenchmarkUvarintSize/4-8 3.74 0.72 -80.75% +BenchmarkUvarintSize/5-8 4.29 0.57 -86.71% +BenchmarkUvarintSize/6-8 4.85 0.58 -88.04% +BenchmarkUvarintSize/7-8 5.43 0.71 -86.92% +BenchmarkUvarintSize/8-8 6.01 0.86 -85.69% +BenchmarkUvarintSize/9-8 6.64 1.00 -84.94% +``` + +# License + +MIT \ No newline at end of file diff --git a/vendor/github.com/dennwc/varint/proto.go b/vendor/github.com/dennwc/varint/proto.go new file mode 100644 index 00000000000..e3b458547f9 --- /dev/null +++ b/vendor/github.com/dennwc/varint/proto.go @@ -0,0 +1,244 @@ +package varint + +// ProtoTag decodes a protobuf's field number and wire type pair +// from buf and returns that value and the number of bytes read (> 0). +// If an error occurred, n = 0 is returned. +func ProtoTag(buf []byte) (num int, typ byte, n int) { + // Same unrolled implementation as in Uvarint. + // + // But this time we can check if the wire type and field num + // are valid when reading the first byte. + // + // Also, the swifts are now different, because first 3 bits + // are for the wire type. + // + // The implementation will stop at 9 bytes, returning an error. + sz := len(buf) + if sz == 0 { + return 0, 0, 0 + } + const ( + bit = 1 << 7 + mask = bit - 1 + step = 7 + + // protobuf + typBits = 3 + typMask = 1<<3 - 1 + ) + if sz >= 9 { // no bound checks + // i == 0 + b := buf[0] + if b == 0 { + return 0, 0, 0 + } + typ = b & typMask + if typ > 5 { + return 0, 0, 0 + } + if b < bit { + num = int(b >> typBits) + if num == 0 { + return 0, 0, 0 + } + n = 1 + return + } + num = int((b & mask) >> typBits) + var s uint = step - typBits + + // i == 1 + b = buf[1] + if b < bit { + num |= int(b) << s + n = 2 + return + } + num |= int(b&mask) << s + s += step + + // i == 2 + b = buf[2] + if b < bit { + num |= int(b) << s + n = 3 + return + } + num |= int(b&mask) << s + s += step + + // i == 3 + b = buf[3] + if b < bit { + num |= int(b) << s + n = 4 + return + } + num |= int(b&mask) << s + s += step + + // i == 4 + b = buf[4] + if b < bit { + num |= int(b) << s + n = 5 + return + } + num |= int(b&mask) << s + s += step + + // i == 5 + b = buf[5] + if b < bit { + num |= int(b) << s + n = 6 + return + } + num |= int(b&mask) << s + s += step + + // i == 6 + b = buf[6] + if b < bit { + num |= int(b) << s + n = 7 + return + } + num |= int(b&mask) << s + s += step + + // i == 7 + b = buf[7] + if b < bit { + num |= int(b) << s + n = 8 + return + } + num |= int(b&mask) << s + s += step + + // i == 8 + b = buf[8] + if b < bit { + num |= int(b) << s + n = 9 + return + } + return 0, 0, 0 // too much + } + + // i == 0 + b := buf[0] + if b == 0 { + return 0, 0, 0 + } + typ = b & typMask + if typ > 5 { + return 0, 0, 0 + } + if b < bit { + num = int(b >> typBits) + if num == 0 { + return 0, 0, 0 + } + n = 1 + return + } else if sz == 1 { + return 0, 0, 0 + } + num = int((b & mask) >> typBits) + var s uint = step - typBits + + // i == 1 + b = buf[1] + if b < bit { + num |= int(b) << s + n = 2 + return + } else if sz == 2 { + return 0, 0, 0 + } + num |= int(b&mask) << s + s += step + + // i == 2 + b = buf[2] + if b < bit { + num |= int(b) << s + n = 3 + return + } else if sz == 3 { + return 0, 0, 0 + } + num |= int(b&mask) << s + s += step + + // i == 3 + b = buf[3] + if b < bit { + num |= int(b) << s + n = 4 + return + } else if sz == 4 { + return 0, 0, 0 + } + num |= int(b&mask) << s + s += step + + // i == 4 + b = buf[4] + if b < bit { + num |= int(b) << s + n = 5 + return + } else if sz == 5 { + return 0, 0, 0 + } + num |= int(b&mask) << s + s += step + + // i == 5 + b = buf[5] + if b < bit { + num |= int(b) << s + n = 6 + return + } else if sz == 6 { + return 0, 0, 0 + } + num |= int(b&mask) << s + s += step + + // i == 6 + b = buf[6] + if b < bit { + num |= int(b) << s + n = 7 + return + } else if sz == 7 { + return 0, 0, 0 + } + num |= int(b&mask) << s + s += step + + // i == 7 + b = buf[7] + if b < bit { + num |= int(b) << s + n = 8 + return + } else if sz == 8 { + return 0, 0, 0 + } + num |= int(b&mask) << s + s += step + + // i == 8 + b = buf[8] + if b < bit { + num |= int(b) << s + n = 9 + return + } + return 0, 0, 0 // too much +} diff --git a/vendor/github.com/dennwc/varint/varint.go b/vendor/github.com/dennwc/varint/varint.go new file mode 100644 index 00000000000..83278c2d7d2 --- /dev/null +++ b/vendor/github.com/dennwc/varint/varint.go @@ -0,0 +1,270 @@ +package varint + +const maxUint64 = uint64(1<<64 - 1) + +// MaxLenN is the maximum length of a varint-encoded N-bit integer. +const ( + MaxLen8 = 2 + MaxLen16 = 3 + MaxLen32 = 5 + MaxLen64 = 10 +) + +// MaxValN is the maximum varint-encoded integer that fits in N bytes. +const ( + MaxVal9 = maxUint64 >> (1 + iota*7) + MaxVal8 + MaxVal7 + MaxVal6 + MaxVal5 + MaxVal4 + MaxVal3 + MaxVal2 + MaxVal1 +) + +// UvarintSize returns the number of bytes necessary to encode a given uint. +func UvarintSize(x uint64) int { + if x <= MaxVal4 { + if x <= MaxVal1 { + return 1 + } else if x <= MaxVal2 { + return 2 + } else if x <= MaxVal3 { + return 3 + } + return 4 + } + if x <= MaxVal5 { + return 5 + } else if x <= MaxVal6 { + return 6 + } else if x <= MaxVal7 { + return 7 + } else if x <= MaxVal8 { + return 8 + } else if x <= MaxVal9 { + return 9 + } + return 10 +} + +// Uvarint decodes a uint64 from buf and returns that value and the +// number of bytes read (> 0). If an error occurred, the value is 0 +// and the number of bytes n is <= 0 meaning: +// +// n == 0: buf too small +// n < 0: value larger than 64 bits (overflow) +// and -n is the number of bytes read +// +func Uvarint(buf []byte) (uint64, int) { + // Fully unrolled implementation of binary.Uvarint. + // + // It will also eliminate bound checks for buffers larger than 9 bytes. + sz := len(buf) + if sz == 0 { + return 0, 0 + } + const ( + step = 7 + bit = 1 << 7 + mask = bit - 1 + ) + if sz >= 10 { // no bound checks + // i == 0 + b := buf[0] + if b < bit { + return uint64(b), 1 + } + x := uint64(b & mask) + var s uint = step + + // i == 1 + b = buf[1] + if b < bit { + return x | uint64(b)< 1 { + return 0, -10 // overflow + } + return x | uint64(b)< 1 { + return 0, -10 // overflow + } + return x | uint64(b)< 100 { +// level.Error(logger).Log("value", value) +// } +// +// NewFilter allows precise control over what happens when a log event is +// emitted without a level key, or if a squelched level is used. Check the +// Option functions for details. +package level diff --git a/vendor/github.com/go-kit/log/level/level.go b/vendor/github.com/go-kit/log/level/level.go new file mode 100644 index 00000000000..c641d985524 --- /dev/null +++ b/vendor/github.com/go-kit/log/level/level.go @@ -0,0 +1,256 @@ +package level + +import ( + "errors" + "strings" + + "github.com/go-kit/log" +) + +// ErrInvalidLevelString is returned whenever an invalid string is passed to Parse. +var ErrInvalidLevelString = errors.New("invalid level string") + +// Error returns a logger that includes a Key/ErrorValue pair. +func Error(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), ErrorValue()) +} + +// Warn returns a logger that includes a Key/WarnValue pair. +func Warn(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), WarnValue()) +} + +// Info returns a logger that includes a Key/InfoValue pair. +func Info(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), InfoValue()) +} + +// Debug returns a logger that includes a Key/DebugValue pair. +func Debug(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), DebugValue()) +} + +// NewFilter wraps next and implements level filtering. See the commentary on +// the Option functions for a detailed description of how to configure levels. +// If no options are provided, all leveled log events created with Debug, +// Info, Warn or Error helper methods are squelched and non-leveled log +// events are passed to next unmodified. +func NewFilter(next log.Logger, options ...Option) log.Logger { + l := &logger{ + next: next, + } + for _, option := range options { + option(l) + } + return l +} + +type logger struct { + next log.Logger + allowed level + squelchNoLevel bool + errNotAllowed error + errNoLevel error +} + +func (l *logger) Log(keyvals ...interface{}) error { + var hasLevel, levelAllowed bool + for i := 1; i < len(keyvals); i += 2 { + if v, ok := keyvals[i].(*levelValue); ok { + hasLevel = true + levelAllowed = l.allowed&v.level != 0 + break + } + } + if !hasLevel && l.squelchNoLevel { + return l.errNoLevel + } + if hasLevel && !levelAllowed { + return l.errNotAllowed + } + return l.next.Log(keyvals...) +} + +// Option sets a parameter for the leveled logger. +type Option func(*logger) + +// Allow the provided log level to pass. +func Allow(v Value) Option { + switch v { + case debugValue: + return AllowDebug() + case infoValue: + return AllowInfo() + case warnValue: + return AllowWarn() + case errorValue: + return AllowError() + default: + return AllowNone() + } +} + +// AllowAll is an alias for AllowDebug. +func AllowAll() Option { + return AllowDebug() +} + +// AllowDebug allows error, warn, info and debug level log events to pass. +func AllowDebug() Option { + return allowed(levelError | levelWarn | levelInfo | levelDebug) +} + +// AllowInfo allows error, warn and info level log events to pass. +func AllowInfo() Option { + return allowed(levelError | levelWarn | levelInfo) +} + +// AllowWarn allows error and warn level log events to pass. +func AllowWarn() Option { + return allowed(levelError | levelWarn) +} + +// AllowError allows only error level log events to pass. +func AllowError() Option { + return allowed(levelError) +} + +// AllowNone allows no leveled log events to pass. +func AllowNone() Option { + return allowed(0) +} + +func allowed(allowed level) Option { + return func(l *logger) { l.allowed = allowed } +} + +// Parse a string to its corresponding level value. Valid strings are "debug", +// "info", "warn", and "error". Strings are normalized via strings.TrimSpace and +// strings.ToLower. +func Parse(level string) (Value, error) { + switch strings.TrimSpace(strings.ToLower(level)) { + case debugValue.name: + return debugValue, nil + case infoValue.name: + return infoValue, nil + case warnValue.name: + return warnValue, nil + case errorValue.name: + return errorValue, nil + default: + return nil, ErrInvalidLevelString + } +} + +// ParseDefault calls Parse and returns the default Value on error. +func ParseDefault(level string, def Value) Value { + v, err := Parse(level) + if err != nil { + return def + } + return v +} + +// ErrNotAllowed sets the error to return from Log when it squelches a log +// event disallowed by the configured Allow[Level] option. By default, +// ErrNotAllowed is nil; in this case the log event is squelched with no +// error. +func ErrNotAllowed(err error) Option { + return func(l *logger) { l.errNotAllowed = err } +} + +// SquelchNoLevel instructs Log to squelch log events with no level, so that +// they don't proceed through to the wrapped logger. If SquelchNoLevel is set +// to true and a log event is squelched in this way, the error value +// configured with ErrNoLevel is returned to the caller. +func SquelchNoLevel(squelch bool) Option { + return func(l *logger) { l.squelchNoLevel = squelch } +} + +// ErrNoLevel sets the error to return from Log when it squelches a log event +// with no level. By default, ErrNoLevel is nil; in this case the log event is +// squelched with no error. +func ErrNoLevel(err error) Option { + return func(l *logger) { l.errNoLevel = err } +} + +// NewInjector wraps next and returns a logger that adds a Key/level pair to +// the beginning of log events that don't already contain a level. In effect, +// this gives a default level to logs without a level. +func NewInjector(next log.Logger, level Value) log.Logger { + return &injector{ + next: next, + level: level, + } +} + +type injector struct { + next log.Logger + level interface{} +} + +func (l *injector) Log(keyvals ...interface{}) error { + for i := 1; i < len(keyvals); i += 2 { + if _, ok := keyvals[i].(*levelValue); ok { + return l.next.Log(keyvals...) + } + } + kvs := make([]interface{}, len(keyvals)+2) + kvs[0], kvs[1] = key, l.level + copy(kvs[2:], keyvals) + return l.next.Log(kvs...) +} + +// Value is the interface that each of the canonical level values implement. +// It contains unexported methods that prevent types from other packages from +// implementing it and guaranteeing that NewFilter can distinguish the levels +// defined in this package from all other values. +type Value interface { + String() string + levelVal() +} + +// Key returns the unique key added to log events by the loggers in this +// package. +func Key() interface{} { return key } + +// ErrorValue returns the unique value added to log events by Error. +func ErrorValue() Value { return errorValue } + +// WarnValue returns the unique value added to log events by Warn. +func WarnValue() Value { return warnValue } + +// InfoValue returns the unique value added to log events by Info. +func InfoValue() Value { return infoValue } + +// DebugValue returns the unique value added to log events by Debug. +func DebugValue() Value { return debugValue } + +var ( + // key is of type interface{} so that it allocates once during package + // initialization and avoids allocating every time the value is added to a + // []interface{} later. + key interface{} = "level" + + errorValue = &levelValue{level: levelError, name: "error"} + warnValue = &levelValue{level: levelWarn, name: "warn"} + infoValue = &levelValue{level: levelInfo, name: "info"} + debugValue = &levelValue{level: levelDebug, name: "debug"} +) + +type level byte + +const ( + levelDebug level = 1 << iota + levelInfo + levelWarn + levelError +) + +type levelValue struct { + name string + level +} + +func (v *levelValue) String() string { return v.name } +func (v *levelValue) levelVal() {} diff --git a/vendor/github.com/go-kit/log/log.go b/vendor/github.com/go-kit/log/log.go new file mode 100644 index 00000000000..62e11adace5 --- /dev/null +++ b/vendor/github.com/go-kit/log/log.go @@ -0,0 +1,179 @@ +package log + +import "errors" + +// Logger is the fundamental interface for all log operations. Log creates a +// log event from keyvals, a variadic sequence of alternating keys and values. +// Implementations must be safe for concurrent use by multiple goroutines. In +// particular, any implementation of Logger that appends to keyvals or +// modifies or retains any of its elements must make a copy first. +type Logger interface { + Log(keyvals ...interface{}) error +} + +// ErrMissingValue is appended to keyvals slices with odd length to substitute +// the missing value. +var ErrMissingValue = errors.New("(MISSING)") + +// With returns a new contextual logger with keyvals prepended to those passed +// to calls to Log. If logger is also a contextual logger created by With, +// WithPrefix, or WithSuffix, keyvals is appended to the existing context. +// +// The returned Logger replaces all value elements (odd indexes) containing a +// Valuer with their generated value for each call to its Log method. +func With(logger Logger, keyvals ...interface{}) Logger { + if len(keyvals) == 0 { + return logger + } + l := newContext(logger) + kvs := append(l.keyvals, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + return &context{ + logger: l.logger, + // Limiting the capacity of the stored keyvals ensures that a new + // backing array is created if the slice must grow in Log or With. + // Using the extra capacity without copying risks a data race that + // would violate the Logger interface contract. + keyvals: kvs[:len(kvs):len(kvs)], + hasValuer: l.hasValuer || containsValuer(keyvals), + sKeyvals: l.sKeyvals, + sHasValuer: l.sHasValuer, + } +} + +// WithPrefix returns a new contextual logger with keyvals prepended to those +// passed to calls to Log. If logger is also a contextual logger created by +// With, WithPrefix, or WithSuffix, keyvals is prepended to the existing context. +// +// The returned Logger replaces all value elements (odd indexes) containing a +// Valuer with their generated value for each call to its Log method. +func WithPrefix(logger Logger, keyvals ...interface{}) Logger { + if len(keyvals) == 0 { + return logger + } + l := newContext(logger) + // Limiting the capacity of the stored keyvals ensures that a new + // backing array is created if the slice must grow in Log or With. + // Using the extra capacity without copying risks a data race that + // would violate the Logger interface contract. + n := len(l.keyvals) + len(keyvals) + if len(keyvals)%2 != 0 { + n++ + } + kvs := make([]interface{}, 0, n) + kvs = append(kvs, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + kvs = append(kvs, l.keyvals...) + return &context{ + logger: l.logger, + keyvals: kvs, + hasValuer: l.hasValuer || containsValuer(keyvals), + sKeyvals: l.sKeyvals, + sHasValuer: l.sHasValuer, + } +} + +// WithSuffix returns a new contextual logger with keyvals appended to those +// passed to calls to Log. If logger is also a contextual logger created by +// With, WithPrefix, or WithSuffix, keyvals is appended to the existing context. +// +// The returned Logger replaces all value elements (odd indexes) containing a +// Valuer with their generated value for each call to its Log method. +func WithSuffix(logger Logger, keyvals ...interface{}) Logger { + if len(keyvals) == 0 { + return logger + } + l := newContext(logger) + // Limiting the capacity of the stored keyvals ensures that a new + // backing array is created if the slice must grow in Log or With. + // Using the extra capacity without copying risks a data race that + // would violate the Logger interface contract. + n := len(l.sKeyvals) + len(keyvals) + if len(keyvals)%2 != 0 { + n++ + } + kvs := make([]interface{}, 0, n) + kvs = append(kvs, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + kvs = append(l.sKeyvals, kvs...) + return &context{ + logger: l.logger, + keyvals: l.keyvals, + hasValuer: l.hasValuer, + sKeyvals: kvs, + sHasValuer: l.sHasValuer || containsValuer(keyvals), + } +} + +// context is the Logger implementation returned by With, WithPrefix, and +// WithSuffix. It wraps a Logger and holds keyvals that it includes in all +// log events. Its Log method calls bindValues to generate values for each +// Valuer in the context keyvals. +// +// A context must always have the same number of stack frames between calls to +// its Log method and the eventual binding of Valuers to their value. This +// requirement comes from the functional requirement to allow a context to +// resolve application call site information for a Caller stored in the +// context. To do this we must be able to predict the number of logging +// functions on the stack when bindValues is called. +// +// Two implementation details provide the needed stack depth consistency. +// +// 1. newContext avoids introducing an additional layer when asked to +// wrap another context. +// 2. With, WithPrefix, and WithSuffix avoid introducing an additional +// layer by returning a newly constructed context with a merged keyvals +// rather than simply wrapping the existing context. +type context struct { + logger Logger + keyvals []interface{} + sKeyvals []interface{} // suffixes + hasValuer bool + sHasValuer bool +} + +func newContext(logger Logger) *context { + if c, ok := logger.(*context); ok { + return c + } + return &context{logger: logger} +} + +// Log replaces all value elements (odd indexes) containing a Valuer in the +// stored context with their generated value, appends keyvals, and passes the +// result to the wrapped Logger. +func (l *context) Log(keyvals ...interface{}) error { + kvs := append(l.keyvals, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + if l.hasValuer { + // If no keyvals were appended above then we must copy l.keyvals so + // that future log events will reevaluate the stored Valuers. + if len(keyvals) == 0 { + kvs = append([]interface{}{}, l.keyvals...) + } + bindValues(kvs[:(len(l.keyvals))]) + } + kvs = append(kvs, l.sKeyvals...) + if l.sHasValuer { + bindValues(kvs[len(kvs)-len(l.sKeyvals):]) + } + return l.logger.Log(kvs...) +} + +// LoggerFunc is an adapter to allow use of ordinary functions as Loggers. If +// f is a function with the appropriate signature, LoggerFunc(f) is a Logger +// object that calls f. +type LoggerFunc func(...interface{}) error + +// Log implements Logger by calling f(keyvals...). +func (f LoggerFunc) Log(keyvals ...interface{}) error { + return f(keyvals...) +} diff --git a/vendor/github.com/go-kit/log/logfmt_logger.go b/vendor/github.com/go-kit/log/logfmt_logger.go new file mode 100644 index 00000000000..a00305298b8 --- /dev/null +++ b/vendor/github.com/go-kit/log/logfmt_logger.go @@ -0,0 +1,62 @@ +package log + +import ( + "bytes" + "io" + "sync" + + "github.com/go-logfmt/logfmt" +) + +type logfmtEncoder struct { + *logfmt.Encoder + buf bytes.Buffer +} + +func (l *logfmtEncoder) Reset() { + l.Encoder.Reset() + l.buf.Reset() +} + +var logfmtEncoderPool = sync.Pool{ + New: func() interface{} { + var enc logfmtEncoder + enc.Encoder = logfmt.NewEncoder(&enc.buf) + return &enc + }, +} + +type logfmtLogger struct { + w io.Writer +} + +// NewLogfmtLogger returns a logger that encodes keyvals to the Writer in +// logfmt format. Each log event produces no more than one call to w.Write. +// The passed Writer must be safe for concurrent use by multiple goroutines if +// the returned Logger will be used concurrently. +func NewLogfmtLogger(w io.Writer) Logger { + return &logfmtLogger{w} +} + +func (l logfmtLogger) Log(keyvals ...interface{}) error { + enc := logfmtEncoderPool.Get().(*logfmtEncoder) + enc.Reset() + defer logfmtEncoderPool.Put(enc) + + if err := enc.EncodeKeyvals(keyvals...); err != nil { + return err + } + + // Add newline to the end of the buffer + if err := enc.EndRecord(); err != nil { + return err + } + + // The Logger interface requires implementations to be safe for concurrent + // use by multiple goroutines. For this implementation that means making + // only one call to l.w.Write() for each call to Log. + if _, err := l.w.Write(enc.buf.Bytes()); err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/go-kit/log/nop_logger.go b/vendor/github.com/go-kit/log/nop_logger.go new file mode 100644 index 00000000000..1047d626c43 --- /dev/null +++ b/vendor/github.com/go-kit/log/nop_logger.go @@ -0,0 +1,8 @@ +package log + +type nopLogger struct{} + +// NewNopLogger returns a logger that doesn't do anything. +func NewNopLogger() Logger { return nopLogger{} } + +func (nopLogger) Log(...interface{}) error { return nil } diff --git a/vendor/github.com/go-kit/log/staticcheck.conf b/vendor/github.com/go-kit/log/staticcheck.conf new file mode 100644 index 00000000000..528438b97d2 --- /dev/null +++ b/vendor/github.com/go-kit/log/staticcheck.conf @@ -0,0 +1 @@ +checks = ["all"] diff --git a/vendor/github.com/go-kit/log/stdlib.go b/vendor/github.com/go-kit/log/stdlib.go new file mode 100644 index 00000000000..0338edbe2ba --- /dev/null +++ b/vendor/github.com/go-kit/log/stdlib.go @@ -0,0 +1,151 @@ +package log + +import ( + "bytes" + "io" + "log" + "regexp" + "strings" +) + +// StdlibWriter implements io.Writer by invoking the stdlib log.Print. It's +// designed to be passed to a Go kit logger as the writer, for cases where +// it's necessary to redirect all Go kit log output to the stdlib logger. +// +// If you have any choice in the matter, you shouldn't use this. Prefer to +// redirect the stdlib log to the Go kit logger via NewStdlibAdapter. +type StdlibWriter struct{} + +// Write implements io.Writer. +func (w StdlibWriter) Write(p []byte) (int, error) { + log.Print(strings.TrimSpace(string(p))) + return len(p), nil +} + +// StdlibAdapter wraps a Logger and allows it to be passed to the stdlib +// logger's SetOutput. It will extract date/timestamps, filenames, and +// messages, and place them under relevant keys. +type StdlibAdapter struct { + Logger + timestampKey string + fileKey string + messageKey string + prefix string + joinPrefixToMsg bool +} + +// StdlibAdapterOption sets a parameter for the StdlibAdapter. +type StdlibAdapterOption func(*StdlibAdapter) + +// TimestampKey sets the key for the timestamp field. By default, it's "ts". +func TimestampKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.timestampKey = key } +} + +// FileKey sets the key for the file and line field. By default, it's "caller". +func FileKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.fileKey = key } +} + +// MessageKey sets the key for the actual log message. By default, it's "msg". +func MessageKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.messageKey = key } +} + +// Prefix configures the adapter to parse a prefix from stdlib log events. If +// you provide a non-empty prefix to the stdlib logger, then your should provide +// that same prefix to the adapter via this option. +// +// By default, the prefix isn't included in the msg key. Set joinPrefixToMsg to +// true if you want to include the parsed prefix in the msg. +func Prefix(prefix string, joinPrefixToMsg bool) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.prefix = prefix; a.joinPrefixToMsg = joinPrefixToMsg } +} + +// NewStdlibAdapter returns a new StdlibAdapter wrapper around the passed +// logger. It's designed to be passed to log.SetOutput. +func NewStdlibAdapter(logger Logger, options ...StdlibAdapterOption) io.Writer { + a := StdlibAdapter{ + Logger: logger, + timestampKey: "ts", + fileKey: "caller", + messageKey: "msg", + } + for _, option := range options { + option(&a) + } + return a +} + +func (a StdlibAdapter) Write(p []byte) (int, error) { + p = a.handlePrefix(p) + + result := subexps(p) + keyvals := []interface{}{} + var timestamp string + if date, ok := result["date"]; ok && date != "" { + timestamp = date + } + if time, ok := result["time"]; ok && time != "" { + if timestamp != "" { + timestamp += " " + } + timestamp += time + } + if timestamp != "" { + keyvals = append(keyvals, a.timestampKey, timestamp) + } + if file, ok := result["file"]; ok && file != "" { + keyvals = append(keyvals, a.fileKey, file) + } + if msg, ok := result["msg"]; ok { + msg = a.handleMessagePrefix(msg) + keyvals = append(keyvals, a.messageKey, msg) + } + if err := a.Logger.Log(keyvals...); err != nil { + return 0, err + } + return len(p), nil +} + +func (a StdlibAdapter) handlePrefix(p []byte) []byte { + if a.prefix != "" { + p = bytes.TrimPrefix(p, []byte(a.prefix)) + } + return p +} + +func (a StdlibAdapter) handleMessagePrefix(msg string) string { + if a.prefix == "" { + return msg + } + + msg = strings.TrimPrefix(msg, a.prefix) + if a.joinPrefixToMsg { + msg = a.prefix + msg + } + return msg +} + +const ( + logRegexpDate = `(?P[0-9]{4}/[0-9]{2}/[0-9]{2})?[ ]?` + logRegexpTime = `(?P