Aws secretsmanager additions

CHANGELOG.md

@@ -66,6 +66,7 @@ Here is an overview of all new **experimental** features:
 
 ### Improvements
 
+- General: Add SecretKey to AWS SecretsManager TriggerAuthentication to allow parsing JSON / Key/Value Pairs in secrets (#5940)
 - TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX))
 
 ### Fixes

apis/keda/v1alpha1/triggerauthentication_types.go

@@ -376,6 +376,8 @@ type AwsSecretManagerSecret struct {
 	VersionID string `json:"versionId,omitempty"`
 	// +optional
 	VersionStage string `json:"versionStage,omitempty"`
+	// +optional
+	SecretKey string `json:"secretKey,omitempty"`
 }
 
 func init() {

config/crd/bases/keda.sh_clustertriggerauthentications.yaml

@@ -183,6 +183,8 @@ spec:
                           type: string
                         parameter:
                           type: string
+                        secretKey:
+                          type: string
                         versionId:
                           type: string
                         versionStage:

config/crd/bases/keda.sh_triggerauthentications.yaml

@@ -182,6 +182,8 @@ spec:
                           type: string
                         parameter:
                           type: string
+                        secretKey:
+                          type: string
                         versionId:
                           type: string
                         versionStage:

pkg/scaling/resolver/aws_secretmanager_handler.go

@@ -18,6 +18,7 @@ package resolver
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -43,9 +44,9 @@ func NewAwsSecretManagerHandler(a *kedav1alpha1.AwsSecretManager) *AwsSecretMana
 	}
 }
 
-// Read fetches the secret value from AWS Secret Manager using the provided secret name, version ID(optional), and version stage(optional).
+// Read fetches the secret value from AWS Secret Manager using the provided secret name, version ID(optional), version stage(optional), and secretKey(optional).
 // It returns the secret value as a string.
-func (ash *AwsSecretManagerHandler) Read(ctx context.Context, logger logr.Logger, secretName, versionID, versionStage string) (string, error) {
+func (ash *AwsSecretManagerHandler) Read(ctx context.Context, logger logr.Logger, secretName, versionID, versionStage string, secretKey string) (string, error) {
 	input := &secretsmanager.GetSecretValueInput{
 		SecretId: aws.String(secretName),
 	}
@@ -60,6 +61,27 @@ func (ash *AwsSecretManagerHandler) Read(ctx context.Context, logger logr.Logger
 		logger.Error(err, "Error getting credentials")
 		return "", err
 	}
+	if secretKey != "" {
+		// Parse the secret string as JSON
+		var secretMap map[string]interface{}
+		err = json.Unmarshal([]byte(*result.SecretString), &secretMap)
+		if err != nil {
+			logger.Error(err, "Error parsing secret string as JSON")
+			return "", err
+		}
+
+		// Check if the specified secret key exists
+		if val, ok := secretMap[secretKey]; ok {
+			// Convert the value to a string and return it
+			if strVal, isString := val.(string); isString {
+				return strVal, nil
+			}
+			logger.Error(nil, "SecretKey value is not a string")
+			return "", fmt.Errorf("SecretKey value is not a string")
+		}
+		logger.Error(nil, "SecretKey Not Found")
+		return "", fmt.Errorf("SecretKey Not Found")
+	}
 	return *result.SecretString, nil
 }
 

pkg/scaling/resolver/scale_resolvers.go

@@ -333,10 +333,10 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge
 					logger.Error(err, "error authenticating to Aws Secret Manager", "triggerAuthRef.Name", triggerAuthRef.Name)
 				} else {
 					for _, secret := range triggerAuthSpec.AwsSecretManager.Secrets {
-						res, err := awsSecretManagerHandler.Read(ctx, logger, secret.Name, secret.VersionID, secret.VersionStage)
+						res, err := awsSecretManagerHandler.Read(ctx, logger, secret.Name, secret.VersionID, secret.VersionStage, secret.SecretKey)
 						if err != nil {
 							logger.Error(err, "error trying to read secret from Aws Secret Manager", "triggerAuthRef.Name", triggerAuthRef.Name,
-								"secret.Name", secret.Name, "secret.Version", secret.VersionID, "secret.VersionStage", secret.VersionStage)
+								"secret.Name", secret.Name, "secret.Version", secret.VersionID, "secret.VersionStage", secret.VersionStage, "secret.SecretKey", secret.SecretKey)
 						} else {
 							result[secret.Parameter] = res
 						}

tests/secret-providers/aws_secretmanager/aws_secretmanager_test.go

@@ -6,6 +6,7 @@
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"os"
 	"testing"
@@ -151,6 +152,31 @@
       name: {{.SecretManagerSecretName}}
 `
 
+triggerAuthenticationSecretKeyTemplate = `apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: {{.TriggerAuthenticationName}}
+  namespace: {{.TestNamespace}}
+spec:
+  awsSecretManager:
+    credentials:
+      accessKey:
+        valueFrom:
+          secretKeyRef:
+            name: {{.AwsCredentialsSecretName}}
+            key: AWS_ACCESS_KEY_ID
+      accessSecretKey:
+        valueFrom:
+          secretKeyRef:
+            name: {{.AwsCredentialsSecretName}}
+            key: AWS_SECRET_ACCESS_KEY
+    region: {{.AwsRegion}}
+    secrets:
+    - parameter: connection
+      name: {{.SecretManagerSecretName}}
+	  secretKey: connectionString
+`
+
 	scaledObjectTemplate = `apiVersion: keda.sh/v1alpha1
 kind: ScaledObject
 metadata:
@@ -257,11 +283,12 @@
 )
 
 func TestAwsSecretManager(t *testing.T) {
+	var useJSONSecretFormat = false
 	require.NotEmpty(t, awsAccessKeyID, "TF_AWS_ACCESS_KEY env variable is required for AWS Secret Manager test")
 	require.NotEmpty(t, awsSecretAccessKey, "TF_AWS_SECRET_KEY env variable is required for AWS Secret Manager test")
 
-	// Create the secret in GCP
-	err := createAWSSecret(t)
+	// Create the secret in AWS
+	err := createAWSSecret(t, useJSONSecretFormat)
 	assert.NoErrorf(t, err, "cannot create AWS Secret Manager secret - %s", err)
 
 	// Create kubernetes resources for PostgreSQL server
@@ -280,7 +307,7 @@
 	assert.True(t, ok, "executing a command on PostreSQL Pod should work; Output: %s, ErrorOutput: %s, Error: %s", out, errOut, err)
 
 	// Create kubernetes resources for testing
-	data, templates := getTemplateData()
+	data, templates := getTemplateData(useJSONSecretFormat)
 
 	KubectlApplyMultipleWithTemplate(t, data, templates)
 	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3),
@@ -292,7 +319,49 @@
 	KubectlDeleteMultipleWithTemplate(t, data, templates)
 	DeleteKubernetesResources(t, testNamespace, data, postgreSQLtemplates)
 
-	// Delete the secret in GCP
+	// Delete the secret in AWS
+	err = deleteAWSSecret(t)
+	assert.NoErrorf(t, err, "cannot delete AWS Secret Manager secret - %s", err)
+}
+
+func TestAwsSecretManagerJSONFormat(t *testing.T) {
+	var useJSONSecretFormat = true
+	require.NotEmpty(t, awsAccessKeyID, "TF_AWS_ACCESS_KEY env variable is required for AWS Secret Manager test")
+	require.NotEmpty(t, awsSecretAccessKey, "TF_AWS_SECRET_KEY env variable is required for AWS Secret Manager test")
+
+	// Create the secret in AWS
+	err := createAWSSecret(t, useJSONSecretFormat) // Create JSON formatted Secret
+	assert.NoErrorf(t, err, "cannot create AWS Secret Manager secret - %s", err)
+
+	// Create kubernetes resources for PostgreSQL server
+	kc := GetKubernetesClient(t)
+	data, postgreSQLtemplates := getPostgreSQLTemplateData()
+
+	CreateKubernetesResources(t, kc, testNamespace, data, postgreSQLtemplates)
+
+	assert.True(t, WaitForStatefulsetReplicaReadyCount(t, kc, postgreSQLStatefulSetName, testNamespace, 1, 60, 3),
+		"replica count should be %d after 3 minutes", 1)
+
+	createTableSQL := "CREATE TABLE task_instance (id serial PRIMARY KEY,state VARCHAR(10));"
+	psqlCreateTableCmd := fmt.Sprintf("psql -U %s -d %s -c \"%s\"", postgreSQLUsername, postgreSQLDatabase, createTableSQL)
+
+	ok, out, errOut, err := WaitForSuccessfulExecCommandOnSpecificPod(t, postgresqlPodName, testNamespace, psqlCreateTableCmd, 60, 3)
+	assert.True(t, ok, "executing a command on PostreSQL Pod should work; Output: %s, ErrorOutput: %s, Error: %s", out, errOut, err)
+
+	// Create kubernetes resources for testing
+	data, templates := getTemplateData(useJSONSecretFormat)
+
+	KubectlApplyMultipleWithTemplate(t, data, templates)
+	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3),
+		"replica count should be %d after 3 minutes", minReplicaCount)
+
+	testScaleOut(t, kc, data)
+
+	// cleanup
+	KubectlDeleteMultipleWithTemplate(t, data, templates)
+	DeleteKubernetesResources(t, testNamespace, data, postgreSQLtemplates)
+
+	// Delete the secret in AWS
 	err = deleteAWSSecret(t)
 	assert.NoErrorf(t, err, "cannot delete AWS Secret Manager secret - %s", err)
 }
@@ -324,14 +393,21 @@
 	}
 }
 
-func getTemplateData() (templateData, []Template) {
-	return data, []Template{
-		{Name: "secretTemplate", Config: secretTemplate},
-		{Name: "awsCredentialsSecretTemplate", Config: awsCredentialsSecretTemplate},
-		{Name: "deploymentTemplate", Config: deploymentTemplate},
-		{Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate},
-		{Name: "scaledObjectTemplate", Config: scaledObjectTemplate},
-	}
+func getTemplateData(useJSONFormat bool) (templateData, []Template) {
+	var triggerConfig string
+    if useJSONFormat {
+        triggerConfig = triggerAuthenticationSecretKeyTemplate
+    } else {
+        triggerConfig = triggerAuthenticationTemplate
+    }
+
+    return data, []Template{
+        {Name: "secretTemplate", Config: secretTemplate},
+        {Name: "awsCredentialsSecretTemplate", Config: awsCredentialsSecretTemplate},
+        {Name: "deploymentTemplate", Config: deploymentTemplate},
+        {Name: "triggerAuthenticationTemplate", Config: triggerConfig},
+        {Name: "scaledObjectTemplate", Config: scaledObjectTemplate},
+    }
 }
 
 func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) {
@@ -342,7 +418,7 @@
 		"replica count should be %d after 3 minutes", maxReplicaCount)
 }
 
-func createAWSSecret(t *testing.T) error {
+func createAWSSecret(t *testing.T, useJSONFormat bool) error {
 	ctx := context.Background()
 
 	// Create AWS configuration
@@ -358,9 +434,24 @@
 
 	// Create a Secrets Manager client
 	client := secretsmanager.NewFromConfig(cfg)
-
+	
 	// Create the secret value
-	secretString := postgreSQLConnectionString
+	var secretString string
+	if(useJSONFormat) {
+		secretObject := map[string]string{
+			"connectionString": postgreSQLConnectionString,
+		}
+		// Convert the map to a JSON string
+		jsonData, err := json.Marshal(secretObject)
+		if err != nil {
+			return fmt.Errorf("Error converting to JSON: %v", err)
+		}
+
+		// Print the JSON string
+		secretString := string(jsonData)
+	} else {
+		secretString := postgreSQLConnectionString
+	}
 	_, err = client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{
 		Name:         &secretManagerSecretName,
 		SecretString: &secretString,