Merge pull request #29 from vimeo/gsm-encoding-fix-unwrapping

Google Secret Manger Enhancements

Author: sergiosalvatore

README.md

@@ -2,10 +2,10 @@
 [![GoDoc](https://godoc.org/github.com/vimeo/pentagon?status.svg)](https://godoc.org/github.com/vimeo/pentagon) [![Go Report Card](https://goreportcard.com/badge/github.com/vimeo/pentagon)](https://goreportcard.com/report/github.com/vimeo/pentagon) 
 
 # Pentagon
-Pentagon is a small application designed to run as a Kubernetes CronJob to periodically copy secrets stored in [Vault](https://www.vaultproject.io) or Google Secrets Manager into equivalent [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/), keeping them synchronized.  Naturally, this should be used with care as "standard" Kubernetes Secrets are simply obfuscated as base64-encoded strings.  However, one can and should use more secure methods of securing secrets including Google's [KMS](https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets) and restricting roles and service accounts appropriately.
+Pentagon is a small application designed to run as a Kubernetes CronJob to periodically copy secrets stored in [Vault](https://www.vaultproject.io) or [Google Secret Manager](https://cloud.google.com/security/products/secret-manager) into equivalent [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/), keeping them synchronized.  Naturally, this should be used with care as "standard" Kubernetes Secrets are simply obfuscated as base64-encoded strings.  However, one can and should use more secure methods of securing secrets including Google's [KMS](https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets) and restricting roles and service accounts appropriately.
 
-## Why not just query Vault?
-That's a good question.  If you have a highly-available Vault setup that is stable and performant and you're able to modify your applications to query Vault, that's a completely reasonable approach to take.  If you don't have such a setup, Pentagon provides a way to cache things securely in Kubernetes secrets which can then be provided to applications without directly introducing a Vault dependency.
+## Why not just query Vault or Google Secret Manager?
+That's a good question.  If you have a highly-available Vault setup that is stable and performant and you're able to modify your applications to query Vault, that's a completely reasonable approach to take.  Similarly, if you are able to modify your application to query Google Secret Manager, that's an entirely valid solution.  If you don't have such a setup, Pentagon provides a way to cache things securely in Kubernetes secrets which can then be provided to applications without directly introducing a dependency on Vault or Google Secret Manager.
 
 ## Configuration
 Pentagon requires a YAML configuration file, the path to which should be passed as the first and only argument to the application.  It is recommended that you store this configuration in a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) and reference it in the CronJob specification.  A sample configuration follows:
@@ -82,6 +82,11 @@ Notice that the `data` object has the `foo` key embedded directly.  Alternativel
 
 Notice the extra `data` element nested inside the outer `data`.  Vault secrets engines can be mounted at arbitrary paths and it does not appear to be possible to reliably detect which engine was used in the API response directly.  In order to properly unwrap the secret data,indicate either `kv` or `kv-v2` as the `vaultEngineType` in the configuration.  In the common case of using only one secrets engine,  simply define the `defaultEngineType` in the `vault` configuration block and the mapping-level `vaultEngineType` will inherit the default.  For compatibility, the unset default value defaults to `kv`.  Note that this differs from the current default that Vault itself uses for the key/value secrets engine.
 
+## Special Things about Google Secret Manager
+Google Secret Manager's API simply returns arbitrary bytes as the value of a secret, making no assumptions about its encoding.  Kubernetes Secrets, on the other hand, can contain multiple key/value pairs.  If you would like a single Google Secret Manager Secret to unwrap into multiple key/value pairs in the Kubernetes Secret, add `gsmEncoding: "json"` to the mapping value.  Then store a JSON document in Google Secret Manager with JSON that will successfully unmarshal to a `map[string]any`.  The key in that map will be used as the key of the Kubernetes Secret.  If that value is a string or number, the value will be stored without any quoting.  If the value is a JSON object or array it will be stored directly as the string serialization of that structure.
+
+In cases where `gsmEncoding` is not set to json, the key's value will default to the name of the secret (`secretName` in the mapping).  If you would like to override this, set `gsmSecretKeyValue` to your preferred key.
+
 ## Return Values
 The application will return 0 on success (when all keys were copied/updated successfully).  A complete list of all possible return values follows:
 

config.go

@@ -173,4 +173,9 @@ type Mapping struct {
 	// GSMEncodingType enables the parsing of JSON secrets with more than one key-value pair when set
 	// to 'json'. For the default behavior, simple values, set to 'string'.
 	GSMEncodingType string `yaml:"gsmEncodingType"`
+
+	// GSMSecretKeyValue allows you to specify the value of the Kubernetes key to
+	// use for this secret's value in cases where gsmEncodingType is *not* json.  If
+	// this is unset, the key name will default to the value of secretName.
+	GSMSecretKeyValue string `yaml:"gsmSecretKeyValue"`
 }

pentagon/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net/url"
 	"os"
@@ -45,7 +44,7 @@ func main() {
 		os.Exit(10)
 	}
 
-	configFile, err := ioutil.ReadFile(os.Args[1])
+	configFile, err := os.ReadFile(os.Args[1])
 	if err != nil {
 		log.Printf("error opening configuration file: %s", err)
 		os.Exit(20)

reflector.go

@@ -167,12 +167,22 @@ func (r *Reflector) getGSMSecret(ctx context.Context, mapping Mapping) (map[stri
 		}
 		casted := make(map[string][]byte, len(unmarshaled))
 		for k, v := range unmarshaled {
+			var stringVal string
+			if err := json.Unmarshal(v, &stringVal); err == nil {
+				casted[k] = []byte(stringVal)
+				continue
+			}
 			casted[k] = v
 		}
 		return casted, nil
 	}
 
-	return map[string][]byte{mapping.SecretName: resp.Payload.Data}, nil
+	keyName := mapping.GSMSecretKeyValue
+	if keyName == "" {
+		keyName = mapping.SecretName
+	}
+
+	return map[string][]byte{keyName: resp.Payload.Data}, nil
 }
 
 func (r *Reflector) createK8sSecret(ctx context.Context, mapping Mapping, data map[string][]byte) error {

reflector_test.go

@@ -2,6 +2,7 @@ package pentagon
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 
 	v1 "k8s.io/api/core/v1"
@@ -101,9 +102,10 @@ func TestReflectorGSM(t *testing.T) {
 
 	err := r.Reflect(ctx, []Mapping{
 		{
-			SourceType: "gsm",
-			Path:       "projects/foo/secrets/bar/versions/latest",
-			SecretName: "foo",
+			SourceType:        "gsm",
+			Path:              "projects/foo/secrets/bar/versions/latest",
+			SecretName:        "foo",
+			GSMSecretKeyValue: "foo-key",
 		},
 	})
 	if err != nil {
@@ -126,12 +128,12 @@ func TestReflectorGSM(t *testing.T) {
 		)
 	}
 
-	if string(secret.Data["foo"]) != "foo_bar_latest" {
+	if string(secret.Data["foo-key"]) != "foo_bar_latest" {
 		t.Fatalf("secret value does not equal foo_bar_latest: %s", string(secret.Data["foo"]))
 	}
 }
 
-func TestReflectorGSMJSON(t *testing.T) {
+func TestReflectorGSMJSONStruct(t *testing.T) {
 	ctx := context.Background()
 	k8sClient := k8sfake.NewSimpleClientset()
 
@@ -174,12 +176,79 @@ func TestReflectorGSMJSON(t *testing.T) {
 		)
 	}
 
-	if string(secret.Data["key1"]) != `{"int": 1, "string": "hello"}` {
-		t.Fatalf("secret value does not equal struct: %s", string(secret.Data["key1"]))
+	data := map[string]any{}
+	if err := json.Unmarshal(secret.Data["key1"], &data); err != nil {
+		t.Fatalf("error unmarshaling secret data for key1: %s", err)
+	}
+	if data["int"] != float64(1) {
+		t.Errorf("secret data for key1 does not contain expected int: %+v", data["int"])
+	}
+	if data["string"] != "hello" {
+		t.Errorf("secret data for key1 does not contain expected string: %v", data["string"])
+	}
+
+	data = map[string]any{}
+	if err := json.Unmarshal(secret.Data["key2"], &data); err != nil {
+		t.Fatalf("error unmarshaling secret data for key1: %s", err)
+	}
+
+	if data["float"] != 3.14 {
+		t.Errorf("secret data for key2 does not contain expected float: %v", data["float"])
+	}
+	if data["bool"] != true {
+		t.Errorf("secret data for key2 does not contain expected bool: %v", data["bool"])
+	}
+}
+
+func TestReflectorGSMJSONUnwrap(t *testing.T) {
+	ctx := context.Background()
+	k8sClient := k8sfake.NewSimpleClientset()
+
+	gsm := gsm.NewMockGSM(map[string][]byte{
+		"projects/foo/secrets/bar/versions/latest": []byte(`{"key1": 1, "key2": "val2\nval3"}`),
+	})
+
+	r := NewReflector(
+		nil,
+		gsm,
+		k8sClient, DefaultNamespace,
+		DefaultLabelValue,
+	)
+
+	err := r.Reflect(ctx, []Mapping{
+		{
+			SourceType:      "gsm",
+			Path:            "projects/foo/secrets/bar/versions/latest",
+			GSMEncodingType: GSMEncodingTypeJSON,
+			SecretName:      "foo",
+		},
+	})
+	if err != nil {
+		t.Fatalf("reflect didn't work: %s", err)
+	}
+
+	// now get the secret out of k8s
+	secrets := k8sClient.CoreV1().Secrets(DefaultNamespace)
+
+	secret, err := secrets.Get(ctx, "foo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("secret should be there: %s", err)
+	}
+
+	if secret.Labels[LabelKey] != DefaultLabelValue {
+		t.Fatalf(
+			"secret pentagon label should be %s is %s",
+			DefaultLabelValue,
+			secret.Labels[LabelKey],
+		)
+	}
+
+	if string(secret.Data["key1"]) != "1" {
+		t.Fatalf("secret value does not equal bare int: %s", string(secret.Data["key1"]))
 	}
 
-	if string(secret.Data["key2"]) != `{"float": 3.14, "bool": true}` {
-		t.Fatalf("secret value does not equal struct: %s", string(secret.Data["key2"]))
+	if string(secret.Data["key2"]) != "val2\nval3" {
+		t.Fatalf("secret value does not equal string: %s", string(secret.Data["key2"]))
 	}
 }