Merge pull request #14 from vimeo/add-backend

Add Vault EngineType

Author: sergiosalvatore

README.md

@@ -16,6 +16,7 @@ vault:
   url: <url to vault>
   authType: # "token" or "gcp-default"
   token: <token value> # if authType == "token" is provided
+  defaultEngineType: # "kv" or "kv-v2" (currently supported)
   role: "vault role" # if left empty, queries the GCP metadata service
   tls: # optional [tls options](https://godoc.org/github.com/hashicorp/vault/api#TLSConfig)
 namespace: <kubernetes namespace for created secrets>
@@ -24,13 +25,59 @@ mappings:
   # mappings from vault paths to kubernetes secret names
   - vaultPath: secret/data/vault-path
     secretName: k8s-secretname
+    vaultEngineType: # optionally "kv" or "kv-v2" to override the defaultEngineType specified above
 ```
 
 ### Labels and Reconciliation
 By default, Pentagon will add a [metadata label](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) with the key `pentagon` and the value `default`.  At the least, this helps identify Pentagon as the creator and maintainer of the secret.
 
 If you set the `label` configuration parameter, you can control the value of the label, allowing multiple Pentagon instances to exist without stepping on each other.  Setting a non-default `label` also enables reconciliation which will cleanup any secrets that were created by Pentagon with a matching label, but are no longer present in the `mappings` configuration.  This provides a simple way to ensure that old secret data does not remain present in your system after its time has passed.
 
+### About Vault Engine Types
+Apparently, different Vault secrets engines have slightly different APIs for returning data.  For instance, here is the response for version 1 of the key/value store:
+
+```json
+{
+  "request_id": "12a0c057-f475-4bbd-6305-e4c07e66805c",
+  "lease_id": "",
+  "renewable": false,
+  "lease_duration": 2764800,
+  "data": {
+    "foo": "world"
+  },
+  "wrap_info": null,
+  "warnings": null,
+  "auth": null
+}
+```
+
+Notice that the `data` object has the `foo` key embedded directly.  Alternatively, here is the response for version 2 of the key/value store:
+
+```json
+{
+  "request_id": "78b921ae-79a8-d7e3-da16-336b634fff22",
+  "lease_id": "",
+  "renewable": false,
+  "lease_duration": 0,
+  "data": {
+    "data": {
+      "foo": "world"
+    },
+    "metadata": {
+      "created_time": "2019-10-01T19:36:25.285387Z",
+      "deletion_time": "",
+      "destroyed": false,
+      "version": 1
+    }
+  },
+  "wrap_info": null,
+  "warnings": null,
+  "auth": null
+}
+```
+
+Notice the extra `data` element nested inside the outer `data`.  Vault secrets engines can be mounted at arbitrary paths and it does not appear to be possible to reliably detect which engine was used in the API response directly.  In order to properly unwrap the secret data,indicate either `kv` or `kv-v2` as the `vaultEngineType` in the configuration.  In the common case of using only one secrets engine,  simply define the `defaultEngineType` in the `vault` configuration block and the mapping-level `vaultEngineType` will inherit the default.  For compatibility, the unset default value defaults to `kv`.  Note that this differs from the current default that Vault itself uses for the key/value secrets engine.
+
 ## Return Values
 The application will return 0 on success (when all keys were copied/updated successfully).  A complete list of all possible return values follows:
 
@@ -70,7 +117,7 @@ spec:
           restartPolicy: OnFailure
           containers:
           - name: pentagon
-            image: vimeo/pentagon:v1.0.0
+            image: vimeo/pentagon:v1.1.0
             args: ["/etc/pentagon/pentagon.yaml"]
             imagePullPolicy: Always
             resources:

config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/hashicorp/vault/api"
+	"github.com/vimeo/pentagon/vault"
 )
 
 // DefaultNamespace is the default kubernetes namespace.
@@ -13,19 +14,6 @@ const DefaultNamespace = "default"
 // created by pentagon.
 const DefaultLabelValue = "default"
 
-// VaultAuthType is a custom type to represent different Vault authentication
-// methods.
-type VaultAuthType string
-
-// VaultAuthTypeToken expects the Token property to be set on the VaultConfig
-// struct with a token to use.
-const VaultAuthTypeToken VaultAuthType = "token"
-
-// VaultAuthTypeGCPDefault expects the Role property of the VaultConfig struct
-// to be populated with the role that vault expects and will use the machine's
-// default service account, running within GCP.
-const VaultAuthTypeGCPDefault VaultAuthType = "gcp-default"
-
 // Config describes the configuration for vaultofsecrets
 type Config struct {
 	// VaultURL is the URL used to connect to vault.
@@ -52,6 +40,19 @@ func (c *Config) SetDefaults() {
 	if c.Label == "" {
 		c.Label = DefaultLabelValue
 	}
+
+	// default to engine type key/value v1 for backward compatibility
+	if c.Vault.DefaultEngineType == "" {
+		c.Vault.DefaultEngineType = vault.EngineTypeKeyValueV1
+	}
+
+	// set all the underlying mapping engine types to their default
+	// if unspecified
+	for _, m := range c.Mappings {
+		if m.VaultEngineType == "" {
+			m.VaultEngineType = c.Vault.DefaultEngineType
+		}
+	}
 }
 
 // Validate checks to make sure that the configuration is valid.
@@ -69,7 +70,13 @@ type VaultConfig struct {
 	URL string `yaml:"url"`
 
 	// AuthType can be "token" or "gcp-default".
-	AuthType VaultAuthType `yaml:"authType"`
+	AuthType vault.AuthType `yaml:"authType"`
+
+	// DefaultEngineType is the type of secrets engine used because the API
+	// responses may differ based on the engine used.  In particular, K/V v2
+	// has an extra layer of data wrapping that differs from v1.
+	// Allowed values are "kv" and "kv-v2".
+	DefaultEngineType vault.EngineType `yaml:"defaultEngineType"`
 
 	// Role is the role used when authenticating with vault.  If this is unset
 	// the role will be discovered by querying the GCP metadata service for
@@ -94,4 +101,9 @@ type Mapping struct {
 	// be written to.  Note that this must be a DNS-1123-compatible name and
 	// match the regex [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*
 	SecretName string `yaml:"secretName"`
+
+	// VaultEngineType is the type of secrets engine mounted at the path of this
+	// Vault secret.  This specifically overrides the DefaultEngineType
+	// specified in VaultConfig.
+	VaultEngineType vault.EngineType `yaml:"vaultEngineType"`
 }

config_test.go

@@ -1,11 +1,15 @@
 package pentagon
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/vimeo/pentagon/vault"
+)
 
 func TestSetDefaults(t *testing.T) {
 	c := &Config{
 		Vault: VaultConfig{
-			AuthType: VaultAuthTypeToken,
+			AuthType: vault.AuthTypeToken,
 		},
 	}
 
@@ -17,6 +21,16 @@ func TestSetDefaults(t *testing.T) {
 	if c.Namespace != DefaultNamespace {
 		t.Fatalf("namespace should be %s, is %s", DefaultNamespace, c.Namespace)
 	}
+
+	if c.Vault.DefaultEngineType != vault.EngineTypeKeyValueV1 {
+		t.Fatalf("unexpected default engine type: %s", c.Vault.DefaultEngineType)
+	}
+
+	for _, m := range c.Mappings {
+		if m.VaultEngineType == "" {
+			t.Fatalf("empty vault engine type for mapping: %+v", m)
+		}
+	}
 }
 
 func TestNoClobber(t *testing.T) {
@@ -56,5 +70,4 @@ func TestValidate(t *testing.T) {
 	if err != nil {
 		t.Fatalf("configuration should have been valid: %s", err)
 	}
-
 }

pentagon/main.go

@@ -15,6 +15,7 @@ import (
 	"k8s.io/client-go/rest"
 
 	"github.com/vimeo/pentagon"
+	"github.com/vimeo/pentagon/vault"
 )
 
 func main() {
@@ -104,9 +105,9 @@ func getVaultClient(vaultConfig pentagon.VaultConfig) (*api.Client, error) {
 	}
 
 	switch vaultConfig.AuthType {
-	case pentagon.VaultAuthTypeToken:
+	case vault.AuthTypeToken:
 		client.SetToken(vaultConfig.Token)
-	case pentagon.VaultAuthTypeGCPDefault:
+	case vault.AuthTypeGCPDefault:
 		// default to using configured Role
 		role := vaultConfig.Role
 

pentagon/main_test.go

@@ -26,7 +26,7 @@ const k8sNamespace = "default"
 const configMapName = "pentagon-config"
 
 type integrationTests struct {
-	vaultInstance vault
+	vaultInstance vaultHelper
 	pj            pentagonJob
 	client        *kubernetes.Clientset
 }
@@ -85,7 +85,7 @@ func TestIntegration(t *testing.T) {
 		t.Fatalf("unable to delete existing jobs: %s", err)
 	}
 
-	vaultInstance := vault{
+	vaultInstance := vaultHelper{
 		client:     clientset,
 		restConfig: restConfig,
 		namespace:  k8sNamespace,

pentagon/vault_test.go

@@ -16,15 +16,15 @@ import (
 )
 
 // helper struct for working with vault instance in k8s
-type vault struct {
+type vaultHelper struct {
 	client     *kubernetes.Clientset
 	restConfig *restclient.Config // this was used to create the client above!
 	namespace  string
 	pod        *v1.Pod
 }
 
 // sets a secret by exec'ing into the pod and running the command line client.
-func (v *vault) setSecret(path string, values map[string]string) error {
+func (v *vaultHelper) setSecret(path string, values map[string]string) error {
 	valuePairs := []string{}
 	for k, v := range values {
 		valuePairs = append(valuePairs, fmt.Sprintf("%s=%s", k, v))
@@ -69,12 +69,12 @@ func (v *vault) setSecret(path string, values map[string]string) error {
 }
 
 // returns the url to the pod.
-func (v *vault) url() string {
+func (v *vaultHelper) url() string {
 	return fmt.Sprintf("http://%s:%d", v.pod.Status.PodIP, vaultPort)
 }
 
 // create a vault instance in k8s.
-func (v *vault) create(wait time.Duration) error {
+func (v *vaultHelper) create(wait time.Duration) error {
 	// see if it exists first
 	pods := v.client.CoreV1().Pods(v.namespace)
 	p, err := pods.Get("vault", metav1.GetOptions{})

reflector.go

@@ -80,10 +80,28 @@ func (r *Reflector) Reflect(mappings []Mapping) error {
 			return fmt.Errorf("secret %s not found", mapping.VaultPath)
 		}
 
+		var k8sSecretData map[string][]byte
+
 		// convert map[string]interface{} to map[string][]byte
-		k8sSecretData, err := r.castData(secretData.Data)
-		if err != nil {
-			return fmt.Errorf("error casting data: %s", err)
+		switch mapping.VaultEngineType {
+		case vault.EngineTypeKeyValueV1:
+			k8sSecretData, err = r.castData(secretData.Data)
+			if err != nil {
+				return fmt.Errorf("error casting data: %s", err)
+			}
+		case vault.EngineTypeKeyValueV2:
+			// there's an extra level of wrapping with the v2 kv secrets engine
+			if unwrapped, ok := secretData.Data["data"].(map[string]interface{}); ok {
+				k8sSecretData, err = r.castData(unwrapped)
+			} else {
+				return fmt.Errorf("key/value v2 interface did not have " +
+					"expected extra wrapping")
+			}
+		default:
+			return fmt.Errorf(
+				"unknown vault engine type: %q",
+				mapping.VaultEngineType,
+			)
 		}
 
 		// create the new Secret