Feature/gh 618 vault dynamic secrets for gcp

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## UNRELEASED
 
+### FEATURES
+
+* Support Vault dynamic secret generation for GCP ([GH-618](https://github.com/ystia/yorc/issues/618))
+
 ### DEPENDENCIES
 
 * The orchestrator requires now at least Terraform google plugin with version constraint `~ 2.20` ([GH-628](https://github.com/ystia/yorc/issues/628))

deployments/artifacts_test.go

@@ -16,13 +16,14 @@ package deployments
 
 import (
 	"context"
+	"strings"
+	"testing"
+
 	"github.com/ystia/yorc/v4/storage"
 	"github.com/ystia/yorc/v4/storage/types"
 	"github.com/ystia/yorc/v4/tosca"
-	"strings"
-	"testing"
 
-	"github.com/hashicorp/consul/testutil"
+	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/stretchr/testify/require"
 
 	"github.com/ystia/yorc/v4/helper/consulutil"

deployments/capabilities_test.go

@@ -16,14 +16,15 @@ package deployments
 
 import (
 	"context"
-	"github.com/ystia/yorc/v4/storage"
-	"github.com/ystia/yorc/v4/storage/types"
-	"github.com/ystia/yorc/v4/tosca"
 	"reflect"
 	"strings"
 	"testing"
 
-	"github.com/hashicorp/consul/testutil"
+	"github.com/ystia/yorc/v4/storage"
+	"github.com/ystia/yorc/v4/storage/types"
+	"github.com/ystia/yorc/v4/tosca"
+
+	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/stretchr/testify/require"
 	"github.com/ystia/yorc/v4/helper/consulutil"
 	"github.com/ystia/yorc/v4/log"

deployments/definition_store_test.go

@@ -17,9 +17,6 @@ package deployments
 import (
 	"context"
 	"fmt"
-	"github.com/ystia/yorc/v4/storage"
-	"github.com/ystia/yorc/v4/storage/types"
-	"github.com/ystia/yorc/v4/tosca"
 	"io/ioutil"
 	stdlog "log"
 	"os"
@@ -29,9 +26,13 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/ystia/yorc/v4/storage"
+	"github.com/ystia/yorc/v4/storage/types"
+	"github.com/ystia/yorc/v4/tosca"
 	"vbom.ml/util/sortorder"
 
-	ctu "github.com/hashicorp/consul/testutil"
+	ctu "github.com/hashicorp/consul/sdk/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 

deployments/nodes_test.go

@@ -17,12 +17,13 @@ package deployments
 import (
 	"context"
 	"fmt"
+	"testing"
+
 	"github.com/ystia/yorc/v4/storage"
 	"github.com/ystia/yorc/v4/storage/types"
 	"github.com/ystia/yorc/v4/tosca"
-	"testing"
 
-	ctu "github.com/hashicorp/consul/testutil"
+	ctu "github.com/hashicorp/consul/sdk/testutil"
 
 	"github.com/stretchr/testify/require"
 	"github.com/ystia/yorc/v4/helper/consulutil"

deployments/requirements_test.go

@@ -16,13 +16,14 @@ package deployments
 
 import (
 	"context"
+	"reflect"
+	"testing"
+
 	"github.com/ystia/yorc/v4/storage"
 	"github.com/ystia/yorc/v4/storage/types"
 	"github.com/ystia/yorc/v4/tosca"
-	"reflect"
-	"testing"
 
-	"github.com/hashicorp/consul/testutil"
+	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/stretchr/testify/require"
 
 	"github.com/ystia/yorc/v4/helper/consulutil"

deployments/store/definition_store_test.go

@@ -16,15 +16,21 @@ package store
 
 import (
 	"context"
-	"github.com/ystia/yorc/v4/storage"
-	"github.com/ystia/yorc/v4/storage/types"
+	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
+	"strconv"
 	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/ystia/yorc/v4/storage"
+	"github.com/ystia/yorc/v4/storage/types"
 
 	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/testutil"
+	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -86,6 +92,7 @@ func newTestConsulInstance(t *testing.T, cfg *config.Configuration) (*testutil.T
 	kv := client.KV()
 	consulutil.InitConsulPublisher(cfg.Consul.PubMaxRoutines, kv)
 
+	waitForConsulReadiness(fmt.Sprintf("http://%s", srv1.HTTPAddr))
 	// Load stores
 	// Load main stores used for deployments, logs, events
 	err = storage.LoadStores(*cfg)
@@ -106,6 +113,41 @@ func storeCommonTypePath(ctx context.Context, t *testing.T, paths []string) {
 
 }
 
+// Duplicated here because of cyclic imports. Orginally placed in testutil/helper.go
+// waits for a known leader and an index of 2 or more to be observed to confirm leader election is done.
+// Inspired by : https://github.com/hashicorp/consul/blob/master/sdk/testutil/server.go#L406
+func waitForConsulReadiness(consulHTTPEndpoint string) {
+	for {
+		leader, index, _ := getConsulLeaderAndIndex(consulHTTPEndpoint)
+		if leader != "" && index >= 2 {
+			return
+		}
+
+		<-time.After(2 * time.Second)
+	}
+}
+
+func getConsulLeaderAndIndex(consulHTTPEndpoint string) (string, int64, error) {
+	// Query the API and check the status code.
+	resp, err := http.Get(fmt.Sprintf("%s/v1/catalog/nodes", consulHTTPEndpoint))
+	if err != nil {
+		return "", -1, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", -1, errors.New(resp.Status)
+	}
+	leader := resp.Header.Get("X-Consul-KnownLeader")
+	if leader != "true" {
+		return "", -1, errors.Errorf("Consul leader status: %#v", leader)
+	}
+	index, err := strconv.ParseInt(resp.Header.Get("X-Consul-Index"), 10, 64)
+	if err != nil {
+		errors.Wrap(err, "bad consul index")
+	}
+	return leader, index, nil
+}
+
 // testTypePath aims to test getLatestCommonsTypesPath by storing some_value with a path constructed by joining :
 // - consulutil.CommonsTypesKVPrefix
 // - some_type/some_value

doc/configuration.rst

@@ -1076,12 +1076,16 @@ Google Cloud Platform location type is ``google`` in lower case.
 +-----------------------------+----------------------------------------------+-----------+----------+----------------------------------------+
 | ``credentials``             | Content of file containing credentials       | string    | no       | Google Application Default Credentials |
 +-----------------------------+----------------------------------------------+-----------+----------+----------------------------------------+
+| ``oauth_access_token``      | OAuth2 access token*                         | string    | no       |                                        |
++-----------------------------+----------------------------------------------+-----------+----------+----------------------------------------+
 | ``region``                  | The region to operate under                  | string    | no       |                                        |
 +-----------------------------+----------------------------------------------+-----------+----------+----------------------------------------+
 
 ``application_credentials`` is the path (accessible to Yorc server) of a file containing service account private keys in JSON format.
 This file can be downloaded from the Google Cloud Console at  `Google Cloud service account file <https://console.cloud.google.com/apis/credentials/serviceaccountkey>`_.
 
+``oauth_access_token`` is a temporary OAuth 2.0 access token used to authenticate requests to GCP APIs. If it is specified, this token will be used over credentials. It is **strongly recommended** to use this configuration field as a secret that will be dynamicly read. In fact, OAuth tokens have a lifetime of one hour that can't be extented or modified. In combination with a correctly configured Vault, a token will be generated for each step of the workflow to be executed on google, so the security is maximum. See vault section for more details.
+
 If no file path is specified in ``application_credentials`` and no file content is specified in ``credentials``, the orchestrator will fall back to using the `Google Application Default Credentials <https://cloud.google.com/docs/authentication/production>`_ if any.
 
 .. _option_infra_aws:

doc/vault.rst

@@ -55,3 +55,16 @@ Bellow are some of the most common ways to get a specific secret using the templ
   * ``{{ with (secret "/secret/yorc/mysecret").Raw }}{{ .Data.myKey }}{{end}}``
   * ``{{ secret "/secret/yorc/mysecret" "data=myKey" | print }}``
   * ``{{ (secret "/secret/yorc/mysecret" "data=myKey").String }}``
+
+Secrets engines support
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Yorc currently support only Google Cloud secret engine but it is planned to support more and more secrets engines in the future. If you try to read in another secret engine, behaviours can be different so do it at your own risk.
+
+Google Cloud
+^^^^^^^^^^^^^^
+Google cloud secret engine allow to dynamically read secrets of two types: service account keys and OAuth 2 tokens. It is **strongly reccomended** to use OAuth 2 tokens over service account key to reduce potential key leak. Moreover, as service account only permits to have 10 different keys, it is not possible to use them for Yorc. For more information and configuration guide of this engine, see its `online documentation <https://www.vaultproject.io/docs/secrets/gcp>`_.
+
+Supposing we have configured token roleset at path *gcp/roleset/yorc-token-roleset*, in order to dynamically read google credentials, we will be using go template as described above. For example:
+
+  * ``{{ (secret "gcp/roleset/yorc-token-roleset" "data=token").String }}`` to read a token 

go.mod

@@ -3,10 +3,8 @@ module github.com/ystia/yorc/v4
 // Makefile should also be updated when changing module major version (for injected variables)
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 // indirect
 	github.com/Masterminds/goutils v1.1.0 // indirect
-	github.com/Masterminds/semver v1.4.2 // indirect
 	github.com/Masterminds/sprig v2.20.0+incompatible // indirect
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/Netflix/go-expect v0.0.0-20190729225929-0e00d9168667
@@ -22,7 +20,6 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible // indirect
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
 	github.com/cheggaaa/pb/v3 v3.0.3
 	github.com/chromedp/cdproto v0.0.0-20200209033844-7e00b02ea7d2 // indirect
@@ -32,34 +29,28 @@ require (
 	github.com/dgraph-io/ristretto v0.0.1
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v0.0.0-20170504205632-89658bed64c2
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/duosecurity/duo_api_golang v0.0.0-20200206192355-a9725220d6ca // indirect
 	github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
-	github.com/fatih/color v1.7.0
-	github.com/fatih/structs v1.1.0 // indirect
+	github.com/fatih/color v1.9.0
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/go-sql-driver/mysql v1.5.0 // indirect
 	github.com/gocql/gocql v0.0.0-20200228163523-cd4b606dd2fb // indirect
-	github.com/golang/snappy v0.0.1 // indirect
 	github.com/google/addlicense v0.0.0-20190107131845-2e5cf00261bf
-	github.com/google/go-github v17.0.0+incompatible // indirect
-	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/googleapis/gnostic v0.3.0 // indirect
-	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
 	github.com/goware/urlx v0.3.1
-	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
-	github.com/hashicorp/consul v1.2.3
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
+	github.com/hashicorp/consul/api v1.4.0
+	github.com/hashicorp/consul/sdk v0.4.0
 	github.com/hashicorp/go-cleanhttp v0.5.1
-	github.com/hashicorp/go-hclog v0.8.0
+	github.com/hashicorp/go-hclog v0.12.0
 	github.com/hashicorp/go-memdb v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/go-plugin v1.0.0
-	github.com/hashicorp/go-rootcerts v1.0.0
-	github.com/hashicorp/serf v0.8.3 // indirect
-	github.com/hashicorp/vault v0.9.0
+	github.com/hashicorp/go-plugin v1.0.1
+	github.com/hashicorp/go-rootcerts v1.0.2
+	github.com/hashicorp/vault v1.3.3
+	github.com/hashicorp/vault/api v1.0.5-0.20200117231345-460d63e36490
 	github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c
 	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
@@ -75,27 +66,17 @@ require (
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/matryer/resync v0.0.0-20161211202428-d39c09a11215
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
-	github.com/mgutz/logxi v0.0.0-20161027140823-aebf8a7d67ab // indirect
-	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.1.2
-	github.com/mitchellh/reflectwalk v1.0.1 // indirect
 	github.com/mkideal/cli v0.0.3 // indirect
 	github.com/mkideal/pkg v0.0.0-20170503154153-3e188c9e7ecc // indirect
 	github.com/moby/moby v0.0.0-20170504205632-89658bed64c2
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/ory/dockertest v3.3.5+incompatible // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml v1.4.0 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.2.1
 	github.com/prometheus/procfs v0.0.8 // indirect
-	github.com/ryanuber/go-glob v1.0.0 // indirect
-	github.com/satori/go.uuid v1.0.0
-	github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
+	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.6
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -112,13 +93,11 @@ require (
 	gopkg.in/AlecAivazis/survey.v1 v1.6.3
 	gopkg.in/cookieo9/resources-go.v2 v2.0.0-20150225115733-d27c04069d0d
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
-	gopkg.in/ory-am/dockertest.v3 v3.3.5 // indirect
-	gopkg.in/yaml.v2 v2.2.4
-	gotest.tools v2.2.0+incompatible // indirect
-	k8s.io/api v0.0.0-20180628040859-072894a440bd
-	k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d
-	k8s.io/client-go v8.0.0+incompatible
-	k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058 // indirect
+	gopkg.in/yaml.v2 v2.2.8
+	k8s.io/api v0.0.0-20190620084959-7cf5895f2711
+	k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719
+	k8s.io/client-go v12.0.0+incompatible
+	k8s.io/utils v0.0.0-20200318093247-d1ab8797c558 // indirect
 	vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc
 )