improving secrets manager

README.md

@@ -2,7 +2,7 @@
 
 Simple, secure, and flexible configuration management.
 
-The cStore CLI provides a command to push config files `$ cstore push service/dev/.env` to remote [storage](docs/STORES.md). The pushed files are replaced by a, `cstore.yml` file, that remembers the storage location, file encryption, and other details making restoration locally or by a service as simple as `$ cstore pull -t dev`.
+The cStore CLI provides a command to push config files to remote [storage](docs/STORES.md) using `$ cstore push service/dev/.env`. The pushed files are replaced by a, `cstore.yml` [file](docs/CATALOG.md), that remembers the storage location, file encryption, and other details making restoration locally or by a service as simple as `$ cstore pull -t dev`.
 
 `*.env` and `*.json` are special file types whose secrets can be [tokenized](docs/SECRETS.md), encrypted, stored separately from the configuration, and injected at runtime.
 
@@ -18,7 +18,7 @@ The cStore CLI provides a command to push config files `$ cstore push service/de
   * Always use encryption when storing secrets.
   * Use your organization's approved vaults for storing secrets.
   * Avoid exporting secrets into the environment when possible.
-  * Realize most mistakes are made by users; so, be careful.
+  * Realize many security mistakes are made by users; so, be careful!
 
 </details>
 
@@ -45,9 +45,9 @@ The cStore CLI provides a command to push config files `$ cstore push service/de
 │           └── fargate.yml
 │           └── docker-compose.yml
 ```
-The `cstore.yml` catalog and hidden `.cstore` ghost files reference the stored `*.env` files. Secrets no longer need to be checked into source control.
+The `cstore.yml` [catalog](docs/CATALOG.md) and hidden `.cstore` ghost files reference the stored `*.env` files. Secrets no longer need to be checked into source control.
 
-When the repository has been cloned or the project shared, running `$ cstore pull` in the same directory as the `cstore.yml` catalog file or any of the `.cstore` ghost files will locate, download, and decrypt the configuration files to their respective original location restoring the project's environment configuration.
+When the repository has been cloned or the project shared, running `$ cstore pull` in the same directory as the `cstore.yml` [catalog](docs/CATALOG.md) or any of the `.cstore` ghost files will locate, download, and decrypt the configuration files to their respective original location restoring the project's environment configuration.
 
 Example: `cstore.yml`
 ```yml
@@ -56,7 +56,6 @@ context: project
 files:
 - path: service/dev/.env
   store: aws-s3
-  isRef: false
   type: env
   data:
     AWS_S3_BUCKET: my-bucket
@@ -71,7 +70,6 @@ files:
   versions: []
 - path: service/prod/.env
   store: aws-parameter
-  isRef: false
   type: env
   data:
     AWS_STORE_KMS_KEY_ID: aws/ssm
@@ -127,14 +125,17 @@ API_URL=https://dev.api.example-service.com
 [email protected]
 ```
 
-Save in one of the following storage solutions.
+Push configs to one of the following storage solutions.
 ```bash
 $ cstore push service/dev/.env -s aws-parameter 
 ```
 ```bash
 $ cstore push service/dev/.env -s aws-s3
 ```
 ```bash
+$ cstore push service/dev/.env -s aws-secret
+```
+```bash
 $ cstore push service/dev/.env -s source-control
 ```
 </details>
@@ -157,6 +158,9 @@ $ cat service/dev/config.json # example
 ```bash
 $ cstore push service/dev/config.json -s aws-s3
 ```
+```bash
+$ cstore push service/dev/config.json -s aws-secret
+```
 
 </details>
 
@@ -251,6 +255,7 @@ $ cstore pull -t dev -g task-def-secrets --store-command refs # AWS Task Definit
 * [Terminology](docs/TERMS.md)
 * [Storage Solutions](docs/STORES.md)
 * [Vault Solutions](docs/VAULTS.md)
+* [Catalog Fields](docs/CATALOG.md)
 
 | Demo |  |
 |---|---|

cli/cmd/helpers.go

@@ -164,28 +164,3 @@ type EnvFormat struct {
 	Name  string `json:"name"`
 	Value string `json:"value"`
 }
-
-func toJsonObjectFormat(file []byte) (bytes.Buffer, error) {
-	reader := bytes.NewReader(file)
-	pairs := gotenv.Parse(reader)
-
-	var buff bytes.Buffer
-
-	env := map[string]string{}
-
-	for key, value := range pairs {
-		env[key] = value
-	}
-
-	b, err := json.MarshalIndent(env, "", "    ")
-	if err != nil {
-		return buff, err
-	}
-
-	_, err = buff.Write(b)
-	if err != nil {
-		return buff, err
-	}
-
-	return buff, nil
-}

cli/cmd/pull.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/turnerlabs/cstore/components/convert"
+
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 	"github.com/turnerlabs/cstore/components/catalog"
@@ -76,7 +78,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 	}
 
 	if len(files) == 0 {
-		return 0, 0, fmt.Errorf("%s is not aware of requested files. Use 'list' command to view available files.", opt.Catalog)
+		return 0, 0, fmt.Errorf("%s is not aware of requested files", opt.Catalog)
 	}
 
 	for _, fileEntry := range files {
@@ -108,13 +110,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 		fileEntryTemp := fileEntry
 		remoteComp, err := remote.InitComponents(&fileEntryTemp, clog, opt, io)
 		if err != nil {
-
-			p := path.BuildPath(root, fileEntry.Path)
-			if len(opt.Version) > 0 {
-				p = fmt.Sprintf("%s (%s)", p, opt.Version)
-			}
-
-			display.Error(fmt.Errorf("Could not retrieve %s! (%s)", p, err), io.UserOutput)
+			display.Error(fmt.Errorf("PullFailedException3: %s (%s)", getPath(root, fileEntry.Path, opt.Version), err), io.UserOutput)
 			continue
 		}
 
@@ -123,13 +119,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 		//----------------------------------------------------
 		file, _, err := remoteComp.Store.Pull(&fileEntry, opt.Version)
 		if err != nil {
-
-			p := path.BuildPath(root, fileEntry.Path)
-			if len(opt.Version) > 0 {
-				p = fmt.Sprintf("%s (%s)", p, opt.Version)
-			}
-
-			display.Error(fmt.Errorf("Could not retrieve %s! (%s)", p, err), io.UserOutput)
+			display.Error(fmt.Errorf("PullFailedException4: %s (%s)", getPath(root, fileEntry.Path, opt.Version), err), io.UserOutput)
 			continue
 		}
 
@@ -147,20 +137,20 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 
 		if opt.InjectSecrets || opt.ModifySecrets {
 			if !fileEntry.SupportsSecrets() {
-				display.Error(fmt.Errorf("Secrets not supported for %s due to incompatible file type.", fileEntry.Path), io.UserOutput)
+				display.Error(fmt.Errorf("IncompatibleFileError: %s secrets not supported", fileEntry.Path), io.UserOutput)
 				continue
 			}
 
 			tokens, err := token.Find(fileWithSecrets, fileEntry.Type, false)
 			if err != nil {
-				display.Error(fmt.Errorf("Failed to find tokens in file %s. (%s)", fileEntry.Path, err), io.UserOutput)
+				display.Error(fmt.Errorf("MissingTokensError: failed to find tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput)
 			}
 
 			for k, t := range tokens {
 
 				value, err := remoteComp.Secrets.Get(clog.Context, t.Secret(), t.Prop)
 				if err != nil {
-					display.Error(fmt.Errorf("Failed to get value for %s/%s for %s! (%s)", t.Secret(), t.Prop, path.BuildPath(root, fileEntry.Path), err), io.UserOutput)
+					display.Error(fmt.Errorf("GetSecretValueError: failed to get value for %s/%s for %s (%s)", t.Secret(), t.Prop, path.BuildPath(root, fileEntry.Path), err), io.UserOutput)
 					continue
 				}
 
@@ -172,14 +162,14 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 				file, err = token.Replace(file, fileEntry.Type, tokens, true)
 
 				if err != nil {
-					display.Error(fmt.Errorf("Failed to replace tokens in file %s. (%s)", fileEntry.Path, err), io.UserOutput)
+					display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput)
 				}
 			}
 
 			if opt.InjectSecrets {
 				fileWithSecrets, err = token.Replace(fileWithSecrets, fileEntry.Type, tokens, false)
 				if err != nil {
-					display.Error(fmt.Errorf("Failed to replace tokens in file %s. (%s)", fileEntry.Path, err), io.UserOutput)
+					display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput)
 				}
 			}
 		}
@@ -190,7 +180,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 		if opt.ExportEnv || len(opt.ExportFormat) > 0 {
 
 			if !compatibleFormat(opt.ExportFormat, fileEntry.Type) {
-				display.Error(fmt.Errorf("File %s is incompatible with export format %s.", fileEntry.Path, opt.ExportFormat), io.UserOutput)
+				display.Error(fmt.Errorf("IncompatibleExportFormat: file %s is incompatible with export format %s", fileEntry.Path, opt.ExportFormat), io.UserOutput)
 				continue
 			}
 
@@ -269,7 +259,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 			}
 		case "json-object":
 			msg = fmt.Sprintf(msg, "JSON object")
-			exportBuffer, err = toJsonObjectFormat(exportBuffer.Bytes())
+			exportBuffer, err = convert.ToJSONObjectFormat(exportBuffer.Bytes())
 			if err != nil {
 				logger.L.Print(err)
 			}
@@ -295,6 +285,15 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
 	return restoredCount, fileCount, nil
 }
 
+func getPath(root, filepath, version string) string {
+
+	if len(version) > 0 {
+		return fmt.Sprintf("%s (%s)", path.BuildPath(root, filepath), version)
+	}
+
+	return path.BuildPath(root, filepath)
+}
+
 func compatibleFormat(format, fileType string) bool {
 
 	switch format {

cli/cmd/push.go

@@ -71,7 +71,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {
 		//- If file is a catalog, link it to this catalog.
 		//-------------------------------------------------
 		if fileEntry.IsRef {
-			fmt.Fprintf(io.UserOutput, "Linking %s   %s \n", fileEntry.Path, checkMark)
+			fmt.Fprintf(io.UserOutput, "Linking %s   %s \n", filePath, checkMark)
 			if err := clog.UpdateEntry(fileEntry); err != nil {
 				display.Error(err, io.UserOutput)
 			}
@@ -107,12 +107,9 @@ func Push(opt cfg.UserOptions, io models.IO) error {
 		//--------------------------------------------------------
 		//- Ensure file has not been modified by another user.
 		//--------------------------------------------------------
-		if lastModified, err := remoteComp.Store.Changed(&fileEntry, file, opt.Version); err != nil {
-			display.Error(fmt.Errorf("Failed to determine when '%s' (%s) was last modified. (%s)", filePath, opt.Version, err), io.UserOutput)
-			continue
-		} else {
+		if lastModified, err := remoteComp.Store.Changed(&fileEntry, file, opt.Version); err == nil {
 			if !fileEntry.IsCurrent(lastModified, clog.Context) {
-				if !prompt.Confirm(fmt.Sprintf("Remote file '%s' was modified on %s. Overwrite?", filePath, lastModified.Format(time.RFC822)), prompt.Warn, io) {
+				if !prompt.Confirm(fmt.Sprintf("Remotely stored data '%s' was modified on %s. Overwrite?", filePath, lastModified.Format("01/02/06")), prompt.Warn, io) {
 					fmt.Fprintf(io.UserOutput, "Skipping %s\n", filePath)
 					continue
 				}
@@ -151,7 +148,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {
 		//-------------------------------------------------
 		if len(opt.Version) > 0 {
 			if !remoteComp.Store.SupportsFeature(store.VersionFeature) {
-				display.Error(fmt.Errorf("%s store does not support %s feature.", remoteComp.Store.Name(), store.VersionFeature), io.UserOutput)
+				display.Error(fmt.Errorf("%s store does not support %s", remoteComp.Store.Name(), store.VersionFeature), io.UserOutput)
 				continue
 			}
 

components/catalog/model.go

@@ -93,7 +93,7 @@ type File struct {
 
 	// IsRef indicates the file is a linked catalog and not a remotely
 	// store file.
-	IsRef bool `yaml:"isRef"`
+	IsRef bool `yaml:"isRef,omitempty"`
 
 	// DeleteAfterPush instructs the local file to be deleted after changes
 	// have been pushed to the remote store to protect secrets
@@ -354,7 +354,7 @@ func (c *Catalog) UpdateEntry(newFile File) error {
 
 	if oldFile, found := c.Files[key]; found {
 		if len(newFile.Store) > 0 && newFile.Store != oldFile.Store {
-			return fmt.Errorf("'cstore purge %s' and then 'cstore push %s' required to change store", newFile.Path, newFile.Path)
+			return fmt.Errorf("AreadyStoredException: Purge %s from %s before pushing to %s", newFile.Path, oldFile.Store, newFile.Store)
 		}
 	}
 

components/catalog/update.go

@@ -29,8 +29,9 @@ func Write(path string, catalog Catalog) error {
 
 	comment := `# This catalog lists files stored remotely based on the files current location.
 # To restore the files, run '$ cstore pull' in the same directory as this catalog file.
-# If this file is deleted without running a purge command, the stores contents will be orphaned 
-# with no way to recover. To get set up, visit https://github.com/turnerlabs/cstore.
+# If this file is deleted without running a purge command, stored data may be orphaned 
+# without a way to recover. To get set up, visit https://github.com/turnerlabs/cstore.
+# To understand the catalog, visit https://github.com/turnerlabs/cstore/docs/CATALOG.md
 `
 	d = append([]byte(comment), d...)
 

components/convert/env.go

@@ -0,0 +1,54 @@
+package convert
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"github.com/subosito/gotenv"
+)
+
+// ToJSONObjectFormat ...
+func ToJSONObjectFormat(file []byte) (bytes.Buffer, error) {
+	reader := bytes.NewReader(file)
+	pairs := gotenv.Parse(reader)
+
+	var buff bytes.Buffer
+
+	env := map[string]string{}
+
+	for key, value := range pairs {
+		env[key] = value
+	}
+
+	b, err := json.MarshalIndent(env, "", "    ")
+	if err != nil {
+		return buff, err
+	}
+
+	_, err = buff.Write(b)
+	if err != nil {
+		return buff, err
+	}
+
+	return buff, nil
+}
+
+// ToENVFileFormat ...
+func ToENVFileFormat(file []byte) (bytes.Buffer, error) {
+	var buff bytes.Buffer
+
+	env := map[string]string{}
+
+	if err := json.Unmarshal(file, &env); err != nil {
+		return buff, err
+	}
+
+	for k, v := range env {
+		if _, err := buff.WriteString(fmt.Sprintf("%s=%s\n", k, v)); err != nil {
+			return buff, err
+		}
+	}
+
+	return buff, nil
+}

components/setting/setting.go

@@ -19,7 +19,7 @@ type IKeyValueStore interface {
 	BuildKey(contextID, group, prop string) string
 }
 
-var promptOnce = map[string]bool{}
+var didPrompt = map[string]bool{}
 
 // Setting ...
 type Setting struct {
@@ -45,6 +45,7 @@ func (s Setting) Key(context string) string {
 
 // Get ...
 func (s Setting) Get(context string, io models.IO) (string, error) {
+
 	value, err := s.Vault.Get(context, s.Group, s.Prop)
 	if err != nil {
 		if err.Error() == contract.ErrSecretNotFound.Error() {
@@ -54,9 +55,10 @@ func (s Setting) Get(context string, io models.IO) (string, error) {
 		}
 	}
 
-	if _, found := promptOnce[s.Prop]; !found && s.Prompt && !s.Silent {
+	if s.Prompt && !s.Silent && !didPrompt[s.Prop] {
+
 		if s.PromptOnce {
-			promptOnce[s.Prop] = true
+			didPrompt[s.Prop] = true
 		}
 
 		formattedKey := s.Vault.BuildKey(context, s.Group, s.Prop)