Skip to content

Commit

Permalink
tokenize file path
Browse files Browse the repository at this point in the history
  • Loading branch information
kendavis2 committed Apr 30, 2020
1 parent 4b69c83 commit b543fbd
Show file tree
Hide file tree
Showing 16 changed files with 100 additions and 67 deletions.
6 changes: 3 additions & 3 deletions cli/cmd/clean.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func cleanCatalog(catalogPath string) {

for _, f := range files {
if f.IsRef {
cleanCatalog(path.BuildPath(root, f.Path))
cleanCatalog(path.BuildPath(root, f.ActualPath()))
}

remoteComp, err := remote.InitComponents(&f, clog, uo, ioStreams)
Expand All @@ -54,7 +54,7 @@ func cleanCatalog(catalogPath string) {
}

if !remoteComp.Store.SupportsFeature(store.SourceControlFeature) {
file := clog.GetFullPath(f.Path)
file := clog.GetFullPath(f.ActualPath())
if err := os.Remove(file); err != nil {
if !os.IsNotExist(err) {
display.Error(fmt.Errorf("failed to delete %s (%s)", file, err), ioStreams.UserOutput)
Expand All @@ -69,7 +69,7 @@ func cleanCatalog(catalogPath string) {
}
}

secretsFile := fmt.Sprintf("%s.secrets", clog.GetFullPath(f.Path))
secretsFile := fmt.Sprintf("%s.secrets", clog.GetFullPath(f.ActualPath()))
if err := os.Remove(secretsFile); err != nil {
if !os.IsNotExist(err) {
display.Error(fmt.Errorf("failed to delete %s (%s)", secretsFile, err), ioStreams.UserOutput)
Expand Down
3 changes: 2 additions & 1 deletion cli/cmd/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"os"

"github.com/turnerlabs/cstore/v4/components/path"
"github.com/turnerlabs/cstore/v4/components/remote"

"github.com/spf13/cobra"
Expand All @@ -29,7 +30,7 @@ var initCmd = &cobra.Command{

for _, filePath := range getFilePathsToPush(clog, uo) {

file, err := localFile.GetBy(clog.GetFullPath(filePath))
file, err := localFile.GetBy(clog.GetFullPath(path.SubstituteTokens(filePath)))
if err != nil {
display.Error(err, ioStreams.UserOutput)
continue
Expand Down
2 changes: 1 addition & 1 deletion cli/cmd/list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func listFilesFor(catalogPath string, opt cfg.UserOptions, io models.IO) (int, e
total := 0

for _, fileEntry := range clog.FilesBy(opt.GetPaths(clog.CWD), opt.TagList, opt.AllTags, opt.Version) {
fullPath := path.BuildPath(basePath, fileEntry.Path)
fullPath := path.BuildPath(basePath, fileEntry.ActualPath())

//-------------------------------------------------
//- If entry is catalog, print child entries.
Expand Down
22 changes: 11 additions & 11 deletions cli/cmd/pull.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
//- Check for a linked catalog with child files.
//----------------------------------------------------
if fileEntry.IsRef {
c, t, err := Pull(path.BuildPath(root, fileEntry.Path), opt, io)
c, t, err := Pull(path.BuildPath(root, fileEntry.ActualPath()), opt, io)
if err != nil {
return 0, 0, err
}
Expand All @@ -111,7 +111,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
fileEntryTemp := fileEntry
remoteComp, err := remote.InitComponents(&fileEntryTemp, clog, opt, io)
if err != nil {
display.Error(fmt.Errorf("PullFailedException3: %s (%s)", getPath(root, fileEntry.Path, opt.Version), err), io.UserOutput)
display.Error(fmt.Errorf("PullFailedException3: %s (%s)", getPath(root, fileEntry.ActualPath(), opt.Version), err), io.UserOutput)
continue
}

Expand All @@ -120,7 +120,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
//----------------------------------------------------
file, _, err := remoteComp.Store.Pull(&fileEntry, opt.Version)
if err != nil {
display.Error(fmt.Errorf("PullFailedException4: %s (%s)", getPath(root, fileEntry.Path, opt.Version), err), io.UserOutput)
display.Error(fmt.Errorf("PullFailedException4: %s (%s)", getPath(root, fileEntry.ActualPath(), opt.Version), err), io.UserOutput)
continue
}

Expand All @@ -138,20 +138,20 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro

if opt.InjectSecrets || opt.ModifySecrets {
if !fileEntry.SupportsSecrets() {
display.Error(fmt.Errorf("IncompatibleFileError: %s secrets not supported", fileEntry.Path), io.UserOutput)
display.Error(fmt.Errorf("IncompatibleFileError: %s secrets not supported", fileEntry.ActualPath()), io.UserOutput)
continue
}

tokens, err := token.Find(fileWithSecrets, fileEntry.Type, false)
if err != nil {
display.Error(fmt.Errorf("MissingTokensError: failed to find tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput)
display.Error(fmt.Errorf("MissingTokensError: failed to find tokens in file %s (%s)", fileEntry.ActualPath(), err), io.UserOutput)
}

for k, t := range tokens {

value, err := remoteComp.Secrets.Get(clog.Context, t.Secret(), t.Prop)
if err != nil {
display.Error(fmt.Errorf("GetSecretValueError: failed to get value for %s/%s for %s (%s)", t.Secret(), t.Prop, path.BuildPath(root, fileEntry.Path), err), io.UserOutput)
display.Error(fmt.Errorf("GetSecretValueError: failed to get value for %s/%s for %s (%s)", t.Secret(), t.Prop, path.BuildPath(root, fileEntry.ActualPath()), err), io.UserOutput)
continue
}

Expand All @@ -163,14 +163,14 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
file, err = token.Replace(file, fileEntry.Type, tokens, true)

if err != nil {
display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput)
display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.ActualPath(), err), io.UserOutput)
}
}

if opt.InjectSecrets {
fileWithSecrets, err = token.Replace(fileWithSecrets, fileEntry.Type, tokens, false)
if err != nil {
display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput)
display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.ActualPath(), err), io.UserOutput)
}
}
}
Expand All @@ -181,7 +181,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
if opt.ExportEnv || len(opt.ExportFormat) > 0 {

if !compatibleFormat(opt.ExportFormat, fileEntry.Type) {
display.Error(fmt.Errorf("IncompatibleExportFormat: file %s is incompatible with export format %s", fileEntry.Path, opt.ExportFormat), io.UserOutput)
display.Error(fmt.Errorf("IncompatibleExportFormat: file %s is incompatible with export format %s", fileEntry.ActualPath(), opt.ExportFormat), io.UserOutput)
continue
}

Expand All @@ -196,7 +196,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
//-----------------------------------------------------
//- Save editable, secret, and alternate files locally.
//-----------------------------------------------------
fullPath := clog.GetFullPath(path.BuildPath(root, fileEntry.Path))
fullPath := clog.GetFullPath(path.BuildPath(root, fileEntry.ActualPath()))

if len(opt.AlternateRestorePath) == 0 {
if err = localFile.Save(fullPath, file); err != nil {
Expand All @@ -219,7 +219,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro
}

fmt.Fprint(io.UserOutput, "Retrieving [")
color.New(color.FgBlue).Fprintf(io.UserOutput, path.BuildPath(root, fileEntry.Path))
color.New(color.FgBlue).Fprintf(io.UserOutput, path.BuildPath(root, fileEntry.ActualPath()))
fmt.Fprint(io.UserOutput, "]")

if len(opt.Version) > 0 {
Expand Down
18 changes: 10 additions & 8 deletions cli/cmd/purge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,16 @@ func Purge(opt cfg.UserOptions, io models.IO) error {
}

fileList := ""

for _, f := range files {
if len(opt.Version) > 0 {
fileList = fmt.Sprintf("%sDelete [%s](%s) from [%s]\n", fileList, f.Path, opt.Version, f.Store)
fileList = fmt.Sprintf("%sDelete [%s](%s) from [%s]\n", fileList, f.ActualPath(), opt.Version, f.Store)
} else {
fileList = fmt.Sprintf("%sDelete [%s] from [%s]\n", fileList, f.Path, f.Store)
fileList = fmt.Sprintf("%sDelete [%s] from [%s]\n", fileList, f.ActualPath(), f.Store)
}
}

if !prompt.Confirm(fmt.Sprintf("File data will be permanently deleted from remote storage! Local files and secrets stored in AWS Secrets Manager will not be affected.\n\n%s \nContinue?", fileList), prompt.Danger, io) {
if !prompt.Confirm(fmt.Sprintf("File data will be permanently deleted from remote storage! Local files will not be affected.\n\n%s \nContinue?", fileList), prompt.Danger, io) {
color.New(color.Bold, color.FgRed).Fprint(ioStreams.UserOutput, "\nOperation Aborted!\n")
os.Exit(0)
}
Expand Down Expand Up @@ -98,7 +99,7 @@ func Purge(opt cfg.UserOptions, io models.IO) error {
//----------------------------------------------------
remoteComp, err := remote.InitComponents(&fileEntryTemp, clog, opt, io)
if err != nil {
display.Error(fmt.Errorf("Purge aborted for %s! (%s)", fileEntry.Path, err), ioStreams.UserOutput)
display.Error(fmt.Errorf("Purge aborted for %s! (%s)", fileEntry.ActualPath(), err), ioStreams.UserOutput)
continue
}

Expand Down Expand Up @@ -132,7 +133,7 @@ func Purge(opt cfg.UserOptions, io models.IO) error {

for _, version := range fileEntry.Versions {
if err = remoteComp.Store.Purge(&fileEntry, version); err != nil {
display.Error(fmt.Errorf("Purge aborted for %s (%s). (%s)", fileEntry.Path, version, err), io.UserOutput)
display.Error(fmt.Errorf("Purge aborted for %s (%s). (%s)", fileEntry.ActualPath(), version, err), io.UserOutput)
undeletedVersions = append(undeletedVersions, version)
continue
}
Expand All @@ -147,7 +148,7 @@ func Purge(opt cfg.UserOptions, io models.IO) error {
//----------------------------------------------------
if len(undeletedVersions) == 0 {
if err = remoteComp.Store.Purge(&fileEntry, none); err != nil {
display.ErrorText(fmt.Sprintf("Purge aborted for %s (%s)", fileEntry.Path, err), io.UserOutput)
display.ErrorText(fmt.Sprintf("Purge aborted for %s (%s)", fileEntry.ActualPath(), err), io.UserOutput)
continue
}

Expand All @@ -158,10 +159,11 @@ func Purge(opt cfg.UserOptions, io models.IO) error {
//----------------------------------------------------
//- Delete the ghost .cstore reference file.
//----------------------------------------------------
fullPath := clog.GetFullPath(path.RemoveFileName(fileEntry.Path))
fullPath := clog.GetFullPath(path.RemoveFileName(fileEntry.ActualPath()))

if len(fullPath) > 0 && !clog.AnyFilesIn(path.RemoveFileName(fileEntry.Path)) {
if err := os.Remove(fmt.Sprintf("%s%s", fullPath, catalog.GhostFile)); err != nil {
display.Error(fmt.Errorf(".cstore file could not be removed for %s! (%s)", fileEntry.Path, err), io.UserOutput)
display.Error(fmt.Errorf(".cstore file could not be removed for %s! (%s)", fileEntry.ActualPath(), err), io.UserOutput)
}
}

Expand Down
22 changes: 11 additions & 11 deletions cli/cmd/push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {
fmt.Fprintln(io.UserOutput)
for _, filePath := range getFilePathsToPush(clog, opt) {

file, err := localFile.GetBy(clog.GetFullPath(filePath))
file, err := localFile.GetBy(clog.GetFullPath(path.SubstituteTokens(filePath)))
if err != nil {
display.Error(err, io.UserOutput)
continue
Expand All @@ -77,7 +77,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {
//-------------------------------------------------
if fileEntry.IsRef {
fmt.Fprint(io.UserOutput, "Linking [")
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path)
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath())
fmt.Fprintln(io.UserOutput, "]")
if err := clog.UpdateEntry(fileEntry); err != nil {
display.Error(err, io.UserOutput)
Expand All @@ -100,7 +100,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {
//- Begin push process.
//--------------------------------------------------
fmt.Fprint(io.UserOutput, "Pushing [")
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path)
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath())
fmt.Fprint(io.UserOutput, "]")

if len(opt.Version) > 0 {
Expand All @@ -124,7 +124,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {
} else {
if !prompt.Confirm(fmt.Sprintf("Overwrite %s with %s?", formatVersion(opt.Version), formatVersion(version)), prompt.Warn, io) {
fmt.Fprint(io.UserOutput, "Skipping [")
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path)
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath())
fmt.Fprintln(io.UserOutput, "]")
continue
}
Expand All @@ -134,7 +134,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {
if !current {
if !prompt.Confirm(fmt.Sprintf("Remotely stored [%s] was modified %s. Overwrite?", filePath, lastModified.Format("01/02/06")), prompt.Warn, io) {
fmt.Fprint(io.UserOutput, "Skipping [")
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path)
color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath())
fmt.Fprintln(io.UserOutput, "]")
continue
}
Expand Down Expand Up @@ -162,7 +162,7 @@ func Push(opt cfg.UserOptions, io models.IO) error {

file = token.RemoveSecrets(file)

if err = localFile.Save(clog.GetFullPath(fileEntry.Path), file); err != nil {
if err = localFile.Save(clog.GetFullPath(fileEntry.ActualPath()), file); err != nil {
logger.L.Print(err)
}
}
Expand Down Expand Up @@ -207,9 +207,9 @@ func Push(opt cfg.UserOptions, io models.IO) error {
}

//---------------------------------------------------------------------
//- Create the ghost .cstore reference file when not in cStore.yml dir.
//- Create the ghost .cstore reference file when not in cstore.yml dir.
//---------------------------------------------------------------------
justThePath := path.RemoveFileName(filePath)
justThePath := path.RemoveFileName(fileEntry.ActualPath())

if len(clog.GetFullPath(justThePath)) > 0 {
if err := catalog.WriteGhost(clog.GetFullPath(justThePath), catalog.Ghost{
Expand All @@ -219,14 +219,14 @@ func Push(opt cfg.UserOptions, io models.IO) error {
}
}

filesPushed = append(filesPushed, fileEntry.Path)
filesPushed = append(filesPushed, fileEntry.ActualPath())

//-------------------------------------------------
//- If user specified, delete local files.
//-------------------------------------------------
if fileEntry.DeleteAfterPush {
os.Remove(clog.GetFullPath(fileEntry.Path))
os.Remove(fmt.Sprintf("%s.secrets", clog.GetFullPath(fileEntry.Path)))
os.Remove(clog.GetFullPath(fileEntry.ActualPath()))
os.Remove(fmt.Sprintf("%s.secrets", clog.GetFullPath(fileEntry.ActualPath())))
}
}

Expand Down
11 changes: 8 additions & 3 deletions components/catalog/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,19 @@ type File struct {
Versions []string `ymal:"versions,omitempty"`
}

// ActualPath ...
func (f File) ActualPath() string {
return path.SubstituteTokens(f.Path)
}

// Key ...
func (f File) Key() string {
return hashPath(f.Path)
}

// ContextKey ...
func (f File) ContextKey(context string) string {
return buildKey(context, hashPath(f.Path))
return buildKey(context, hashPath(f.ActualPath()))
}

// SupportsSecrets ...
Expand Down Expand Up @@ -324,7 +329,7 @@ func (c Catalog) FilesBy(paths, tags []string, allTags bool, version string) map
// AnyFilesIn ...
func (c Catalog) AnyFilesIn(dir string) bool {
for _, f := range c.Files {
if path.RemoveFileName(f.Path) == dir {
if path.RemoveFileName(f.ActualPath()) == dir {
return true
}
}
Expand Down Expand Up @@ -357,7 +362,7 @@ func (c *Catalog) UpdateEntry(newFile File) error {

if oldFile, found := c.Files[key]; found {
if len(newFile.Store) > 0 && newFile.Store != oldFile.Store {
return fmt.Errorf("AreadyStoredException: Purge %s from %s before pushing to %s", newFile.Path, oldFile.Store, newFile.Store)
return fmt.Errorf("AreadyStoredException: Purge %s from %s before pushing to %s", newFile.ActualPath(), oldFile.Store, newFile.Store)
}
}

Expand Down
25 changes: 25 additions & 0 deletions components/path/substitute.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package path

import (
"os"
"strings"
"regexp"
)

// SubstituteTokens ...
func SubstituteTokens(path string) string {

re := regexp.MustCompile(`\${([A-Z_]+)}`)

matches := re.FindAllStringSubmatch(path, -1)

for _, m := range matches {
env, found := os.LookupEnv(m[1])

if found {
path = strings.Replace(path, m[0], env, -1)
}
}

return path
}
Loading

0 comments on commit b543fbd

Please sign in to comment.