From 89e931c72a7680ca9e8aca0dd478dada62eaa151 Mon Sep 17 00:00:00 2001 From: Bryce Lowe Date: Sun, 21 Aug 2022 19:18:50 -0700 Subject: [PATCH 1/3] feature: add recursion add recursion as an option to the list and update functions --- go.mod | 40 ++++++++++++++++++++++++++++++++++++++++ go.sum | 5 ----- main.go | 4 +++- pkg/output/modules.go | 4 ++-- pkg/scan/scanner.go | 37 +++++++++++++++++++++++++++++++++++-- vendor/modules.txt | 2 -- 6 files changed, 80 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 7b1b3dc7..7a4376b7 100644 --- a/go.mod +++ b/go.mod @@ -17,18 +17,58 @@ require ( require ( cloud.google.com/go v0.74.0 // indirect cloud.google.com/go/storage v1.12.0 // indirect + github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/aws/aws-sdk-go v1.37.0 // indirect + github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect + github.com/emirpasic/gods v1.12.0 // indirect github.com/gliderlabs/ssh v0.3.1 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-test/deep v1.0.7 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/google/go-cmp v0.5.4 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.2.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl/v2 v2.10.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect + github.com/klauspost/compress v1.11.2 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/ulikunitz/xz v0.5.8 // indirect + github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/zclconf/go-cty v1.8.3 // indirect + go.opencensus.io v0.22.5 // indirect + golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect + golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect + golang.org/x/mod v0.4.0 // indirect + golang.org/x/net v0.0.0-20210326060303-6b1517762897 // indirect + golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 // indirect + golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect + golang.org/x/text v0.3.5 // indirect golang.org/x/tools v0.0.0-20201211025543-abf6a1d87e11 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/api v0.36.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc // indirect + google.golang.org/grpc v1.34.0 // indirect + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect ) diff --git a/go.sum b/go.sum index 8a3a2c54..f33bb333 100644 --- a/go.sum +++ b/go.sum @@ -56,7 +56,6 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= -github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= @@ -204,7 +203,6 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jstemmer/go-junit-report v1.0.0 h1:8X1gzZpR+nVQLAht+L/foqOeX2l9DTZoaIPbEQHxsds= github.com/jstemmer/go-junit-report v1.0.0/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -244,7 +242,6 @@ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTS github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/ff/v3 v3.3.0 h1:PaKe7GW8orVFh8Unb5jNHS+JZBwWUMa2se0HM6/BI24= github.com/peterbourgon/ff/v3 v3.3.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -427,7 +424,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 h1:RX8C8PRZc2hTIod4ds8ij+/4RQX3AqhYj3uOHmyaz4E= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -608,7 +604,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 3d211caf..6d0a9c1f 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ var ( All bool GenerateSed bool IncludePrereleaseVersions bool + Recursive bool } ) @@ -78,6 +79,7 @@ func main() { checkFlagSet.BoolVar(&config.All, "a", config.All, "(alias for -all)") checkFlagSet.BoolVar(&config.All, "all", config.All, "include modules without updates") listFlagSet.Var(&config.ModuleNames, "module", "include this module (may be specified repeatedly. by default, all modules are included)") + listFlagSet.BoolVar(&config.Recursive, "recursive", config.Recursive, "recursively list all modules under paths") checkFlagSet.Var(&config.ModuleNames, "module", "include this module (may be specified repeatedly. by default, all modules are included)") checkFlagSet.Var(&config.RegistryHeaders, "H", "(alias for -registry-header)") checkFlagSet.Var(&config.RegistryHeaders, "registry-header", fmt.Sprintf("extra HTTP headers for requests to Terraform module registries (%s, may be specified repeatedly)", config.RegistryHeaders.Help())) @@ -160,7 +162,7 @@ func main() { } func scanForModuleCalls() []scan.Result { - scanResults, err := scan.Scan(config.Paths) + scanResults, err := scan.Scan(config.Paths, config.Recursive) if err != nil { log.Fatal(err) } diff --git a/pkg/output/modules.go b/pkg/output/modules.go index e7d080ef..a303d605 100644 --- a/pkg/output/modules.go +++ b/pkg/output/modules.go @@ -80,12 +80,12 @@ func (m Modules) WriteMarkdownWide(w io.Writer) error { func (m Modules) WriteMarkdown(w io.Writer) error { table := tablewriter.NewWriter(w) - table.SetHeader([]string{"Type", "Name", "Constraint", "Version", "Source"}) + table.SetHeader([]string{"Type", "Name", "Constraint", "Version", "Source", "Path"}) table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) table.SetCenterSeparator("|") rows := make([][]string, 0, len(m)) for _, item := range m { - row := []string{item.Type, item.Name, item.VersionConstraint, item.Version, item.Source} + row := []string{item.Type, item.Name, item.VersionConstraint, item.Version, item.Source, item.Path} rows = append(rows, row) } sort.Slice(rows, func(i, j int) bool { diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index 8b3357b8..f9dd139b 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -2,8 +2,9 @@ package scan import ( "fmt" - "github.com/hashicorp/terraform-config-inspect/tfconfig" + "io/fs" + "path/filepath" ) type Result struct { @@ -11,7 +12,17 @@ type Result struct { Path string } -func Scan(paths []string) ([]Result, error) { +func Scan(paths []string, recursive bool) ([]Result, error) { + // recurse through the provided paths in a non-overlapping way + if recursive { + recursivePaths, err := loadSubPaths(paths) + if err != nil { + return nil, err + } + + paths = recursivePaths + } + var out []Result for _, path := range paths { module, err := tfconfig.LoadModule(path) @@ -30,3 +41,25 @@ func Scan(paths []string) ([]Result, error) { } return out, nil } + +func loadSubPaths(paths []string) ([]string, error) { + combinedPaths := make([]string, 0) + + for _, path := range paths { + err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + // don't worry about anything that isn't a directory + if !d.IsDir() { + return nil + } + + combinedPaths = append(combinedPaths, path) + return nil + }) + + if err != nil { + return paths, err + } + } + + return combinedPaths, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2805b655..382d8c02 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -519,7 +519,5 @@ google.golang.org/protobuf/types/pluginpb # gopkg.in/warnings.v0 v0.1.2 ## explicit gopkg.in/warnings.v0 -# gopkg.in/yaml.v2 v2.4.0 -## explicit; go 1.15 # gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ## explicit From 11a15ce28aced8142998391bf0aac8775b2594c4 Mon Sep 17 00:00:00 2001 From: Bryce Lowe Date: Mon, 22 Aug 2022 11:02:36 -0700 Subject: [PATCH 2/3] fix: add check missed check flag --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 6d0a9c1f..89d5bb48 100644 --- a/main.go +++ b/main.go @@ -84,6 +84,7 @@ func main() { checkFlagSet.Var(&config.RegistryHeaders, "H", "(alias for -registry-header)") checkFlagSet.Var(&config.RegistryHeaders, "registry-header", fmt.Sprintf("extra HTTP headers for requests to Terraform module registries (%s, may be specified repeatedly)", config.RegistryHeaders.Help())) checkFlagSet.BoolVar(&config.GenerateSed, "sed", config.GenerateSed, "generate sed statements for upgrade") + checkFlagSet.BoolVar(&config.Recursive, "recursive", config.Recursive, "recursively check all modules under paths") cmdList := &ffcli.Command{ Name: "list", From 52d569abb674373f118a80a6125e2c8e509691a2 Mon Sep 17 00:00:00 2001 From: Bryce Lowe Date: Mon, 22 Aug 2022 11:07:53 -0700 Subject: [PATCH 3/3] fix: ensure only unique paths are returned from loadSubPaths --- pkg/scan/scanner.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index f9dd139b..e65793ff 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -44,6 +44,7 @@ func Scan(paths []string, recursive bool) ([]Result, error) { func loadSubPaths(paths []string) ([]string, error) { combinedPaths := make([]string, 0) + pathCheck := make(map[string]bool) for _, path := range paths { err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { @@ -52,7 +53,12 @@ func loadSubPaths(paths []string) ([]string, error) { return nil } - combinedPaths = append(combinedPaths, path) + // make sure we only capture a path once + if _, ok := pathCheck[path]; !ok { + pathCheck[path] = true + combinedPaths = append(combinedPaths, path) + } + return nil })