Add ruby subcommand for working with ruby files (#571)

* Initial add of the ruby subcommand

Signed-off-by: James Petersen <[email protected]>

* use gh package to interact with github

use the gh package to interact with github and download the gemspec

Signed-off-by: James Petersen <[email protected]>

* add caching for gemspecs

Signed-off-by: James Petersen <[email protected]>

* add version checking for ruby versions

Signed-off-by: James Petersen <[email protected]>

* extract package version from fetch archive uris

Signed-off-by: James Petersen <[email protected]>

* add code search capability

Signed-off-by: James Petersen <[email protected]>

* refactor now that we know what we are doing

Signed-off-by: James Petersen <[email protected]>

* add ability to search multiple strings

Signed-off-by: James Petersen <[email protected]>

* add ruby tests

Signed-off-by: James Petersen <[email protected]>

* add some more tests

Signed-off-by: James Petersen <[email protected]>

* go fmt told me to

Signed-off-by: James Petersen <[email protected]>

* fix conflicts

Signed-off-by: James Petersen <[email protected]>

* go fmt told me to

Signed-off-by: James Petersen <[email protected]>

* fix golangci-lint errors

Signed-off-by: James Petersen <[email protected]>

* plumb ctx and adress feedback

Signed-off-by: James Petersen <[email protected]>

* convert to table test

Signed-off-by: James Petersen <[email protected]>

* more table testing

Signed-off-by: James Petersen <[email protected]>

* organize code search caching

Signed-off-by: James Petersen <[email protected]>

* move ruby subcommands to their own files

Signed-off-by: James Petersen <[email protected]>

* Update pkg/ruby/code_search.go

Co-authored-by: Josh Wolf <[email protected]>
Signed-off-by: James Petersen <[email protected]>

* use t.Run to run tests

Signed-off-by: James Petersen <[email protected]>

* address feedback

Signed-off-by: James Petersen <[email protected]>

---------

Signed-off-by: James Petersen <[email protected]>
Co-authored-by: Josh Wolf <[email protected]>

pkg/cli/commands.go

@@ -50,6 +50,7 @@ func New() *cobra.Command {
 		cmdImage(),
 		cmdIndex(),
 		cmdLint(),
+		cmdRuby(),
 		cmdLs(),
 		cmdSVG(),
 		cmdText(),

pkg/cli/ruby.go

@@ -0,0 +1,66 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func cmdRuby() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "ruby",
+		Short: "Work with ruby packages",
+		Long: `Work with ruby packages
+
+The ruby subcommand is intended to work with all ruby packages inside the wolfi
+repo. The main uses right now are to check if the ruby version can be upgraded,
+and run Github code searches for Github repos pulled from melange yaml files.
+
+This command takes a path to the wolfi-dev/os repository as an argument. The
+path can either be the directory itself to discover all files using ruby-* or
+a specific melange yaml to work with.
+
+NOTE: This is currently restricted to ruby code housed on Github as that is the
+      majority. There are some on Gitlab and adding Gitlab API support is TODO.
+`,
+		SilenceErrors: true,
+		Hidden:        false,
+		Example: `
+# Run a search query over all ruby-3.2 package in the current directory
+wolfictl ruby code-search . --ruby-version 3.2 --search-term 'language:ruby racc'
+
+# Check if all ruby-3.2 packages in the current directory can be upgraded to ruby-3.3
+wolfictl ruby check-upgrade . --ruby-version 3.2 --ruby-upgrade-version 3.3
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) == 0 {
+				cmd.Help() //nolint:errcheck
+				return
+			}
+		},
+	}
+
+	cmd.AddCommand(
+		cmdRubyCodeSearch(),
+		cmdRubyCheckUpgrade(),
+	)
+	return cmd
+}
+
+type rubyParams struct {
+	version string
+	noCache bool
+}
+
+func (p *rubyParams) addFlagsTo(cmd *cobra.Command) {
+	cmd.Flags().StringVarP(&p.version, "ruby-version", "r", "", "ruby version to search for")
+	cmd.Flags().BoolVar(&p.noCache, "no-cache", false, "do not use cached results")
+}
+
+func resolvePath(args []string) (path string, isDir bool, err error) {
+	if f, err := os.Stat(args[0]); err == nil {
+		return args[0], f.IsDir(), nil
+	}
+	return "", false, fmt.Errorf("%s does not exist", args[0])
+}

pkg/cli/ruby_check_upgrade.go

@@ -0,0 +1,92 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/oauth2"
+	"golang.org/x/time/rate"
+
+	http2 "github.com/wolfi-dev/wolfictl/pkg/http"
+	"github.com/wolfi-dev/wolfictl/pkg/ruby"
+)
+
+func cmdRubyCheckUpgrade() *cobra.Command {
+	p := &rubyParams{}
+	var upgradeVersion string
+	cmd := &cobra.Command{
+		Use:   "check-upgrade",
+		Short: "Check if gemspec for restricts a gem from upgrading to a specified ruby version.",
+		Long: `
+NOTE: This is currently restricted to ruby code housed on Github as that is the
+      majority. There are some on Gitlab and adding Gitlab API support is TODO.
+`,
+		SilenceErrors: true,
+		Hidden:        false,
+		Aliases:       []string{"cu"},
+		Example: `
+# Check if all ruby-3.2 packages in the current directory can be upgraded to ruby-3.3
+wolfictl ruby check-upgrade . --ruby-version 3.2 --ruby-upgrade-version 3.3
+`,
+		Args: cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			path, isDir, err := resolvePath(args)
+			if err != nil {
+				return fmt.Errorf("could not resolve path: %w", err)
+			}
+
+			if p.version == "" && isDir {
+				return fmt.Errorf("directory specified, but no --ruby-version to search for")
+			}
+
+			if upgradeVersion == "" {
+				return fmt.Errorf("no ruby upgrade version specified (--ruby-upgrade-version, -u)")
+			}
+
+			client := &http2.RLHTTPClient{
+				Client: oauth2.NewClient(context.Background(), ghTokenSource{}),
+
+				// 1 request every (n) second(s) to avoid DOS'ing server.
+				// https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits
+				Ratelimiter: rate.NewLimiter(rate.Every(5*time.Second), 1),
+			}
+
+			opts := ruby.Options{
+				RubyVersion:       p.version,
+				RubyUpdateVersion: upgradeVersion,
+				Path:              path,
+				Client:            client,
+				NoCache:           p.noCache,
+			}
+
+			pkgs, err := opts.DiscoverRubyPackages(ctx)
+			if err != nil {
+				return fmt.Errorf("could not discover ruby packages: %w", err)
+			}
+
+			checkUpdateError := false
+			for i := range pkgs {
+				// Check gemspec for version constraints
+				err = opts.CheckUpgrade(ctx, &pkgs[i])
+				if err != nil {
+					fmt.Printf("❌ %s: %s\n", pkgs[i].Name, err.Error())
+					checkUpdateError = true
+				} else {
+					fmt.Printf("✅ %s\n", pkgs[i].Name)
+				}
+			}
+
+			if checkUpdateError {
+				return fmt.Errorf("errors checking ruby upgrade")
+			}
+			return nil
+		},
+	}
+
+	p.addFlagsTo(cmd)
+	cmd.Flags().StringVarP(&upgradeVersion, "ruby-upgrade-version", "u", "", "ruby version to check for updates")
+	return cmd
+}

pkg/cli/ruby_code_search.go

@@ -0,0 +1,101 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/oauth2"
+	"golang.org/x/time/rate"
+
+	http2 "github.com/wolfi-dev/wolfictl/pkg/http"
+	"github.com/wolfi-dev/wolfictl/pkg/ruby"
+)
+
+func cmdRubyCodeSearch() *cobra.Command {
+	p := &rubyParams{}
+	var searchTerms []string
+	cmd := &cobra.Command{
+		Use:   "code-search",
+		Short: "Run Github search queries for ruby packages.",
+		Long: `
+NOTE: Due to limitations of GitHub Code Search, the search terms are only matched
+      against the default branch rather than the tag from which the package is
+      built. Hopefully this gets better in the future but it could lead to false
+      negatives if upgrade work has been committed to the main branch but a release
+      has not been cut yet.
+
+      https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code
+
+NOTE: This is currently restricted to ruby code housed on Github as that is the
+      majority. There are some on Gitlab and adding Gitlab API support is TODO.
+`,
+		SilenceErrors: true,
+		Hidden:        false,
+		Aliases:       []string{"cs", "search"},
+		Example: `
+# Run a search query over all ruby-3.2 package in the current directory
+wolfictl ruby code-search . --ruby-version 3.2 --search-terms 'language:ruby racc'
+`,
+		Args: cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			path, isDir, err := resolvePath(args)
+			if err != nil {
+				return fmt.Errorf("could not resolve path: %w", err)
+			}
+
+			if p.version == "" && isDir {
+				return fmt.Errorf("directory specified, but no --ruby-version to search for")
+			}
+
+			client := &http2.RLHTTPClient{
+				Client: oauth2.NewClient(context.Background(), ghTokenSource{}),
+
+				// 1 request every (n) second(s) to avoid DOS'ing server.
+				// https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits
+				Ratelimiter: rate.NewLimiter(rate.Every(5*time.Second), 1),
+			}
+
+			opts := ruby.Options{
+				RubyVersion: p.version,
+				Path:        path,
+				Client:      client,
+				NoCache:     p.noCache,
+			}
+
+			pkgs, err := opts.DiscoverRubyPackages(ctx)
+			if err != nil {
+				return fmt.Errorf("could not discover ruby packages: %w", err)
+			}
+
+			codeSearchError := false
+			for i := range pkgs {
+				// Check gemspec for version constraints
+				var localErr string
+				for _, term := range searchTerms {
+					err = opts.CodeSearch(ctx, &pkgs[i], term)
+					if err != nil {
+						localErr += fmt.Sprintf(" |query='%s': %v", term, err)
+					}
+				}
+				if localErr != "" {
+					fmt.Printf("⚠️ %s: %s\n", pkgs[i].Name, localErr)
+					codeSearchError = true
+				} else {
+					fmt.Printf("✅ %s\n", pkgs[i].Name)
+				}
+			}
+
+			if codeSearchError {
+				return fmt.Errorf("errors checking ruby upgrade")
+			}
+			return nil
+		},
+	}
+
+	p.addFlagsTo(cmd)
+	cmd.Flags().StringArrayVarP(&searchTerms, "search-terms", "s", []string{}, "GitHub code search term")
+	return cmd
+}

pkg/gh/repository.go

@@ -0,0 +1,32 @@
+package gh
+
+import (
+	"context"
+
+	"github.com/google/go-github/v58/github"
+)
+
+func (o GitOptions) ListRepositoryFiles(ctx context.Context, owner, repo, path, ref string) ([]*github.RepositoryContent, error) {
+	opts := github.RepositoryContentGetOptions{
+		Ref: ref,
+	}
+
+	_, directoryContents, _, err := o.GithubClient.Repositories.GetContents(ctx, owner, repo, path, &opts)
+	if err != nil {
+		return nil, err
+	}
+	return directoryContents, nil
+}
+
+func (o GitOptions) RepositoryFilesContents(ctx context.Context, owner, repo, file, ref string) (*github.RepositoryContent, error) {
+	opts := github.RepositoryContentGetOptions{
+		Ref: ref,
+	}
+
+	fileContent, _, _, err := o.GithubClient.Repositories.GetContents(ctx, owner, repo, file, &opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return fileContent, nil
+}

pkg/gh/search.go

@@ -0,0 +1,44 @@
+package gh
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/google/go-github/v58/github"
+)
+
+// SearchCode does a rate-limited search using the Github Code Search API. It
+// does not currently do paginated search, that would be a good feature to add,
+// but is not needed for it's immediate use case.
+func (o GitOptions) SearchCode(ctx context.Context, query string) (*github.CodeSearchResult, error) {
+	options := &github.SearchOptions{
+		TextMatch: true,
+	}
+	var result *github.CodeSearchResult
+
+	for {
+		rs, resp, err := o.GithubClient.Search.Code(ctx, query, options)
+
+		// if no err return result
+		if err == nil {
+			result = rs
+			break
+		}
+
+		// if err is rate limit, delay and try again
+		githubErr := github.CheckResponse(resp.Response)
+		if githubErr != nil {
+			rateLimited, delay := o.checkRateLimiting(githubErr)
+			if !rateLimited {
+				return nil, githubErr
+			}
+			fmt.Printf("retrying after %v second delay due to rate limiting\n", delay.Seconds())
+			time.Sleep(delay)
+		} else {
+			// if err is not rate limit, return err
+			return nil, err
+		}
+	}
+	return result, nil
+}

pkg/git/git.go

@@ -66,6 +66,10 @@ func GetRemoteURL(repo *git.Repository) (*URL, error) {
 
 // ParseGitURL returns owner, repo name, errors
 func ParseGitURL(rawURL string) (*URL, error) {
+	if rawURL == "" {
+		return nil, fmt.Errorf("no URL provided")
+	}
+
 	gitURL := &URL{}
 
 	rawURL = strings.TrimSuffix(rawURL, ".git")