From 6c99b9e2ff937b183a2deb30294bccbf9701b36b Mon Sep 17 00:00:00 2001 From: Dentrax Date: Sat, 13 Jul 2024 13:47:27 +0300 Subject: [PATCH] scan: ability to override repository Signed-off-by: Dentrax --- .github/actions/action.yaml | 2 +- docs/cmd/wolfictl_scan.md | 37 ++++++++++++++++++++++------------- docs/man/man1/wolfictl-scan.1 | 27 +++++++++++++++++++++---- pkg/cli/scan.go | 28 ++++++++++++++++++++------ 4 files changed, 69 insertions(+), 25 deletions(-) diff --git a/.github/actions/action.yaml b/.github/actions/action.yaml index 8ba21813..1e2524e8 100644 --- a/.github/actions/action.yaml +++ b/.github/actions/action.yaml @@ -8,7 +8,7 @@ runs: repository: wolfi-dev/wolfictl path: wolfictl-setup-gha - - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version: '1.21' check-latest: true diff --git a/docs/cmd/wolfictl_scan.md b/docs/cmd/wolfictl_scan.md index 40a9f026..d65f5e9c 100644 --- a/docs/cmd/wolfictl_scan.md +++ b/docs/cmd/wolfictl_scan.md @@ -36,8 +36,8 @@ By default, the command will print all vulnerabilities found in the package(s) to stdout. You can filter the vulnerabilities shown using existing local advisory data. To do this, you must first clone the advisory data from the advisories repository for the distro whose packages you are scanning. You -specify the path to each local advisories repository using the ---advisories-repo-dir flag for each repository. Then, you can use the +specify the path to the local advisories repository using the +--advisories-repo-dir flag for the repository. Then, you can use the "--advisory-filter" flag to specify which set of advisories to use for filtering. The following sets of advisories are available: @@ -113,18 +113,27 @@ wolfictl scan package1 package2 --remote ### Options ``` - -a, --advisories-repo-dir strings local directory for advisory data - -f, --advisory-filter string exclude vulnerability matches that are referenced from the specified set of advisories (resolved|all|concluded) - --build-log treat input as a package build log file (or a directory that contains a packages.log file) - --disable-sbom-cache don't use the SBOM cache - --distro string distro to use during vulnerability matching (default "wolfi") - -h, --help help for scan - --local-file-grype-db string import a local grype db file - -o, --output string output format (outline|json), defaults to outline - -r, --remote treat input(s) as the name(s) of package(s) in the Wolfi package repository to download and scan the latest versions of - --require-zero exit 1 if any vulnerabilities are found - -s, --sbom treat input(s) as SBOM(s) of APK(s) instead of as actual APK(s) - --use-cpes turn on all CPE matching in Grype + -a, --advisories-repo-dir string directory containing the advisories repository + -f, --advisory-filter string exclude vulnerability matches that are referenced from the specified set of advisories (resolved|all|concluded) + --build-log treat input as a package build log file (or a directory that contains a packages.log file) + --disable-sbom-cache don't use the SBOM cache + --distro string distro to use during vulnerability matching (default "wolfi") + -h, --help help for scan + --local-file-grype-db string import a local grype db file + -o, --output string output format (outline|json), defaults to outline + -r, --remote treat input(s) as the name(s) of package(s) in the Wolfi package repository to download and scan the latest versions of + --repository string URL of the APK package repository (default "https://packages.wolfi.dev/os") + --require-zero exit 1 if any vulnerabilities are found + -s, --sbom treat input(s) as SBOM(s) of APK(s) instead of as actual APK(s) + --use-cpes turn on all CPE matching in Grype + -v, --verbose count logging verbosity (v = info, vv = debug, default is none) +``` + +### Options inherited from parent commands + +``` + --log-level string log level (e.g. debug, info, warn, error) (default "info") + --log-policy strings log policy (e.g. builtin:stderr, /tmp/log/foo) (default [builtin:stderr]) ``` ### SEE ALSO diff --git a/docs/man/man1/wolfictl-scan.1 b/docs/man/man1/wolfictl-scan.1 index 29ae3bf4..5bb06200 100644 --- a/docs/man/man1/wolfictl-scan.1 +++ b/docs/man/man1/wolfictl-scan.1 @@ -53,8 +53,8 @@ By default, the command will print all vulnerabilities found in the package(s) to stdout. You can filter the vulnerabilities shown using existing local advisory data. To do this, you must first clone the advisory data from the advisories repository for the distro whose packages you are scanning. You -specify the path to each local advisories repository using the -\-\-advisories\-repo\-dir flag for each repository. Then, you can use the +specify the path to the local advisories repository using the +\-\-advisories\-repo\-dir flag for the repository. Then, you can use the "\-\-advisory\-filter" flag to specify which set of advisories to use for filtering. The following sets of advisories are available: @@ -130,8 +130,8 @@ found and the \-\-require\-zero flag is specified. .SH OPTIONS .PP -\fB\-a\fP, \fB\-\-advisories\-repo\-dir\fP=[] - local directory for advisory data +\fB\-a\fP, \fB\-\-advisories\-repo\-dir\fP="" + directory containing the advisories repository .PP \fB\-f\fP, \fB\-\-advisory\-filter\fP="" @@ -165,6 +165,11 @@ found and the \-\-require\-zero flag is specified. \fB\-r\fP, \fB\-\-remote\fP[=false] treat input(s) as the name(s) of package(s) in the Wolfi package repository to download and scan the latest versions of +.PP +\fB\-\-repository\fP=" +\[la]https://packages.wolfi.dev/os"\[ra] + URL of the APK package repository + .PP \fB\-\-require\-zero\fP[=false] exit 1 if any vulnerabilities are found @@ -177,6 +182,20 @@ found and the \-\-require\-zero flag is specified. \fB\-\-use\-cpes\fP[=false] turn on all CPE matching in Grype +.PP +\fB\-v\fP, \fB\-\-verbose\fP[=0] + logging verbosity (v = info, vv = debug, default is none) + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB\-\-log\-level\fP="info" + log level (e.g. debug, info, warn, error) + +.PP +\fB\-\-log\-policy\fP=[builtin:stderr] + log policy (e.g. builtin:stderr, /tmp/log/foo) + .SH EXAMPLE diff --git a/pkg/cli/scan.go b/pkg/cli/scan.go index c17f2c0f..7d2f9d84 100644 --- a/pkg/cli/scan.go +++ b/pkg/cli/scan.go @@ -14,6 +14,7 @@ import ( "strings" "chainguard.dev/apko/pkg/apk/apk" + "chainguard.dev/apko/pkg/apk/auth" sbomSyft "github.com/anchore/syft/syft/sbom" "github.com/chainguard-dev/clog" "github.com/charmbracelet/lipgloss" @@ -25,7 +26,6 @@ import ( "github.com/wolfi-dev/wolfictl/pkg/configs" v2 "github.com/wolfi-dev/wolfictl/pkg/configs/advisory/v2" rwos "github.com/wolfi-dev/wolfictl/pkg/configs/rwfs/os" - "github.com/wolfi-dev/wolfictl/pkg/index" "github.com/wolfi-dev/wolfictl/pkg/sbom" "github.com/wolfi-dev/wolfictl/pkg/scan" "github.com/wolfi-dev/wolfictl/pkg/versions" @@ -357,6 +357,7 @@ type scanParams struct { disableSBOMCache bool triageWithGoVulnCheck bool remoteScanning bool + remoteRepository string useCPEMatching bool verbosity int } @@ -374,6 +375,7 @@ func (p *scanParams) addFlagsTo(cmd *cobra.Command) { cmd.Flags().BoolVar(&p.triageWithGoVulnCheck, "govulncheck", false, "EXPERIMENTAL: triage vulnerabilities in Go binaries using govulncheck") _ = cmd.Flags().MarkHidden("govulncheck") //nolint:errcheck cmd.Flags().BoolVarP(&p.remoteScanning, "remote", "r", false, "treat input(s) as the name(s) of package(s) in the Wolfi package repository to download and scan the latest versions of") + cmd.Flags().StringVar(&p.remoteRepository, "repository", "https://packages.wolfi.dev/os", "URL of the APK package repository") cmd.Flags().BoolVar(&p.useCPEMatching, "use-cpes", false, "turn on all CPE matching in Grype") addVerboseFlag(&p.verbosity, cmd) } @@ -403,7 +405,7 @@ func (p *scanParams) resolveInputsToScan(ctx context.Context, args []string) (in } for _, arg := range args { - targetPaths, cleanup, err := resolveInputForRemoteTarget(ctx, arg) + targetPaths, cleanup, err := resolveInputForRemoteTarget(ctx, arg, p.remoteRepository) if err != nil { return nil, nil, fmt.Errorf("failed to resolve input %q for remote scanning: %w", arg, err) } @@ -606,6 +608,18 @@ func resolveInputFileFromArg(inputFilePath string) (*os.File, error) { } } +// getAPKIndexURL returns the URL of the APKINDEX.tar.gz file for the given +// repository and architecture. If the repository URL already points to an +// APKINDEX.tar.gz file, it will be returned as-is. User input may or may not +// have included the architecture or the APKINDEX.tar.gz suffix, so construct +// the full URL to provide better UX. +func getAPKIndexURL(repositoryURL, arch string) string { + if strings.HasSuffix(repositoryURL, "/x86_64/APKINDEX.tar.gz") || strings.HasSuffix(repositoryURL, "/aarch64/APKINDEX.tar.gz") { + return repositoryURL + } + return fmt.Sprintf("%s/%s/APKINDEX.tar.gz", repositoryURL, arch) +} + // resolveInputForRemoteTarget takes the given input string, which is expected // to be the name of a Wolfi package (or subpackage), and it queries the Wolfi // APK repository to find the latest version of the package for each @@ -615,13 +629,14 @@ func resolveInputFileFromArg(inputFilePath string) (*os.File, error) { // For example, given the input value "calico", this function will find the // latest version of the package (e.g. "calico-3.26.3-r3.apk") and download it // for each architecture. -func resolveInputForRemoteTarget(ctx context.Context, input string) (downloadedAPKFilePaths []string, cleanup func() error, err error) { +func resolveInputForRemoteTarget(ctx context.Context, input, repository string) (downloadedAPKFilePaths []string, cleanup func() error, err error) { logger := clog.FromContext(ctx) archesFound := 0 for _, arch := range []string{"x86_64", "aarch64"} { - const apkRepositoryURL = "https://packages.wolfi.dev/os" - apkindex, err := index.Index(arch, apkRepositoryURL) + // Since index.Index function doesn't respect the `$HTTP_AUTH`, use + // fetchAPKIndex function instead. + apkindex, _, err := fetchAPKIndex(ctx, getAPKIndexURL(repository, arch)) if err != nil { return nil, nil, fmt.Errorf("getting APKINDEX: %w", err) } @@ -651,7 +666,7 @@ func resolveInputForRemoteTarget(ctx context.Context, input string) (downloadedA break } } - downloadURL := fmt.Sprintf("%s/%s/%s", apkRepositoryURL, arch, latestPkg.Filename()) + downloadURL := fmt.Sprintf("%s/%s/%s", repository, arch, latestPkg.Filename()) apkTempFileName := fmt.Sprintf("%s-%s-%s-*.apk", arch, input, latestVersion) tmpFile, err := os.CreateTemp("", apkTempFileName) @@ -665,6 +680,7 @@ func resolveInputForRemoteTarget(ctx context.Context, input string) (downloadedA return nil, nil, fmt.Errorf("creating request for %q: %w", downloadURL, err) } + auth.DefaultAuthenticators.AddAuth(ctx, req) logger.Debug("downloading APK", "url", downloadURL) resp, err := http.DefaultClient.Do(req) if err != nil {