From 3659796e5a3447ac53e0bc0be2c58328a579b2c1 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 15 Jan 2025 08:07:34 -0500 Subject: [PATCH] analysis: option to skip dependencies Add an option to the analysis system that skips the downloading of a module's dependencies. The module is still unzipped, but the analysis program that is run on it won't be able to get type information, and in fact cannot use the analysis framework at all, since that assumes types. However, if the program only needs the module's source, for example to match regular expressions, then it can run much faster. Change-Id: I54902804c4578a20940786cb28eced4dc6594a0d Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/642876 Reviewed-by: Zvonimir Pavlinovic LUCI-TryBot-Result: Go LUCI --- cmd/ejobs/main.go | 7 +++++-- internal/analysis/analysis.go | 2 ++ internal/worker/analysis.go | 2 ++ internal/worker/scan.go | 4 ++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/ejobs/main.go b/cmd/ejobs/main.go index 9ab8b80..41de5b3 100644 --- a/cmd/ejobs/main.go +++ b/cmd/ejobs/main.go @@ -47,6 +47,7 @@ var ( var ( minImporters int // for start + noDeps bool // for start waitInterval time.Duration // for wait force bool // for results errs bool // for results @@ -69,6 +70,7 @@ var commands = []command{ func(fs *flag.FlagSet) { fs.IntVar(&minImporters, "min", -1, "run on modules with at least this many importers (<0: use server default of 10)") + fs.BoolVar(&noDeps, "nodeps", false, "do not download dependencies for modules") }, }, {"wait", "JOBID", @@ -269,7 +271,7 @@ func doWait(ctx context.Context, args []string) error { func doStart(ctx context.Context, args []string) error { // Validate arguments. if len(args) == 0 { - return errors.New("wrong number of args: want [-min N] BINARY [ARG1 ARG2 ...]") + return errors.New("wrong number of args: want [-min N] [-nodeps] BINARY [ARG1 ARG2 ...]") } binaryFile := args[0] if fi, err := os.Stat(binaryFile); err != nil { @@ -300,7 +302,8 @@ func doStart(ctx context.Context, args []string) error { if err != nil { return err } - u := fmt.Sprintf("%s/analysis/enqueue?binary=%s&user=%s", workerURL, filepath.Base(binaryFile), os.Getenv("USER")) + u := fmt.Sprintf("%s/analysis/enqueue?binary=%s&user=%s&nodeps=%t", + workerURL, filepath.Base(binaryFile), os.Getenv("USER"), noDeps) if len(binaryArgs) > 0 { u += fmt.Sprintf("&args=%s", url.QueryEscape(strings.Join(binaryArgs, " "))) } diff --git a/internal/analysis/analysis.go b/internal/analysis/analysis.go index d44735e..50cf2a5 100644 --- a/internal/analysis/analysis.go +++ b/internal/analysis/analysis.go @@ -37,6 +37,7 @@ type ScanParams struct { Serve bool // serve results back to client instead of writing them to BigQuery JobID string // ID of job, if non-empty SkipInit bool // if true, do not initialize non-module Go projects + NoDeps bool // if true, do not download module dependencies } type EnqueueParams struct { @@ -48,6 +49,7 @@ type EnqueueParams struct { Suffix string // appended to task queue IDs to generate unique tasks User string // user initiating enqueue SkipInit bool // if true, do not initialize non-module Go projects + NoDeps bool // if true, do not download module dependencies } // Request implements queue.Task so it can be put on a TaskQueue. diff --git a/internal/worker/analysis.go b/internal/worker/analysis.go index c50651f..1828464 100644 --- a/internal/worker/analysis.go +++ b/internal/worker/analysis.go @@ -262,6 +262,7 @@ func (s *analysisServer) scanInternal(ctx context.Context, req *analysis.ScanReq proxyClient: s.proxyClient, insecure: req.Insecure, init: !req.SkipInit, + noDeps: req.NoDeps, } if err := prepareModule(ctx, args); err != nil { return nil, err @@ -496,6 +497,7 @@ func createAnalysisQueueTasks(params *analysis.EnqueueParams, jobID string, bina Insecure: params.Insecure, JobID: jobID, SkipInit: params.SkipInit, + NoDeps: params.NoDeps, }, }) } diff --git a/internal/worker/scan.go b/internal/worker/scan.go index 745ae65..4447121 100644 --- a/internal/worker/scan.go +++ b/internal/worker/scan.go @@ -244,6 +244,7 @@ type prepareModuleArgs struct { proxyClient *proxy.Client // for downloading insecure bool // run without sandbox init bool // run `go mod init` and `go mod tidy` on modules with no go.mod file + noDeps bool // do not download dependencies } // prepareModule prepares a module for scanning, and takes other actions that increase @@ -254,6 +255,9 @@ func prepareModule(ctx context.Context, args prepareModuleArgs) error { log.Debugf(ctx, "download error: %v (%[1]T)", err) return err } + if args.noDeps { + return nil + } hasGoMod := fileExists(filepath.Join(args.dir, "go.mod")) if !args.init || hasGoMod {