analysis: option to skip dependencies

Add an option to the analysis system that skips the downloading of
a module's dependencies. The module is still unzipped, but the analysis
program that is run on it won't be able to get type information, and
in fact cannot use the analysis framework at all, since that assumes
types.

However, if the program only needs the module's source, for example
to match regular expressions, then it can run much faster.

Change-Id: I54902804c4578a20940786cb28eced4dc6594a0d
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/642876
Reviewed-by: Zvonimir Pavlinovic <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

cmd/ejobs/main.go

@@ -47,6 +47,7 @@ var (
 
 var (
 	minImporters int           // for start
+	noDeps       bool          // for start
 	waitInterval time.Duration // for wait
 	force        bool          // for results
 	errs         bool          // for results
@@ -69,6 +70,7 @@ var commands = []command{
 		func(fs *flag.FlagSet) {
 			fs.IntVar(&minImporters, "min", -1,
 				"run on modules with at least this many importers (<0: use server default of 10)")
+			fs.BoolVar(&noDeps, "nodeps", false, "do not download dependencies for modules")
 		},
 	},
 	{"wait", "JOBID",
@@ -269,7 +271,7 @@ func doWait(ctx context.Context, args []string) error {
 func doStart(ctx context.Context, args []string) error {
 	// Validate arguments.
 	if len(args) == 0 {
-		return errors.New("wrong number of args: want [-min N] BINARY [ARG1 ARG2 ...]")
+		return errors.New("wrong number of args: want [-min N] [-nodeps] BINARY [ARG1 ARG2 ...]")
 	}
 	binaryFile := args[0]
 	if fi, err := os.Stat(binaryFile); err != nil {
@@ -300,7 +302,8 @@ func doStart(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
-	u := fmt.Sprintf("%s/analysis/enqueue?binary=%s&user=%s", workerURL, filepath.Base(binaryFile), os.Getenv("USER"))
+	u := fmt.Sprintf("%s/analysis/enqueue?binary=%s&user=%s&nodeps=%t",
+		workerURL, filepath.Base(binaryFile), os.Getenv("USER"), noDeps)
 	if len(binaryArgs) > 0 {
 		u += fmt.Sprintf("&args=%s", url.QueryEscape(strings.Join(binaryArgs, " ")))
 	}

internal/analysis/analysis.go

@@ -37,6 +37,7 @@ type ScanParams struct {
 	Serve         bool   // serve results back to client instead of writing them to BigQuery
 	JobID         string // ID of job, if non-empty
 	SkipInit      bool   // if true, do not initialize non-module Go projects
+	NoDeps        bool   // if true, do not download module dependencies
 }
 
 type EnqueueParams struct {
@@ -48,6 +49,7 @@ type EnqueueParams struct {
 	Suffix   string // appended to task queue IDs to generate unique tasks
 	User     string // user initiating enqueue
 	SkipInit bool   // if true, do not initialize non-module Go projects
+	NoDeps   bool   // if true, do not download module dependencies
 }
 
 // Request implements queue.Task so it can be put on a TaskQueue.

internal/worker/analysis.go

@@ -262,6 +262,7 @@ func (s *analysisServer) scanInternal(ctx context.Context, req *analysis.ScanReq
 		proxyClient: s.proxyClient,
 		insecure:    req.Insecure,
 		init:        !req.SkipInit,
+		noDeps:      req.NoDeps,
 	}
 	if err := prepareModule(ctx, args); err != nil {
 		return nil, err
@@ -496,6 +497,7 @@ func createAnalysisQueueTasks(params *analysis.EnqueueParams, jobID string, bina
 				Insecure:      params.Insecure,
 				JobID:         jobID,
 				SkipInit:      params.SkipInit,
+				NoDeps:        params.NoDeps,
 			},
 		})
 	}

internal/worker/scan.go

@@ -244,6 +244,7 @@ type prepareModuleArgs struct {
 	proxyClient         *proxy.Client // for downloading
 	insecure            bool          // run without sandbox
 	init                bool          // run `go mod init` and `go mod tidy` on modules with no go.mod file
+	noDeps              bool          // do not download dependencies
 }
 
 // prepareModule prepares a module for scanning, and takes other actions that increase
@@ -254,6 +255,9 @@ func prepareModule(ctx context.Context, args prepareModuleArgs) error {
 		log.Debugf(ctx, "download error: %v (%[1]T)", err)
 		return err
 	}
+	if args.noDeps {
+		return nil
+	}
 
 	hasGoMod := fileExists(filepath.Join(args.dir, "go.mod"))
 	if !args.init || hasGoMod {