cmd/go: do context propagation for tracing downloads

This change does context propagation (and only context propagation)
necessary to add context to modfetch.Download and pkg.LoadImport.
This was done by adding context to their callers, and then
adding context to all call-sites, and then repeating adding
context to callers of those enclosing functions and their
callers until none were left. In some cases the call graph expansion
was pruned by using context.TODOs.

The next CL will add a span to Download. I kept it out of this
change to avoid making it any larger (and harder to review)
than it needs to be.

Updates #38714

Change-Id: I5bf2d599aafef67334c384dfccd5e255198c85b4
Reviewed-on: https://go-review.googlesource.com/c/go/+/248327
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

Author: matloob

src/cmd/go/internal/get/get.go

@@ -246,9 +246,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int)
 	load1 := func(path string, mode int) *load.Package {
 		if parent == nil {
 			mode := 0 // don't do module or vendor resolution
-			return load.LoadImport(path, base.Cwd, nil, stk, nil, mode)
+			return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode)
 		}
-		return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
+		return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
 	}
 
 	p := load1(arg, mode)

src/cmd/go/internal/list/list.go

@@ -20,6 +20,7 @@ import (
 	"cmd/go/internal/cache"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modinfo"
 	"cmd/go/internal/modload"
 	"cmd/go/internal/str"
 	"cmd/go/internal/work"
@@ -349,7 +350,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
 		fm := template.FuncMap{
 			"join":    strings.Join,
 			"context": context,
-			"module":  modload.ModuleInfo,
+			"module":  func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) },
 		}
 		tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt)
 		if err != nil {
@@ -389,7 +390,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
 			base.Fatalf("go list -m: not using modules")
 		}
 
-		modload.InitMod() // Parses go.mod and sets cfg.BuildMod.
+		modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod.
 		if cfg.BuildMod == "vendor" {
 			const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)"
 

src/cmd/go/internal/load/pkg.go

@@ -42,10 +42,10 @@ var (
 	ModBinDir            func() string                                                                            // return effective bin directory
 	ModLookup            func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import
 	ModPackageModuleInfo func(path string) *modinfo.ModulePublic                                                  // return module info for Package struct
-	ModImportPaths       func(args []string) []*search.Match                                                      // expand import paths
+	ModImportPaths       func(ctx context.Context, args []string) []*search.Match                                 // expand import paths
 	ModPackageBuildInfo  func(main string, deps []string) string                                                  // return module info to embed in binary
 	ModInfoProg          func(info string, isgccgo bool) []byte                                                   // wrap module info in .go code for binary
-	ModImportFromFiles   func([]string)                                                                           // update go.mod to add modules for imports in these files
+	ModImportFromFiles   func(context.Context, []string)                                                          // update go.mod to add modules for imports in these files
 	ModDirImportPath     func(string) string                                                                      // return effective import path for directory
 )
 
@@ -553,7 +553,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package {
 		})
 		packageDataCache.Delete(p.ImportPath)
 	}
-	return LoadImport(arg, base.Cwd, nil, stk, nil, 0)
+	return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0)
 }
 
 // dirToImportPath returns the pseudo-import path we use for a package
@@ -605,11 +605,11 @@ const (
 // LoadImport does not set tool flags and should only be used by
 // this package, as part of a bigger load operation, and by GOPATH-based "go get".
 // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function.
-func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
-	return loadImport(nil, path, srcDir, parent, stk, importPos, mode)
+func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
+	return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode)
 }
 
-func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
+func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
 	if path == "" {
 		panic("LoadImport called with empty package path")
 	}
@@ -657,7 +657,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS
 		// Load package.
 		// loadPackageData may return bp != nil even if an error occurs,
 		// in order to return partial information.
-		p.load(path, stk, importPos, bp, err)
+		p.load(ctx, path, stk, importPos, bp, err)
 
 		if !cfg.ModulesEnabled && path != cleanImport(path) {
 			p.Error = &PackageError{
@@ -1591,7 +1591,7 @@ func (p *Package) DefaultExecName() string {
 // load populates p using information from bp, err, which should
 // be the result of calling build.Context.Import.
 // stk contains the import stack, not including path itself.
-func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) {
+func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) {
 	p.copyBuild(bp)
 
 	// The localPrefix is the path we interpret ./ imports relative to.
@@ -1800,7 +1800,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position
 		if path == "C" {
 			continue
 		}
-		p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
+		p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
 
 		path = p1.ImportPath
 		importPaths[i] = path
@@ -2073,7 +2073,7 @@ func PackageList(roots []*Package) []*Package {
 // TestPackageList returns the list of packages in the dag rooted at roots
 // as visited in a depth-first post-order traversal, including the test
 // imports of the roots. This ignores errors in test packages.
-func TestPackageList(roots []*Package) []*Package {
+func TestPackageList(ctx context.Context, roots []*Package) []*Package {
 	seen := map[*Package]bool{}
 	all := []*Package{}
 	var walk func(*Package)
@@ -2089,7 +2089,7 @@ func TestPackageList(roots []*Package) []*Package {
 	}
 	walkTest := func(root *Package, path string) {
 		var stk ImportStack
-		p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport)
+		p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport)
 		if p1.Error == nil {
 			walk(p1)
 		}
@@ -2112,7 +2112,7 @@ func TestPackageList(roots []*Package) []*Package {
 // TODO(jayconrod): delete this function and set flags automatically
 // in LoadImport instead.
 func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
-	p := LoadImport(path, srcDir, parent, stk, importPos, mode)
+	p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode)
 	setToolFlags(p)
 	return p
 }
@@ -2153,12 +2153,12 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package {
 			// We need to test whether the path is an actual Go file and not a
 			// package path or pattern ending in '.go' (see golang.org/issue/34653).
 			if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
-				return []*Package{GoFilesPackage(patterns)}
+				return []*Package{GoFilesPackage(ctx, patterns)}
 			}
 		}
 	}
 
-	matches := ImportPaths(patterns)
+	matches := ImportPaths(ctx, patterns)
 	var (
 		pkgs    []*Package
 		stk     ImportStack
@@ -2174,7 +2174,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package {
 			if pkg == "" {
 				panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern()))
 			}
-			p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0)
+			p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0)
 			p.Match = append(p.Match, m.Pattern())
 			p.Internal.CmdlinePkg = true
 			if m.IsLiteral() {
@@ -2228,9 +2228,9 @@ func setToolFlags(pkgs ...*Package) {
 	}
 }
 
-func ImportPaths(args []string) []*search.Match {
+func ImportPaths(ctx context.Context, args []string) []*search.Match {
 	if ModInit(); cfg.ModulesEnabled {
-		return ModImportPaths(args)
+		return ModImportPaths(ctx, args)
 	}
 	return search.ImportPaths(args)
 }
@@ -2281,7 +2281,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package {
 // GoFilesPackage creates a package for building a collection of Go files
 // (typically named on the command line). The target is named p.a for
 // package p or named after the first Go file for package main.
-func GoFilesPackage(gofiles []string) *Package {
+func GoFilesPackage(ctx context.Context, gofiles []string) *Package {
 	ModInit()
 
 	for _, f := range gofiles {
@@ -2329,7 +2329,7 @@ func GoFilesPackage(gofiles []string) *Package {
 	ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil }
 
 	if cfg.ModulesEnabled {
-		ModImportFromFiles(gofiles)
+		ModImportFromFiles(ctx, gofiles)
 	}
 
 	var err error
@@ -2345,7 +2345,7 @@ func GoFilesPackage(gofiles []string) *Package {
 	pkg := new(Package)
 	pkg.Internal.Local = true
 	pkg.Internal.CmdlineFiles = true
-	pkg.load("command-line-arguments", &stk, nil, bp, err)
+	pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err)
 	pkg.Internal.LocalPrefix = dirToImportPath(dir)
 	pkg.ImportPath = "command-line-arguments"
 	pkg.Target = ""

src/cmd/go/internal/load/test.go

@@ -108,7 +108,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
 	stk.Push(p.ImportPath + " (test)")
 	rawTestImports := str.StringList(p.TestImports)
 	for i, path := range p.TestImports {
-		p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport)
+		p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport)
 		if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath {
 			// Same error that loadPackage returns (via reusePackage) in pkg.go.
 			// Can't change that code, because that code is only for loading the
@@ -127,7 +127,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
 	pxtestNeedsPtest := false
 	rawXTestImports := str.StringList(p.XTestImports)
 	for i, path := range p.XTestImports {
-		p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport)
+		p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport)
 		if p1.ImportPath == p.ImportPath {
 			pxtestNeedsPtest = true
 		} else {
@@ -244,7 +244,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
 		if dep == ptest.ImportPath {
 			pmain.Internal.Imports = append(pmain.Internal.Imports, ptest)
 		} else {
-			p1 := loadImport(pre, dep, "", nil, &stk, nil, 0)
+			p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0)
 			pmain.Internal.Imports = append(pmain.Internal.Imports, p1)
 		}
 	}

src/cmd/go/internal/modcmd/download.go

@@ -90,7 +90,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
 	if len(args) == 0 {
 		args = []string{"all"}
 	} else if modload.HasModRoot() {
-		modload.InitMod() // to fill Target
+		modload.InitMod(ctx) // to fill Target
 		targetAtLatest := modload.Target.Path + "@latest"
 		targetAtUpgrade := modload.Target.Path + "@upgrade"
 		targetAtPatch := modload.Target.Path + "@patch"
@@ -126,7 +126,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
 			return
 		}
 		m.Sum = modfetch.Sum(mod)
-		m.Dir, err = modfetch.Download(mod)
+		m.Dir, err = modfetch.Download(ctx, mod)
 		if err != nil {
 			m.Error = err.Error()
 			return

src/cmd/go/internal/modcmd/init.go

@@ -51,6 +51,6 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) {
 	if strings.Contains(modload.CmdModModule, "@") {
 		base.Fatalf("go mod init: module path must not contain '@'")
 	}
-	modload.InitMod() // does all the hard work
+	modload.InitMod(ctx) // does all the hard work
 	modload.WriteGoMod()
 }

src/cmd/go/internal/modcmd/tidy.go

@@ -40,7 +40,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) {
 		base.Fatalf("go mod tidy: no arguments allowed")
 	}
 
-	modload.LoadALL()
+	modload.LoadALL(ctx)
 	modload.TidyBuildList()
 	modload.TrimGoSum()
 	modload.WriteGoMod()

src/cmd/go/internal/modcmd/vendor.go

@@ -48,7 +48,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
 	if len(args) != 0 {
 		base.Fatalf("go mod vendor: vendor takes no arguments")
 	}
-	pkgs := modload.LoadVendor()
+	pkgs := modload.LoadVendor(ctx)
 
 	vdir := filepath.Join(modload.ModRoot(), "vendor")
 	if err := os.RemoveAll(vdir); err != nil {

src/cmd/go/internal/modcmd/why.go

@@ -76,7 +76,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) {
 		}
 		mods := modload.ListModules(ctx, args, listU, listVersions)
 		byModule := make(map[module.Version][]string)
-		for _, path := range loadALL() {
+		for _, path := range loadALL(ctx) {
 			m := modload.PackageModule(path)
 			if m.Path != "" {
 				byModule[m] = append(byModule[m], path)
@@ -105,8 +105,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) {
 			sep = "\n"
 		}
 	} else {
-		matches := modload.ImportPaths(args) // resolve to packages
-		loadALL()                            // rebuild graph, from main module (not from named packages)
+		matches := modload.ImportPaths(ctx, args) // resolve to packages
+		loadALL(ctx)                              // rebuild graph, from main module (not from named packages)
 		sep := ""
 		for _, m := range matches {
 			for _, path := range m.Pkgs {

src/cmd/go/internal/modconv/convert_test.go

@@ -6,6 +6,7 @@ package modconv
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"internal/testenv"
 	"io/ioutil"
@@ -146,6 +147,8 @@ func TestConvertLegacyConfig(t *testing.T) {
 		},
 	}
 
+	ctx := context.Background()
+
 	for _, tt := range tests {
 		t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) {
 			f, err := modfile.Parse("golden", []byte(tt.gomod), nil)
@@ -157,7 +160,7 @@ func TestConvertLegacyConfig(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers})
+			dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers})
 			if err != nil {
 				t.Fatal(err)
 			}

src/cmd/go/internal/modfetch/fetch.go

@@ -7,6 +7,7 @@ package modfetch
 import (
 	"archive/zip"
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -34,7 +35,7 @@ var downloadCache par.Cache
 // Download downloads the specific module version to the
 // local download cache and returns the name of the directory
 // corresponding to the root of the module's file tree.
-func Download(mod module.Version) (dir string, err error) {
+func Download(ctx context.Context, mod module.Version) (dir string, err error) {
 	if cfg.GOMODCACHE == "" {
 		// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
 		// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.