cmd/go: add global ignore mechanism

This CL adds the ignore directive which enables users to tell the Go
Command to skip traversing into a given directory.

This behaves similar to how '_' or 'testdata' are currently treated.
This mainly has benefits for go list and go mod tidy.
This does not affect what is packed into a module.

Fixes: #42965
Change-Id: I232e27c1a065bb6eb2d210dbddad0208426a1fdd
Reviewed-on: https://go-review.googlesource.com/c/go/+/643355
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Matloob <matloob@golang.org>
Reviewed-by: Michael Matloob <matloob@google.com>

Author: samthanawalla

src/cmd/go/alldocs.go

@@ -2927,8 +2927,7 @@ func PackagesAndErrors(ctx context.Context, opts PackageOpts, patterns []string)
 		}
 		matches, _ = modload.LoadPackages(ctx, modOpts, patterns...)
 	} else {
-		noModRoots := []string{}
-		matches = search.ImportPaths(patterns, noModRoots)
+		matches = search.ImportPaths(patterns)
 	}
 
 	var (

src/cmd/go/internal/load/pkg.go

@@ -90,9 +90,13 @@ like "v1.2.3" or a closed interval like "[v1.1.0,v1.1.9]". Note that
 The -tool=path and -droptool=path flags add and drop a tool declaration
 for the given path.
 
+The -ignore=path and -dropignore=path flags add and drop a ignore declaration
+for the given path.
+
 The -godebug, -dropgodebug, -require, -droprequire, -exclude, -dropexclude,
--replace, -dropreplace, -retract, -dropretract, -tool, and -droptool editing
-flags may be repeated, and the changes are applied in the order given.
+-replace, -dropreplace, -retract, -dropretract, -tool, -droptool, -ignore,
+and -dropignore editing flags may be repeated, and the changes are applied
+in the order given.
 
 The -print flag prints the final go.mod in its text format instead of
 writing it back to go.mod.
@@ -147,6 +151,10 @@ writing it back to go.mod. The JSON output corresponds to these Go types:
 		Path string
 	}
 
+	type Ignore struct {
+		Path string
+	}
+
 Retract entries representing a single version (not an interval) will have
 the "Low" and "High" fields set to the same value.
 
@@ -190,6 +198,8 @@ func init() {
 	cmdEdit.Flag.Var(flagFunc(flagDropRetract), "dropretract", "")
 	cmdEdit.Flag.Var(flagFunc(flagTool), "tool", "")
 	cmdEdit.Flag.Var(flagFunc(flagDropTool), "droptool", "")
+	cmdEdit.Flag.Var(flagFunc(flagIgnore), "ignore", "")
+	cmdEdit.Flag.Var(flagFunc(flagDropIgnore), "dropignore", "")
 
 	base.AddBuildFlagsNX(&cmdEdit.Flag)
 	base.AddChdirFlag(&cmdEdit.Flag)
@@ -546,6 +556,24 @@ func flagDropTool(arg string) {
 	})
 }
 
+// flagIgnore implements the -ignore flag.
+func flagIgnore(arg string) {
+	edits = append(edits, func(f *modfile.File) {
+		if err := f.AddIgnore(arg); err != nil {
+			base.Fatalf("go: -ignore=%s: %v", arg, err)
+		}
+	})
+}
+
+// flagDropIgnore implements the -dropignore flag.
+func flagDropIgnore(arg string) {
+	edits = append(edits, func(f *modfile.File) {
+		if err := f.DropIgnore(arg); err != nil {
+			base.Fatalf("go: -dropignore=%s: %v", arg, err)
+		}
+	})
+}
+
 // fileJSON is the -json output data structure.
 type fileJSON struct {
 	Module    editModuleJSON
@@ -556,6 +584,7 @@ type fileJSON struct {
 	Replace   []replaceJSON
 	Retract   []retractJSON
 	Tool      []toolJSON
+	Ignore    []ignoreJSON
 }
 
 type editModuleJSON struct {
@@ -584,6 +613,10 @@ type toolJSON struct {
 	Path string
 }
 
+type ignoreJSON struct {
+	Path string
+}
+
 // editPrintJSON prints the -json output.
 func editPrintJSON(modFile *modfile.File) {
 	var f fileJSON
@@ -614,6 +647,9 @@ func editPrintJSON(modFile *modfile.File) {
 	for _, t := range modFile.Tool {
 		f.Tool = append(f.Tool, toolJSON{t.Path})
 	}
+	for _, i := range modFile.Ignore {
+		f.Ignore = append(f.Ignore, ignoreJSON{i.Path})
+	}
 	data, err := json.MarshalIndent(&f, "", "\t")
 	if err != nil {
 		base.Fatalf("go: internal error: %v", err)

src/cmd/go/internal/modcmd/edit.go

@@ -94,6 +94,7 @@ type modFileIndex struct {
 	require      map[module.Version]requireMeta
 	replace      map[module.Version]module.Version
 	exclude      map[module.Version]bool
+	ignore       []string
 }
 
 type requireMeta struct {
@@ -455,7 +456,11 @@ func indexModFile(data []byte, modFile *modfile.File, mod module.Version, needsF
 	for _, x := range modFile.Exclude {
 		i.exclude[x.Mod] = true
 	}
-
+	if modFile.Ignore != nil {
+		for _, x := range modFile.Ignore {
+			i.ignore = append(i.ignore, x.Path)
+		}
+	}
 	return i
 }
 
@@ -539,6 +544,7 @@ type modFileSummary struct {
 	module     module.Version
 	goVersion  string
 	toolchain  string
+	ignore     []string
 	pruning    modPruning
 	require    []module.Version
 	retract    []retraction
@@ -714,6 +720,11 @@ func rawGoModSummary(m module.Version) (*modFileSummary, error) {
 		if f.Toolchain != nil {
 			summary.toolchain = f.Toolchain.Name
 		}
+		if f.Ignore != nil {
+			for _, i := range f.Ignore {
+				summary.ignore = append(summary.ignore, i.Path)
+			}
+		}
 		if len(f.Require) > 0 {
 			summary.require = make([]module.Version, 0, len(f.Require)+1)
 			for _, req := range f.Require {

src/cmd/go/internal/modload/modfile.go

@@ -74,15 +74,16 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 	)
 
 	q := par.NewQueue(runtime.GOMAXPROCS(0))
-
+	ignorePatternsMap := parseIgnorePatterns(ctx, treeCanMatch, modules)
 	walkPkgs := func(root, importPathRoot string, prune pruning) {
 		_, span := trace.StartSpan(ctx, "walkPkgs "+root)
 		defer span.Done()
 
 		// If the root itself is a symlink to a directory,
 		// we want to follow it (see https://go.dev/issue/50807).
 		// Add a trailing separator to force that to happen.
-		root = str.WithFilePathSeparator(filepath.Clean(root))
+		cleanRoot := filepath.Clean(root)
+		root = str.WithFilePathSeparator(cleanRoot)
 		err := fsys.WalkDir(root, func(pkgDir string, d fs.DirEntry, err error) error {
 			if err != nil {
 				m.AddError(err)
@@ -91,6 +92,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 
 			want := true
 			elem := ""
+			relPkgDir := filepath.ToSlash(pkgDir[len(root):])
 
 			// Don't use GOROOT/src but do walk down into it.
 			if pkgDir == root {
@@ -102,10 +104,15 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 				_, elem = filepath.Split(pkgDir)
 				if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
 					want = false
+				} else if ignorePatternsMap[cleanRoot] != nil && ignorePatternsMap[cleanRoot].ShouldIgnore(relPkgDir) {
+					if cfg.BuildX {
+						fmt.Fprintf(os.Stderr, "# ignoring directory %s\n", pkgDir)
+					}
+					want = false
 				}
 			}
 
-			name := path.Join(importPathRoot, filepath.ToSlash(pkgDir[len(root):]))
+			name := path.Join(importPathRoot, relPkgDir)
 			if !treeCanMatch(name) {
 				want = false
 			}
@@ -303,3 +310,43 @@ func MatchInModule(ctx context.Context, pattern string, m module.Version, tags m
 	}
 	return match
 }
+
+// parseIgnorePatterns collects all ignore patterns associated with the
+// provided list of modules.
+// It returns a map of module root -> *search.IgnorePatterns.
+func parseIgnorePatterns(ctx context.Context, treeCanMatch func(string) bool, modules []module.Version) map[string]*search.IgnorePatterns {
+	ignorePatternsMap := make(map[string]*search.IgnorePatterns)
+	for _, mod := range modules {
+		if gover.IsToolchain(mod.Path) || !treeCanMatch(mod.Path) {
+			continue
+		}
+		var modRoot string
+		var ignorePatterns []string
+		if MainModules.Contains(mod.Path) {
+			modRoot = MainModules.ModRoot(mod)
+			if modRoot == "" {
+				continue
+			}
+			modIndex := MainModules.Index(mod)
+			if modIndex == nil {
+				continue
+			}
+			ignorePatterns = modIndex.ignore
+		} else if cfg.BuildMod != "vendor" {
+			// Skip getting ignore patterns for vendored modules because they
+			// do not have go.mod files.
+			var err error
+			modRoot, _, err = fetch(ctx, mod)
+			if err != nil {
+				continue
+			}
+			summary, err := goModSummary(mod)
+			if err != nil {
+				continue
+			}
+			ignorePatterns = summary.ignore
+		}
+		ignorePatternsMap[modRoot] = search.NewIgnorePatterns(ignorePatterns)
+	}
+	return ignorePatternsMap
+}