cmd/go: add file with list of all counters we collect

Maintain a list of counters we collect and test that it hasn't
changed. If it has, fail a test and have the user update the list. The
update process will print a reminder to update the list of collected
counters.

Also run go mod vendor to pull in
golang.org/x/telemetry/counter/countertest.

For #58894

Change-Id: I661a9c3d67cb33f42a5519f4639af7aa05c3821d
Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest
Reviewed-on: https://go-review.googlesource.com/c/go/+/564555
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Bryan Mills <bcmills@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>

Author: matloob

src/cmd/go/counters_test.go

@@ -0,0 +1,89 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main_test
+
+import (
+	"cmd/go/internal/base"
+	"flag"
+	"internal/diff"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+)
+
+var update = flag.Bool("update", false, "if true update testdata/counternames.txt")
+
+func TestCounterNamesUpToDate(t *testing.T) {
+	if !*update {
+		t.Parallel()
+	}
+
+	var counters []string
+	// -C is a special case because it's handled by handleChdirFlag rather than
+	// standard flag processing with FlagSets.
+	// cmd/go/subcommand:unknown is also a special case: it's used when the subcommand
+	// doesn't match any of the known commands.
+	counters = append(counters, "cmd/go/flag:C", "cmd/go/subcommand:unknown")
+	counters = append(counters, flagscounters("cmd/go/flag:", *flag.CommandLine)...)
+
+	for _, cmd := range base.Go.Commands {
+		counters = append(counters, cmdcounters(nil, cmd)...)
+	}
+	cstr := []byte(strings.Join(counters, "\n") + "\n")
+
+	const counterNamesFile = "testdata/counters.txt"
+	old, err := os.ReadFile(counterNamesFile)
+	if err != nil {
+		t.Fatalf("error reading %s: %v", counterNamesFile, err)
+	}
+	diff := diff.Diff(counterNamesFile, old, "generated counter names", cstr)
+	if diff == nil {
+		t.Logf("%s is up to date.", counterNamesFile)
+		return
+	}
+
+	if *update {
+		if err := os.WriteFile(counterNamesFile, cstr, 0666); err != nil {
+			t.Fatal(err)
+		}
+		t.Logf("wrote %d bytes to %s", len(cstr), counterNamesFile)
+		t.Logf("don't forget to file a proposal to update the list of collected counters")
+	} else {
+		t.Logf("\n%s", diff)
+		t.Errorf("%s is stale. To update, run 'go generate cmd/go'.", counterNamesFile)
+	}
+}
+
+func flagscounters(prefix string, flagSet flag.FlagSet) []string {
+	var counters []string
+	flagSet.VisitAll(func(f *flag.Flag) {
+		counters = append(counters, prefix+f.Name)
+	})
+	return counters
+}
+
+func cmdcounters(previous []string, cmd *base.Command) []string {
+	const subcommandPrefix = "cmd/go/subcommand:"
+	const flagPrefix = "cmd/go/flag:"
+	var counters []string
+	previousComponent := strings.Join(previous, "-")
+	if len(previousComponent) > 0 {
+		previousComponent += "-"
+	}
+	if cmd.Runnable() {
+		counters = append(counters, subcommandPrefix+previousComponent+cmd.Name())
+	}
+	counters = append(counters, flagscounters(flagPrefix+previousComponent+cmd.Name()+"-", cmd.Flag)...)
+	if len(previous) != 0 {
+		counters = append(counters, subcommandPrefix+previousComponent+"help-"+cmd.Name())
+	}
+	counters = append(counters, subcommandPrefix+"help-"+previousComponent+cmd.Name())
+
+	for _, subcmd := range cmd.Commands {
+		counters = append(counters, cmdcounters(append(slices.Clone(previous), cmd.Name()), subcmd)...)
+	}
+	return counters
+}

src/cmd/go/go_test.go

@@ -44,6 +44,8 @@ import (
 	"cmd/internal/sys"
 
 	cmdgo "cmd/go"
+
+	"golang.org/x/telemetry/counter/countertest"
 )
 
 func init() {
@@ -153,6 +155,15 @@ func TestMain(m *testing.M) {
 			web.EnableTestHooks(interceptors)
 		}
 
+		cmdgo.TelemetryStart = func() {
+			// TODO(matloob): we'll ideally want to call telemetry.Start here
+			// but it calls counter.Open, which we don't want to do because
+			// we want to call countertest.Open.
+			if telemetryDir := os.Getenv("TESTGO_TELEMETRY_DIR"); telemetryDir != "" {
+				countertest.Open(telemetryDir)
+			}
+		}
+
 		cmdgo.Main()
 		os.Exit(0)
 	}

src/cmd/go/main.go

@@ -3,6 +3,7 @@
 // license that can be found in the LICENSE file.
 
 //go:generate go test cmd/go -v -run=^TestDocsUpToDate$ -fixdocs
+//go:generate go test cmd/go -v -run=^TestCounterNamesUpToDate$ -update
 
 package main
 
@@ -91,7 +92,7 @@ var _ = go11tag
 
 func main() {
 	log.SetFlags(0)
-	counter.Open() // Open the telemetry counter file so counters can be written to it.
+	TelemetryStart() // Open the telemetry counter file so counters can be written to it.
 	handleChdirFlag()
 	toolchain.Select()
 
@@ -153,7 +154,6 @@ func main() {
 
 	cmd, used := lookupCmd(args)
 	cfg.CmdName = strings.Join(args[:used], " ")
-	counter.Inc("cmd/go/subcommand:" + strings.ReplaceAll(cfg.CmdName, " ", "-"))
 	if len(cmd.Commands) > 0 {
 		if used >= len(args) {
 			help.PrintUsage(os.Stderr, cmd)
@@ -162,6 +162,7 @@ func main() {
 		}
 		if args[used] == "help" {
 			// Accept 'go mod help' and 'go mod help foo' for 'go help mod' and 'go help mod foo'.
+			counter.Inc("cmd/go/subcommand:" + strings.ReplaceAll(cfg.CmdName, " ", "-") + "-" + strings.Join(args[used:], "-"))
 			help.Help(os.Stdout, append(slices.Clip(args[:used]), args[used+1:]...))
 			base.Exit()
 		}
@@ -173,10 +174,12 @@ func main() {
 		if cmdName == "" {
 			cmdName = args[0]
 		}
+		counter.Inc("cmd/go/subcommand:unknown")
 		fmt.Fprintf(os.Stderr, "go %s: unknown command\nRun 'go help%s' for usage.\n", cmdName, helpArg)
 		base.SetExitStatus(2)
 		base.Exit()
 	}
+	counter.Inc("cmd/go/subcommand:" + strings.ReplaceAll(cfg.CmdName, " ", "-"))
 	invoke(cmd, args[used-1:])
 	base.Exit()
 }
@@ -241,7 +244,7 @@ func invoke(cmd *base.Command, args []string) {
 	} else {
 		base.SetFromGOFLAGS(&cmd.Flag)
 		cmd.Flag.Parse(args[1:])
-		counter.CountFlags("cmd/go/flag:"+cmd.Name()+"-", cmd.Flag)
+		counter.CountFlags("cmd/go/flag:"+strings.ReplaceAll(cfg.CmdName, " ", "-")+"-", cmd.Flag)
 		args = cmd.Flag.Args()
 	}
 
@@ -326,7 +329,7 @@ func handleChdirFlag() {
 		_, dir, _ = strings.Cut(a, "=")
 		os.Args = slices.Delete(os.Args, used, used+1)
 	}
-	counter.Inc("cmd/go:flag-C")
+	counter.Inc("cmd/go/flag:C")
 
 	if err := os.Chdir(dir); err != nil {
 		base.Fatalf("go: %v", err)

src/cmd/go/script_test.go

@@ -13,6 +13,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	_ "embed"
 	"flag"
 	"internal/testenv"
 	"internal/txtar"
@@ -21,6 +22,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -29,6 +31,8 @@ import (
 	"cmd/go/internal/script"
 	"cmd/go/internal/script/scripttest"
 	"cmd/go/internal/vcweb/vcstest"
+
+	"golang.org/x/telemetry/counter/countertest"
 )
 
 var testSum = flag.String("testsum", "", `may be tidy, listm, or listall. If set, TestScript generates a go.sum file at the beginning of each test and updates test files if they pass.`)
@@ -124,7 +128,7 @@ func TestScript(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			initScriptDirs(t, s)
+			telemetryDir := initScriptDirs(t, s)
 			if err := s.ExtractFiles(a); err != nil {
 				t.Fatal(err)
 			}
@@ -154,6 +158,7 @@ func TestScript(t *testing.T) {
 			// will work better seeing the full path relative to cmd/go
 			// (where the "go test" command is usually run).
 			scripttest.Run(t, engine, s, file, bytes.NewReader(a.Comment))
+			checkCounters(t, telemetryDir)
 		})
 	}
 }
@@ -177,7 +182,7 @@ func tbFromContext(ctx context.Context) (testing.TB, bool) {
 
 // initScriptState creates the initial directory structure in s for unpacking a
 // cmd/go script.
-func initScriptDirs(t testing.TB, s *script.State) {
+func initScriptDirs(t testing.TB, s *script.State) (telemetryDir string) {
 	must := func(err error) {
 		if err != nil {
 			t.Helper()
@@ -188,6 +193,10 @@ func initScriptDirs(t testing.TB, s *script.State) {
 	work := s.Getwd()
 	must(s.Setenv("WORK", work))
 
+	telemetryDir = filepath.Join(work, "telemetry")
+	must(os.MkdirAll(telemetryDir, 0777))
+	must(s.Setenv("TESTGO_TELEMETRY_DIR", filepath.Join(work, "telemetry")))
+
 	must(os.MkdirAll(filepath.Join(work, "tmp"), 0777))
 	must(s.Setenv(tempEnvName(), filepath.Join(work, "tmp")))
 
@@ -196,6 +205,7 @@ func initScriptDirs(t testing.TB, s *script.State) {
 	gopathSrc := filepath.Join(gopath, "src")
 	must(os.MkdirAll(gopathSrc, 0777))
 	must(s.Chdir(gopathSrc))
+	return telemetryDir
 }
 
 func scriptEnv(srv *vcstest.Server, srvCertFile string) ([]string, error) {
@@ -357,3 +367,53 @@ func updateSum(t testing.TB, e *script.Engine, s *script.State, archive *txtar.A
 	}
 	return rewrite
 }
+
+func readCounters(t *testing.T, telemetryDir string) map[string]uint64 {
+	localDir := filepath.Join(telemetryDir, "local")
+	dirents, err := os.ReadDir(localDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil // The Go command didn't ever run so the local dir wasn't created
+		}
+		t.Fatalf("reading telemetry local dir: %v", err)
+	}
+	totals := map[string]uint64{}
+	for _, dirent := range dirents {
+		if dirent.IsDir() || !strings.HasSuffix(dirent.Name(), ".count") {
+			// not a counter file
+			continue
+		}
+		counters, _, err := countertest.ReadFile(filepath.Join(localDir, dirent.Name()))
+		if err != nil {
+			t.Fatalf("reading counter file: %v", err)
+		}
+		for k, v := range counters {
+			totals[k] += v
+		}
+	}
+
+	return totals
+}
+
+//go:embed testdata/counters.txt
+var countersTxt string
+
+var (
+	allowedCountersOnce sync.Once
+	allowedCounters     = map[string]bool{} // Set of allowed counters.
+)
+
+func checkCounters(t *testing.T, telemetryDir string) {
+	allowedCountersOnce.Do(func() {
+		for _, counter := range strings.Fields(countersTxt) {
+			allowedCounters[counter] = true
+		}
+	})
+	counters := readCounters(t, telemetryDir)
+	for name := range counters {
+		if !allowedCounters[name] {
+			t.Fatalf("incremented counter %q is not in testdata/counters.txt. "+
+				"Please update counters_test.go to produce an entry for it.", name)
+		}
+	}
+}

src/cmd/go/telemetry.go

@@ -0,0 +1,13 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !cmd_go_bootstrap
+
+package main
+
+import "golang.org/x/telemetry"
+
+var TelemetryStart = func() {
+	telemetry.Start(telemetry.Config{Upload: true})
+}

src/cmd/go/telemetry_bootstrap.go

@@ -0,0 +1,9 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build cmd_go_bootstrap
+
+package main
+
+var TelemetryStart = func() {}