internal/relui: extend VersionTasks feature to automate goimports stdlib regeneration

Add a new task in addSingleReleaseWorkflow that creates the stdlib regeneration CL for each Go release using func CreateUpdateStdlibIndexCL in VersionTasks.
CreateUpdateStdlibIndexCL fetches the latest go binaries, downloads the latest version of tools repo, and submit a CL whose diff is created by regenerating stdlib with `go generate ./internal/imports`.

For golang/go#54377.

Change-Id: I3d95a88def984dd677d49f38587bed298b3aeaea
Reviewed-on: https://go-review.googlesource.com/c/build/+/442157
Reviewed-by: Heschi Kreinick <heschi@google.com>
Run-TryBot: Dylan Le <dungtuanle@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

Author: Dung Le

cmd/relui/main.go

@@ -113,10 +113,6 @@ func main() {
 	gerritClient := &task.RealGerritClient{
 		Client: gerrit.NewClient("https://go-review.googlesource.com", gerrit.OAuth2Auth(creds.TokenSource)),
 	}
-	versionTasks := &task.VersionTasks{
-		Gerrit:    gerritClient,
-		GoProject: "go",
-	}
 	commTasks := task.CommunicationTasks{
 		AnnounceMailTasks: task.AnnounceMailTasks{
 			SendMail:           task.NewSendGridMailClient(*sendgridAPIKey).SendMail,
@@ -196,6 +192,11 @@ func main() {
 		RepoName:      "go",
 		ApproveAction: relui.ApproveActionDep(dbPool),
 	}
+	versionTasks := &task.VersionTasks{
+		Gerrit:         gerritClient,
+		GoProject:      "go",
+		CreateBuildlet: coordinator.CreateBuildlet,
+	}
 	if err := relui.RegisterReleaseWorkflows(ctx, dh, buildTasks, milestoneTasks, versionTasks, commTasks); err != nil {
 		log.Fatalf("RegisterReleaseWorkflows: %v", err)
 	}

internal/relui/buildrelease_test.go

@@ -56,6 +56,31 @@ func TestSecurity(t *testing.T) {
 	})
 }
 
+const fakeGo = `#!/bin/bash -eu
+
+case "$1" in
+"get")
+  ls go.mod go.sum >/dev/null
+  for i in "${@:2}"; do
+    echo -e "// pretend we've upgraded to $i" >> go.mod
+    echo "$i h1:asdasd" | tr '@' ' ' >> go.sum
+  done
+  ;;
+"mod")
+  ls go.mod go.sum >/dev/null
+  echo "tidied!" >> go.mod
+  ;;
+"generate")
+  mkdir -p internal/imports
+  cd internal/imports && echo "package imports" >> zstdlib.go
+  ;;
+*)
+  echo unexpected command $@
+  exit 1
+  ;;
+esac
+`
+
 type releaseTestDeps struct {
 	ctx            context.Context
 	cancel         context.CancelFunc
@@ -142,12 +167,30 @@ esac
 	base := goRepo.Commit(goFiles)
 	goRepo.Tag("go1.17", base)
 	dlRepo := task.NewFakeRepo(t, "dl")
-	fakeGerrit := task.NewFakeGerrit(t, goRepo, dlRepo)
+	toolsRepo := task.NewFakeRepo(t, "tools")
+	toolsRepo1 := toolsRepo.Commit(map[string]string{
+		"go.mod":                       "module golang.org/x/tools\n",
+		"go.sum":                       "\n",
+		"internal/imports/mkstdlib.go": "package imports\nconst C=1",
+	})
+	toolsRepo.Tag("master", toolsRepo1)
+	fakeGerrit := task.NewFakeGerrit(t, goRepo, dlRepo, toolsRepo)
 
 	gerrit := &reviewerCheckGerrit{FakeGerrit: fakeGerrit}
+	goServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		task.ServeTarball("dl/go1.19.linux-amd64.tar.gz", map[string]string{
+			"go/bin/go": fakeGo,
+		}, w, r)
+	}))
+	t.Cleanup(goServer.Close)
 	versionTasks := &task.VersionTasks{
-		Gerrit:    gerrit,
-		GoProject: "go",
+		Gerrit:         gerrit,
+		GerritURL:      fakeGerrit.GerritURL(),
+		GoProject:      "go",
+		CreateBuildlet: fakeBuildlets.CreateBuildlet,
+		LatestGoBinaries: func(context.Context) (string, error) {
+			return goServer.URL + "/dl/go1.19.linux-amd64.tar.gz", nil
+		},
 	}
 	milestoneTasks := &task.MilestoneTasks{
 		Client:    &fakeGitHub{},

internal/relui/workflows.go

@@ -392,6 +392,9 @@ func addSingleReleaseWorkflow(
 	uploaded := wf.Action1(wd, "Upload artifacts to CDN", build.uploadArtifacts, signedAndTestedArtifacts, wf.After(tagged))
 	pushed := wf.Action3(wd, "Push issues", milestone.PushIssues, milestones, nextVersion, kindVal, wf.After(tagged))
 	versionPublished = wf.Task2(wd, "Publish to website", build.publishArtifacts, nextVersion, signedAndTestedArtifacts, wf.After(uploaded, pushed))
+	if kind == task.KindMajor {
+		wf.Task3(wd, fmt.Sprintf("Mail update stdlib index CL for 1.%d", major), version.CreateUpdateStdlibIndexCL, wf.Const("master"), coordinators, versionPublished)
+	}
 	wf.Output(wd, "Released version", versionPublished)
 	return versionPublished
 }

internal/task/fakes.go

@@ -363,6 +363,24 @@ func (repo *FakeRepo) Tag(tag, commit string) {
 	repo.tags[tag] = commit
 }
 
+// GetRepoContent returns the content of repo based on the value of commit:
+// - commit is "master": return content of the most recent revision
+// - commit is tag: return content of the repo associating with the commit that the tag maps to
+// - commit is neither "master" or tag: return content of the repo associated with that commit
+func (repo *FakeRepo) GetRepoContent(commit string) (map[string]string, error) {
+	rev := commit
+	if commit == "master" {
+		l := len(repo.history)
+		if l == 0 {
+			return nil, fmt.Errorf("repo %v history is empty", repo.name)
+		}
+		rev = repo.history[l-1]
+	} else if val, ok := repo.tags[commit]; ok {
+		rev = val
+	}
+	return repo.content[rev], nil
+}
+
 var _ GerritClient = (*FakeGerrit)(nil)
 
 func (g *FakeGerrit) ListProjects(ctx context.Context) ([]string, error) {
@@ -394,11 +412,15 @@ func (g *FakeGerrit) ReadFile(ctx context.Context, project string, commit string
 	if err != nil {
 		return nil, err
 	}
-	content := repo.content[commit][file]
-	if content == "" {
+	repoContent, err := repo.GetRepoContent(commit)
+	if err != nil {
+		return nil, err
+	}
+	fileContent := repoContent[file]
+	if fileContent == "" {
 		return nil, fmt.Errorf("commit/file not found %v at %v: %w", file, commit, gerrit.ErrResourceNotExist)
 	}
-	return []byte(content), nil
+	return []byte(fileContent), nil
 }
 
 func (g *FakeGerrit) ListTags(ctx context.Context, project string) ([]string, error) {
@@ -475,5 +497,10 @@ func (g *FakeGerrit) serveHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	rev := strings.TrimSuffix(parts[3], ".tar.gz")
-	ServeTarball("", repo.content[rev], w, r)
+	repoContent, err := repo.GetRepoContent(rev)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	ServeTarball("", repoContent, w, r)
 }

internal/task/version.go

@@ -6,15 +6,19 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/build/buildlet"
 	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal/workflow"
 	"golang.org/x/net/context"
 )
 
 // VersionTasks contains tasks related to versioning the release.
 type VersionTasks struct {
-	Gerrit    GerritClient
-	GoProject string
+	Gerrit           GerritClient
+	GerritURL        string
+	GoProject        string
+	CreateBuildlet   func(context.Context, string) (buildlet.RemoteClient, error)
+	LatestGoBinaries func(context.Context) (string, error)
 }
 
 func (t *VersionTasks) GetCurrentMajor(ctx context.Context) (int, error) {
@@ -138,3 +142,58 @@ func (t *VersionTasks) ReadBranchHead(ctx *workflow.TaskContext, branch string)
 func (t *VersionTasks) TagRelease(ctx *workflow.TaskContext, version, commit string) error {
 	return t.Gerrit.Tag(ctx, t.GoProject, version, commit)
 }
+
+func (t *VersionTasks) CreateUpdateStdlibIndexCL(ctx *workflow.TaskContext, branch string, reviewers []string, version string) (string, error) {
+	var files = make(map[string]string) // Map key is relative path, and map value is file content.
+
+	binaries, err := t.LatestGoBinaries(ctx)
+	if err != nil {
+		return "", err
+	}
+	bc, err := t.CreateBuildlet(ctx, "linux-amd64-longtest")
+	if err != nil {
+		return "", err
+	}
+	defer bc.Close()
+	if err := bc.PutTarFromURL(ctx, binaries, ""); err != nil {
+		return "", err
+	}
+	toolsTarURL := fmt.Sprintf("%s/%s/+archive/%s.tar.gz", t.GerritURL, "tools", "master")
+	if err := bc.PutTarFromURL(ctx, toolsTarURL, "tools"); err != nil {
+		return "", err
+	}
+	writer := &LogWriter{Logger: ctx}
+	go writer.Run(ctx)
+	remoteErr, execErr := bc.Exec(ctx, "go/bin/go", buildlet.ExecOpts{
+		Dir:    "tools",
+		Args:   []string{"generate", "./internal/imports"},
+		Output: writer,
+	})
+	if execErr != nil {
+		return "", fmt.Errorf("Exec failed: %v", execErr)
+	}
+	if remoteErr != nil {
+		return "", fmt.Errorf("Command failed: %v", remoteErr)
+	}
+	tgz, err := bc.GetTar(context.Background(), "tools/internal/imports")
+	if err != nil {
+		return "", err
+	}
+	defer tgz.Close()
+	tools, err := tgzToMap(tgz)
+	if err != nil {
+		return "", err
+	}
+	files["internal/imports/zstdlib.go"] = tools["zstdlib.go"]
+
+	major, err := t.GetCurrentMajor(ctx)
+	if err != nil {
+		return "", err
+	}
+	changeInput := gerrit.ChangeInput{
+		Project: "tools",
+		Subject: fmt.Sprintf("internal/imports: update stdlib index for %v", major),
+		Branch:  "master",
+	}
+	return t.Gerrit.CreateAutoSubmitChange(ctx, changeInput, reviewers, files)
+}