gomod-zip: use golang.org/x/mod/zip for creating zip file

cmd/gomod-zip/zip.go

@@ -17,23 +17,16 @@ limitations under the License.
 package main
 
 import (
-	"archive/zip"
 	"bytes"
 	"flag"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
-	"path"
-	"strings"
+	"path/filepath"
 
 	"github.com/golang/glog"
-)
-
-const (
-	// MaxZipFile is the maximum size of downloaded zip file
-	MaxZipFile = 500 << 20
+	"golang.org/x/mod/modfile"
+	modzip "golang.org/x/mod/zip"
 )
 
 func Usage() {
@@ -66,179 +59,54 @@ func main() {
 		glog.Fatalf("pseudo-version cannot be empty")
 	}
 
-	// create a zip file using git archive, and remove it after using it
-	depPseudoVersion := fmt.Sprintf("%s@%s", *packageName, *pseudoVersion)
-	zipFileName := fmt.Sprintf("%s/src/%s/%s.zip", os.Getenv("GOPATH"), *packageName, *pseudoVersion)
-	prefix := fmt.Sprintf("%s/", depPseudoVersion)
-	gitArchive := exec.Command("git", "archive", "--format=zip", "--prefix", prefix, "-o", zipFileName, "HEAD")
-	gitArchive.Dir = fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), *packageName)
-	gitArchive.Stdout = os.Stdout
-	gitArchive.Stderr = os.Stderr
-	if err := gitArchive.Run(); err != nil {
-		glog.Fatalf("unable to run git archive for %s at %s: %v", *packageName, *pseudoVersion, err)
-	}
-	defer os.Remove(zipFileName)
-
-	archive, err := ioutil.ReadFile(zipFileName)
-	if err != nil {
-		glog.Fatalf("error reading zip file %s: %v", zipFileName, err)
-	}
+	packagePath := fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), *packageName)
+	cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), *packageName)
 
-	dl := ioutil.NopCloser(bytes.NewReader(archive))
-	defer dl.Close()
-
-	// This is taken from https://github.com/golang/go/blob/b373d31c25e58d0b69cff3521b915f0c06fa6ac8/src/cmd/go/internal/modfetch/coderepo.go#L459.
-	// Spool to local file.
-	f, err := ioutil.TempFile("", "gomodzip-")
+	moduleFile, err := getModuleFile(packagePath, *pseudoVersion)
 	if err != nil {
-		dl.Close()
-		glog.Fatalf("error creating temp file: %v", err)
-	}
-	defer os.Remove(f.Name())
-	defer f.Close()
-
-	maxSize := int64(MaxZipFile)
-	lr := &io.LimitedReader{R: dl, N: maxSize + 1}
-	if _, err := io.Copy(f, lr); err != nil {
-		dl.Close()
-		glog.Fatalf("error reading from %s: %v", f.Name(), err)
+		glog.Fatalf("error getting module file: %v", err)
 	}
 
-	if lr.N <= 0 {
-		glog.Fatalf("downloaded zip file too large")
-	}
-	size := (maxSize + 1) - lr.N
-	if _, err := f.Seek(0, 0); err != nil {
-		glog.Fatal(err)
+	if err := createZipArchive(packagePath, moduleFile, cacheDir); err != nil {
+		glog.Fatalf("error creating zip archive: %v", err)
 	}
+}
 
-	// The zip file created by go mod download has extra normalization over
-	// the zip file created by git archive. The normalization process is done below.
-	//
-	// While the normalization can also be achieved via a simple zip command, the zip file
-	// created by go mod download has the `00-00-1980 00:00` timestamp in the file header
-	// for all files in the zip archive. This is not a valid UNIX timestamp and cannot be
-	// set easily. This is, however, valid in MSDOS. The `archive/zip` package uses the
-	// MSDOS version so we create the zip file using this package.
-	zr, err := zip.NewReader(f, size)
+func getModuleFile(packagePath, version string) (*modfile.File, error) {
+	goModPath := filepath.Join(packagePath, "go.mod")
+	file, err := os.Open(goModPath)
 	if err != nil {
-		glog.Fatalf("error reading %s: %v", f.Name(), err)
+		return nil, fmt.Errorf("error opening %s: %v", goModPath, err)
 	}
+	defer file.Close()
 
-	packagedZipPath := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v/%s.zip", os.Getenv("GOPATH"), *packageName, *pseudoVersion)
-	dst, err := os.OpenFile(packagedZipPath, os.O_CREATE|os.O_WRONLY, 0755)
+	moduleBytes, err := ioutil.ReadAll(file)
 	if err != nil {
-		glog.Fatalf("failed to create zip file at %s: %v", packagedZipPath, err)
-	}
-	defer dst.Close()
-	zw := zip.NewWriter(dst)
-
-	isSubmodule := map[string]bool{}
-	for _, zf := range zr.File {
-		if zf.Name == prefix+"go.mod" {
-			continue
-		}
-		dir, file := path.Split(zf.Name)
-		if file == "go.mod" {
-			isSubmodule[dir] = true
-		}
+		return nil, fmt.Errorf("error reading %s: %v", goModPath, err)
 	}
 
-	inSubmodule := func(name string) bool {
-		for {
-			dir, _ := path.Split(name)
-			if len(dir) == 0 {
-				return false
-			}
-			if isSubmodule[dir] {
-				return true
-			}
-			name = strings.TrimSuffix(dir, "/")
-		}
+	moduleFile, err := modfile.Parse(packagePath, moduleBytes, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing module file: %v", err)
 	}
 
-	for _, zf := range zr.File {
-		// Skip symlinks (golang.org/issue/27093)
-		if !zf.FileInfo().Mode().IsRegular() {
-			continue
-		}
-		// drop directory dummy entries
-		if strings.HasSuffix(zf.Name, "/") {
-			continue
-		}
-		// all file paths should have module@version/ prefix
-		if !strings.HasPrefix(zf.Name, prefix) {
-			continue
-		}
-		// inserted by hg archive.
-		// not correct to drop from other version control systems, but too bad.
-		name := strings.TrimPrefix(zf.Name, prefix)
-		if name == ".hg_archival.txt" {
-			continue
-		}
-		// don't consider vendored packages
-		if isVendoredPackage(name) {
-			continue
-		}
-		// don't consider submodules
-		if inSubmodule(zf.Name) {
-			continue
-		}
-		// make sure we have lower-case go.mod
-		base := path.Base(name)
-		if strings.ToLower(base) == "go.mod" && base != "go.mod" {
-			glog.Fatalf("zip file contains %s, want all lower-case go.mod", zf.Name)
-		}
-
-		size := int64(zf.UncompressedSize64)
-		if size < 0 || maxSize < size {
-			glog.Fatalf("module source tree too big")
-		}
-		maxSize -= size
-
-		rc, err := zf.Open()
-		if err != nil {
-			glog.Fatalf("unable to open file %s: %v", zf.Name, err)
-		}
-		w, err := zw.Create(zf.Name)
-		if err != nil {
-			glog.Fatal(err)
-		}
-		lr := &io.LimitedReader{R: rc, N: size + 1}
-		if _, err := io.Copy(w, lr); err != nil {
-			glog.Fatal(err)
-		}
-		if lr.N <= 0 {
-			glog.Fatalf("individual file too large")
-		}
+	if moduleFile.Module == nil {
+		return nil, fmt.Errorf("parsed module should not be nil")
 	}
 
-	if err := zw.Close(); err != nil {
-		glog.Fatal(err)
-	}
+	moduleFile.Module.Mod.Version = version
+	return moduleFile, nil
 }
 
-func isVendoredPackage(name string) bool {
-	var i int
-	if strings.HasPrefix(name, "vendor/") {
-		i += len("vendor/")
-	} else if j := strings.Index(name, "/vendor/"); j >= 0 {
-		// This offset looks incorrect; this should probably be
-		//
-		// 	i = j + len("/vendor/")
-		//
-		// (See https://golang.org/issue/31562.)
-		//
-		// Unfortunately, we can't fix it without invalidating checksums.
-		// Fortunately, the error appears to be strictly conservative: we'll retain
-		// vendored packages that we should have pruned, but we won't prune
-		// non-vendored packages that we should have retained.
-		//
-		// Since this defect doesn't seem to break anything, it's not worth fixing
-		// for now.
-		i += len("/vendor/")
-	} else {
-		return false
+func createZipArchive(packagePath string, moduleFile *modfile.File, outputDirectory string) error {
+	zipFilePath := filepath.Join(outputDirectory, moduleFile.Module.Mod.Version+".zip")
+	var zipContents bytes.Buffer
+
+	if err := modzip.CreateFromDir(&zipContents, moduleFile.Module.Mod, packagePath); err != nil {
+		return fmt.Errorf("create zip from dir: %w", err)
+	}
+	if err := ioutil.WriteFile(zipFilePath, zipContents.Bytes(), 0644); err != nil {
+		return fmt.Errorf("writing zip file: %w", err)
 	}
-	return strings.Contains(name[i:], "/")
+	return nil
 }

go.mod

@@ -12,9 +12,9 @@ require (
 	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
 	github.com/renstrom/dedent v1.0.0
 	github.com/shurcooL/go v0.0.0-20171108033853-004faa6b0118
+	golang.org/x/mod v0.4.3-0.20210409134425-858fdbee9c24
 	golang.org/x/oauth2 v0.0.0-20171205225816-ea8c6730ed5b
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	golang.org/x/sys v0.0.0-20181003145944-af653ce8b74f // indirect
 	google.golang.org/appengine v1.0.1-0.20171031194329-9d8544a6b2c7 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3
 	gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect

go.sum

@@ -57,17 +57,32 @@ github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro
 github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.4.3-0.20210409134425-858fdbee9c24 h1:XWBCOnD7qf8cYkORdr1AfVspwadsirSDgThkrje7nWs=
+golang.org/x/mod v0.4.3-0.20210409134425-858fdbee9c24/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20171205225816-ea8c6730ed5b h1:8f+ehCD4q1R24kvqzQ1Q1g1hLEbKSa+Bvk+9Eeb0Iw0=
 golang.org/x/oauth2 v0.0.0-20171205225816-ea8c6730ed5b/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181003145944-af653ce8b74f h1:zAtpFwFDtnvBWPPelq8CSiqRN1wrIzMUk9dwzbpjpNM=
-golang.org/x/sys v0.0.0-20181003145944-af653ce8b74f/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.0.1-0.20171031194329-9d8544a6b2c7 h1:8IYBJdOzN2qmTJQHPibS4j37MIXxlqmDSbMmY/RWm9s=
 google.golang.org/appengine v1.0.1-0.20171031194329-9d8544a6b2c7/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=