cmd/go/internal/modfetch: switch to golang.org/x/mod/zip

zip.Create is now used to filter and translate zip files from VCS tools.

zip.Unzip is now used instead of Unzip.

Fixes #35290

Change-Id: I4aa41b2e96bf147c09db43d1d189b8393cafb06f
Reviewed-on: https://go-review.googlesource.com/c/go/+/204917
Run-TryBot: Jay Conrod <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Bryan C. Mills <[email protected]>

Author: Jay Conrod

src/cmd/go.sum

@@ -7,8 +7,6 @@ golang.org/x/arch v0.0.0-20190815191158-8a70ba74b3a1/go.mod h1:flIaEI6LNU6xOCD5P
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.1.1-0.20191101203923-a222b9651630 h1:QsMqsRXZFQT+jRZnwpEDIwGHWg0UY9ZrpWiplCOEK5I=
-golang.org/x/mod v0.1.1-0.20191101203923-a222b9651630/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

src/cmd/go/internal/modfetch/coderepo.go

@@ -6,6 +6,7 @@ package modfetch
 
 import (
 	"archive/zip"
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -21,6 +22,7 @@ import (
 	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/semver"
+	modzip "golang.org/x/mod/zip"
 )
 
 // A codeRepo implements modfetch.Repo using an underlying codehost.Repo.
@@ -900,13 +902,12 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error {
 		return err
 	}
 
-	zw := zip.NewWriter(dst)
+	var files []modzip.File
 	if subdir != "" {
 		subdir += "/"
 	}
 	haveLICENSE := false
 	topPrefix := ""
-	haveGoMod := make(map[string]bool)
 	for _, zf := range zr.File {
 		if topPrefix == "" {
 			i := strings.Index(zf.Name, "/")
@@ -918,106 +919,61 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error {
 		if !strings.HasPrefix(zf.Name, topPrefix) {
 			return fmt.Errorf("zip file contains more than one top-level directory")
 		}
-		dir, file := path.Split(zf.Name)
-		if file == "go.mod" {
-			haveGoMod[dir] = true
-		}
-	}
-	root := topPrefix + subdir
-	inSubmodule := func(name string) bool {
-		for {
-			dir, _ := path.Split(name)
-			if len(dir) <= len(root) {
-				return false
-			}
-			if haveGoMod[dir] {
-				return true
-			}
-			name = dir[:len(dir)-1]
-		}
-	}
-
-	for _, zf := range zr.File {
-		if !zf.FileInfo().Mode().IsRegular() {
-			// Skip symlinks (golang.org/issue/27093).
-			continue
-		}
-
-		if topPrefix == "" {
-			i := strings.Index(zf.Name, "/")
-			if i < 0 {
-				return fmt.Errorf("missing top-level directory prefix")
-			}
-			topPrefix = zf.Name[:i+1]
-		}
-		if strings.HasSuffix(zf.Name, "/") { // drop directory dummy entries
-			continue
-		}
-		if !strings.HasPrefix(zf.Name, topPrefix) {
-			return fmt.Errorf("zip file contains more than one top-level directory")
-		}
 		name := strings.TrimPrefix(zf.Name, topPrefix)
 		if !strings.HasPrefix(name, subdir) {
 			continue
 		}
-		if name == ".hg_archival.txt" {
-			// Inserted by hg archive.
-			// Not correct to drop from other version control systems, but too bad.
-			continue
-		}
 		name = strings.TrimPrefix(name, subdir)
-		if isVendoredPackage(name) {
+		if name == "" || strings.HasSuffix(name, "/") {
 			continue
 		}
-		if inSubmodule(zf.Name) {
-			continue
-		}
-		base := path.Base(name)
-		if strings.ToLower(base) == "go.mod" && base != "go.mod" {
-			return fmt.Errorf("zip file contains %s, want all lower-case go.mod", zf.Name)
-		}
+		files = append(files, zipFile{name: name, f: zf})
 		if name == "LICENSE" {
 			haveLICENSE = true
 		}
-		size := int64(zf.UncompressedSize64)
-		if size < 0 || maxSize < size {
-			return fmt.Errorf("module source tree too big")
-		}
-		maxSize -= size
-
-		rc, err := zf.Open()
-		if err != nil {
-			return err
-		}
-		w, err := zw.Create(r.modPrefix(version) + "/" + name)
-		if err != nil {
-			return err
-		}
-		lr := &io.LimitedReader{R: rc, N: size + 1}
-		if _, err := io.Copy(w, lr); err != nil {
-			return err
-		}
-		if lr.N <= 0 {
-			return fmt.Errorf("individual file too large")
-		}
 	}
 
 	if !haveLICENSE && subdir != "" {
 		data, err := r.code.ReadFile(rev, "LICENSE", codehost.MaxLICENSE)
 		if err == nil {
-			w, err := zw.Create(r.modPrefix(version) + "/LICENSE")
-			if err != nil {
-				return err
-			}
-			if _, err := w.Write(data); err != nil {
-				return err
-			}
+			files = append(files, dataFile{name: "LICENSE", data: data})
 		}
 	}
 
-	return zw.Close()
+	return modzip.Create(dst, module.Version{Path: r.modPath, Version: version}, files)
+}
+
+type zipFile struct {
+	name string
+	f    *zip.File
+}
+
+func (f zipFile) Path() string                 { return f.name }
+func (f zipFile) Lstat() (os.FileInfo, error)  { return f.f.FileInfo(), nil }
+func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
+
+type dataFile struct {
+	name string
+	data []byte
+}
+
+func (f dataFile) Path() string                { return f.name }
+func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil }
+func (f dataFile) Open() (io.ReadCloser, error) {
+	return ioutil.NopCloser(bytes.NewReader(f.data)), nil
 }
 
+type dataFileInfo struct {
+	f dataFile
+}
+
+func (fi dataFileInfo) Name() string       { return path.Base(fi.f.name) }
+func (fi dataFileInfo) Size() int64        { return int64(len(fi.f.data)) }
+func (fi dataFileInfo) Mode() os.FileMode  { return 0644 }
+func (fi dataFileInfo) ModTime() time.Time { return time.Time{} }
+func (fi dataFileInfo) IsDir() bool        { return false }
+func (fi dataFileInfo) Sys() interface{}   { return nil }
+
 // hasPathPrefix reports whether the path s begins with the
 // elements in prefix.
 func hasPathPrefix(s, prefix string) bool {

src/cmd/go/internal/modfetch/fetch.go

@@ -25,6 +25,7 @@ import (
 
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/sumdb/dirhash"
+	modzip "golang.org/x/mod/zip"
 )
 
 var downloadCache par.Cache
@@ -113,8 +114,7 @@ func download(mod module.Version, dir string) (err error) {
 		}
 	}()
 
-	modpath := mod.Path + "@" + mod.Version
-	if err := Unzip(tmpDir, zipfile, modpath, 0); err != nil {
+	if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
 		fmt.Fprintf(os.Stderr, "-> %s\n", err)
 		return err
 	}
@@ -266,6 +266,45 @@ func downloadZip(mod module.Version, zipfile string) (err error) {
 	return nil
 }
 
+// makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
+// and its transitive contents.
+func makeDirsReadOnly(dir string) {
+	type pathMode struct {
+		path string
+		mode os.FileMode
+	}
+	var dirs []pathMode // in lexical order
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err == nil && info.Mode()&0222 != 0 {
+			if info.IsDir() {
+				dirs = append(dirs, pathMode{path, info.Mode()})
+			}
+		}
+		return nil
+	})
+
+	// Run over list backward to chmod children before parents.
+	for i := len(dirs) - 1; i >= 0; i-- {
+		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
+	}
+}
+
+// RemoveAll removes a directory written by Download or Unzip, first applying
+// any permission changes needed to do so.
+func RemoveAll(dir string) error {
+	// Module cache has 0555 directories; make them writable in order to remove content.
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil // ignore errors walking in file system
+		}
+		if info.IsDir() {
+			os.Chmod(path, 0777)
+		}
+		return nil
+	})
+	return os.RemoveAll(dir)
+}
+
 var GoSumFile string // path to go.sum; set by package modload
 
 type modSum struct {