Update to Go 1.17

Go 1.17 now lazy-loads dependencies when `go.mod` is version `go17`. This does not work for us for new projects started with `hugo mod init`.

Before this commit, starting a project with Go 1.17 with `hugo mod init` and then start adding dependencies with transitive dependenies to `config.toml` would treat the transitive dependencies as new, and you would potentially get a too recent version of those.

Note that this does not effect existing projects, where all dependencies are already recorded in `go.mod`.

Fixes #8930

2 files changed, 89 insertions, 18 deletions

diff --git a/modules/client.go b/modules/client.go
index 73c3242a8..8d9bf8762 100644
--- a/modules/client.go
+++ b/modules/client.go
@@ -318,7 +318,7 @@ func (c *Client) Get(args ...string) error {
 			}
 			var args []string
 			if update {
-				args = []string{"-u"}
+				args = append(args, "-u")
 			}
 			args = append(args, m.Path)
 			if err := c.get(args...); err != nil {
@@ -333,6 +333,18 @@ func (c *Client) Get(args ...string) error {
 }
 
 func (c *Client) get(args ...string) error {
+	var hasD bool
+	for _, arg := range args {
+		if arg == "-d" {
+			hasD = true
+			break
+		}
+	}
+	if !hasD {
+		// go get without the -d flag does not make sense to us, as
+		// it will try to build and install go packages.
+		args = append([]string{"-d"}, args...)
+	}
 	if err := c.runGo(context.Background(), c.logger.Out(), append([]string{"get"}, args...)...); err != nil {
 		errors.Wrapf(err, "failed to get %q", args)
 	}
@@ -425,31 +437,83 @@ func (c *Client) listGoMods() (goModules, error) {
 		return nil, nil
 	}
 
-	out := ioutil.Discard
-	err := c.runGo(context.Background(), out, "mod", "download")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to download modules")
+	downloadModules := func(modules ...string) error {
+		args := []string{"mod", "download"}
+		args = append(args, modules...)
+		out := ioutil.Discard
+		err := c.runGo(context.Background(), out, args...)
+		if err != nil {
+			return errors.Wrap(err, "failed to download modules")
+		}
+		return nil
 	}
 
-	b := &bytes.Buffer{}
-	err = c.runGo(context.Background(), b, "list", "-m", "-json", "all")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to list modules")
+	if err := downloadModules(); err != nil {
+		return nil, err
 	}
 
-	var modules goModules
+	listAndDecodeModules := func(handle func(m *goModule) error, modules ...string) error {
+		b := &bytes.Buffer{}
+		args := []string{"list", "-m", "-json"}
+		if len(modules) > 0 {
+			args = append(args, modules...)
+		} else {
+			args = append(args, "all")
+		}
+		err := c.runGo(context.Background(), b, args...)
+		if err != nil {
+			return errors.Wrap(err, "failed to list modules")
+		}
+
+		dec := json.NewDecoder(b)
+		for {
+			m := &goModule{}
+			if err := dec.Decode(m); err != nil {
+				if err == io.EOF {
+					break
+				}
+				return errors.Wrap(err, "failed to decode modules list")
+			}
 
-	dec := json.NewDecoder(b)
-	for {
-		m := &goModule{}
-		if err := dec.Decode(m); err != nil {
-			if err == io.EOF {
-				break
+			if err := handle(m); err != nil {
+				return err
 			}
-			return nil, errors.Wrap(err, "failed to decode modules list")
 		}
+		return nil
+	}
 
+	var modules goModules
+	err := listAndDecodeModules(func(m *goModule) error {
 		modules = append(modules, m)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// From Go 1.17, go lazy loads transitive dependencies.
+	// That does not work for us.
+	// So, download these modules and update the Dir in the modules list.
+	var modulesToDownload []string
+	for _, m := range modules {
+		if m.Dir == "" {
+			modulesToDownload = append(modulesToDownload, fmt.Sprintf("%s@%s", m.Path, m.Version))
+		}
+	}
+
+	if len(modulesToDownload) > 0 {
+		if err := downloadModules(modulesToDownload...); err != nil {
+			return nil, err
+		}
+		err := listAndDecodeModules(func(m *goModule) error {
+			if mm := modules.GetByPath(m.Path); mm != nil {
+				mm.Dir = m.Dir
+			}
+			return nil
+		}, modulesToDownload...)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return modules, err
diff --git a/modules/collect.go b/modules/collect.go
index 29bf057a6..8798f45b1 100644
--- a/modules/collect.go
+++ b/modules/collect.go
@@ -252,15 +252,22 @@ func (c *collector) add(owner *moduleAdapter, moduleImport Import, disabled bool
 	}
 
 	if moduleDir == "" {
+		var versionQuery string
 		mod = c.gomods.GetByPath(modulePath)
 		if mod != nil {
 			moduleDir = mod.Dir
+			versionQuery = mod.Version
 		}
 
 		if moduleDir == "" {
 			if c.GoModulesFilename != "" && isProbablyModule(modulePath) {
 				// Try to "go get" it and reload the module configuration.
-				if err := c.Get(modulePath); err != nil {
+				if versionQuery == "" {
+					// See https://golang.org/ref/mod#version-queries
+					// This will select the latest release-version (not beta etc.).
+					versionQuery = "upgrade"
+				}
+				if err := c.Get(fmt.Sprintf("%s@%s", modulePath, versionQuery)); err != nil {
 					return nil, err
 				}
 				if err := c.loadModules(); err != nil {