7 files changed, 1962 insertions, 0 deletions

diff --git a/modules/client.go b/modules/client.go
new file mode 100644
index 000000000..ac09721dc
--- /dev/null
+++ b/modules/client.go
@@ -0,0 +1,570 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/gohugoio/hugo/hugofs/files"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"strings"
+	"time"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/rogpeppe/go-internal/module"
+
+	"github.com/gohugoio/hugo/common/hugio"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+var (
+	fileSeparator = string(os.PathSeparator)
+)
+
+const (
+	goBinaryStatusOK goBinaryStatus = iota
+	goBinaryStatusNotFound
+	goBinaryStatusTooOld
+)
+
+// The "vendor" dir is reserved for Go Modules.
+const vendord = "_vendor"
+
+const (
+	goModFilename = "go.mod"
+	goSumFilename = "go.sum"
+)
+
+// NewClient creates a new Client that can be used to manage the Hugo Components
+// in a given workingDir.
+// The Client will resolve the dependencies recursively, but needs the top
+// level imports to start out.
+func NewClient(cfg ClientConfig) *Client {
+	fs := cfg.Fs
+
+	n := filepath.Join(cfg.WorkingDir, goModFilename)
+	goModEnabled, _ := afero.Exists(fs, n)
+	var goModFilename string
+	if goModEnabled {
+		goModFilename = n
+	}
+
+	env := os.Environ()
+	mcfg := cfg.ModuleConfig
+
+	config.SetEnvVars(&env,
+		"PWD", cfg.WorkingDir,
+		"GOPROXY", mcfg.Proxy,
+		"GOPRIVATE", mcfg.Private,
+		"GONOPROXY", mcfg.NoProxy)
+
+	if cfg.CacheDir != "" {
+		// Module cache stored below $GOPATH/pkg
+		config.SetEnvVars(&env, "GOPATH", cfg.CacheDir)
+
+	}
+
+	logger := cfg.Logger
+	if logger == nil {
+		logger = loggers.NewWarningLogger()
+	}
+
+	return &Client{
+		fs:                fs,
+		ignoreVendor:      cfg.IgnoreVendor,
+		workingDir:        cfg.WorkingDir,
+		themesDir:         cfg.ThemesDir,
+		logger:            logger,
+		moduleConfig:      mcfg,
+		environ:           env,
+		GoModulesFilename: goModFilename}
+}
+
+// Client contains most of the API provided by this package.
+type Client struct {
+	fs     afero.Fs
+	logger *loggers.Logger
+
+	// Ignore any _vendor directory.
+	ignoreVendor bool
+
+	// Absolute path to the project dir.
+	workingDir string
+
+	// Absolute path to the project's themes dir.
+	themesDir string
+
+	// The top level module config
+	moduleConfig Config
+
+	// Environment variables used in "go get" etc.
+	environ []string
+
+	// Set when Go modules are initialized in the current repo, that is:
+	// a go.mod file exists.
+	GoModulesFilename string
+
+	// Set if we get a exec.ErrNotFound when running Go, which is most likely
+	// due to being run on a system without Go installed. We record it here
+	// so we can give an instructional error at the end if module/theme
+	// resolution fails.
+	goBinaryStatus goBinaryStatus
+}
+
+// Graph writes a module dependenchy graph to the given writer.
+func (c *Client) Graph(w io.Writer) error {
+	mc, coll := c.collect(true)
+	if coll.err != nil {
+		return coll.err
+	}
+	for _, module := range mc.AllModules {
+		if module.Owner() == nil {
+			continue
+		}
+
+		prefix := ""
+		if module.Disabled() {
+			prefix = "DISABLED "
+		}
+		dep := pathVersion(module.Owner()) + " " + pathVersion(module)
+		if replace := module.Replace(); replace != nil {
+			if replace.Version() != "" {
+				dep += " => " + pathVersion(replace)
+			} else {
+				// Local dir.
+				dep += " => " + replace.Dir()
+			}
+
+		}
+		fmt.Fprintln(w, prefix+dep)
+	}
+
+	return nil
+}
+
+// Tidy can be used to remove unused dependencies from go.mod and go.sum.
+func (c *Client) Tidy() error {
+	tc, coll := c.collect(false)
+	if coll.err != nil {
+		return coll.err
+	}
+
+	if coll.skipTidy {
+		return nil
+	}
+
+	return c.tidy(tc.AllModules, false)
+}
+
+// Vendor writes all the module dependencies to a _vendor folder.
+//
+// Unlike Go, we support it for any level.
+//
+// We, by default, use the /_vendor folder first, if found. To disable,
+// run with
+//    hugo --ignoreVendor
+//
+// Given a module tree, Hugo will pick the first module for a given path,
+// meaning that if the top-level module is vendored, that will be the full
+// set of dependencies.
+func (c *Client) Vendor() error {
+	vendorDir := filepath.Join(c.workingDir, vendord)
+	if err := c.rmVendorDir(vendorDir); err != nil {
+		return err
+	}
+
+	// Write the modules list to modules.txt.
+	//
+	// On the form:
+	//
+	// # github.com/alecthomas/chroma v0.6.3
+	//
+	// This is how "go mod vendor" does it. Go also lists
+	// the packages below it, but that is currently not applicable to us.
+	//
+	var modulesContent bytes.Buffer
+
+	tc, coll := c.collect(true)
+	if coll.err != nil {
+		return coll.err
+	}
+
+	for _, t := range tc.AllModules {
+		if t.Owner() == nil {
+			// This is the project.
+			continue
+		}
+		// We respect the --ignoreVendor flag even for the vendor command.
+		if !t.IsGoMod() && !t.Vendor() {
+			// We currently do not vendor components living in the
+			// theme directory, see https://github.com/gohugoio/hugo/issues/5993
+			continue
+		}
+
+		fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version())
+
+		dir := t.Dir()
+
+		for _, mount := range t.Mounts() {
+			if err := hugio.CopyDir(c.fs, filepath.Join(dir, mount.Source), filepath.Join(vendorDir, t.Path(), mount.Source), nil); err != nil {
+				return errors.Wrap(err, "failed to copy module to vendor dir")
+			}
+		}
+
+		// Include the resource cache if present.
+		resourcesDir := filepath.Join(dir, files.FolderResources)
+		_, err := c.fs.Stat(resourcesDir)
+		if err == nil {
+			if err := hugio.CopyDir(c.fs, resourcesDir, filepath.Join(vendorDir, t.Path(), files.FolderResources), nil); err != nil {
+				return errors.Wrap(err, "failed to copy resources to vendor dir")
+			}
+		}
+
+		// Also include any theme.toml or config.* files in the root.
+		configFiles, _ := afero.Glob(c.fs, filepath.Join(dir, "config.*"))
+		configFiles = append(configFiles, filepath.Join(dir, "theme.toml"))
+		for _, configFile := range configFiles {
+			if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil {
+				if !os.IsNotExist(err) {
+					return err
+				}
+			}
+		}
+	}
+
+	if modulesContent.Len() > 0 {
+		if err := afero.WriteFile(c.fs, filepath.Join(vendorDir, vendorModulesFilename), modulesContent.Bytes(), 0666); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Get runs "go get" with the supplied arguments.
+func (c *Client) Get(args ...string) error {
+	if err := c.runGo(context.Background(), os.Stdout, append([]string{"get"}, args...)...); err != nil {
+		errors.Wrapf(err, "failed to get %q", args)
+	}
+	return nil
+}
+
+// Init initializes this as a Go Module with the given path.
+// If path is empty, Go will try to guess.
+// If this succeeds, this project will be marked as Go Module.
+func (c *Client) Init(path string) error {
+	err := c.runGo(context.Background(), os.Stdout, "mod", "init", path)
+	if err != nil {
+		return errors.Wrap(err, "failed to init modules")
+	}
+
+	c.GoModulesFilename = filepath.Join(c.workingDir, goModFilename)
+
+	return nil
+}
+
+func (c *Client) isProbablyModule(path string) bool {
+	return module.CheckPath(path) == nil
+}
+
+func (c *Client) listGoMods() (goModules, error) {
+	if c.GoModulesFilename == "" {
+		return nil, nil
+	}
+
+	out := ioutil.Discard
+	err := c.runGo(context.Background(), out, "mod", "download")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to download modules")
+	}
+
+	b := &bytes.Buffer{}
+	err = c.runGo(context.Background(), b, "list", "-m", "-json", "all")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list modules")
+	}
+
+	var modules goModules
+
+	dec := json.NewDecoder(b)
+	for {
+		m := &goModule{}
+		if err := dec.Decode(m); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, errors.Wrap(err, "failed to decode modules list")
+		}
+
+		modules = append(modules, m)
+	}
+
+	return modules, err
+
+}
+
+func (c *Client) rewriteGoMod(name string, isGoMod map[string]bool) error {
+	data, err := c.rewriteGoModRewrite(name, isGoMod)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := afero.WriteFile(c.fs, filepath.Join(c.workingDir, name), data, 0666); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]byte, error) {
+	if name == goModFilename && c.GoModulesFilename == "" {
+		// Already checked.
+		return nil, nil
+	}
+
+	modlineSplitter := getModlineSplitter(name == goModFilename)
+
+	b := &bytes.Buffer{}
+	f, err := c.fs.Open(filepath.Join(c.workingDir, name))
+	if err != nil {
+		if os.IsNotExist(err) {
+			// It's been deleted.
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	var dirty bool
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		var doWrite bool
+
+		if parts := modlineSplitter(line); parts != nil {
+			modname, modver := parts[0], parts[1]
+			modver = strings.TrimSuffix(modver, "/"+goModFilename)
+			modnameVer := modname + " " + modver
+			doWrite = isGoMod[modnameVer]
+		} else {
+			doWrite = true
+		}
+
+		if doWrite {
+			fmt.Fprintln(b, line)
+		} else {
+			dirty = true
+		}
+	}
+
+	if !dirty {
+		// Nothing changed
+		return nil, nil
+	}
+
+	return b.Bytes(), nil
+
+}
+
+func (c *Client) rmVendorDir(vendorDir string) error {
+	modulestxt := filepath.Join(vendorDir, vendorModulesFilename)
+
+	if _, err := c.fs.Stat(vendorDir); err != nil {
+		return nil
+	}
+
+	_, err := c.fs.Stat(modulestxt)
+	if err != nil {
+		// If we have a _vendor dir without modules.txt it sounds like
+		// a _vendor dir created by others.
+		return errors.New("found _vendor dir without modules.txt, skip delete")
+	}
+
+	return c.fs.RemoveAll(vendorDir)
+}
+
+func (c *Client) runGo(
+	ctx context.Context,
+	stdout io.Writer,
+	args ...string) error {
+
+	if c.goBinaryStatus != 0 {
+		return nil
+	}
+
+	stderr := new(bytes.Buffer)
+	cmd := exec.CommandContext(ctx, "go", args...)
+
+	cmd.Env = c.environ
+	cmd.Dir = c.workingDir
+	cmd.Stdout = stdout
+	cmd.Stderr = io.MultiWriter(stderr, os.Stderr)
+
+	if err := cmd.Run(); err != nil {
+		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
+			c.goBinaryStatus = goBinaryStatusNotFound
+			return nil
+		}
+
+		_, ok := err.(*exec.ExitError)
+		if !ok {
+			return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err)
+		}
+
+		// Too old Go version
+		if strings.Contains(stderr.String(), "flag provided but not defined") {
+			c.goBinaryStatus = goBinaryStatusTooOld
+			return nil
+		}
+
+		return errors.Errorf("go command failed: %s", stderr)
+
+	}
+
+	return nil
+}
+
+func (c *Client) tidy(mods Modules, goModOnly bool) error {
+	isGoMod := make(map[string]bool)
+	for _, m := range mods {
+		if m.Owner() == nil {
+			continue
+		}
+		if m.IsGoMod() {
+			// Matching the format in go.mod
+			pathVer := m.Path() + " " + m.Version()
+			isGoMod[pathVer] = true
+		}
+	}
+
+	if err := c.rewriteGoMod(goModFilename, isGoMod); err != nil {
+		return err
+	}
+
+	if goModOnly {
+		return nil
+	}
+
+	if err := c.rewriteGoMod(goSumFilename, isGoMod); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ClientConfig configures the module Client.
+type ClientConfig struct {
+	Fs           afero.Fs
+	Logger       *loggers.Logger
+	IgnoreVendor bool
+	WorkingDir   string
+	ThemesDir    string // Absolute directory path
+	CacheDir     string // Module cache
+	ModuleConfig Config
+}
+
+type goBinaryStatus int
+
+type goModule struct {
+	Path     string         // module path
+	Version  string         // module version
+	Versions []string       // available module versions (with -versions)
+	Replace  *goModule      // replaced by this module
+	Time     *time.Time     // time version was created
+	Update   *goModule      // available update, if any (with -u)
+	Main     bool           // is this the main module?
+	Indirect bool           // is this module only an indirect dependency of main module?
+	Dir      string         // directory holding files for this module, if any
+	GoMod    string         // path to go.mod file for this module, if any
+	Error    *goModuleError // error loading module
+}
+
+type goModuleError struct {
+	Err string // the error itself
+}
+
+type goModules []*goModule
+
+func (modules goModules) GetByPath(p string) *goModule {
+	if modules == nil {
+		return nil
+	}
+
+	for _, m := range modules {
+		if strings.EqualFold(p, m.Path) {
+			return m
+		}
+	}
+
+	return nil
+}
+
+func (modules goModules) GetMain() *goModule {
+	for _, m := range modules {
+		if m.Main {
+			return m
+		}
+	}
+
+	return nil
+}
+
+func getModlineSplitter(isGoMod bool) func(line string) []string {
+	if isGoMod {
+		return func(line string) []string {
+			if strings.HasPrefix(line, "require (") {
+				return nil
+			}
+			if !strings.HasPrefix(line, "require") && !strings.HasPrefix(line, "\t") {
+				return nil
+			}
+			line = strings.TrimPrefix(line, "require")
+			line = strings.TrimSpace(line)
+			line = strings.TrimSuffix(line, "// indirect")
+
+			return strings.Fields(line)
+		}
+	}
+
+	return func(line string) []string {
+		return strings.Fields(line)
+	}
+}
+
+func pathVersion(m Module) string {
+	versionStr := m.Version()
+	if m.Vendor() {
+		versionStr += "+vendor"
+	}
+	if versionStr == "" {
+		return m.Path()
+	}
+	return fmt.Sprintf("%s@%s", m.Path(), versionStr)
+}
diff --git a/modules/client_test.go b/modules/client_test.go
new file mode 100644
index 000000000..d8301514d
--- /dev/null
+++ b/modules/client_test.go
@@ -0,0 +1,117 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/gohugoio/hugo/common/hugo"
+
+	"github.com/gohugoio/hugo/htesting"
+
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestClient(t *testing.T) {
+	if hugo.GoMinorVersion() < 12 {
+		// https://github.com/golang/go/issues/26794
+		// There were some concurrent issues with Go modules in < Go 12.
+		t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib")
+	}
+
+	t.Parallel()
+
+	modName := "hugo-modules-basic-test"
+	modPath := "github.com/gohugoio/tests/" + modName
+	modConfig := DefaultModuleConfig
+	modConfig.Imports = []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}}
+
+	assert := require.New(t)
+
+	workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName)
+	assert.NoError(err)
+	defer clean()
+
+	client := NewClient(ClientConfig{
+		Fs:           hugofs.Os,
+		WorkingDir:   workingDir,
+		ModuleConfig: modConfig,
+	})
+
+	// Test Init
+	assert.NoError(client.Init(modPath))
+
+	// Test Collect
+	mc, err := client.Collect()
+	assert.NoError(err)
+	assert.Equal(4, len(mc.AllModules))
+	for _, m := range mc.AllModules {
+		assert.NotNil(m)
+	}
+
+	// Test Graph
+	var graphb bytes.Buffer
+	assert.NoError(client.Graph(&graphb))
+
+	expect := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0
+github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0
+github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0
+`
+
+	assert.Equal(expect, graphb.String())
+
+	// Test Vendor
+	assert.NoError(client.Vendor())
+	graphb.Reset()
+	assert.NoError(client.Graph(&graphb))
+	expectVendored := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0+vendor
+github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0+vendor
+github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor
+`
+	assert.Equal(expectVendored, graphb.String())
+
+	// Test the ignoreVendor setting
+	clientIgnoreVendor := NewClient(ClientConfig{
+		Fs:           hugofs.Os,
+		WorkingDir:   workingDir,
+		ModuleConfig: modConfig,
+		IgnoreVendor: true,
+	})
+
+	graphb.Reset()
+	assert.NoError(clientIgnoreVendor.Graph(&graphb))
+	assert.Equal(expect, graphb.String())
+
+	// Test Tidy
+	assert.NoError(client.Tidy())
+
+}
+
+func TestGetModlineSplitter(t *testing.T) {
+
+	assert := require.New(t)
+
+	gomodSplitter := getModlineSplitter(true)
+
+	assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gomodSplitter("\tgithub.com/BurntSushi/toml v0.3.1"))
+	assert.Equal([]string{"github.com/cpuguy83/go-md2man", "v1.0.8"}, gomodSplitter("\tgithub.com/cpuguy83/go-md2man v1.0.8 // indirect"))
+	assert.Nil(gomodSplitter("require ("))
+
+	gosumSplitter := getModlineSplitter(false)
+	assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gosumSplitter("github.com/BurntSushi/toml v0.3.1"))
+
+}
diff --git a/modules/collect.go b/modules/collect.go
new file mode 100644
index 000000000..f57b4d04b
--- /dev/null
+++ b/modules/collect.go
@@ -0,0 +1,574 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cast"
+
+	"github.com/gohugoio/hugo/common/maps"
+
+	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+
+	"github.com/gohugoio/hugo/hugofs/files"
+
+	"github.com/rogpeppe/go-internal/module"
+
+	"github.com/pkg/errors"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+)
+
+var ErrNotExist = errors.New("module does not exist")
+
+const vendorModulesFilename = "modules.txt"
+
+// IsNotExist returns whether an error means that a module could not be found.
+func IsNotExist(err error) bool {
+	return errors.Cause(err) == ErrNotExist
+}
+
+// CreateProjectModule creates modules from the given config.
+// This is used in tests only.
+func CreateProjectModule(cfg config.Provider) (Module, error) {
+	workingDir := cfg.GetString("workingDir")
+	var modConfig Config
+
+	mod := createProjectModule(nil, workingDir, modConfig)
+	if err := ApplyProjectConfigDefaults(cfg, mod); err != nil {
+		return nil, err
+	}
+
+	return mod, nil
+}
+
+func (h *Client) Collect() (ModulesConfig, error) {
+	mc, coll := h.collect(true)
+	return mc, coll.err
+
+}
+
+func (h *Client) collect(tidy bool) (ModulesConfig, *collector) {
+	c := &collector{
+		Client: h,
+	}
+
+	c.collect()
+	if c.err != nil {
+		return ModulesConfig{}, c
+	}
+
+	if !c.skipTidy && tidy {
+		if err := h.tidy(c.modules, true); err != nil {
+			c.err = err
+			return ModulesConfig{}, c
+		}
+	}
+
+	// TODO(bep) consider --ignoreVendor vs removing from go.mod
+	var activeMods Modules
+	for _, mod := range c.modules {
+		if !mod.Config().HugoVersion.IsValid() {
+			h.logger.WARN.Printf(`Module %q is not compatible with this Hugo version; run "hugo mod graph" for more information.`, mod.Path())
+		}
+		if !mod.Disabled() {
+			activeMods = append(activeMods, mod)
+		}
+	}
+
+	return ModulesConfig{
+		AllModules:        c.modules,
+		ActiveModules:     activeMods,
+		GoModulesFilename: c.GoModulesFilename,
+	}, c
+
+}
+
+type ModulesConfig struct {
+	// All modules, including any disabled.
+	AllModules Modules
+
+	// All active modules.
+	ActiveModules Modules
+
+	// Set if this is a Go modules enabled project.
+	GoModulesFilename string
+}
+
+type collected struct {
+	// Pick the first and prevent circular loops.
+	seen map[string]bool
+
+	// Maps module path to a _vendor dir. These values are fetched from
+	// _vendor/modules.txt, and the first (top-most) will win.
+	vendored map[string]vendoredModule
+
+	// Set if a Go modules enabled project.
+	gomods goModules
+
+	// Ordered list of collected modules, including Go Modules and theme
+	// components stored below /themes.
+	modules Modules
+}
+
+// Collects and creates a module tree.
+type collector struct {
+	*Client
+
+	// Store away any non-fatal error and return at the end.
+	err error
+
+	// Set to disable any Tidy operation in the end.
+	skipTidy bool
+
+	*collected
+}
+
+func (c *collector) initModules() error {
+	c.collected = &collected{
+		seen:     make(map[string]bool),
+		vendored: make(map[string]vendoredModule),
+	}
+
+	// We may fail later if we don't find the mods.
+	return c.loadModules()
+}
+
+func (c *collector) isSeen(path string) bool {
+	key := pathKey(path)
+	if c.seen[key] {
+		return true
+	}
+	c.seen[key] = true
+	return false
+}
+
+func (c *collector) getVendoredDir(path string) (vendoredModule, bool) {
+	v, found := c.vendored[path]
+	return v, found
+}
+
+func (c *collector) add(owner *moduleAdapter, moduleImport Import, disabled bool) (*moduleAdapter, error) {
+	var (
+		mod       *goModule
+		moduleDir string
+		version   string
+		vendored  bool
+	)
+
+	modulePath := moduleImport.Path
+	var realOwner Module = owner
+
+	if !c.ignoreVendor {
+		if err := c.collectModulesTXT(owner); err != nil {
+			return nil, err
+		}
+
+		// Try _vendor first.
+		var vm vendoredModule
+		vm, vendored = c.getVendoredDir(modulePath)
+		if vendored {
+			moduleDir = vm.Dir
+			realOwner = vm.Owner
+			version = vm.Version
+
+			if owner.projectMod {
+				// We want to keep the go.mod intact with the versions and all.
+				c.skipTidy = true
+			}
+
+		}
+	}
+
+	if moduleDir == "" {
+		mod = c.gomods.GetByPath(modulePath)
+		if mod != nil {
+			moduleDir = mod.Dir
+		}
+
+		if moduleDir == "" {
+
+			if c.GoModulesFilename != "" && c.isProbablyModule(modulePath) {
+				// Try to "go get" it and reload the module configuration.
+				if err := c.Get(modulePath); err != nil {
+					return nil, err
+				}
+				if err := c.loadModules(); err != nil {
+					return nil, err
+				}
+
+				mod = c.gomods.GetByPath(modulePath)
+				if mod != nil {
+					moduleDir = mod.Dir
+				}
+			}
+
+			// Fall back to /themes/<mymodule>
+			if moduleDir == "" {
+				moduleDir = filepath.Join(c.themesDir, modulePath)
+
+				if found, _ := afero.Exists(c.fs, moduleDir); !found {
+					c.err = c.wrapModuleNotFound(errors.Errorf(`module %q not found; either add it as a Hugo Module or store it in %q.`, modulePath, c.themesDir))
+					return nil, nil
+				}
+			}
+		}
+	}
+
+	if found, _ := afero.Exists(c.fs, moduleDir); !found {
+		c.err = c.wrapModuleNotFound(errors.Errorf("%q not found", moduleDir))
+		return nil, nil
+	}
+
+	if !strings.HasSuffix(moduleDir, fileSeparator) {
+		moduleDir += fileSeparator
+	}
+
+	ma := &moduleAdapter{
+		dir:      moduleDir,
+		vendor:   vendored,
+		disabled: disabled,
+		gomod:    mod,
+		version:  version,
+		// This may be the owner of the _vendor dir
+		owner: realOwner,
+	}
+
+	if mod == nil {
+		ma.path = modulePath
+	}
+
+	if err := ma.validateAndApplyDefaults(c.fs); err != nil {
+		return nil, err
+	}
+
+	if !moduleImport.IgnoreConfig {
+		if err := c.applyThemeConfig(ma); err != nil {
+			return nil, err
+		}
+	}
+
+	if err := c.applyMounts(moduleImport, ma); err != nil {
+		return nil, err
+	}
+
+	c.modules = append(c.modules, ma)
+	return ma, nil
+
+}
+
+func (c *collector) addAndRecurse(owner *moduleAdapter, disabled bool) error {
+	moduleConfig := owner.Config()
+	if owner.projectMod {
+		if err := c.applyMounts(Import{}, owner); err != nil {
+			return err
+		}
+	}
+
+	for _, moduleImport := range moduleConfig.Imports {
+		disabled := disabled || moduleImport.Disabled
+
+		if !c.isSeen(moduleImport.Path) {
+			tc, err := c.add(owner, moduleImport, disabled)
+			if err != nil {
+				return err
+			}
+			if tc == nil {
+				continue
+			}
+			if err := c.addAndRecurse(tc, disabled); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error {
+	mounts := moduleImport.Mounts
+
+	if !mod.projectMod && len(mounts) == 0 {
+		modConfig := mod.Config()
+		mounts = modConfig.Mounts
+		if len(mounts) == 0 {
+			// Create default mount points for every component folder that
+			// exists in the module.
+			for _, componentFolder := range files.ComponentFolders {
+				sourceDir := filepath.Join(mod.Dir(