Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

1 files changed, 110 insertions, 88 deletions

diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
index e852e7f1d..6ad871564 100644
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -22,6 +22,8 @@ import (
 	"strings"
 	"sync"
 
+	radix "github.com/hashicorp/go-immutable-radix"
+
 	"github.com/gohugoio/hugo/output"
 	"github.com/gohugoio/hugo/parser/metadecoders"
 
@@ -33,7 +35,6 @@ import (
 
 	"github.com/bep/gitmap"
 	"github.com/gohugoio/hugo/config"
-	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/publisher"
 
@@ -199,10 +200,11 @@ func (h *HugoSites) IsMultihost() bool {
 	return h != nil && h.multihost
 }
 
-func (h *HugoSites) LanguageSet() map[string]bool {
-	set := make(map[string]bool)
-	for _, s := range h.Sites {
-		set[s.language.Lang] = true
+// TODO(bep) consolidate
+func (h *HugoSites) LanguageSet() map[string]int {
+	set := make(map[string]int)
+	for i, s := range h.Sites {
+		set[s.language.Lang] = i
 	}
 	return set
 }
@@ -222,14 +224,6 @@ func (h *HugoSites) PrintProcessingStats(w io.Writer) {
 	helpers.ProcessingStatsTable(w, stats...)
 }
 
-func (h *HugoSites) langSite() map[string]*Site {
-	m := make(map[string]*Site)
-	for _, s := range h.Sites {
-		m[s.language.Lang] = s
-	}
-	return m
-}
-
 // GetContentPage finds a Page with content given the absolute filename.
 // Returns nil if none found.
 func (h *HugoSites) GetContentPage(filename string) page.Page {
@@ -265,7 +259,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 	langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...)
 
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to create language config")
 	}
 
 	var contentChangeTracker *contentChangeMap
@@ -288,8 +282,11 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 	}
 
 	h.init.data.Add(func() (interface{}, error) {
-		err := h.loadData(h.PathSpec.BaseFs.Data.Fs)
-		return err, nil
+		err := h.loadData(h.PathSpec.BaseFs.Data.Dirs)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load data")
+		}
+		return nil, nil
 	})
 
 	h.init.translations.Add(func() (interface{}, error) {
@@ -303,7 +300,10 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 
 	h.init.gitInfo.Add(func() (interface{}, error) {
 		err := h.loadGitInfo()
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load Git info")
+		}
+		return nil, nil
 	})
 
 	for _, s := range sites {
@@ -311,7 +311,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 	}
 
 	if err := applyDeps(cfg, sites...); err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "add site dependencies")
 	}
 
 	h.Deps = sites[0].Deps
@@ -319,7 +319,12 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 	// Only needed in server mode.
 	// TODO(bep) clean up the running vs watching terms
 	if cfg.Running {
-		contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)}
+		contentChangeTracker = &contentChangeMap{
+			pathSpec:      h.PathSpec,
+			symContent:    make(map[string]map[string]bool),
+			leafBundles:   radix.New(),
+			branchBundles: make(map[string]bool),
+		}
 		h.ContentChanges = contentChangeTracker
 	}
 
@@ -371,7 +376,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error {
 
 			siteConfig, err := loadSiteConfig(s.language)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "load site config")
 			}
 			s.siteConfigConfig = siteConfig
 			s.siteRefLinker, err = newSiteRefLinker(s.language, s)
@@ -388,17 +393,17 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error {
 			var err error
 			d, err = deps.New(cfg)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "create deps")
 			}
 
 			d.OutputFormatsConfig = s.outputFormatsConfig
 
 			if err := onCreated(d); err != nil {
-				return err
+				return errors.Wrap(err, "on created")
 			}
 
 			if err = d.LoadResources(); err != nil {
-				return err
+				return errors.Wrap(err, "load resources")
 			}
 
 		} else {
@@ -418,7 +423,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error {
 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 	sites, err := createSitesFromConfig(cfg)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "from config")
 	}
 	return newHugoSites(cfg, sites...)
 }
@@ -800,41 +805,45 @@ func (h *HugoSites) Pages() page.Pages {
 	return h.Sites[0].AllPages()
 }
 
-func (h *HugoSites) loadData(fs afero.Fs) (err error) {
-	spec := source.NewSourceSpec(h.PathSpec, fs)
-	fileSystem := spec.NewFilesystem("")
+func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) {
+	spec := source.NewSourceSpec(h.PathSpec, nil)
+
 	h.data = make(map[string]interface{})
-	for _, r := range fileSystem.Files() {
-		if err := h.handleDataFile(r); err != nil {
+	for _, fi := range fis {
+		fileSystem := spec.NewFilesystemFromFileMetaInfo(fi)
+		files, err := fileSystem.Files()
+		if err != nil {
 			return err
 		}
+		for _, r := range files {
+			if err := h.handleDataFile(r); err != nil {
+				return err
+			}
+		}
 	}
 
 	return
 }
 
-func (h *HugoSites) handleDataFile(r source.ReadableFile) error {
+func (h *HugoSites) handleDataFile(r source.File) error {
 	var current map[string]interface{}
 
-	f, err := r.Open()
+	f, err := r.FileInfo().Meta().Open()
 	if err != nil {
-		return errors.Wrapf(err, "Failed to open data file %q:", r.LogicalName())
+		return errors.Wrapf(err, "data: failed to open %q:", r.LogicalName())
 	}
 	defer f.Close()
 
 	// Crawl in data tree to insert data
 	current = h.data
 	keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator)
-	// The first path element is the virtual folder (typically theme name), which is
-	// not part of the key.
-	if len(keyParts) > 1 {
-		for _, key := range keyParts[1:] {
-			if key != "" {
-				if _, ok := current[key]; !ok {
-					current[key] = make(map[string]interface{})
-				}
-				current = current[key].(map[string]interface{})
+
+	for _, key := range keyParts {
+		if key != "" {
+			if _, ok := current[key]; !ok {
+				current[key] = make(map[string]interface{})
 			}
+			current = current[key].(map[string]interface{})
 		}
 	}
 
@@ -848,15 +857,10 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error {
 	}
 
 	// filepath.Walk walks the files in lexical order, '/' comes before '.'
-	// this warning could happen if
-	// 1. A theme uses the same key; the main data folder wins
-	// 2. A sub folder uses the same key: the sub folder wins
 	higherPrecedentData := current[r.BaseFileName()]
 
 	switch data.(type) {
 	case nil:
-		// hear the crickets?
-
 	case map[string]interface{}:
 
 		switch higherPrecedentData.(type) {
@@ -868,7 +872,11 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error {
 			higherPrecedentMap := higherPrecedentData.(map[string]interface{})
 			for key, value := range data.(map[string]interface{}) {
 				if _, exists := higherPrecedentMap[key]; exists {
-					h.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path())
+					// this warning could happen if
+					// 1. A theme uses the same key; the main data folder wins
+					// 2. A sub folder uses the same key: the sub folder wins
+					// TODO(bep) figure out a way to detect 2) above and make that a WARN
+					h.Log.INFO.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path())
 				} else {
 					higherPrecedentMap[key] = value
 				}
@@ -896,12 +904,12 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error {
 }
 
 func (h *HugoSites) errWithFileContext(err error, f source.File) error {
-	rfi, ok := f.FileInfo().(hugofs.RealFilenameInfo)
+	fim, ok := f.FileInfo().(hugofs.FileMetaInfo)
 	if !ok {
 		return err
 	}
 
-	realFilename := rfi.RealFilename()
+	realFilename := fim.Meta().Filename()
 
 	err, _ = herrors.WithFileContextForFile(
 		err,
@@ -913,8 +921,8 @@ func (h *HugoSites) errWithFileContext(err error, f source.File) error {
 	return err
 }
 
-func (h *HugoSites) readData(f source.ReadableFile) (interface{}, error) {
-	file, err := f.Open()
+func (h *HugoSites) readData(f source.File) (interface{}, error) {
+	file, err := f.FileInfo().Meta().Open()
 	if err != nil {
 		return nil, errors.Wrap(err, "readData: failed to open data file")
 	}
@@ -939,9 +947,14 @@ func (h *HugoSites) findPagesByShortcode(shortcode string) page.Pages {
 
 // Used in partial reloading to determine if the change is in a bundle.
 type contentChangeMap struct {
-	mu       sync.RWMutex
-	branches []string
-	leafs    []string
+	mu sync.RWMutex
+
+	// Holds directories with leaf bundles.
+	leafBundles    *radix.Tree
+	leafBundlesTxn *radix.Txn
+
+	// Holds directories with branch bundles.
+	branchBundles map[string]bool
 
 	pathSpec *helpers.PathSpec
 
@@ -950,9 +963,22 @@ type contentChangeMap struct {
 	// locations in /content -- which is really cool, but also means we have to
 	// go an extra mile to handle changes.
 	// This map is only used in watch mode.
-	// It maps either file to files or the real dir to a set of content directories where it is in use.
-	symContent   map[string]map[string]bool
+	// It maps either file to files or the real dir to a set of content directories
+	// where it is in use.
 	symContentMu sync.Mutex
+	symContent   map[string]map[string]bool
+}
+
+func (m *contentChangeMap) start() {
+	m.mu.Lock()
+	m.leafBundlesTxn = m.leafBundles.Txn()
+	m.mu.Unlock()
+}
+
+func (m *contentChangeMap) stop() {
+	m.mu.Lock()
+	m.leafBundles = m.leafBundlesTxn.Commit()
+	m.mu.Unlock()
 }
 
 func (m *contentChangeMap) add(filename string, tp bundleDirType) {
@@ -961,68 +987,63 @@ func (m *contentChangeMap) add(filename string, tp bundleDirType) {
 	dir = strings.TrimPrefix(dir, ".")
 	switch tp {
 	case bundleBranch:
-		m.branches = append(m.branches, dir)
+		m.branchBundles[dir] = true
 	case bundleLeaf:
-		m.leafs = append(m.leafs, dir)
+		m.leafBundlesTxn.Insert([]byte(dir), true)
 	default:
 		panic("invalid bundle type")
 	}
 	m.mu.Unlock()
 }
 
-// Track the addition of bundle dirs.
-func (m *contentChangeMap) handleBundles(b *bundleDirs) {
-	for _, bd := range b.bundles {
-		m.add(bd.fi.Path(), bd.tp)
-	}
-}
-
-// resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant.
-// It also removes the entry from the map. It will be re-added again by the partial
-// build if it still is a bundle.
 func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
 
 	// Bundles share resources, so we need to start from the virtual root.
-	relPath := m.pathSpec.RelContentDir(filename)
-	dir, name := filepath.Split(relPath)
+	relFilename := m.pathSpec.RelContentDir(filename)
+	dir, name := filepath.Split(relFilename)
 	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
 		dir += helpers.FilePathSeparator
 	}
 
-	fileTp, isContent := classifyBundledFile(name)
-
-	// This may be a member of a bundle. Start with branch bundles, the most specific.
-	if fileTp == bundleBranch || (fileTp == bundleNot && !isContent) {
-		for i, b := range m.branches {
-			if b == dir {
-				m.branches = append(m.branches[:i], m.branches[i+1:]...)
-				return dir, b, bundleBranch
-			}
-		}
+	if _, found := m.branchBundles[dir]; found {
+		delete(m.branchBundles, dir)
+		return dir, dir, bundleBranch
 	}
 
-	// And finally the leaf bundles, which can contain anything.
-	for i, l := range m.leafs {
-		if strings.HasPrefix(dir, l) {
-			m.leafs = append(m.leafs[:i], m.leafs[i+1:]...)
-			return dir, l, bundleLeaf
-		}
+	if key, _, found := m.leafBundles.Root().LongestPrefix([]byte(dir)); found {
+		m.leafBundlesTxn.Delete(key)
+		dir = string(key)
+		return dir, dir, bundleLeaf
 	}
 
+	fileTp, isContent := classifyBundledFile(name)
 	if isContent && fileTp != bundleNot {
 		// A new bundle.
 		return dir, dir, fileTp
 	}
 
-	// Not part of any bundle
 	return dir, filename, bundleNot
+
 }
 
-func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) {
+func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) {
+	meta := fim.Meta()
+	if !meta.IsSymlink() {
+		return
+	}
 	m.symContentMu.Lock()
+
+	from, to := meta.Filename(), meta.OriginalFilename()
+	if fim.IsDir() {
+		if !strings.HasSuffix(from, helpers.FilePathSeparator) {
+			from += helpers.FilePathSeparator
+		}
+	}
+
 	mm, found := m.symContent[from]
+
 	if !found {
 		mm = make(map[string]bool)
 		m.symContent[from] = mm
@@ -1044,5 +1065,6 @@ func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string {
 	}
 
 	sort.Strings(dirs)
+
 	return dirs
 }