Add support for theme composition and inheritance

This commit adds support for theme composition and inheritance in Hugo.

With this, it helps thinking about a theme as a set of ordered components:

```toml
theme = ["my-shortcodes", "base-theme", "hyde"]
```

The theme definition example above in `config.toml` creates a theme with the 3 components with presedence from left to right.

So, Hugo will, for any given file, data entry etc., look first in the project, and then in `my-shortcode`, `base-theme` and lastly `hyde`.

Hugo uses two different algorithms to merge the filesystems, depending on the file type:

* For `i18n` and `data` files, Hugo merges deeply using the translation id and data key inside the files.
* For `static`, `layouts` (templates) and `archetypes` files, these are merged on file level. So the left-most file will be chosen.

The name used in the `theme` definition above must match a folder in `/your-site/themes`, e.g. `/your-site/themes/my-shortcodes`. There are  plans to improve on this and get a URL scheme so this can be resolved automatically.

Also note that a component that is part of a theme can have its own configuration file, e.g. `config.toml`. There are currently some restrictions to what a theme component can configure:

* `params` (global and per language)
* `menu` (global and per language)
* `outputformats` and `mediatypes`

The same rules apply here: The left-most param/menu etc. with the same ID will win. There are some hidden and experimental namespace support in the above, which we will work to improve in the future, but theme authors are encouraged to create their own namespaces to avoid naming conflicts.

A final note: Themes/components can also have a `theme` definition in their `config.toml` and similar, which is the "inheritance" part of this commit's title. This is currently not supported by the Hugo theme site. We will have to wait for some "auto dependency" feature to be implemented for that to happen, but this can be a powerful feature if you want to create your own theme-variant based on others.

Fixes #4460
Fixes #4450

1 files changed, 25 insertions, 308 deletions

diff --git a/helpers/pathspec.go b/helpers/pathspec.go
index b18408590..847029f44 100644
--- a/helpers/pathspec.go
+++ b/helpers/pathspec.go
@@ -14,354 +14,71 @@
 package helpers
 
 import (
-	"fmt"
 	"strings"
 
-	"github.com/spf13/afero"
-
-	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/cast"
+	"github.com/gohugoio/hugo/hugolib/filesystems"
+	"github.com/gohugoio/hugo/hugolib/paths"
 )
 
 // PathSpec holds methods that decides how paths in URLs and files in Hugo should look like.
 type PathSpec struct {
-	BaseURL
-
-	// If the baseURL contains a base path, e.g. https://example.com/docs, then "/docs" will be the BasePath.
-	// This will not be set if canonifyURLs is enabled.
-	BasePath string
-
-	disablePathToLower bool
-	removePathAccents  bool
-	uglyURLs           bool
-	canonifyURLs       bool
-
-	Language  *Language
-	Languages Languages
-
-	// pagination path handling
-	paginatePath string
-
-	theme string
-
-	// Directories
-	contentDir     string
-	themesDir      string
-	layoutDir      string
-	workingDir     string
-	staticDirs     []string
-	absContentDirs []types.KeyValueStr
-
-	PublishDir string
-
-	// The PathSpec looks up its config settings in both the current language
-	// and then in the global Viper config.
-	// Some settings, the settings listed below, does not make sense to be set
-	// on per-language-basis. We have no good way of protecting against this
-	// other than a "white-list". See language.go.
-	defaultContentLanguageInSubdir bool
-	defaultContentLanguage         string
-	multilingual                   bool
+	*paths.Paths
+	*filesystems.BaseFs
 
 	ProcessingStats *ProcessingStats
 
 	// The file systems to use
 	Fs *hugofs.Fs
 
-	// The fine grained filesystems in play (resources, content etc.).
-	BaseFs *hugofs.BaseFs
-
 	// The config provider to use
 	Cfg config.Provider
 }
 
-func (p PathSpec) String() string {
-	return fmt.Sprintf("PathSpec, language %q, prefix %q, multilingual: %T", p.Language.Lang, p.getLanguagePrefix(), p.multilingual)
-}
-
-// NewPathSpec creats a new PathSpec from the given filesystems and Language.
+// NewPathSpec creats a new PathSpec from the given filesystems and language.
 func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
+	return NewPathSpecWithBaseBaseFsProvided(fs, cfg, nil)
+}
 
-	baseURLstr := cfg.GetString("baseURL")
-	baseURL, err := newBaseURLFromString(baseURLstr)
-
-	if err != nil {
-		return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err)
-	}
-
-	var staticDirs []string
-
-	for i := -1; i <= 10; i++ {
-		staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
-	}
-
-	var (
-		lang      string
-		language  *Language
-		languages Languages
-	)
-
-	if l, ok := cfg.(*Language); ok {
-		language = l
-		lang = l.Lang
-
-	}
-
-	if l, ok := cfg.Get("languagesSorted").(Languages); ok {
-		languages = l
-	}
-
-	defaultContentLanguage := cfg.GetString("defaultContentLanguage")
-
-	// We will eventually pull out this badly placed path logic.
-	contentDir := cfg.GetString("contentDir")
-	workingDir := cfg.GetString("workingDir")
-	resourceDir := cfg.GetString("resourceDir")
-	publishDir := cfg.GetString("publishDir")
-
-	if len(languages) == 0 {
-		// We have some old tests that does not test the entire chain, hence
-		// they have no languages. So create one so we get the proper filesystem.
-		languages = Languages{&Language{Lang: "en", ContentDir: contentDir}}
-	}
+// NewPathSpecWithBaseBaseFsProvided creats a new PathSpec from the given filesystems and language.
+// If an existing BaseFs is provided, parts of that is reused.
+func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.Provider, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) {
 
-	absPuslishDir := AbsPathify(workingDir, publishDir)
-	if !strings.HasSuffix(absPuslishDir, FilePathSeparator) {
-		absPuslishDir += FilePathSeparator
-	}
-	// If root, remove the second '/'
-	if absPuslishDir == "//" {
-		absPuslishDir = FilePathSeparator
-	}
-	absResourcesDir := AbsPathify(workingDir, resourceDir)
-	if !strings.HasSuffix(absResourcesDir, FilePathSeparator) {
-		absResourcesDir += FilePathSeparator
-	}
-	if absResourcesDir == "//" {
-		absResourcesDir = FilePathSeparator
-	}
-
-	contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages)
+	p, err := paths.New(fs, cfg)
 	if err != nil {
 		return nil, err
 	}
 
-	// Make sure we don't have any overlapping content dirs. That will never work.
-	for i, d1 := range absContentDirs {
-		for j, d2 := range absContentDirs {
-			if i == j {
-				continue
-			}
-			if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) {
-				return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
-			}
+	var options []func(*filesystems.BaseFs) error
+	if baseBaseFs != nil {
+		options = []func(*filesystems.BaseFs) error{
+			filesystems.WithBaseFs(baseBaseFs),
 		}
 	}
-
-	resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir)
-	publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir)
-
-	baseFs := &hugofs.BaseFs{
-		ContentFs:   contentFs,
-		ResourcesFs: resourcesFs,
-		PublishFs:   publishFs,
+	bfs, err := filesystems.NewBase(p, options...)
+	if err != nil {
+		return nil, err
 	}
 
 	ps := &PathSpec{
-		Fs:                             fs,
-		BaseFs:                         baseFs,
-		Cfg:                            cfg,
-		disablePathToLower:             cfg.GetBool("disablePathToLower"),
-		removePathAccents:              cfg.GetBool("removePathAccents"),
-		uglyURLs:                       cfg.GetBool("uglyURLs"),
-		canonifyURLs:                   cfg.GetBool("canonifyURLs"),
-		multilingual:                   cfg.GetBool("multilingual"),
-		Language:                       language,
-		Languages:                      languages,
-		defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"),
-		defaultContentLanguage:         defaultContentLanguage,
-		paginatePath:                   cfg.GetString("paginatePath"),
-		BaseURL:                        baseURL,
-		contentDir:                     contentDir,
-		themesDir:                      cfg.GetString("themesDir"),
-		layoutDir:                      cfg.GetString("layoutDir"),
-		workingDir:                     workingDir,
-		staticDirs:                     staticDirs,
-		absContentDirs:                 absContentDirs,
-		theme:                          cfg.GetString("theme"),
-		ProcessingStats:                NewProcessingStats(lang),
+		Paths:           p,
+		BaseFs:          bfs,
+		Fs:              fs,
+		Cfg:             cfg,
+		ProcessingStats: NewProcessingStats(p.Lang()),
 	}
 
-	if !ps.canonifyURLs {
-		basePath := ps.BaseURL.url.Path
+	if !ps.CanonifyURLs {
+		basePath := ps.BaseURL.Path()
 		if basePath != "" && basePath != "/" {
 			ps.BasePath = basePath
 		}
 	}
 
-	// TODO(bep) remove this, eventually
-	ps.PublishDir = absPuslishDir
-
 	return ps, nil
 }
 
-func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
-
-	if id >= 0 {
-		key = fmt.Sprintf("%s%d", key, id)
-	}
-
-	var out []string
-
-	sd := cfg.Get(key)
-
-	if sds, ok := sd.(string); ok {
-		out = []string{sds}
-	} else if sd != nil {
-		out = cast.ToStringSlice(sd)
-	}
-
-	return out
-}
-
-func createContentFs(fs afero.Fs,
-	workingDir,
-	defaultContentLanguage string,
-	languages Languages) (afero.Fs, []types.KeyValueStr, error) {
-
-	var contentLanguages Languages
-	var contentDirSeen = make(map[string]bool)
-	languageSet := make(map[string]bool)
-
-	// The default content language needs to be first.
-	for _, language := range languages {
-		if language.Lang == defaultContentLanguage {
-			contentLanguages = append(contentLanguages, language)
-			contentDirSeen[language.ContentDir] = true
-		}
-		languageSet[language.Lang] = true
-	}
-
-	for _, language := range languages {
-		if contentDirSeen[language.ContentDir] {
-			continue
-		}
-		if language.ContentDir == "" {
-			language.ContentDir = defaultContentLanguage
-		}
-		contentDirSeen[language.ContentDir] = true
-		contentLanguages = append(contentLanguages, language)
-
-	}
-
-	var absContentDirs []types.KeyValueStr
-
-	fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
-	return fs, absContentDirs, err
-
-}
-
-func createContentOverlayFs(source afero.Fs,
-	workingDir string,
-	languages Languages,
-	languageSet map[string]bool,
-	absContentDirs *[]types.KeyValueStr) (afero.Fs, error) {
-	if len(languages) == 0 {
-		return source, nil
-	}
-
-	language := languages[0]
-
-	contentDir := language.ContentDir
-	if contentDir == "" {
-		panic("missing contentDir")
-	}
-
-	absContentDir := AbsPathify(workingDir, language.ContentDir)
-	if !strings.HasSuffix(absContentDir, FilePathSeparator) {
-		absContentDir += FilePathSeparator
-	}
-
-	// If root, remove the second '/'
-	if absContentDir == "//" {
-		absContentDir = FilePathSeparator
-	}
-
-	if len(absContentDir) < 6 {
-		return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort)
-	}
-
-	*absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir})
-
-	overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
-	if len(languages) == 1 {
-		return overlay, nil
-	}
-
-	base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)
-	if err != nil {
-		return nil, err
-	}
-
-	return hugofs.NewLanguageCompositeFs(base, overlay), nil
-
-}
-
-// RelContentDir tries to create a path relative to the content root from
-// the given filename. The return value is the path and language code.
-func (p *PathSpec) RelContentDir(filename string) (string, string) {
-	for _, dir := range p.absContentDirs {
-		if strings.HasPrefix(filename, dir.Value) {
-			rel := strings.TrimPrefix(filename, dir.Value)
-			return strings.TrimPrefix(rel, FilePathSeparator), dir.Key
-		}
-	}
-	// Either not a content dir or already relative.
-	return filename, ""
-}
-
-// ContentDirs returns all the content dirs (absolute paths).
-func (p *PathSpec) ContentDirs() []types.KeyValueStr {
-	return p.absContentDirs
-}
-
-// PaginatePath returns the configured root path used for paginator pages.
-func (p *PathSpec) PaginatePath() string {
-	return p.paginatePath
-}
-
-// ContentDir returns the configured workingDir.
-func (p *PathSpec) ContentDir() string {
-	return p.contentDir
-}
-
-// WorkingDir returns the configured workingDir.
-func (p *PathSpec) WorkingDir() string {
-	return p.workingDir
-}
-
-// StaticDirs returns the relative static dirs for the current configuration.
-func (p *PathSpec) StaticDirs() []string {
-	return p.staticDirs
-}
-
-// LayoutDir returns the relative layout dir in the current configuration.
-func (p *PathSpec) LayoutDir() string {
-	return p.layoutDir
-}
-
-// Theme returns the theme name if set.
-func (p *PathSpec) Theme() string {
-	return p.theme
-}
-
-// Theme returns the theme relative theme dir.
-func (p *PathSpec) ThemesDir() string {
-	return p.themesDir
-}
-
 // PermalinkForBaseURL creates a permalink from the given link and baseURL.
 func (p *PathSpec) PermalinkForBaseURL(link, baseURL string) string {
 	link = strings.TrimPrefix(link, "/")