Add support for theme composition and inheritance

This commit adds support for theme composition and inheritance in Hugo.

With this, it helps thinking about a theme as a set of ordered components:

```toml
theme = ["my-shortcodes", "base-theme", "hyde"]
```

The theme definition example above in `config.toml` creates a theme with the 3 components with presedence from left to right.

So, Hugo will, for any given file, data entry etc., look first in the project, and then in `my-shortcode`, `base-theme` and lastly `hyde`.

Hugo uses two different algorithms to merge the filesystems, depending on the file type:

* For `i18n` and `data` files, Hugo merges deeply using the translation id and data key inside the files.
* For `static`, `layouts` (templates) and `archetypes` files, these are merged on file level. So the left-most file will be chosen.

The name used in the `theme` definition above must match a folder in `/your-site/themes`, e.g. `/your-site/themes/my-shortcodes`. There are  plans to improve on this and get a URL scheme so this can be resolved automatically.

Also note that a component that is part of a theme can have its own configuration file, e.g. `config.toml`. There are currently some restrictions to what a theme component can configure:

* `params` (global and per language)
* `menu` (global and per language)
* `outputformats` and `mediatypes`

The same rules apply here: The left-most param/menu etc. with the same ID will win. There are some hidden and experimental namespace support in the above, which we will work to improve in the future, but theme authors are encouraged to create their own namespaces to avoid naming conflicts.

A final note: Themes/components can also have a `theme` definition in their `config.toml` and similar, which is the "inheritance" part of this commit's title. This is currently not supported by the Hugo theme site. We will have to wait for some "auto dependency" feature to be implemented for that to happen, but this can be a powerful feature if you want to create your own theme-variant based on others.

Fixes #4460
Fixes #4450

8 files changed, 28 insertions, 405 deletions

diff --git a/source/content_directory_test.go b/source/content_directory_test.go
index ed00af625..7f050e0da 100644
--- a/source/content_directory_test.go
+++ b/source/content_directory_test.go
@@ -20,7 +20,6 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
 
@@ -52,9 +51,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
 	}
 
 	for i, test := range tests {
-
-		v := viper.New()
-		v.Set("contentDir", "content")
+		v := newTestConfig()
 		v.Set("ignoreFiles", test.ignoreFilesRegexpes)
 		fs := hugofs.NewMem(v)
 		ps, err := helpers.NewPathSpec(fs, v)
diff --git a/source/dirs.go b/source/dirs.go
deleted file mode 100644
index 49a849453..000000000
--- a/source/dirs.go
+++ /dev/null
@@ -1,194 +0,0 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package source
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/spf13/afero"
-
-	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugofs"
-	jww "github.com/spf13/jwalterweatherman"
-)
-
-// Dirs holds the source directories for a given build.
-// In case where there are more than one of a kind, the order matters:
-// It will be used to construct a union filesystem, so the right-most directory
-// will "win" on duplicates. Typically, the theme version will be the first.
-type Dirs struct {
-	logger   *jww.Notepad
-	pathSpec *helpers.PathSpec
-
-	staticDirs    []string
-	AbsStaticDirs []string
-
-	Language *helpers.Language
-}
-
-// NewDirs creates a new dirs with the given configuration and filesystem.
-func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, error) {
-	ps, err := helpers.NewPathSpec(fs, cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	var l *helpers.Language
-	if language, ok := cfg.(*helpers.Language); ok {
-		l = language
-	}
-
-	d := &Dirs{Language: l, pathSpec: ps, logger: logger}
-
-	return d, d.init(cfg)
-
-}
-
-func (d *Dirs) init(cfg config.Provider) error {
-
-	var (
-		statics []string
-	)
-
-	if d.pathSpec.Theme() != "" {
-		statics = append(statics, filepath.Join(d.pathSpec.ThemesDir(), d.pathSpec.Theme(), "static"))
-	}
-
-	_, isLanguage := cfg.(*helpers.Language)
-	languages, hasLanguages := cfg.Get("languagesSorted").(helpers.Languages)
-
-	if !isLanguage && !hasLanguages {
-		return errors.New("missing languagesSorted in config")
-	}
-
-	if !isLanguage {
-		// Merge all the static dirs.
-		for _, l := range languages {
-			addend, err := d.staticDirsFor(l)
-			if err != nil {
-				return err
-			}
-
-			statics = append(statics, addend...)
-		}
-	} else {
-		addend, err := d.staticDirsFor(cfg)
-		if err != nil {
-			return err
-		}
-
-		statics = append(statics, addend...)
-	}
-
-	d.staticDirs = removeDuplicatesKeepRight(statics)
-	d.AbsStaticDirs = make([]string, len(d.staticDirs))
-	for i, di := range d.staticDirs {
-		d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator
-	}
-
-	return nil
-}
-
-func (d *Dirs) staticDirsFor(cfg config.Provider) ([]string, error) {
-	var statics []string
-	ps, err := helpers.NewPathSpec(d.pathSpec.Fs, cfg)
-	if err != nil {
-		return statics, err
-	}
-
-	statics = append(statics, ps.StaticDirs()...)
-
-	return statics, nil
-}
-
-// CreateStaticFs will create a union filesystem with the static paths configured.
-// Any missing directories will be logged as warnings.
-func (d *Dirs) CreateStaticFs() (afero.Fs, error) {
-	var (
-		source   = d.pathSpec.Fs.Source
-		absPaths []string
-	)
-
-	for _, staticDir := range d.AbsStaticDirs {
-		if _, err := source.Stat(staticDir); os.IsNotExist(err) {
-			d.logger.WARN.Printf("Unable to find Static Directory: %s", staticDir)
-		} else {
-			absPaths = append(absPaths, staticDir)
-		}
-
-	}
-
-	if len(absPaths) == 0 {
-		return nil, nil
-	}
-
-	return d.createOverlayFs(absPaths), nil
-
-}
-
-// IsStatic returns whether the given filename is located in one of the static
-// source dirs.
-func (d *Dirs) IsStatic(filename string) bool {
-	for _, absPath := range d.AbsStaticDirs {
-		if strings.HasPrefix(filename, absPath) {
-			return true
-		}
-	}
-	return false
-}
-
-// MakeStaticPathRelative creates a relative path from the given filename.
-// It will return an empty string if the filename is not a member of dirs.
-func (d *Dirs) MakeStaticPathRelative(filename string) string {
-	for _, currentPath := range d.AbsStaticDirs {
-		if strings.HasPrefix(filename, currentPath) {
-			return strings.TrimPrefix(filename, currentPath)
-		}
-	}
-
-	return ""
-
-}
-
-func (d *Dirs) createOverlayFs(absPaths []string) afero.Fs {
-	source := d.pathSpec.Fs.Source
-
-	if len(absPaths) == 1 {
-		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
-	}
-
-	base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
-	overlay := d.createOverlayFs(absPaths[1:])
-
-	return afero.NewCopyOnWriteFs(base, overlay)
-}
-
-func removeDuplicatesKeepRight(in []string) []string {
-	seen := make(map[string]bool)
-	var out []string
-	for i := len(in) - 1; i >= 0; i-- {
-		v := in[i]
-		if seen[v] {
-			continue
-		}
-		out = append([]string{v}, out...)
-		seen[v] = true
-	}
-
-	return out
-}
diff --git a/source/dirs_test.go b/source/dirs_test.go
deleted file mode 100644
index 46236120e..000000000
--- a/source/dirs_test.go
+++ /dev/null
@@ -1,185 +0,0 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package source
-
-import (
-	"testing"
-
-	"github.com/gohugoio/hugo/helpers"
-
-	"fmt"
-
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-
-	"github.com/gohugoio/hugo/config"
-	"github.com/spf13/afero"
-
-	jww "github.com/spf13/jwalterweatherman"
-
-	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/viper"
-	"github.com/stretchr/testify/require"
-)
-
-var logger = jww.NewNotepad(jww.LevelInfo, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
-
-func TestStaticDirs(t *testing.T) {
-	assert := require.New(t)
-
-	tests := []struct {
-		setup    func(cfg config.Provider, fs *hugofs.Fs) config.Provider
-		expected []string
-	}{
-
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-			return cfg
-		}, []string{"s1"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", []string{"s2", "s1", "s2"})
-			return cfg
-		}, []string{"s1", "s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("theme", "mytheme")
-			cfg.Set("themesDir", "themes")
-			cfg.Set("staticDir", []string{"s1", "s2"})
-			return cfg
-		}, []string{filepath.FromSlash("themes/mytheme/static"), "s1", "s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir", []string{"l1s1", "l1s2"})
-			return l1
-
-		}, []string{"l1s1", "l1s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
-			return l1
-
-		}, []string{"s1", "l1s1", "l1s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", []string{"s1", "s2"})
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
-			return l1
-
-		}, []string{"s1", "s2", "l1s1", "l1s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
-			l2 := helpers.NewLanguage("nn", cfg)
-			l2.Set("staticDir3", []string{"l2s1", "l2s2"})
-			l2.Set("staticDir", []string{"l2"})
-
-			cfg.Set("languagesSorted", helpers.Languages{l1, l2})
-			return cfg
-
-		}, []string{"s1", "l1s1", "l1s2", "l2", "l2s1", "l2s2"}},
-	}
-
-	for i, test := range tests {
-		msg := fmt.Sprintf("Test %d", i)
-		v := viper.New()
-		v.Set("contentDir", "content")
-
-		fs := hugofs.NewMem(v)
-		cfg := test.setup(v, fs)
-		cfg.Set("workingDir", filepath.FromSlash("/work"))
-		_, isLanguage := cfg.(*helpers.Language)
-		if !isLanguage && !cfg.IsSet("languagesSorted") {
-			cfg.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(cfg)})
-		}
-		dirs, err := NewDirs(fs, cfg, logger)
-		assert.NoError(err)
-		assert.Equal(test.expected, dirs.staticDirs, msg)
-		assert.Len(dirs.AbsStaticDirs, len(dirs.staticDirs))
-
-		for i, d := range dirs.staticDirs {
-			abs := dirs.AbsStaticDirs[i]
-			assert.Equal(filepath.Join("/work", d)+helpers.FilePathSeparator, abs)
-			assert.True(dirs.IsStatic(filepath.Join(abs, "logo.png")))
-			rel := dirs.MakeStaticPathRelative(filepath.Join(abs, "logo.png"))
-			assert.Equal("logo.png", rel)
-		}
-
-		assert.False(dirs.IsStatic(filepath.FromSlash("/some/other/dir/logo.png")))
-
-	}
-
-}
-
-func TestStaticDirsFs(t *testing.T) {
-	assert := require.New(t)
-	v := viper.New()
-	fs := hugofs.NewMem(v)
-	v.Set("workingDir", filepath.FromSlash("/work"))
-	v.Set("theme", "mytheme")
-	v.Set("themesDir", "themes")
-	v.Set("contentDir", "content")
-	v.Set("staticDir", []string{"s1", "s2"})
-	v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)})
-
-	writeToFs(t, fs.Source, "/work/s1/f1.txt", "s1-f1")
-	writeToFs(t, fs.Source, "/work/s2/f2.txt", "s2-f2")
-	writeToFs(t, fs.Source, "/work/s1/f2.txt", "s1-f2")
-	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f1.txt", "theme-f1")
-	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f3.txt", "theme-f3")
-
-	dirs, err := NewDirs(fs, v, logger)
-	assert.NoError(err)
-
-	sfs, err := dirs.CreateStaticFs()
-	assert.NoError(err)
-
-	assert.Equal("s1-f1", readFileFromFs(t, sfs, "f1.txt"))
-	assert.Equal("s2-f2", readFileFromFs(t, sfs, "f2.txt"))
-	assert.Equal("theme-f3", readFileFromFs(t, sfs, "f3.txt"))
-
-}
-
-func TestRemoveDuplicatesKeepRight(t *testing.T) {
-	in := []string{"a", "b", "c", "a"}
-	out := removeDuplicatesKeepRight(in)
-
-	require.Equal(t, []string{"b", "c", "a"}, out)
-}
-
-func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {
-	if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {
-		t.Fatalf("Failed to write file: %s", err)
-	}
-}
-
-func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
-	filename = filepath.FromSlash(filename)
-	b, err := afero.ReadFile(fs, filename)
-	if err != nil {
-		afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
-			fmt.Println("    ", path, " ", info)
-			return nil
-		})
-		t.Fatalf("Failed to read file: %s", err)
-	}
-	return string(b)
-}
diff --git a/source/fileInfo.go b/source/fileInfo.go
index 9adb96df4..31885bfd4 100644
--- a/source/fileInfo.go
+++ b/source/fileInfo.go
@@ -220,7 +220,7 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
 
 // Open implements ReadableFile.
 func (fi *FileInfo) Open() (io.ReadCloser, error) {
-	f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename())
+	f, err := fi.sp.SourceFs.Open(fi.Filename())
 	return f, err
 }
 
diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go
index ec2a17c65..9d3566240 100644
--- a/source/fileInfo_test.go
+++ b/source/fileInfo_test.go
@@ -19,8 +19,6 @@ import (
 
 	"github.com/gohugoio/hugo/helpers"
 
-	"github.com/spf13/viper"
-
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
@@ -72,14 +70,13 @@ func TestFileInfoLanguage(t *testing.T) {
 
 	m := afero.NewMemMapFs()
 	lfs := hugofs.NewLanguageFs("sv", langs, m)
-	v := viper.New()
-	v.Set("contentDir", "content")
+	v := newTestConfig()
 
 	fs := hugofs.NewFrom(m, v)
 
 	ps, err := helpers.NewPathSpec(fs, v)
 	assert.NoError(err)
-	s := SourceSpec{Fs: lfs, PathSpec: ps}
+	s := SourceSpec{SourceFs: lfs, PathSpec: ps}
 	s.Languages = map[string]interface{}{
 		"en": true,
 	}
diff --git a/source/filesystem.go b/source/filesystem.go
index 50075e3c4..3f4bf0ff1 100644
--- a/source/filesystem.go
+++ b/source/filesystem.go
@@ -79,16 +79,13 @@ func (f *Filesystem) captureFiles() {
 		return err
 	}
 
-	if f.Fs == nil {
+	if f.SourceFs == nil {
 		panic("Must have a fs")
 	}
-	err := helpers.SymbolicWalk(f.Fs, f.Base, walker)
+	err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker)
 
 	if err != nil {
 		jww.ERROR.Println(err)
-		if err == helpers.ErrPathTooShort {
-			panic("The root path is too short. If this is a test, make sure to init the content paths.")
-		}
 	}
 
 }
@@ -100,7 +97,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
 			jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
 			return false, nil
 		}
-		linkfi, err := f.Fs.Stat(link)
+		linkfi, err := f.SourceFs.Stat(link)
 		if err != nil {
 			jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
 			return false, nil
diff --git a/source/filesystem_test.go b/source/filesystem_test.go
index 82f02d404..ee86c1487 100644
--- a/source/filesystem_test.go
+++ b/source/filesystem_test.go
@@ -19,7 +19,6 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/helpers"
-
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/viper"
@@ -69,9 +68,19 @@ func TestUnicodeNorm(t *testing.T) {
 
 }
 
-func newTestSourceSpec() SourceSpec {
+func newTestConfig() *viper.Viper {
 	v := viper.New()
 	v.Set("contentDir", "content")
-	ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v)
-	return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps}
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
+	return v
+}
+
+func newTestSourceSpec() *SourceSpec {
+	v := newTestConfig()
+	fs := hugofs.NewMem(v)
+	ps, _ := helpers.NewPathSpec(fs, v)
+	return NewSourceSpec(ps, fs.Source)
 }
diff --git a/source/sourceSpec.go b/source/sourceSpec.go
index 634306e5f..144d86ca3 100644
--- a/source/sourceSpec.go
+++ b/source/sourceSpec.go
@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	"github.com/gohugoio/hugo/langs"
 	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -29,7 +30,7 @@ import (
 type SourceSpec struct {
 	*helpers.PathSpec
 
-	Fs afero.Fs
+	SourceFs afero.Fs
 
 	// This is set if the ignoreFiles config is set.
 	ignoreFilesRe []*regexp.Regexp
@@ -52,7 +53,7 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec {
 	}
 
 	if len(languages) == 0 {
-		l := helpers.NewDefaultLanguage(cfg)
+		l := langs.NewDefaultLanguage(cfg)
 		languages[l.Lang] = l
 		defaultLang = l.Lang
 	}
@@ -71,12 +72,13 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec {
 		}
 	}
 
-	return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
+	return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
+
 }
 
 func (s *SourceSpec) IgnoreFile(filename string) bool {
 	if filename == "" {
-		if _, ok := s.Fs.(*afero.OsFs); ok {
+		if _, ok := s.SourceFs.(*afero.OsFs); ok {
 			return true
 		}
 		return false
@@ -108,7 +110,7 @@ func (s *SourceSpec) IgnoreFile(filename string) bool {
 }
 
 func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
-	fi, err := helpers.LstatIfPossible(s.Fs, filename)
+	fi, err := helpers.LstatIfPossible(s.SourceFs, filename)
 	if err != nil {
 		return false, err
 	}
@@ -119,7 +121,7 @@ func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
 
 	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
 		link, err := filepath.EvalSymlinks(filename)
-		fi, err = helpers.LstatIfPossible(s.Fs, link)
+		fi, err = helpers.LstatIfPossible(s.SourceFs, link)
 		if err != nil {
 			return false, err
 		}