From d392893cd73dc00c927f342778f6dca9628d328e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 9 Jun 2021 10:58:18 +0200 Subject: Misc config loading fixes The main motivation behind this is simplicity and correctnes, but the new small config library is also faster: ``` BenchmarkDefaultConfigProvider/Viper-16 252418 4546 ns/op 2720 B/op 30 allocs/op BenchmarkDefaultConfigProvider/Custom-16 450756 2651 ns/op 1008 B/op 6 allocs/op ``` Fixes #8633 Fixes #8618 Fixes #8630 Updates #8591 Closes #6680 Closes #5192 --- cache/filecache/filecache_config.go | 7 +- cache/filecache/filecache_config_test.go | 5 +- commands/commandeer.go | 2 - commands/commands_test.go | 9 +- commands/config.go | 5 +- commands/new_site.go | 4 +- commands/server_test.go | 4 +- common/maps/maps.go | 77 ++- common/maps/maps_test.go | 2 +- common/maps/params.go | 148 +++++ common/maps/params_test.go | 87 +++ config/commonConfig_test.go | 4 +- config/compositeConfig.go | 113 ++++ config/compositeConfig_test.go | 40 ++ config/configLoader.go | 23 +- config/configProvider.go | 5 + config/configProvider_test.go | 3 +- config/defaultConfigProvider.go | 372 +++++++++++ config/defaultConfigProvider_test.go | 315 ++++++++++ config/docshelper.go | 45 ++ config/privacy/privacyConfig_test.go | 3 +- config/services/servicesConfig_test.go | 4 +- create/content_test.go | 5 +- deploy/deployConfig_test.go | 4 +- docs/config.toml | 2 +- docs/content/en/getting-started/configuration.md | 20 + docs/data/docs.json | 59 ++ docs/layouts/shortcodes/code-toggle.html | 59 +- go.mod | 2 +- go.sum | 1 + helpers/content_test.go | 5 +- helpers/general_test.go | 4 +- helpers/path_test.go | 4 - helpers/testhelpers_test.go | 21 +- hugofs/fs_test.go | 9 +- hugofs/rootmapping_fs_test.go | 4 +- hugolib/config.go | 680 ++++++++++----------- hugolib/config_test.go | 372 ++++++----- hugolib/filesystems/basefs_test.go | 10 +- hugolib/hugo_modules_test.go | 30 +- hugolib/hugo_sites.go | 15 +- hugolib/hugo_sites_build_errors_test.go | 4 +- hugolib/image_test.go | 4 +- hugolib/js_test.go | 11 +- hugolib/minify_publisher_test.go | 4 +- hugolib/page__meta.go | 2 +- hugolib/page_test.go | 5 +- hugolib/pagebundler_test.go | 18 +- hugolib/pages_capture.go | 2 +- hugolib/paths/paths_test.go | 7 +- hugolib/resource_chain_babel_test.go | 6 +- hugolib/resource_chain_test.go | 18 +- hugolib/robotstxt_test.go | 4 +- hugolib/shortcode_test.go | 7 +- hugolib/site.go | 22 +- hugolib/site_output_test.go | 12 +- hugolib/site_test.go | 5 +- hugolib/template_test.go | 5 +- hugolib/testhelpers_test.go | 26 +- langs/config.go | 8 +- langs/i18n/i18n_test.go | 7 +- langs/language.go | 84 +-- langs/language_test.go | 7 +- markup/asciidocext/convert_test.go | 18 +- markup/blackfriday/convert_test.go | 6 +- markup/highlight/config_test.go | 5 +- markup/markup_config/config.go | 3 +- markup/markup_config/config_test.go | 6 +- markup/markup_test.go | 8 +- markup/mmark/convert_test.go | 4 +- markup/org/convert_test.go | 5 +- media/mediaType.go | 4 +- minifiers/config.go | 4 +- minifiers/config_test.go | 6 +- minifiers/minifiers_test.go | 15 +- modules/collect.go | 2 +- modules/npm/package_builder.go | 8 +- output/outputFormat.go | 6 +- publisher/htmlElementsCollector_test.go | 5 +- related/inverted_index.go | 11 +- resources/page/pagemeta/page_frontmatter_test.go | 13 +- resources/page/pagination_test.go | 4 +- resources/page/testhelpers_test.go | 4 +- resources/resource_metadata.go | 2 +- .../resource_transformers/htesting/testhelpers.go | 4 +- resources/resource_transformers/js/options.go | 7 +- resources/testhelpers_test.go | 6 +- source/filesystem_test.go | 8 +- tpl/cast/docshelper.go | 8 +- tpl/collections/collections_test.go | 4 +- tpl/data/init_test.go | 4 +- tpl/data/resources_test.go | 8 +- tpl/encoding/encoding.go | 3 +- tpl/hugo/init_test.go | 5 +- tpl/images/images_test.go | 4 +- tpl/os/os_test.go | 9 +- tpl/path/path_test.go | 4 +- tpl/resources/resources.go | 2 +- tpl/site/init_test.go | 5 +- tpl/strings/init_test.go | 5 +- tpl/strings/strings_test.go | 4 +- tpl/tplimpl/template_funcs_test.go | 6 +- tpl/transform/remarshal_test.go | 10 +- tpl/transform/transform_test.go | 16 +- tpl/transform/unmarshal_test.go | 9 +- tpl/urls/init_test.go | 5 +- tpl/urls/urls_test.go | 5 +- 107 files changed, 2115 insertions(+), 1016 deletions(-) create mode 100644 config/compositeConfig.go create mode 100644 config/compositeConfig_test.go create mode 100644 config/defaultConfigProvider.go create mode 100644 config/defaultConfigProvider_test.go create mode 100644 config/docshelper.go diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go index 0c6b569c1..801799e36 100644 --- a/cache/filecache/filecache_config.go +++ b/cache/filecache/filecache_config.go @@ -19,6 +19,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" @@ -123,6 +125,9 @@ func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { _, isOsFs := fs.(*afero.OsFs) for k, v := range m { + if _, ok := v.(maps.Params); !ok { + continue + } cc := defaultCacheConfig dc := &mapstructure.DecoderConfig{ @@ -137,7 +142,7 @@ func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { } if err := decoder.Decode(v); err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to decode filecache config") } if cc.Dir == "" { diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go index cd1d2c82a..acc127e67 100644 --- a/cache/filecache/filecache_config_test.go +++ b/cache/filecache/filecache_config_test.go @@ -25,7 +25,6 @@ import ( "github.com/gohugoio/hugo/config" qt "github.com/frankban/quicktest" - "github.com/spf13/viper" ) func TestDecodeConfig(t *testing.T) { @@ -178,8 +177,8 @@ dir = "/" c.Assert(err, qt.Not(qt.IsNil)) } -func newTestConfig() *viper.Viper { - cfg := viper.New() +func newTestConfig() config.Provider { + cfg := config.New() cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject")) cfg.Set("contentDir", "content") cfg.Set("dataDir", "data") diff --git a/commands/commandeer.go b/commands/commandeer.go index 024651aad..06e27cf18 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -410,7 +410,5 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { } config.Set("cacheDir", cacheDir) - cfg.Logger.Infoln("Using config file:", config.ConfigFileUsed()) - return nil } diff --git a/commands/commands_test.go b/commands/commands_test.go index a0eeb157b..99ffea48c 100644 --- a/commands/commands_test.go +++ b/commands/commands_test.go @@ -20,6 +20,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/htesting" "github.com/spf13/afero" @@ -29,7 +31,6 @@ import ( "github.com/gohugoio/hugo/common/types" "github.com/spf13/cobra" - "github.com/spf13/viper" qt "github.com/frankban/quicktest" ) @@ -166,7 +167,7 @@ func TestFlags(t *testing.T) { name: "ignoreVendor as bool", args: []string{"server", "--ignoreVendor"}, check: func(c *qt.C, cmd *serverCmd) { - cfg := viper.New() + cfg := config.New() cmd.flagsToConfig(cfg) c.Assert(cfg.Get("ignoreVendor"), qt.Equals, true) }, @@ -176,7 +177,7 @@ func TestFlags(t *testing.T) { name: "ignoreVendorPaths", args: []string{"server", "--ignoreVendorPaths=github.com/**"}, check: func(c *qt.C, cmd *serverCmd) { - cfg := viper.New() + cfg := config.New() cmd.flagsToConfig(cfg) c.Assert(cfg.Get("ignoreVendorPaths"), qt.Equals, "github.com/**") }, @@ -216,7 +217,7 @@ func TestFlags(t *testing.T) { c.Assert(sc.serverPort, qt.Equals, 1366) c.Assert(sc.environment, qt.Equals, "testing") - cfg := viper.New() + cfg := config.New() sc.flagsToConfig(cfg) c.Assert(cfg.GetString("publishDir"), qt.Equals, "/tmp/mydestination") c.Assert(cfg.GetString("contentDir"), qt.Equals, "mycontent") diff --git a/commands/config.go b/commands/config.go index 37bf45e3c..7ab429308 100644 --- a/commands/config.go +++ b/commands/config.go @@ -22,13 +22,14 @@ import ( "sort" "strings" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/modules" "github.com/spf13/cobra" - "github.com/spf13/viper" ) var _ cmder = (*configCmd)(nil) @@ -81,7 +82,7 @@ func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error { return err } - allSettings := cfg.Cfg.(*viper.Viper).AllSettings() + allSettings := cfg.Cfg.Get("").(maps.Params) // We need to clean up this, but we store objects in the config that // isn't really interesting to the end user, so filter these. diff --git a/commands/new_site.go b/commands/new_site.go index 6fac2c22c..71097b8ff 100644 --- a/commands/new_site.go +++ b/commands/new_site.go @@ -19,6 +19,7 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/parser/metadecoders" _errors "github.com/pkg/errors" @@ -29,7 +30,6 @@ import ( "github.com/gohugoio/hugo/parser" "github.com/spf13/cobra" jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/viper" ) var _ cmder = (*newSiteCmd)(nil) @@ -123,7 +123,7 @@ func (n *newSiteCmd) newSite(cmd *cobra.Command, args []string) error { forceNew, _ := cmd.Flags().GetBool("force") - return n.doNewSite(hugofs.NewDefault(viper.New()), createpath, forceNew) + return n.doNewSite(hugofs.NewDefault(config.New()), createpath, forceNew) } func createConfig(fs *hugofs.Fs, inpath string, kind string) (err error) { diff --git a/commands/server_test.go b/commands/server_test.go index cc2a6d92c..05d21a516 100644 --- a/commands/server_test.go +++ b/commands/server_test.go @@ -22,10 +22,10 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" qt "github.com/frankban/quicktest" - "github.com/spf13/viper" ) func TestServer(t *testing.T) { @@ -101,7 +101,7 @@ func TestFixURL(t *testing.T) { t.Run(test.TestName, func(t *testing.T) { b := newCommandsBuilder() s := b.newServerCmd() - v := viper.New() + v := config.New() baseURL := test.CLIBaseURL v.Set("baseURL", test.CfgBaseURL) s.serverAppend = test.AppendPort diff --git a/common/maps/maps.go b/common/maps/maps.go index 41d9b6e15..5fb079009 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -18,53 +18,65 @@ import ( "strings" "github.com/gobwas/glob" - "github.com/spf13/cast" ) -// ToLower makes all the keys in the given map lower cased and will do so -// recursively. -// Notes: -// * This will modify the map given. -// * Any nested map[interface{}]interface{} will be converted to Params. -func ToLower(m Params) { - for k, v := range m { - var retyped bool - switch v.(type) { - case map[interface{}]interface{}: - var p Params = cast.ToStringMap(v) - v = p - ToLower(p) - retyped = true - case map[string]interface{}: - var p Params = v.(map[string]interface{}) - v = p - ToLower(p) - retyped = true +// ToStringMapE converts in to map[string]interface{}. +func ToStringMapE(in interface{}) (map[string]interface{}, error) { + switch vv := in.(type) { + case Params: + return vv, nil + case map[string]string: + var m = map[string]interface{}{} + for k, v := range vv { + m[k] = v } + return m, nil - lKey := strings.ToLower(k) - if retyped || k != lKey { - delete(m, k) - m[lKey] = v - } + default: + return cast.ToStringMapE(in) } } -func ToStringMapE(in interface{}) (map[string]interface{}, error) { - switch in.(type) { - case Params: - return in.(Params), nil - default: - return cast.ToStringMapE(in) +// ToParamsAndPrepare converts in to Params and prepares it for use. +// See PrepareParams. +func ToParamsAndPrepare(in interface{}) (Params, bool) { + m, err := ToStringMapE(in) + if err != nil { + return nil, false } + PrepareParams(m) + return m, true } +// ToStringMap converts in to map[string]interface{}. func ToStringMap(in interface{}) map[string]interface{} { m, _ := ToStringMapE(in) return m } +// ToStringMapStringE converts in to map[string]string. +func ToStringMapStringE(in interface{}) (map[string]string, error) { + m, err := ToStringMapE(in) + if err != nil { + return nil, err + } + return cast.ToStringMapStringE(m) +} + +// ToStringMapString converts in to map[string]string. +func ToStringMapString(in interface{}) map[string]string { + m, _ := ToStringMapStringE(in) + return m +} + +// ToStringMapBool converts in to bool. +func ToStringMapBool(in interface{}) map[string]bool { + m, _ := ToStringMapE(in) + return cast.ToStringMapBool(m) +} + +// ToSliceStringMap converts in to []map[string]interface{}. func ToSliceStringMap(in interface{}) ([]map[string]interface{}, error) { switch v := in.(type) { case []map[string]interface{}: @@ -127,9 +139,8 @@ func (KeyRenamer) keyPath(k1, k2 string) string { k1, k2 = strings.ToLower(k1), strings.ToLower(k2) if k1 == "" { return k2 - } else { - return k1 + "/" + k2 } + return k1 + "/" + k2 } func (r KeyRenamer) renamePath(parentKeyPath string, m map[string]interface{}) { diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go index 7e527aac5..dbe97a15a 100644 --- a/common/maps/maps_test.go +++ b/common/maps/maps_test.go @@ -67,7 +67,7 @@ func TestToLower(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprint(i), func(t *testing.T) { // ToLower modifies input. - ToLower(test.input) + PrepareParams(test.input) if !reflect.DeepEqual(test.expected, test.input) { t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input) } diff --git a/common/maps/params.go b/common/maps/params.go index 4c881093c..7e94d593b 100644 --- a/common/maps/params.go +++ b/common/maps/params.go @@ -14,6 +14,7 @@ package maps import ( + "fmt" "strings" "github.com/spf13/cast" @@ -29,6 +30,95 @@ func (p Params) Get(indices ...string) interface{} { return v } +// Set overwrites values in p with values in pp for common or new keys. +// This is done recursively. +func (p Params) Set(pp Params) { + for k, v := range pp { + vv, found := p[k] + if !found { + p[k] = v + } else { + switch vvv := vv.(type) { + case Params: + if pv, ok := v.(Params); ok { + vvv.Set(pv) + } else { + p[k] = v + } + default: + p[k] = v + } + } + } +} + +// Merge transfers values from pp to p for new keys. +// This is done recursively. +func (p Params) Merge(pp Params) { + p.merge("", pp) +} + +func (p Params) merge(ps ParamsMergeStrategy, pp Params) { + ns, found := p.GetMergeStrategy() + + var ms = ns + if !found && ps != "" { + ms = ps + } + + noUpdate := ms == ParamsMergeStrategyNone + noUpdate = noUpdate || (ps != "" && ps == ParamsMergeStrategyShallow) + + for k, v := range pp { + + if k == mergeStrategyKey { + continue + } + vv, found := p[k] + + if found { + // Key matches, if both sides are Params, we try to merge. + if vvv, ok := vv.(Params); ok { + if pv, ok := v.(Params); ok { + vvv.merge(ms, pv) + } + + } + + } else if !noUpdate { + p[k] = v + + } + + } +} + +func (p Params) GetMergeStrategy() (ParamsMergeStrategy, bool) { + if v, found := p[mergeStrategyKey]; found { + if s, ok := v.(ParamsMergeStrategy); ok { + return s, true + } + } + return ParamsMergeStrategyShallow, false +} + +func (p Params) DeleteMergeStrategy() bool { + if _, found := p[mergeStrategyKey]; found { + delete(p, mergeStrategyKey) + return true + } + return false +} + +func (p Params) SetDefaultMergeStrategy(s ParamsMergeStrategy) { + switch s { + case ParamsMergeStrategyDeep, ParamsMergeStrategyNone, ParamsMergeStrategyShallow: + default: + panic(fmt.Sprintf("invalid merge strategy %q", s)) + } + p[mergeStrategyKey] = s +} + func getNested(m map[string]interface{}, indices []string) (interface{}, string, map[string]interface{}) { if len(indices) == 0 { return nil, "", nil @@ -108,3 +198,61 @@ func GetNestedParamFn(keyStr, separator string, lookupFn func(key string) interf return nil, "", nil, nil } + +// ParamsMergeStrategy tells what strategy to use in Params.Merge. +type ParamsMergeStrategy string + +const ( + // Do not merge. + ParamsMergeStrategyNone ParamsMergeStrategy = "none" + // Only add new keys. + ParamsMergeStrategyShallow ParamsMergeStrategy = "shallow" + // Add new keys, merge existing. + ParamsMergeStrategyDeep ParamsMergeStrategy = "deep" + + mergeStrategyKey = "_merge" +) + +func toMergeStrategy(v interface{}) ParamsMergeStrategy { + s := ParamsMergeStrategy(cast.ToString(v)) + switch s { + case ParamsMergeStrategyDeep, ParamsMergeStrategyNone, ParamsMergeStrategyShallow: + return s + default: + return ParamsMergeStrategyDeep + } +} + +// PrepareParams +// * makes all the keys in the given map lower cased and will do so +// * This will modify the map given. +// * Any nested map[interface{}]interface{} will be converted to Params. +// * Any _merge value will be converted to proper type and value. +func PrepareParams(m Params) { + for k, v := range m { + var retyped bool + lKey := strings.ToLower(k) + if lKey == mergeStrategyKey { + v = toMergeStrategy(v) + retyped = true + } else { + switch v.(type) { + case map[interface{}]interface{}: + var p Params = cast.ToStringMap(v) + v = p + PrepareParams(p) + retyped = true + case map[string]interface{}: + var p Params = v.(map[string]interface{}) + v = p + PrepareParams(p) + retyped = true + } + } + + if retyped || k != lKey { + delete(m, k) + m[lKey] = v + } + } +} diff --git a/common/maps/params_test.go b/common/maps/params_test.go index df8cbf8d6..8859bb86b 100644 --- a/common/maps/params_test.go +++ b/common/maps/params_test.go @@ -69,3 +69,90 @@ func TestGetNestedParamFnNestedNewKey(t *testing.T) { c.Assert(nestedKey, qt.Equals, "new") c.Assert(owner, qt.DeepEquals, nested) } + +func TestParamsSetAndMerge(t *testing.T) { + c := qt.New(t) + + createParamsPair := func() (Params, Params) { + p1 := Params{"a": "av", "c": "cv", "nested": Params{"al2": "al2v", "cl2": "cl2v"}} + p2 := Params{"b": "bv", "a": "abv", "nested": Params{"bl2": "bl2v", "al2": "al2bv"}, mergeStrategyKey: ParamsMergeStrategyDeep} + return p1, p2 + } + + p1, p2 := createParamsPair() + + p1.Set(p2) + + c.Assert(p1, qt.DeepEquals, Params{ + "a": "abv", + "c": "cv", + "nested": Params{ + "al2": "al2bv", + "cl2": "cl2v", + "bl2": "bl2v", + }, + "b": "bv", + mergeStrategyKey: ParamsMergeStrategyDeep, + }) + + p1, p2 = createParamsPair() + + p1.Merge(p2) + + // Default is to do a shallow merge. + c.Assert(p1, qt.DeepEquals, Params{ + "c": "cv", + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + }, + "b": "bv", + "a": "av", + }) + + p1, p2 = createParamsPair() + p1.SetDefaultMergeStrategy(ParamsMergeStrategyNone) + p1.Merge(p2) + p1.DeleteMergeStrategy() + + c.Assert(p1, qt.DeepEquals, Params{ + "a": "av", + "c": "cv", + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + }, + }) + + p1, p2 = createParamsPair() + p1.SetDefaultMergeStrategy(ParamsMergeStrategyShallow) + p1.Merge(p2) + p1.DeleteMergeStrategy() + + c.Assert(p1, qt.DeepEquals, Params{ + "a": "av", + "c": "cv", + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + }, + "b": "bv", + }) + + p1, p2 = createParamsPair() + p1.SetDefaultMergeStrategy(ParamsMergeStrategyDeep) + p1.Merge(p2) + p1.DeleteMergeStrategy() + + c.Assert(p1, qt.DeepEquals, Params{ + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + "bl2": "bl2v", + }, + "b": "bv", + "a": "av", + "c": "cv", + }) + +} diff --git a/config/commonConfig_test.go b/config/commonConfig_test.go index d4273277a..55767913f 100644 --- a/config/commonConfig_test.go +++ b/config/commonConfig_test.go @@ -21,14 +21,12 @@ import ( "github.com/gohugoio/hugo/common/types" qt "github.com/frankban/quicktest" - - "github.com/spf13/viper" ) func TestBuild(t *testing.T) { c := qt.New(t) - v := viper.New() + v := New() v.Set("build", map[string]interface{}{ "useResourceCacheWhen": "always", }) diff --git a/config/compositeConfig.go b/config/compositeConfig.go new file mode 100644 index 000000000..c68419533 --- /dev/null +++ b/config/compositeConfig.go @@ -0,0 +1,113 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/gohugoio/hugo/common/maps" +) + +// NewCompositeConfig creates a new composite Provider with a read-only base +// and a writeable layer. +func NewCompositeConfig(base, layer Provider) Provider { + return &compositeConfig{ + base: base, + layer: layer, + } +} + +// compositeConfig contains a read only config base with +// a possibly writeable config layer on top. +type compositeConfig struct { + base Provider + layer Provider +} + +func (c *compositeConfig) GetBool(key string) bool { + if c.layer.IsSet(key) { + return c.layer.GetBool(key) + } + return c.base.GetBool(key) +} + +func (c *compositeConfig) GetInt(key string) int { + if c.layer.IsSet(key) { + return c.layer.GetInt(key) + } + return c.base.GetInt(key) +} + +func (c *compositeConfig) Merge(key string, value interface{}) { + c.layer.Merge(key, value) +} + +func (c *compositeConfig) GetParams(key string) maps.Params { + if c.layer.IsSet(key) { + return c.layer.GetParams(key) + } + return c.base.GetParams(key) +} + +func (c *compositeConfig) GetStringMap(key string) map[string]interface{} { + if c.layer.IsSet(key) { + return c.layer.GetStringMap(key) + } + return c.base.GetStringMap(key) +} + +func (c *compositeConfig) GetStringMapString(key string) map[string]string { + if c.layer.IsSet(key) { + return c.layer.GetStringMapString(key) + } + return c.base.GetStringMapString(key) +} + +func (c *compositeConfig) GetStringSlice(key string) []string { + if c.layer.IsSet(key) { + return c.layer.GetStringSlice(key) + } + return c.base.GetStringSlice(key) +} + +func (c *compositeConfig) Get(key string) interface{} { + if c.layer.IsSet(key) { + return c.layer.Get(key) + } + return c.base.Get(key) +} + +func (c *compositeConfig) IsSet(key string) bool { + if c.layer.IsSet(key) { + return true + } + return c.base.IsSet(key) +} + +func (c *compositeConfig) GetString(key string) string { + if c.layer.IsSet(key) { + return c.layer.GetString(key) + } + return c.base.GetString(key) +} + +func (c *compositeConfig) Set(key string, value interface{}) { + c.layer.Set(key, value) +} + +func (c *compositeConfig) WalkParams(walkFn func(params ...KeyParams) bool) { + panic("not supported") +} + +func (c *compositeConfig) SetDefaultMergeStrategy() { + panic("not supported") +} diff --git a/config/compositeConfig_test.go b/config/compositeConfig_test.go new file mode 100644 index 000000000..60644102f --- /dev/null +++ b/config/compositeConfig_test.go @@ -0,0 +1,40 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestCompositeConfig(t *testing.T) { + c := qt.New(t) + + c.Run("Set and get", func(c *qt.C) { + base, layer := New(), New() + cfg := NewCompositeConfig(base, layer) + + layer.Set("a1", "av") + base.Set("b1", "bv") + cfg.Set("c1", "cv") + + c.Assert(cfg.Get("a1"), qt.Equals, "av") + c.Assert(cfg.Get("b1"), qt.Equals, "bv") + c.Assert(cfg.Get("c1"), qt.Equals, "cv") + c.Assert(cfg.IsSet("c1"), qt.IsTrue) + c.Assert(layer.IsSet("c1"), qt.IsTrue) + c.Assert(base.IsSet("c1"), qt.IsFalse) + }) +} diff --git a/config/configLoader.go b/config/configLoader.go index 6d94f0b79..0998b1bef 100644 --- a/config/configLoader.go +++ b/config/configLoader.go @@ -20,7 +20,6 @@ import ( "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/afero" - "github.com/spf13/viper" ) var ( @@ -43,15 +42,11 @@ func IsValidConfigFilename(filename string) bool { // FromConfigString creates a config from the given YAML, JSON or TOML config. This is useful in tests. func FromConfigString(config, configType string) (Provider, error) { - v := newViper() m, err := readConfig(metadecoders.FormatFromString(configType), []byte(config)) if err != nil { return nil, err } - - v.MergeConfigMap(m) - - return v, nil + return NewFrom(m), nil } // FromFile loads the configuration from the given filename. @@ -60,15 +55,7 @@ func FromFile(fs afero.Fs, filename string) (Provider, error) { if err != nil { return nil, err } - - v := newViper() - - err = v.MergeConfigMap(m) - if err != nil { - return nil, err - } - - return v, nil + return NewFrom(m), nil } // FromFileToMap is the same as FromFile, but it returns the config values @@ -116,9 +103,3 @@ func init() { func RenameKeys(m map[string]interface{}) { keyAliases.Rename(m) } - -func newViper() *viper.Viper { - v := viper.New() - - return v -} diff --git a/config/configProvider.go b/config/configProvider.go index 928bf948a..92206ca9e 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -14,6 +14,7 @@ package config import ( + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" ) @@ -22,11 +23,15 @@ type Provider interface { GetString(key string) string GetInt(key string) int GetBool(key string) bool + GetParams(key string) maps.Params GetStringMap(key string) map[string]interface{} GetStringMapString(key string) map[string]string GetStringSlice(key string) []string Get(key string) interface{} Set(key string, value interface{}) + Merge(key string, value interface{}) + SetDefaultMergeStrategy() + WalkParams(walkFn func(params ...KeyParams) bool) IsSet(key string) bool } diff --git a/config/configProvider_test.go b/config/configProvider_test.go index d9fff56b6..0afba1e58 100644 --- a/config/configProvider_test.go +++ b/config/configProvider_test.go @@ -17,12 +17,11 @@ import ( "testing" qt "github.com/frankban/quicktest" - "github.com/spf13/viper" ) func TestGetStringSlicePreserveString(t *testing.T) { c := qt.New(t) - cfg := viper.New() + cfg := New() s := "This is a string" sSlice := []string{"This", "is", "a", "slice"} diff --git a/config/defaultConfigProvider.go b/config/defaultConfigProvider.go new file mode 100644 index 000000000..d9c9db7f1 --- /dev/null +++ b/config/defaultConfigProvider.go @@ -0,0 +1,372 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "sort" + "strings" + "sync" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/common/maps" +) + +var ( + + // ConfigRootKeysSet contains all of the config map root keys. + // TODO(bep) use this for something (docs etc.) + ConfigRootKeysSet = map[string]bool{ + "build": true, + "caches": true, + "frontmatter": true, + "languages": true, + "imaging": true, + "markup": true, + "mediatypes": true, + "menus": true, + "minify": true, + "module": true, + "outputformats": true, + "params": true, + "permalinks": true, + "related": true, + "sitemap": true, + "taxonomies": true, + } + + // ConfigRootKeys is a sorted version of ConfigRootKeysSet. + ConfigRootKeys []string +) + +func init() { + for k := range ConfigRootKeysSet { + ConfigRootKeys = append(ConfigRootKeys, k) + } + sort.Strings(ConfigRootKeys) +} + +// New creates a Provider backed by an empty maps.Params. +func New() Provider { + return &defaultConfigProvider{ + root: make(maps.Params), + } +} + +// NewFrom creates a Provider backed by params. +func NewFrom(params maps.Params) Provider { + maps.PrepareParams(params) + return &defaultConfigProvider{ + root: params, + } +} + +// defaultConfigProvider is a Provider backed by a map where all keys are lower case. +// All methods are thread safe. +type defaultConfigProvider struct { + mu sync.RWMutex + root maps.Params + + keyCache sync.Map +} + +func (c *defaultConfigProvider) Get(k string) interface{} { + if k == "" { + return c.root + } + c.mu.RLock() + key, m := c.getNestedKeyAndMap(strings.ToLower(k), false) + if m == nil { + return nil + } + v := m[key] + c.mu.RUnlock() + return v +} + +func (c *defaultConfigProvider) GetBool(k string) bool { + v := c.Get(k) + return cast.ToBool(v) +} + +func (c *defaultConfigProvider) GetInt(k string) int { + v := c.Get(k) + return cast.ToInt(v) +} + +func (c *defaultConfigProvider) IsSet(k string) bool { + var found bool + c.mu.RLock() + key, m := c.getNestedKeyAndMap(strings.ToLower(k), false) + if m != nil { + _, found = m[key] + } + c.mu.RUnlock() + return found +} + +func (c *defaultConfigProvider) GetString(k string) string { + v := c.Get(k) + return cast.ToString(v) +} + +func (c *defaultConfigProvider) GetParams(k string) maps.Params { + v := c.Get(k) + if v == nil { + return nil + } + return v.(maps.Params) +} + +func (c *defaultConfigProvider) GetStringMap(k string) map[string]interface{} { + v := c.Get(k) + return maps.ToStringMap(v) +} + +func (c *defaultConfigProvider) GetStringMapString(k string) map[string]string { + v := c.Get(k) + return maps.ToStringMapString(v) +} + +func (c *defaultConfigProvider) GetStringSlice(k string) []string { + v := c.Get(k) + return cast.ToStringSlice(v) +} + +func (c *defaultConfigProvider) Set(k string, v interface{}) { + c.mu.Lock() + defer c.mu.Unlock() + + k = strings.ToLower(k) + + if k == "" { + if p, ok := maps.ToParamsAndPrepare(v); ok { + // Set the values directly in root. + c.root.Set(p) + } else { + c.root[k] = v + } + + return + } + + switch vv := v.(type) { + case map[string]interface{}: + var p maps.Params = vv + v = p + maps.PrepareParams(p) + } + + key, m := c.getNestedKeyAndMap(k, true) + + if existing, found := m[key]; found { + if p1, ok := existing.(maps.Params); ok { + if p2, ok := v.(maps.Params); ok { + p1.Set(p2) + return + } + } + } + + m[key] = v +} + +func (c *defaultConfigProvider) Merge(k string, v interface{}) { + c.mu.Lock() + defer c.mu.Unlock() + k = strings.ToLower(k) + + if k == "" { + rs, f := c.root.GetMergeStrategy() + if f && rs == maps.ParamsMergeStrategyNone { + // The user has set a "no merge" strategy on this, + // nothing more to do. + return + } + + if p, ok := maps.ToParamsAndPrepare(v); ok { + // As there may be keys in p not in root, we need to handle + // those as a special case. + for kk, vv := range p { + if pp, ok := vv.(maps.Params); ok { + if ppp, ok := c.root[kk]; ok { + ppp.(maps.Params).Merge(pp) + } else { + // We need to use the default merge strategy for + // this key. + np := make(maps.Params) + strategy := c.determineMergeStrategy(KeyParams{Key: "", Params: c.root}, KeyParams{Key: kk, Params: np}) + np.SetDefaultMergeStrategy(strategy) + np.Merge(pp) + if len(np) > 0 { + c.root[kk] = np + } + } + } + } + // Merge the rest. + c.root.Merge(p) + } else { + panic(fmt.Sprintf("unsupported type %T received in Merge", v)) + } + + return + } + + switch vv := v.(type) { + case map[string]interface{}: + var p maps.Params = vv + v = p + maps.PrepareParams(p) + } + + key, m := c.getNestedKeyAndMap(k, true) + + if existing, found := m[key]; found { + if p1, ok := existing.(maps.Params); ok { + if p2, ok := v.(maps.Params); ok { + p1.Merge(p2) + } + } + } else { + m[key] = v + } +} + +func (c *defaultConfigProvider) WalkParams(walkFn func(params ...KeyParams) bool) { + var walk func(params ...KeyParams) + walk = func(params ...KeyParams) { + if walkFn(params...) { + return + } + p1 := params[len(params)-1] + i := len(params) + for k, v := range p1.Params { + if p2, ok := v.(maps.Params); ok { + paramsplus1 := make([]KeyParams, i+1) + copy(paramsplus1, params) + paramsplus1[i] = KeyParams{Key: k, Params: p2} + walk(paramsplus1...) + } + } + } + walk(KeyParams{Key: "", Params: c.root}) +} + +func (c *defaultConfigProvider) determineMergeStrategy(params ...KeyParams) maps.ParamsMergeStrategy { + if len(params) == 0 { + return maps.ParamsMergeStrategyNone + } + + var ( + strategy maps.ParamsMergeStrategy + prevIsRoot bool + curr = params[len(params)-1] + ) + + if len(params) > 1 { + prev := params[len(params)-2] + prevIsRoot = prev.Key == "" + + // Inherit from parent (but not from the root unless it's set by user). + s, found := prev.Params.GetMergeStrategy() + if !prevIsRoot && !found { + panic("invalid state, merge strategy not set on parent") + } + if found || !prevIsRoot { + strategy = s + } + } + + switch curr.Key { + case "": + // Don't set a merge strategy on the root unless set by user. + // This will be handled as a special case. + case "params": + strategy = maps.ParamsMergeStrategyDeep + case "outputformats", "mediatypes": + if prevIsRoot { + strategy = maps.ParamsMergeStrategyShallow + } + case "menus": + isMenuKey := prevIsRoot + if !isMenuKey { + // Can also be set below languages. + // root > languages > en > menus + if len(params) == 4 && params[1].Key == "languages" { + isMenuKey = true + } + } + if isMenuKey { + strategy = maps.ParamsMergeStrategyShallow + } + default: + if strategy == "" { + strategy = maps.ParamsMergeStrategyNone + } + } + + return strategy +} + +type KeyParams struct { + Key string + Params maps.Params +} + +func (c *defaultConfigProvider) SetDefaultMergeStrategy() { + c.WalkParams(func(params ...KeyParams) bool { + if len(params) == 0 { + return false + } + p := params[len(params)-1].Params + var found bool + if _, found = p.GetMergeStrategy(); found { + // Set by user. + return false + } + strategy := c.determineMergeStrategy(params...) + if strategy != "" { + p.SetDefaultMergeStrategy(strategy) + } + return false + }) + +} + +func (c *defaultConfigProvider) getNestedKeyAndMap(key string, create bool) (string, maps.Params) { + var parts []string + v, ok := c.keyCache.Load(key) + if ok { + parts = v.([]string) + } else { + parts = strings.Split(key, ".") + c.keyCache.Store(key, parts) + } + current := c.root + for i := 0; i < len(parts)-1; i++ { + next, found := current[parts[i]] + if !found { + if create { + next = make(maps.Params) + current[parts[i]] = next + } else { + return "", nil + } + } + current = next.(maps.Params) + } + return parts[len(parts)-1], current +} diff --git a/config/defaultConfigProvider_test.go b/config/defaultConfigProvider_test.go new file mode 100644 index 000000000..834165d96 --- /dev/null +++ b/config/defaultConfigProvider_test.go @@ -0,0 +1,315 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/common/para" + + "github.com/gohugoio/hugo/common/maps" + + qt "github.com/frankban/quicktest" +) + +func TestDefaultConfigProvider(t *testing.T) { + c := qt.New(t) + + c.Run("Set and get", func(c *qt.C) { + cfg := New() + var k string + var v interface{} + + k, v = "foo", "bar" + cfg.Set(k, v) + c.Assert(cfg.Get(k), qt.Equals, v) + c.Assert(cfg.Get(strings.ToUpper(k)), qt.Equals, v) + c.Assert(cfg.GetString(k), qt.Equals, v) + + k, v = "foo", 42 + cfg.Set(k, v) + c.Assert(cfg.Get(k), qt.Equals, v) + c.Assert(cfg.GetInt(k), qt.Equals, v) + + c.Assert(cfg.Get(""), qt.DeepEquals, maps.Params{ + "foo": 42, + }) + }) + + c.Run("Set and get map", func(c *qt.C) { + cfg := New() + + cfg.Set("foo", map[string]interface{}{ + "bar": "baz", + }) + + c.Assert(cfg.Get("foo"), qt.DeepEquals, maps.Params{ + "bar": "baz", + }) + + c.Assert(cfg.GetStringMap("foo"), qt.DeepEquals, map[string]interface{}{"bar": string("baz")}) + c.Assert(cfg.GetStringMapString("foo"), qt.DeepEquals, map[string]string{"bar": string("baz")}) + }) + + c.Run("Set and get nested", func(c *qt.C) { + cfg := New() + + cfg.Set("a", map[string]interface{}{ + "B": "bv", + }) + cfg.Set("a.c", "cv") + + c.Assert(cfg.Get("a"), qt.DeepEquals, maps.Params{ + "b": "bv", + "c": "cv", + }) + c.Assert(cfg.Get("a.c"), qt.Equals, "cv") + + cfg.Set("b.a", "av") + c.Assert(cfg.Get("b"), qt.DeepEquals, maps.Params{ + "a": "av", + }) + + cfg.Set("b", map[string]interface{}{ + "b": "bv", + }) + + c.Assert(cfg.Get("b"), qt.DeepEquals, maps.Params{ + "a": "av", + "b": "bv", + }) + + cfg = New() + + cfg.Set("a", "av") + + cfg.Set("", map[string]interface{}{ + "a": "av2", + "b": "bv2", + }) + + c.Assert(cfg.Get(""), qt.DeepEquals, maps.Params{ + "a": "av2", + "b": "bv2", + }) + + cfg = New() + + cfg.Set("a", "av") + + cfg.Set("", map[string]interface{}{ + "b": "bv2", + }) + + c.Assert(cfg.Get(""), qt.DeepEquals, maps.Params{ + "a": "av", + "b": "bv2", + }) + + cfg = New() + + cfg.Set("", map[string]interface{}{ + "foo": map[string]interface{}{ + "a": "av", + }, + }) + + cfg.Set("", map[string]interface{}{ + "foo": map[string]interface{}{ + "b": "bv2", + }, + }) + + c.Assert(cfg.Get("foo"), qt.DeepEquals, maps.Params{ + "a": "av", + "b": "bv2", + }) + }) + + c.Run("Merge default strategy", func(c *qt.C) { + cfg := New() + + cfg.Set("a", map[string]interface{}{ + "B": "bv", + }) + + cfg.Merge("a", map[string]interface{}{ + "B": "bv2", + "c": "cv2", + }) + + c.Assert(cfg.Get("a"), qt.DeepEquals, maps.Params{ + "b": "bv", + "c": "cv2", + }) + + cfg = New() + + cfg.Set("a", "av") + + cfg.Merge("", map[string]interface{}{ + "a": "av2", + "b": "bv2", + }) + + c.Assert(cfg.Get(""), qt.DeepEquals, maps.Params{ + "a": "av", + "b": "bv2", + }) + }) + + c.Run("Merge shallow", func(c *qt.C) { + cfg := New() + + cfg.Set("a", map[string]interface{}{ + "_merge": "shallow", + "B": "bv", + "c": map[string]interface{}{ + "b": "bv", + }, + }) + + cfg.Merge("a", map[string]interface{}{ + "c": map[string]interface{}{ + "d": "dv2", + }, + "e": "ev2", + }) + + c.Assert(cfg.Get("a"), qt.DeepEquals, maps.Params{ + "e": "ev2", + "_merge": maps.ParamsMergeStrategyShallow, + "b": "bv", + "c": maps.Params{ + "b": "bv", + }, + }) + }) + + c.Run("IsSet", func(c *qt.C) { + cfg := New() + + cfg.Set("a", map[string]interface{}{ + "B": "bv", + }) + + c.Assert(cfg.IsSet("A"), qt.IsTrue) + c.Assert(cfg.IsSet("a.b"), qt.IsTrue) + c.Assert(cfg.IsSet("z"), qt.IsFalse) + }) + + c.Run("Para", func(c *qt.C) { + cfg := New() + p := para.New(4) + r, _ := p.Start(context.Background()) + + setAndGet := func(k string, v int) error { + vs := strconv.Itoa(v) + cfg.Set(k, v) + err := errors.New("get failed") + if cfg.Get(k) != v { + return err + } + if cfg.GetInt(k) != v { + return err + } + if cfg.GetString(k) != vs { + return err + } + if !cfg.IsSet(k) { + return err + } + return nil + } + + for i := 0; i < 20; i++ { + i := i + r.Run(func() error { + const v = 42 + k := fmt.Sprintf("k%d", i) + if err := setAndGet(k, v); err != nil { + return err + } + + m := maps.Params{ + "new": 42, + } + + cfg.Merge("", m) + + return nil + }) + } + + c.Assert(r.Wait(), qt.IsNil) + }) +} + +func BenchmarkDefaultConfigProvider(b *testing.B) { + type cfger interface { + Get(key string) interface{} + Set(key string, value interface{}) + IsSet(key string) bool + } + + newMap := func() map[string]interface{} { + return map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": 32, + "d": 43, + }, + }, + "b": 62, + } + } + + runMethods := func(b *testing.B, cfg cfger) { + m := newMap() + cfg.Set("mymap", m) + cfg.Set("num", 32) + if !(cfg.IsSet("mymap") && cfg.IsSet("mymap.a") && cfg.IsSet("mymap.a.b") && cfg.IsSet("mymap.a.b.c")) { + b.Fatal("IsSet failed") + } + + if cfg.Get("num") != 32 { + b.Fatal("Get failed") + } + + if cfg.Get("mymap.a.b.c") != 32 { + b.Fatal("Get failed") + } + } + + b.Run("Viper", func(b *testing.B) { + v := viper.New() + for i := 0; i < b.N; i++ { + runMethods(b, v) + } + }) + + b.Run("Custom", func(b *testing.B) { + cfg := New() + for i := 0; i < b.N; i++ { + runMethods(b, cfg) + } + }) +} diff --git a/config/docshelper.go b/config/docshelper.go new file mode 100644 index 000000000..336a0dc16 --- /dev/null +++ b/config/docshelper.go @@ -0,0 +1,45 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/docshelper" +) + +// This is is just some helpers used to create some JSON used in the Hugo docs. +func init() { + docsProvider := func() docshelper.DocProvider { + + cfg := New() + for _, configRoot := range ConfigRootKeys { + cfg.Set(configRoot, make(maps.Params)) + } + lang := maps.Params{ + "en": maps.Params{ + "menus": maps.Params{}, + "params": maps.Params{}, + }, + } + cfg.Set("languages", lang) + cfg.SetDefaultMergeStrategy() + + configHelpers := map[string]interface{}{ + "mergeStrategy": cfg.Get(""), + } + return docshelper.DocProvider{"config": configHelpers} + } + + docshelper.AddDocProviderFunc(docsProvider) +} diff --git a/config/privacy/privacyConfig_test.go b/config/privacy/privacyConfig_test.go index 0fb599c0a..c17ce713d 100644 --- a/config/privacy/privacyConfig_test.go +++ b/config/privacy/privacyConfig_test.go @@ -18,7 +18,6 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/config" - "github.com/spf13/viper" ) func TestDecodeConfigFromTOML(t *testing.T) { @@ -94,7 +93,7 @@ PrivacyENhanced = true func TestDecodeConfigDefault(t *testing.T) { c := qt.New(t) - pc, err := DecodeConfig(viper.New()) + pc, err := DecodeConfig(config.New()) c.Assert(err, qt.IsNil) c.Assert(pc, qt.Not(qt.IsNil)) c.Assert(pc.YouTube.PrivacyEnhanced, qt.Equals, false) diff --git a/config/services/servicesConfig_test.go b/config/services/servicesConfig_test.go index 6e979b999..d7a52ba4f 100644 --- a/config/services/servicesConfig_test.go +++ b/config/services/servicesConfig_test.go @@ -18,7 +18,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/config" - "github.com/spf13/viper" + ) func TestDecodeConfigFromTOML(t *testing.T) { @@ -55,7 +55,7 @@ disableInlineCSS = true func TestUseSettingsFromRootIfSet(t *testing.T) { c := qt.New(t) - cfg := viper.New() + cfg := config.New() cfg.Set("disqusShortname", "root_short") cfg.Set("googleAnalytics", "ga_root") diff --git a/create/content_test.go b/create/content_test.go index 37dbf98bd..38ff7de8d 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -20,6 +20,8 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugolib" @@ -30,7 +32,6 @@ import ( "github.com/gohugoio/hugo/create" "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" - "github.com/spf13/viper" ) func TestNewContent(t *testing.T) { @@ -245,7 +246,7 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { return string(b) } -func newTestCfg(c *qt.C, mm afero.Fs) (*viper.Viper, *hugofs.Fs) { +func newTestCfg(c *qt.C, mm afero.Fs) (config.Provider, *hugofs.Fs) { cfg := ` theme = "mytheme" diff --git a/deploy/deployConfig_test.go b/deploy/deployConfig_test.go index 413b1211b..e30f31c32 100644 --- a/deploy/deployConfig_test.go +++ b/deploy/deployConfig_test.go @@ -21,7 +21,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/config" - "github.com/spf13/viper" + ) func TestDecodeConfigFromTOML(t *testing.T) { @@ -164,7 +164,7 @@ Pattern = "[" # invalid regular expression func TestDecodeConfigDefault(t *testing.T) { c := qt.New(t) - dcfg, err := decodeConfig(viper.New()) + dcfg, err := decodeConfig(config.New()) c.Assert(err, qt.IsNil) c.Assert(len(dcfg.Targets), qt.Equals, 0) c.Assert(len(dcfg.Matchers), qt.Equals, 0) diff --git a/docs/config.toml b/docs/config.toml index efa1b0573..d21f6f172 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -7,7 +7,7 @@ footnotereturnlinkcontents = "↩" languageCode = "en-us" title = "Hugo" - ignoreErrors = ["error-remote-getjson"] + ignoreErrors = ["error-remote-getjson", "err-missing-instagram-accesstoken"] googleAnalytics = "UA-7131036-4" diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index 48a4af0c9..fd8c65d04 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -80,6 +80,26 @@ Considering the structure above, when running `hugo --environment staging`, Hugo {{% note %}} Default environments are __development__ with `hugo server` and __production__ with `hugo`. {{%/ note %}} + +## Merge Configuration from Themes + +{{< new-in "0.84.0" >}} The configuration merge described below was improved in Hugo 0.84.0 and made fully configurable. The big change/improvement was that we now, by default, do deep merging of `params` maps from themes. + +The configuration value for `_merge` can be one of: + +none +: No merge. + +shallow +: Only add values for new keys. + +shallow +: Add values for new keys, merge existing. + +Note that you don't need to be so verbose as in the default setup below; a `_merge` value higher up will be inherited if not set. + +{{< code-toggle config="mergeStrategy" skipHeader=true />}} + ## All Configuration Settings The following is the full list of Hugo-defined variables with their default diff --git a/docs/data/docs.json b/docs/data/docs.json index 2b17ad8af..b927f8d0a 100644 --- a/docs/data/docs.json +++ b/docs/data/docs.json @@ -1587,6 +1587,65 @@ "preserveTOC": false } }, + "mergeStrategy": { + "build": { + "_merge": "none" + }, + "caches": { + "_merge": "none" + }, + "frontmatter": { + "_merge": "none" + }, + "imaging": { + "_merge": "none" + }, + "languages": { + "_merge": "none", + "en": { + "_merge": "none", + "menus": { + "_merge": "shallow" + }, + "params": { + "_merge": "deep" + } + } + }, + "markup": { + "_merge": "none" + }, + "mediatypes": { + "_merge": "shallow" + }, + "menus": { + "_merge": "shallow" + }, + "minify": { + "_merge": "none" + }, + "module": { + "_merge": "none" + }, + "outputformats": { + "_merge": "shallow" + }, + "params": { + "_merge": "deep" + }, + "permalinks": { + "_merge": "none" + }, + "related": { + "_merge": "none" + }, + "sitemap": { + "_merge": "none" + }, + "taxonomies": { + "_merge": "none" + } + }, "minify": { "minifyOutput": false, "disableHTML": false, diff --git a/docs/layouts/shortcodes/code-toggle.html b/docs/layouts/shortcodes/code-toggle.html index da4b00719..0b92d520d 100644 --- a/docs/layouts/shortcodes/code-toggle.html +++ b/docs/layouts/shortcodes/code-toggle.html @@ -1,34 +1,41 @@ {{ $file := .Get "file" }} {{ $code := "" }} {{ with .Get "config" }} -{{ $file = $file | default "config" }} -{{ $sections := (split . ".") }} -{{ $configSection := index $.Site.Data.docs.config $sections }} -{{ $code = dict $sections $configSection }} + {{ $file = $file | default "config" }} + {{ $sections := (split . ".") }} + {{ $configSection := index $.Site.Data.docs.config $sections }} + {{ $code = dict $sections $configSection }} + {{ if $.Get "skipHeader"}} + {{ $code = $configSection }} + {{ end }} {{ else }} -{{ $code = $.Inner }} + {{ $code = $.Inner }} {{ end }} {{ $langs := (slice "yaml" "toml" "json") }}
-
- {{- with $file -}} -
{{ . }}.
- {{- end -}} - {{ range $langs }} -   - {{ end }} -
-
- {{ range $langs }} -
- {{ highlight ($code | transform.Remarshal . | safeHTML) . ""}} -
- {{ if ne ($.Get "copy") "false" }} - - {{/* Functionality located within filesaver.js The copy here is located in the css with .copy class so it can be replaced with JS on success */}} - {{end}} - {{ end }} -
- +
+ {{- with $file -}} +
+ {{ . }}. +
+ {{- end -}} + {{ range $langs }} + +   + {{ end }} +
+
+ {{ range $langs }} +
+ {{ highlight ($code | transform.Remarshal . | safeHTML) . ""}} +
+ {{ if ne ($.Get "copy") "false" }} + + {{/* Functionality located within filesaver.js The copy here is located in the css with .copy class so it can be replaced with JS on success */}} + {{end}} + {{ end }} +
+
diff --git a/go.mod b/go.mod index ca89a0ca6..f0c9f1fa7 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/spf13/fsync v0.9.0 github.com/spf13/jwalterweatherman v1.1.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.1 + github.com/spf13/viper v1.7.0 github.com/tdewolff/minify/v2 v2.9.16 github.com/yuin/goldmark v1.3.5 github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 diff --git a/go.sum b/go.sum index ccdc49b69..d9cd59c34 100644 --- a/go.sum +++ b/go.sum @@ -642,6 +642,7 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= diff --git a/helpers/content_test.go b/helpers/content_test.go index 573c1c90d..515b788f1 100644 --- a/helpers/content_test.go +++ b/helpers/content_test.go @@ -19,12 +19,11 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/config" "github.com/spf13/afero" "github.com/gohugoio/hugo/common/loggers" - "github.com/spf13/viper" - qt "github.com/frankban/quicktest" ) @@ -103,7 +102,7 @@ func TestBytesToHTML(t *testing.T) { } func TestNewContentSpec(t *testing.T) { - cfg := viper.New() + cfg := config.New() c := qt.New(t) cfg.Set("summaryLength", 32) diff --git a/helpers/general_test.go b/helpers/general_test.go index 518188c17..bfabcbef4 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/common/loggers" @@ -29,7 +29,7 @@ import ( func TestResolveMarkup(t *testing.T) { c := qt.New(t) - cfg := viper.New() + cfg := config.New() spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs()) c.Assert(err, qt.IsNil) diff --git a/helpers/path_test.go b/helpers/path_test.go index e1ac90b2f..c95951832 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -31,7 +31,6 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/spf13/afero" - "github.com/spf13/viper" ) func TestMakePath(t *testing.T) { @@ -490,8 +489,6 @@ func TestExists(t *testing.T) { } func TestAbsPathify(t *testing.T) { - defer viper.Reset() - type test struct { inPath, workingDir, expected string } @@ -511,7 +508,6 @@ func TestAbsPathify(t *testing.T) { } for i, d := range data { - viper.Reset() // todo see comment in AbsPathify ps := newTestDefaultPathSpec("workingDir", d.workingDir) diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 4a6ebd0b1..7d63e4d88 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -2,24 +2,24 @@ package helpers import ( "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config" "github.com/spf13/afero" - "github.com/spf13/viper" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/modules" ) -func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { +func newTestPathSpec(fs *hugofs.Fs, v config.Provider) *PathSpec { l := langs.NewDefaultLanguage(v) ps, _ := NewPathSpec(fs, l, nil) return ps } func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { - v := viper.New() + v := config.New() fs := hugofs.NewMem(v) - cfg := newTestCfgFor(fs) + cfg := newTestCfg() for i := 0; i < len(configKeyValues); i += 2 { cfg.Set(configKeyValues[i].(string), configKeyValues[i+1]) @@ -27,15 +27,8 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { return newTestPathSpec(fs, cfg) } -func newTestCfgFor(fs *hugofs.Fs) *viper.Viper { - v := newTestCfg() - v.SetFs(fs.Source) - - return v -} - -func newTestCfg() *viper.Viper { - v := viper.New() +func newTestCfg() config.Provider { + v := config.New() v.Set("contentDir", "content") v.Set("dataDir", "data") v.Set("i18nDir", "i18n") @@ -56,7 +49,7 @@ func newTestCfg() *viper.Viper { } func newTestContentSpec() *ContentSpec { - v := viper.New() + v := config.New() spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs()) if err != nil { panic(err) diff --git a/hugofs/fs_test.go b/hugofs/fs_test.go index a343bbd1f..8d52267af 100644 --- a/hugofs/fs_test.go +++ b/hugofs/fs_test.go @@ -16,15 +16,16 @@ package hugofs import ( "testing" + "github.com/gohugoio/hugo/config" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting/hqt" "github.com/spf13/afero" - "github.com/spf13/viper" ) func TestNewDefault(t *testing.T) { c := qt.New(t) - v := viper.New() + v := config.New() f := NewDefault(v) c.Assert(f.Source, qt.Not(qt.IsNil)) @@ -35,7 +36,7 @@ func TestNewDefault(t *testing.T) { func TestNewMem(t *testing.T) { c := qt.New(t) - v := viper.New() + v := config.New() f := NewMem(v) c.Assert(f.Source, qt.Not(qt.IsNil)) @@ -48,7 +49,7 @@ func TestNewMem(t *testing.T) { func TestWorkingDir(t *testing.T) { c := qt.New(t) - v := viper.New() + v := config.New() v.Set("workingDir", "/a/b/") diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index fc2ddeb62..db9ed25cd 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -20,7 +20,7 @@ import ( "sort" "testing" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/config" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" @@ -29,7 +29,7 @@ import ( func TestLanguageRootMapping(t *testing.T) { c := qt.New(t) - v := viper.New() + v := config.New() v.Set("contentDir", "content") fs := NewBaseFileDecorator(afero.NewMemMapFs()) diff --git a/hugolib/config.go b/hugolib/config.go index eaa6710ae..f559d7fd3 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -43,34 +43,136 @@ import ( "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" - "github.com/spf13/viper" ) -// SiteConfig represents the config in .Site.Config. -type SiteConfig struct { - // This contains all privacy related settings that can be used to - // make the YouTube template etc. GDPR compliant. - Privacy privacy.Config +var ErrNoConfigFile = errors.New("Unable to locate config file or config directory. Perhaps you need to create a new site.\n Run `hugo help new` for details.\n") - // Services contains config for services such as Google Analytics etc. - Services services.Config -} +// LoadConfig loads Hugo configuration into a new Viper and then adds +// a set of defaults. +func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (config.Provider, []string, error) { -func loadSiteConfig(cfg config.Provider) (scfg SiteConfig, err error) { - privacyConfig, err := privacy.DecodeConfig(cfg) + if d.Environment == "" { + d.Environment = hugo.EnvironmentProduction + } + + if len(d.Environ) == 0 { + d.Environ = os.Environ() + } + + var configFiles []string + + l := configLoader{ConfigSourceDescriptor: d, cfg: config.New()} + + if err := l.applyConfigDefaults(); err != nil { + return l.cfg, configFiles, err + } + + for _, name := range d.configFilenames() { + var filename string + filename, err := l.loadConfig(name) + if err == nil { + configFiles = append(configFiles, filename) + } else if err != ErrNoConfigFile { + return nil, nil, err + } + } + + if d.AbsConfigDir != "" { + dirnames, err := l.loadConfigFromConfigDir() + if err == nil { + configFiles = append(configFiles, dirnames...) + } else if err != ErrNoConfigFile { + return nil, nil, err + } + } + + // TODO(bep) improve this. This is currently needed to get the merge correctly. + if l.cfg.IsSet("languages") { + langs := l.cfg.GetParams("languages") + for _, lang := range langs { + langp := lang.(maps.Params) + if _, ok := langp["menus"]; !ok { + langp["menus"] = make(maps.Params) + } + if _, ok := langp["params"]; !ok { + langp["params"] = make(maps.Params) + } + } + + } + l.cfg.SetDefaultMergeStrategy() + + // We create languages based on the settings, so we need to make sure that + // all configuration is loaded/set before doing that. + for _, d := range doWithConfig { + if err := d(l.cfg); err != nil { + return l.cfg, configFiles, err + } + } + + // We made thi