From d392893cd73dc00c927f342778f6dca9628d328e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 9 Jun 2021 10:58:18 +0200 Subject: Misc config loading fixes The main motivation behind this is simplicity and correctnes, but the new small config library is also faster: ``` BenchmarkDefaultConfigProvider/Viper-16 252418 4546 ns/op 2720 B/op 30 allocs/op BenchmarkDefaultConfigProvider/Custom-16 450756 2651 ns/op 1008 B/op 6 allocs/op ``` Fixes #8633 Fixes #8618 Fixes #8630 Updates #8591 Closes #6680 Closes #5192 --- hugolib/config.go | 680 +++++++++++++++----------------- hugolib/config_test.go | 372 +++++++++-------- hugolib/filesystems/basefs_test.go | 10 +- hugolib/hugo_modules_test.go | 30 +- hugolib/hugo_sites.go | 15 +- hugolib/hugo_sites_build_errors_test.go | 4 +- hugolib/image_test.go | 4 +- hugolib/js_test.go | 11 +- hugolib/minify_publisher_test.go | 4 +- hugolib/page__meta.go | 2 +- hugolib/page_test.go | 5 +- hugolib/pagebundler_test.go | 18 +- hugolib/pages_capture.go | 2 +- hugolib/paths/paths_test.go | 7 +- hugolib/resource_chain_babel_test.go | 6 +- hugolib/resource_chain_test.go | 18 +- hugolib/robotstxt_test.go | 4 +- hugolib/shortcode_test.go | 7 +- hugolib/site.go | 22 +- hugolib/site_output_test.go | 12 +- hugolib/site_test.go | 5 +- hugolib/template_test.go | 5 +- hugolib/testhelpers_test.go | 26 +- 23 files changed, 601 insertions(+), 668 deletions(-) (limited to 'hugolib') diff --git a/hugolib/config.go b/hugolib/config.go index eaa6710ae..f559d7fd3 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -43,34 +43,136 @@ import ( "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" - "github.com/spf13/viper" ) -// SiteConfig represents the config in .Site.Config. -type SiteConfig struct { - // This contains all privacy related settings that can be used to - // make the YouTube template etc. GDPR compliant. - Privacy privacy.Config +var ErrNoConfigFile = errors.New("Unable to locate config file or config directory. Perhaps you need to create a new site.\n Run `hugo help new` for details.\n") - // Services contains config for services such as Google Analytics etc. - Services services.Config -} +// LoadConfig loads Hugo configuration into a new Viper and then adds +// a set of defaults. +func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (config.Provider, []string, error) { -func loadSiteConfig(cfg config.Provider) (scfg SiteConfig, err error) { - privacyConfig, err := privacy.DecodeConfig(cfg) + if d.Environment == "" { + d.Environment = hugo.EnvironmentProduction + } + + if len(d.Environ) == 0 { + d.Environ = os.Environ() + } + + var configFiles []string + + l := configLoader{ConfigSourceDescriptor: d, cfg: config.New()} + + if err := l.applyConfigDefaults(); err != nil { + return l.cfg, configFiles, err + } + + for _, name := range d.configFilenames() { + var filename string + filename, err := l.loadConfig(name) + if err == nil { + configFiles = append(configFiles, filename) + } else if err != ErrNoConfigFile { + return nil, nil, err + } + } + + if d.AbsConfigDir != "" { + dirnames, err := l.loadConfigFromConfigDir() + if err == nil { + configFiles = append(configFiles, dirnames...) + } else if err != ErrNoConfigFile { + return nil, nil, err + } + } + + // TODO(bep) improve this. This is currently needed to get the merge correctly. + if l.cfg.IsSet("languages") { + langs := l.cfg.GetParams("languages") + for _, lang := range langs { + langp := lang.(maps.Params) + if _, ok := langp["menus"]; !ok { + langp["menus"] = make(maps.Params) + } + if _, ok := langp["params"]; !ok { + langp["params"] = make(maps.Params) + } + } + + } + l.cfg.SetDefaultMergeStrategy() + + // We create languages based on the settings, so we need to make sure that + // all configuration is loaded/set before doing that. + for _, d := range doWithConfig { + if err := d(l.cfg); err != nil { + return l.cfg, configFiles, err + } + } + + // We made this a Glob pattern in Hugo 0.75, we don't need both. + if l.cfg.GetBool("ignoreVendor") { + helpers.Deprecated("--ignoreVendor", "--ignoreVendorPaths **", false) + l.cfg.Set("ignoreVendorPaths", "**") + } + + // Some settings are used before we're done collecting all settings, + // so apply OS environment both before and after. + if err := l.applyOsEnvOverrides(d.Environ); err != nil { + return l.cfg, configFiles, err + } + + modulesConfig, err := l.loadModulesConfig() if err != nil { - return + return l.cfg, configFiles, err } - servicesConfig, err := services.DecodeConfig(cfg) + // Need to run these after the modules are loaded, but before + // they are finalized. + collectHook := func(m *modules.ModulesConfig) error { + // We don't need the merge strategy configuration anymore, + // remove it so it doesn't accidentaly show up in other settings. + l.cfg.WalkParams(func(params ...config.KeyParams) bool { + params[len(params)-1].Params.DeleteMergeStrategy() + return false + }) + + if err := l.loadLanguageSettings(nil); err != nil { + return err + } + + mods := m.ActiveModules + + // Apply default project mounts. + if err := modules.ApplyProjectConfigDefaults(l.cfg, mods[0]); err != nil { + return err + } + + return nil + } + + _, modulesConfigFiles, err := l.collectModules(modulesConfig, l.cfg, collectHook) if err != nil { - return + return l.cfg, configFiles, err } - scfg.Privacy = privacyConfig - scfg.Services = servicesConfig + configFiles = append(configFiles, modulesConfigFiles...) - return + if err := l.applyOsEnvOverrides(d.Environ); err != nil { + return l.cfg, configFiles, err + } + + if err = l.applyConfigAliases(); err != nil { + return l.cfg, configFiles, err + } + + return l.cfg, configFiles, err +} + +// LoadConfigDefault is a convenience method to load the default "config.toml" config. +func LoadConfigDefault(fs afero.Fs) (config.Provider, error) { + v, _, err := LoadConfig(ConfigSourceDescriptor{Fs: fs, Filename: "config.toml"}) + return v, err } // ConfigSourceDescriptor describes where to find the config (e.g. config.toml etc.). @@ -98,6 +200,13 @@ type ConfigSourceDescriptor struct { Environ []string } +func (d ConfigSourceDescriptor) configFileDir() string { + if d.Path != "" { + return d.Path + } + return d.WorkingDir +} + func (d ConfigSourceDescriptor) configFilenames() []string { if d.Filename == "" { return []string{"config"} @@ -105,185 +214,219 @@ func (d ConfigSourceDescriptor) configFilenames() []string { return strings.Split(d.Filename, ",") } -func (d ConfigSourceDescriptor) configFileDir() string { - if d.Path != "" { - return d.Path - } - return d.WorkingDir +// SiteConfig represents the config in .Site.Config. +type SiteConfig struct { + // This contains all privacy related settings that can be used to + // make the YouTube template etc. GDPR compliant. + Privacy privacy.Config + + // Services contains config for services such as Google Analytics etc. + Services services.Config } -// LoadConfigDefault is a convenience method to load the default "config.toml" config. -func LoadConfigDefault(fs afero.Fs) (*viper.Viper, error) { - v, _, err := LoadConfig(ConfigSourceDescriptor{Fs: fs, Filename: "config.toml"}) - return v, err +type configLoader struct { + cfg config.Provider + ConfigSourceDescriptor } -var ErrNoConfigFile = errors.New("Unable to locate config file or config directory. Perhaps you need to create a new site.\n Run `hugo help new` for details.\n") +// Handle some legacy values. +func (l configLoader) applyConfigAliases() error { + aliases := []types.KeyValueStr{{Key: "taxonomies", Value: "indexes"}} -// LoadConfig loads Hugo configuration into a new Viper and then adds -// a set of defaults. -func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (*viper.Viper, []string, error) { - if d.Environment == "" { - d.Environment = hugo.EnvironmentProduction + for _, alias := range aliases { + if l.cfg.IsSet(alias.Key) { + vv := l.cfg.Get(alias.Key) + l.cfg.Set(alias.Value, vv) + } } - if len(d.Environ) == 0 { - d.Environ = os.Environ() - } + return nil +} - var configFiles []string +func (l configLoader) applyConfigDefaults() error { + defaultSettings := maps.Params{ + "cleanDestinationDir": false, + "watch": false, + "resourceDir": "resources", + "publishDir": "public", + "themesDir": "themes", + "buildDrafts": false, + "buildFuture": false, + "buildExpired": false, + "environment": hugo.EnvironmentProduction, + "uglyURLs": false, + "verbose": false, + "ignoreCache": false, + "canonifyURLs": false, + "relativeURLs": false, + "removePathAccents": false, + "titleCaseStyle": "AP", + "taxonomies": map[string]string{"tag": "tags", "category": "categories"}, + "permalinks": make(map[string]string), + "sitemap": config.Sitemap{Priority: -1, Filename: "sitemap.xml"}, + "disableLiveReload": false, + "pluralizeListTitles": true, + "forceSyncStatic": false, + "footnoteAnchorPrefix": "", + "footnoteReturnLinkContents": "", + "newContentEditor": "", + "paginate": 10, + "paginatePath": "page", + "summaryLength": 70, + "rssLimit": -1, + "sectionPagesMenu": "", + "disablePathToLower": false, + "hasCJKLanguage": false, + "enableEmoji": false, + "pygmentsCodeFencesGuessSyntax": false, + "defaultContentLanguage": "en", + "defaultContentLanguageInSubdir": false, + "enableMissingTranslationPlaceholders": false, + "enableGitInfo": false, + "ignoreFiles": make([]string, 0), + "disableAliases": false, + "debug": false, + "disableFastRender": false, + "timeout": "30s", + "enableInlineShortcodes": false, + } + + l.cfg.Merge("", defaultSettings) - v := viper.New() - l := configLoader{ConfigSourceDescriptor: d} + return nil +} - for _, name := range d.configFilenames() { - var filename string - filename, err := l.loadConfig(name, v) - if err == nil { - configFiles = append(configFiles, filename) - } else if err != ErrNoConfigFile { - return nil, nil, err - } +func (l configLoader) applyOsEnvOverrides(environ []string) error { + if len(environ) == 0 { + return nil } - if d.AbsConfigDir != "" { - dirnames, err := l.loadConfigFromConfigDir(v) - if err == nil { - configFiles = append(configFiles, dirnames...) - } else if err != ErrNoConfigFile { - return nil, nil, err - } - } + const delim = "__env__delim" - if err := loadDefaultSettingsFor(v); err != nil { - return v, configFiles, err - } + // Extract all that start with the HUGO prefix. + // The delimiter is the following rune, usually "_". + const hugoEnvPrefix = "HUGO" + var hugoEnv []types.KeyValueStr + for _, v := range environ { + key, val := config.SplitEnvVar(v) + if strings.HasPrefix(key, hugoEnvPrefix) { + delimiterAndKey := strings.TrimPrefix(key, hugoEnvPrefix) + if len(delimiterAndKey) < 2 { + continue + } + // Allow delimiters to be case sensitive. + // It turns out there isn't that many allowed special + // chars in environment variables when used in Bash and similar, + // so variables on the form HUGOxPARAMSxFOO=bar is one option. + key := strings.ReplaceAll(delimiterAndKey[1:], delimiterAndKey[:1], delim) + key = strings.ToLower(key) + hugoEnv = append(hugoEnv, types.KeyValueStr{ + Key: key, + Value: val, + }) - // We create languages based on the settings, so we need to make sure that - // all configuration is loaded/set before doing that. - for _, d := range doWithConfig { - if err := d(v); err != nil { - return v, configFiles, err } } - // This is invoked both after we load the main config and at the end - // to support OS env override of config options used in the module collector. - applyOsEnvOverrides := func() error { - if d.Environ == nil { - return nil - } - - const delim = "__env__delim" - - // Extract all that start with the HUGO prefix. - // The delimiter is the following rune, usually "_". - const hugoEnvPrefix = "HUGO" - var hugoEnv []types.KeyValueStr - for _, v := range d.Environ { - key, val := config.SplitEnvVar(v) - if strings.HasPrefix(key, hugoEnvPrefix) { - delimiterAndKey := strings.TrimPrefix(key, hugoEnvPrefix) - if len(delimiterAndKey) < 2 { - continue - } - // Allow delimiters to be case sensitive. - // It turns out there isn't that many allowed special - // chars in environment variables when used in Bash and similar, - // so variables on the form HUGOxPARAMSxFOO=bar is one option. - key := strings.ReplaceAll(delimiterAndKey[1:], delimiterAndKey[:1], delim) - key = strings.ToLower(key) - hugoEnv = append(hugoEnv, types.KeyValueStr{ - Key: key, - Value: val, - }) - - } + for _, env := range hugoEnv { + existing, nestedKey, owner, err := maps.GetNestedParamFn(env.Key, delim, l.cfg.Get) + if err != nil { + return err } - for _, env := range hugoEnv { - existing, nestedKey, owner, err := maps.GetNestedParamFn(env.Key, delim, v.Get) + if existing != nil { + val, err := metadecoders.Default.UnmarshalStringTo(env.Value, existing) if err != nil { - return err + continue } - if existing != nil { - val, err := metadecoders.Default.UnmarshalStringTo(env.Value, existing) - if err != nil { - continue - } - - if owner != nil { - owner[nestedKey] = val - } else { - v.Set(env.Key, val) - } - } else if nestedKey != "" { - owner[nestedKey] = env.Value + if owner != nil { + owner[nestedKey] = val } else { - v.Set(strings.ReplaceAll(env.Key, delim, "."), env.Value) + l.cfg.Set(env.Key, val) } + } else if nestedKey != "" { + owner[nestedKey] = env.Value + } else { + // The container does not exist yet. + l.cfg.Set(strings.ReplaceAll(env.Key, delim, "."), env.Value) } + } - return nil + return nil +} +func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provider, hookBeforeFinalize func(m *modules.ModulesConfig) error) (modules.Modules, []string, error) { + workingDir := l.WorkingDir + if workingDir == "" { + workingDir = v1.GetString("workingDir") } - if err := applyOsEnvOverrides(); err != nil { - return v, configFiles, err - } + themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir")) - // We made this a Glob pattern in Hugo 0.75, we don't need both. - if v.GetBool("ignoreVendor") { - helpers.Deprecated("--ignoreVendor", "--ignoreVendorPaths **", false) - v.Set("ignoreVendorPaths", "**") + var ignoreVendor glob.Glob + if s := v1.GetString("ignoreVendorPaths"); s != "" { + ignoreVendor, _ = hglob.GetGlob(hglob.NormalizePath(s)) } - modulesConfig, err := l.loadModulesConfig(v) + filecacheConfigs, err := filecache.DecodeConfig(l.Fs, v1) if err != nil { - return v, configFiles, err + return nil, nil, err } - // Need to run these after the modules are loaded, but before - // they are finalized. - collectHook := func(m *modules.ModulesConfig) error { - if err := loadLanguageSettings(v, nil); err != nil { - return err - } + v1.Set("filecacheConfigs", filecacheConfigs) - mods := m.ActiveModules + var configFilenames []string - // Apply default project mounts. - if err := modules.ApplyProjectConfigDefaults(v, mods[0]); err != nil { - return err + hook := func(m *modules.ModulesConfig) error { + for _, tc := range m.ActiveModules { + if tc.ConfigFilename() != "" { + if tc.Watch() { + configFilenames = append(configFilenames, tc.ConfigFilename()) + } + + // Merge from theme config into v1 based on configured + // merge strategy. + v1.Merge("", tc.Cfg().Get("")) + + } + } + + if hookBeforeFinalize != nil { + return hookBeforeFinalize(m) } return nil } - _, modulesConfigFiles, err := l.collectModules(modulesConfig, v, collectHook) + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: l.Fs, + Logger: l.Logger, + HookBeforeFinalize: hook, + WorkingDir: workingDir, + ThemesDir: themesDir, + CacheDir: filecacheConfigs.CacheDirModules(), + ModuleConfig: modConfig, + IgnoreVendor: ignoreVendor, + }) - if err == nil && len(modulesConfigFiles) > 0 { - configFiles = append(configFiles, modulesConfigFiles...) - } + v1.Set("modulesClient", modulesClient) - if err := applyOsEnvOverrides(); err != nil { - return v, configFiles, err - } + moduleConfig, err := modulesClient.Collect() - return v, configFiles, err -} + // Avoid recreating these later. + v1.Set("allModules", moduleConfig.ActiveModules) -func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { - _, err := langs.LoadLanguageSettings(cfg, oldLangs) - return err -} + if moduleConfig.GoModulesFilename != "" { + // We want to watch this for changes and trigger rebuild on version + // changes etc. + configFilenames = append(configFilenames, moduleConfig.GoModulesFilename) + } -type configLoader struct { - ConfigSourceDescriptor + return moduleConfig.ActiveModules, configFilenames, err } -func (l configLoader) loadConfig(configName string, v *viper.Viper) (string, error) { +func (l configLoader) loadConfig(configName string) (string, error) { baseDir := l.configFileDir() var baseFilename string if filepath.IsAbs(configName) { @@ -318,24 +461,13 @@ func (l configLoader) loadConfig(configName string, v *viper.Viper) (string, err return "", l.wrapFileError(err, filename) } - if err = v.MergeConfigMap(m); err != nil { - return "", l.wrapFileError(err, filename) - } + // Set overwrites keys of the same name, recursively. + l.cfg.Set("", m) return filename, nil } -func (l configLoader) wrapFileError(err error, filename string) error { - err, _ = herrors.WithFileContextForFile( - err, - filename, - filename, - l.Fs, - herrors.SimpleLineMatcher) - return err -} - -func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error) { +func (l configLoader) loadConfigFromConfigDir() ([]string, error) { sourceFs := l.Fs configDir := l.AbsConfigDir @@ -421,9 +553,8 @@ func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error) // Migrate menu => menus etc. config.RenameKeys(root) - if err := v.MergeConfigMap(root); err != nil { - return l.wrapFileError(err, path) - } + // Set will overwrite keys with the same name, recursively. + l.cfg.Set("", root) return nil }) @@ -436,8 +567,13 @@ func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error) return dirnames, nil } -func (l configLoader) loadModulesConfig(v1 *viper.Viper) (modules.Config, error) { - modConfig, err := modules.DecodeConfig(v1) +func (l configLoader) loadLanguageSettings(oldLangs langs.Languages) error { + _, err := langs.LoadLanguageSettings(l.cfg, oldLangs) + return err +} + +func (l configLoader) loadModulesConfig() (modules.Config, error) { + modConfig, err := modules.DecodeConfig(l.cfg) if err != nil { return modules.Config{}, err } @@ -445,211 +581,29 @@ func (l configLoader) loadModulesConfig(v1 *viper.Viper) (modules.Config, error) return modConfig, nil } -func (l configLoader) collectModules(modConfig modules.Config, v1 *viper.Viper, hookBeforeFinalize func(m *modules.ModulesConfig) error) (modules.Modules, []string, error) { - workingDir := l.WorkingDir - if workingDir == "" { - workingDir = v1.GetString("workingDir") - } - - themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir")) - - var ignoreVendor glob.Glob - if s := v1.GetString("ignoreVendorPaths"); s != "" { - ignoreVendor, _ = hglob.GetGlob(hglob.NormalizePath(s)) - } - - filecacheConfigs, err := filecache.DecodeConfig(l.Fs, v1) +func (configLoader) loadSiteConfig(cfg config.Provider) (scfg SiteConfig, err error) { + privacyConfig, err := privacy.DecodeConfig(cfg) if err != nil { - return nil, nil, err - } - - v1.Set("filecacheConfigs", filecacheConfigs) - - var configFilenames []string - - hook := func(m *modules.ModulesConfig) error { - for _, tc := range m.ActiveModules { - if tc.ConfigFilename() != "" { - if tc.Watch() { - configFilenames = append(configFilenames, tc.ConfigFilename()) - } - if err := l.applyThemeConfig(v1, tc); err != nil { - return err - } - } - } - - if hookBeforeFinalize != nil { - return hookBeforeFinalize(m) - } - - return nil - } - - modulesClient := modules.NewClient(modules.ClientConfig{ - Fs: l.Fs, - Logger: l.Logger, - HookBeforeFinalize: hook, - WorkingDir: workingDir, - ThemesDir: themesDir, - CacheDir: filecacheConfigs.CacheDirModules(), - ModuleConfig: modConfig, - IgnoreVendor: ignoreVendor, - }) - - v1.Set("modulesClient", modulesClient) - - moduleConfig, err := modulesClient.Collect() - - // Avoid recreating these later. - v1.Set("allModules", moduleConfig.ActiveModules) - - if moduleConfig.GoModulesFilename != "" { - // We want to watch this for changes and trigger rebuild on version - // changes etc. - configFilenames = append(configFilenames, moduleConfig.GoModulesFilename) - } - - return moduleConfig.ActiveModules, configFilenames, err -} - -func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme modules.Module) error { - const ( - paramsKey = "params" - languagesKey = "languages" - menuKey = "menus" - ) - - v2 := theme.Cfg() - - for _, key := range []string{paramsKey, "outputformats", "mediatypes"} { - l.mergeStringMapKeepLeft("", key, v1, v2) - } - - // Only add params and new menu entries, we do not add language definitions. - if v1.IsSet(languagesKey) && v2.IsSet(languagesKey) { - v1Langs := v1.GetStringMap(languagesKey) - for k := range v1Langs { - langParamsKey := languagesKey + "." + k + "." + paramsKey - l.mergeStringMapKeepLeft(paramsKey, langParamsKey, v1, v2) - } - v2Langs := v2.GetStringMap(languagesKey) - for k := range v2Langs { - if k == "" { - continue - } - - langMenuKey := languagesKey + "." + k + "." + menuKey - if v2.IsSet(langMenuKey) { - // Only add if not in the main config. - v2menus := v2.GetStringMap(langMenuKey) - for k, v := range v2menus { - menuEntry := menuKey + "." + k - menuLangEntry := langMenuKey + "." + k - if !v1.IsSet(menuEntry) && !v1.IsSet(menuLangEntry) { - v1.Set(menuLangEntry, v) - } - } - } - } - } - - // Add menu definitions from theme not found in project - if v2.IsSet(menuKey) { - v2menus := v2.GetStringMap(menuKey) - for k, v := range v2menus { - menuEntry := menuKey + "." + k - if !v1.IsSet(menuEntry) { - v1.SetDefault(menuEntry, v) - } - } - } - - return nil -} - -func (configLoader) mergeStringMapKeepLeft(rootKey, key string, v1, v2 config.Provider) { - if !v2.IsSet(key) { return } - if !v1.IsSet(key) && !(rootKey != "" && rootKey != key && v1.IsSet(rootKey)) { - v1.Set(key, v2.Get(key)) + servicesConfig, err := services.DecodeConfig(cfg) + if err != nil { return } - m1 := v1.GetStringMap(key) - m2 := v2.GetStringMap(key) + scfg.Privacy = privacyConfig + scfg.Services = servicesConfig - for k, v := range m2 { - if _, found := m1[k]; !found { - if rootKey != "" && v1.IsSet(rootKey+"."+k) { - continue - } - m1[k] = v - } - } + return } -func loadDefaultSettingsFor(v *viper.Viper) error { - v.RegisterAlias("indexes", "taxonomies") - - /* - - TODO(bep) from 0.56 these are configured as module mounts. - v.SetDefault("contentDir", "content") - v.SetDefault("layoutDir", "layouts") - v.SetDefault("assetDir", "assets") - v.SetDefault("staticDir", "static") - v.SetDefault("dataDir", "data") - v.SetDefault("i18nDir", "i18n") - v.SetDefault("archetypeDir", "archetypes") - */ - - v.SetDefault("cleanDestinationDir", false) - v.SetDefault("watch", false) - v.SetDefault("resourceDir", "resources") - v.SetDefault("publishDir", "public") - v.SetDefault("themesDir", "themes") - v.SetDefault("buildDrafts", false) - v.SetDefault("buildFuture", false) - v.SetDefault("buildExpired", false) - v.SetDefault("environment", hugo.EnvironmentProduction) - v.SetDefault("uglyURLs", false) - v.SetDefault("verbose", false) - v.SetDefault("ignoreCache", false) - v.SetDefault("canonifyURLs", false) - v.SetDefault("relativeURLs", false) - v.SetDefault("removePathAccents", false) - v.SetDefault("titleCaseStyle", "AP") - v.SetDefault("taxonomies", map[string]string{"tag": "tags", "category": "categories"}) - v.SetDefault("permalinks", make(map[string]string)) - v.SetDefault("sitemap", config.Sitemap{Priority: -1, Filename: "sitemap.xml"}) - v.SetDefault("disableLiveReload", false) - v.SetDefault("pluralizeListTitles", true) - v.SetDefault("forceSyncStatic", false) - v.SetDefault("footnoteAnchorPrefix", "") - v.SetDefault("footnoteReturnLinkContents", "") - v.SetDefault("newContentEditor", "") - v.SetDefault("paginate", 10) - v.SetDefault("paginatePath", "page") - v.SetDefault("summaryLength", 70) - v.SetDefault("rssLimit", -1) - v.SetDefault("sectionPagesMenu", "") - v.SetDefault("disablePathToLower", false) - v.SetDefault("hasCJKLanguage", false) - v.SetDefault("enableEmoji", false) - v.SetDefault("pygmentsCodeFencesGuessSyntax", false) - v.SetDefault("defaultContentLanguage", "en") - v.SetDefault("defaultContentLanguageInSubdir", false) - v.SetDefault("enableMissingTranslationPlaceholders", false) - v.SetDefault("enableGitInfo", false) - v.SetDefault("ignoreFiles", make([]string, 0)) - v.SetDefault("disableAliases", false) - v.SetDefault("debug", false) - v.SetDefault("disableFastRender", false) - v.SetDefault("timeout", "30s") - v.SetDefault("enableInlineShortcodes", false) - - return nil +func (l configLoader) wrapFileError(err error, filename string) error { + err, _ = herrors.WithFileContextForFile( + err, + filename, + filename, + l.Fs, + herrors.SimpleLineMatcher) + return err } diff --git a/hugolib/config_test.go b/hugolib/config_test.go index fb81c927e..77ac9b92f 100644 --- a/hugolib/config_test.go +++ b/hugolib/config_test.go @@ -17,11 +17,15 @@ import ( "bytes" "fmt" "path/filepath" + "strings" "testing" + "github.com/gohugoio/hugo/media" + "github.com/google/go-cmp/cmp" + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/maps" "github.com/spf13/afero" - "github.com/spf13/viper" ) func TestLoadConfig(t *testing.T) { @@ -77,12 +81,7 @@ func TestLoadConfigFromTheme(t *testing.T) { c := qt.New(t) - mainConfigBasic := ` -theme = "test-theme" -baseURL = "https://example.com/" - -` - mainConfig := ` + mainConfigTemplate := ` theme = "test-theme" baseURL = "https://example.com/" @@ -90,9 +89,12 @@ baseURL = "https://example.com/" date = ["date","publishDate"] [params] +MERGE_PARAMS p1 = "p1 main" -p2 = "p2 main" -top = "top" +[params.b] +b1 = "b1 main" +[params.b.c] +bc1 = "bc1 main" [mediaTypes] [mediaTypes."text/m1"] @@ -130,7 +132,14 @@ expiryDate = ["date"] [params] p1 = "p1 theme" p2 = "p2 theme" -p3 = "p3 theme" +[params.b] +b1 = "b1 theme" +b2 = "b2 theme" +[params.b.c] +bc1 = "bc1 theme" +bc2 = "bc2 theme" +[params.b.c.d] +bcd1 = "bcd1 theme" [mediaTypes] [mediaTypes."text/m1"] @@ -176,190 +185,137 @@ name = "menu-theme" ` - b := newTestSitesBuilder(t) - b.WithConfigFile("toml", mainConfig).WithThemeConfigFile("toml", themeConfig) - b.CreateSites().Build(BuildCfg{}) - - got := b.Cfg.(*viper.Viper).AllSettings() - - b.AssertObject(` -map[string]interface {}{ - "p1": "p1 main", - "p2": "p2 main", - "p3": "p3 theme", - "top": "top", -}`, got["params"]) - - b.AssertObject(` -map[string]interface {}{ - "date": []interface {}{ - "date", - "publishDate", - }, -}`, got["frontmatter"]) - - b.AssertObject(` -map[string]interface {}{ - "text/m1": map[string]interface {}{ - "suffixes": []interface {}{ - "m1main", - }, - }, - "text/m2": map[string]interface {}{ - "suffixes": []interface {}{ - "m2theme", - }, - }, -}`, got["mediatypes"]) - - b.AssertObject(` -map[string]interface {}{ - "o1": map[string]interface {}{ - "basename": "o1main", - "mediatype": Type{ - MainType: "text", - SubType: "m1", - Delimiter: ".", - FirstSuffix: SuffixInfo{ - Suffix: "m1main", - FullSuffix: ".m1main", - }, - }, - }, - "o2": map[string]interface {}{ - "basename": "o2theme", - "mediatype": Type{ - MainType: "text", - SubType: "m2", - Delimiter: ".", - FirstSuffix: SuffixInfo{ - Suffix: "m2theme", - FullSuffix: ".m2theme", - }, - }, - }, -}`, got["outputformats"]) - - b.AssertObject(`map[string]interface {}{ - "en": map[string]interface {}{ - "languagename": "English", - "menus": map[string]interface {}{ - "theme": []map[string]interface {}{ - map[string]interface {}{ - "name": "menu-lang-en-theme", - }, - }, - }, - "params": map[string]interface {}{ - "pl1": "p1-en-main", - "pl2": "p2-en-theme", - }, - }, - "nb": map[string]interface {}{ - "languagename": "Norsk", - "menus": map[string]interface {}{ - "theme": []map[string]interface {}{ - map[string]interface {}{ - "name": "menu-lang-nb-theme", - }, - }, - }, - "params": map[string]interface {}{ - "pl1": "p1-nb-main", - "pl2": "p2-nb-theme", - }, - }, -} -`, got["languages"]) - - b.AssertObject(` -map[string]interface {}{ - "main": []map[string]interface {}{ - map[string]interface {}{ - "name": "menu-main-main", - }, - }, - "thememenu": []map[string]interface {}{ - map[string]interface {}{ - "name": "menu-theme", - }, - }, - "top": []map[string]interface {}{ - map[string]interface {}{ - "name": "menu-top-main", - }, - }, -} -`, got["menus"]) + buildForStrategy := func(t testing.TB, s string) *sitesBuilder { + mainConfig := strings.ReplaceAll(mainConfigTemplate, "MERGE_PARAMS", s) + b := newTestSitesBuilder(t) + b.WithConfigFile("toml", mainConfig).WithThemeConfigFile("toml", themeConfig) + return b.CreateSites().Build(BuildCfg{}) + } - c.Assert(got["baseurl"], qt.Equals, "https://example.com/") + c.Run("Merge default", func(c *qt.C) { + b := buildForStrategy(c, "") + + got := b.Cfg.Get("").(maps.Params) + + b.Assert(got["params"], qt.DeepEquals, maps.Params{ + "b": maps.Params{ + "b1": "b1 main", + "c": maps.Params{ + "bc1": "bc1 main", + "bc2": "bc2 theme", + "d": maps.Params{"bcd1": string("bcd1 theme")}, + }, + "b2": "b2 theme", + }, + "p2": "p2 theme", + "p1": "p1 main", + }) + + b.Assert(got["mediatypes"], qt.DeepEquals, maps.Params{ + "text/m2": maps.Params{ + "suffixes": []interface{}{ + "m2theme", + }, + }, + "text/m1": maps.Params{ + "suffixes": []interface{}{ + "m1main", + }, + }, + }) + + var eq = qt.CmpEquals( + cmp.Comparer(func(m1, m2 media.Type) bool { + if m1.SubType != m2.SubType { + return false + } + return m1.FirstSuffix == m2.FirstSuffix + }), + ) + + mediaTypes := b.H.Sites[0].mediaTypesConfig + m1, _ := mediaTypes.GetByType("text/m1") + m2, _ := mediaTypes.GetByType("text/m2") + + b.Assert(got["outputformats"], eq, maps.Params{ + "o1": maps.Params{ + "mediatype": m1, + "basename": "o1main", + }, + "o2": maps.Params{ + "basename": "o2theme", + "mediatype": m2, + }, + }) + + b.Assert(got["languages"], qt.DeepEquals, maps.Params{ + "en": maps.Params{ + "languagename": "English", + "params": maps.Params{ + "pl2": "p2-en-theme", + "pl1": "p1-en-main", + }, + "menus": maps.Params{ + "main": []map[string]interface{}{ + { + "name": "menu-lang-en-main", + }, + }, + "theme": []map[string]interface{}{ + { + "name": "menu-lang-en-theme", + }, + }, + }, + }, + "nb": maps.Params{ + "languagename": "Norsk", + "params": maps.Params{ + "top": "top-nb-theme", + "pl1": "p1-nb-main", + "pl2": "p2-nb-theme", + }, + "menus": maps.Params{ + "main": []map[string]interface{}{ + { + "name": "menu-lang-nb-main", + }, + }, + "theme": []map[string]interface{}{ + { + "name": "menu-lang-nb-theme", + }, + }, + "top": []map[string]interface{}{ + { + "name": "menu-lang-nb-top", + }, + }, + }, + }, + }) + + c.Assert(got["baseurl"], qt.Equals, "https://example.com/") + }) + + c.Run("Merge shallow", func(c *qt.C) { + b := buildForStrategy(c, fmt.Sprintf("_merge=%q", "shallow")) + + got := b.Cfg.Get("").(maps.Params) + + // Shallow merge, only add new keys to params. + b.Assert(got["params"], qt.DeepEquals, maps.Params{ + "p1": "p1 main", + "b": maps.Params{ + "b1": "b1 main", + "c": maps.Params{ + "bc1": "bc1 main", + }, + }, + "p2": "p2 theme", + }) + }) - if true { - return - } - // Test variants with only values from theme - b = newTestSitesBuilder(t) - b.WithConfigFile("toml", mainConfigBasic).WithThemeConfigFile("toml", themeConfig) - b.CreateSites().Build(BuildCfg{}) - - got = b.Cfg.(*viper.Viper).AllSettings() - - b.AssertObject(`map[string]interface {}{ - "p1": "p1 theme", - "p2": "p2 theme", - "p3": "p3 theme", - "test-theme": map[string]interface {}{ - "p1": "p1 theme", - "p2": "p2 theme", - "p3": "p3 theme", - }, -}`, got["params"]) - - c.Assert(got["languages"], qt.IsNil) - b.AssertObject(` -map[string]interface {}{ - "text/m1": map[string]interface {}{ - "suffix": "m1theme", - }, - "text/m2": map[string]interface {}{ - "suffix": "m2theme", - }, -}`, got["mediatypes"]) - - b.AssertObject(` -map[string]interface {}{ - "o1": map[string]interface {}{ - "basename": "o1theme", - "mediatype": Type{ - MainType: "text", - SubType: "m1", - Suffix: "m1theme", - Delimiter: ".", - }, - }, - "o2": map[string]interface {}{ - "basename": "o2theme", - "mediatype": Type{ - MainType: "text", - SubType: "m2", - Suffix: "m2theme", - Delimiter: ".", - }, - }, -}`, got["outputformats"]) - b.AssertObject(` -map[string]interface {}{ - "main": []interface {}{ - map[string]interface {}{ - "name": "menu-main-theme", - }, - }, - "thememenu": []interface {}{ - map[string]interface {}{ - "name": "menu-theme", - }, - }, -}`, got["menu"]) } func TestPrivacyConfig(t *testing.T) { @@ -490,7 +446,12 @@ intSlice = [5,7,9] floatSlice = [3.14, 5.19] stringSlice = ["a", "b"] +[outputFormats] +[outputFormats.ofbase] +mediaType = "text/plain" + [params] +paramWithNoEnvOverride="nooverride" [params.api_config] api_key="default_key" another_key="default another_key" @@ -504,9 +465,16 @@ quality = 75 b.WithSourceFile("themes/mytheme/config.toml", ` +[outputFormats] +[outputFormats.oftheme] +mediaType = "text/plain" +[outputFormats.ofbase] +mediaType = "application/xml" + [params] [params.mytheme_section] theme_param="themevalue" +theme_param_nooverride="nooverride" [params.mytheme_section2] theme_param="themevalue2" @@ -530,14 +498,16 @@ theme_param="themevalue2" "HUGOxPARAMSxMYTHEME_SECTION2xTHEME_PARAM", "themevalue2_changed", "HUGO_PARAMS_EMPTY", ``, "HUGO_PARAMS_HTML", ``, - // + // Issue #8618 "HUGO_SERVICES_GOOGLEANALYTICS_ID", `gaid`, + "HUGO_PARAMS_A_B_C", "abc", ) b.Build(BuildCfg{}) cfg := b.H.Cfg - scfg := b.H.Sites[0].siteConfigConfig.Services + s := b.H.Sites[0] + scfg := s.siteConfigConfig.Services c.Assert(cfg.Get("environment"), qt.Equals, "test") c.Assert(cfg.GetBool("enablegitinfo"), qt.Equals, false) @@ -551,9 +521,23 @@ theme_param="themevalue2" c.Assert(cfg.Get("params.api_config.api_key"), qt.Equals, "new_key") c.Assert(cfg.Get("params.api_config.another_key"), qt.Equals, "default another_key") c.Assert(cfg.Get("params.mytheme_section.theme_param"), qt.Equals, "themevalue_changed") + c.Assert(cfg.Get("params.mytheme_section.theme_param_nooverride"), qt.Equals, "nooverride") c.Assert(cfg.Get("params.mytheme_section2.theme_param"), qt.Equals, "themevalue2_changed") c.Assert(cfg.Get("params.empty"), qt.Equals, ``) c.Assert(cfg.Get("params.html"), qt.Equals, ``) + params := cfg.Get("params").(maps.Params) + c.Assert(params["paramwithnoenvoverride"], qt.Equals, "nooverride") + c.Assert(cfg.Get("params.paramwithnoenvoverride"), qt.Equals, "nooverride") c.Assert(scfg.GoogleAnalytics.ID, qt.Equals, "gaid") + c.Assert(cfg.Get("params.a.b"), qt.DeepEquals, maps.Params{ + "c": "abc", + }) + + ofBase, _ := s.outputFormatsConfig.GetByName("ofbase") + ofTheme, _ := s.outputFormatsConfig.GetByName("oftheme") + + c.Assert(ofBase.MediaType, qt.Equals, media.TextType) + c.Assert(ofTheme.MediaType, qt.Equals, media.TextType) + } diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index 139d0c20e..a119d4c17 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -33,7 +33,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/paths" "github.com/gohugoio/hugo/modules" - "github.com/spf13/viper" + ) func initConfig(fs afero.Fs, cfg config.Provider) error { @@ -76,7 +76,7 @@ func initConfig(fs afero.Fs, cfg config.Provider) error { func TestNewBaseFs(t *testing.T) { c := qt.New(t) - v := viper.New() + v := config.New() fs := hugofs.NewMem(v) @@ -181,8 +181,8 @@ theme = ["atheme"] } } -func createConfig() *viper.Viper { - v := viper.New() +func createConfig() config.Provider { + v := config.New() v.Set("contentDir", "mycontent") v.Set("i18nDir", "myi18n") v.Set("staticDir", "mystatic") @@ -453,7 +453,7 @@ func countFilesAndGetFilenames(fs afero.Fs, dirname string) (int, []string, erro return counter, filenames, nil } -func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) { +func setConfigAndWriteSomeFilesTo(fs afero.Fs, v config.Provider, key, val string, num int) { workingDir := v.GetString("workingDir") v.Set(key, val) fs.Mkdir(val, 0755) diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 5ebf8bd0b..96355f08b 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/modules/npm" "github.com/gohugoio/hugo/common/loggers" @@ -37,7 +38,6 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/testmodBuilder/mods" - "github.com/spf13/viper" ) func TestHugoModulesVariants(t *testing.T) { @@ -45,7 +45,7 @@ func TestHugoModulesVariants(t *testing.T) { t.Skip("skip (relative) long running modules test when running locally") } - config := ` + tomlConfig := ` baseURL="https://example.org" workingDir = %q @@ -56,7 +56,7 @@ path="github.com/gohugoio/hugoTestModule2" ` createConfig := func(workingDir, moduleOpts string) string { - return fmt.Sprintf(config, workingDir, moduleOpts) + return fmt.Sprintf(tomlConfig, workingDir, moduleOpts) } newTestBuilder := func(t testing.TB, moduleOpts string) (*sitesBuilder, func()) { @@ -65,7 +65,7 @@ path="github.com/gohugoio/hugoTestModule2" b.Assert(err, qt.IsNil) workingDir := filepath.Join(tempDir, "myhugosite") b.Assert(os.MkdirAll(workingDir, 0777), qt.IsNil) - b.Fs = hugofs.NewDefault(viper.New()) + b.Fs = hugofs.NewDefault(config.New()) b.WithWorkingDir(workingDir).WithConfigFile("toml", createConfig(workingDir, moduleOpts)) b.WithTemplates( "index.html", ` @@ -333,7 +333,7 @@ func TestHugoModulesMatrix(t *testing.T) { for _, m := range testmods[:2] { c := qt.New(t) - v := viper.New() + v := config.New() workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-test") c.Assert(err, qt.IsNil) @@ -671,7 +671,7 @@ func TestModulesSymlinks(t *testing.T) { c := qt.New(t) // We need to use the OS fs for this. - cfg := viper.New() + cfg := config.New() fs := hugofs.NewFrom(hugofs.Os, cfg) workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mod-sym") @@ -839,13 +839,13 @@ workingDir = %q ` - config := fmt.Sprintf(configTemplate, workingDir) + tomlConfig := fmt.Sprintf(configTemplate, workingDir) b := newTestSitesBuilder(t).Running() - b.Fs = hugofs.NewDefault(viper.New()) + b.Fs = hugofs.NewDefault(config.New()) - b.WithWorkingDir(workingDir).WithConfigFile("toml", config) + b.WithWorkingDir(workingDir).WithConfigFile("toml", tomlConfig) b.WithTemplatesAdded("index.html", ` {{ .Title }} {{ .Content }} @@ -960,16 +960,16 @@ workingDir = %q %s ` - config := fmt.Sprintf(configTemplate, workingDir, mounts) - config = strings.Replace(config, "WORKING_DIR", workingDir, -1) + tomlConfig := fmt.Sprintf(configTemplate, workingDir, mounts) + tomlConfig = strings.Replace(tomlConfig, "WORKING_DIR", workingDir, -1) b := newTestSitesBuilder(c).Running() - b.Fs = hugofs.NewDefault(viper.New()) + b.Fs = hugofs.NewDefault(config.New()) os.MkdirAll(filepath.Join(workingDir, "content", "blog"), 0777) - b.WithWorkingDir(workingDir).WithConfigFile("toml", config) + b.WithWorkingDir(workingDir).WithConfigFile("toml", tomlConfig) return test{ b: b, @@ -1064,7 +1064,7 @@ func TestSiteWithGoModButNoModules(t *testing.T) { workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-no-mod") c.Assert(err, qt.IsNil) - cfg := viper.New() + cfg := config.New() cfg.Set("workingDir", workDir) fs := hugofs.NewFrom(hugofs.Os, cfg) @@ -1090,7 +1090,7 @@ func TestModuleAbsMount(t *testing.T) { absContentDir, clean2, err := htesting.CreateTempDir(hugofs.Os, "hugo-content") c.Assert(err, qt.IsNil) - cfg := viper.New() + cfg := config.New() cfg.Set("workingDir", workDir) fs := hugofs.NewFrom(hugofs.Os, cfg) diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 0607bde1c..d380cf737 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -374,7 +374,8 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { s.h = h } - if err := applyDeps(cfg, sites...); err != nil { + var l configLoader + if err := l.applyDeps(cfg, sites...); err != nil { return nil, errors.Wrap(err, "add site dependencies") } @@ -407,7 +408,7 @@ func (h *HugoSites) loadGitInfo() error { return nil } -func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { +func (l configLoader) applyDeps(cfg deps.DepsCfg, sites ...*Site) error { if cfg.TemplateProvider == nil { cfg.TemplateProvider = tplimpl.DefaultTemplateProvider } @@ -446,7 +447,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { d.Site = s.Info - siteConfig, err := loadSiteConfig(s.language) + siteConfig, err := l.loadSiteConfig(s.language) if err != nil { return errors.Wrap(err, "load site config") } @@ -607,11 +608,12 @@ func (h *HugoSites) withSite(fn func(s *Site) error) error { func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error { oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages) - if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil { + l := configLoader{cfg: h.Cfg} + if err := l.loadLanguageSettings(oldLangs); err != nil { return err } - depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: cfg} + depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: l.cfg} sites, err := createSitesFromConfig(depsCfg) if err != nil { @@ -629,7 +631,8 @@ func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error { s.h = h } - if err := applyDeps(depsCfg, sites...); err != nil { + var cl configLoader + if err := cl.applyDeps(depsCfg, sites...); err != nil { return err } diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index efe013c39..8b23e7ac7 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -5,9 +5,7 @@ import ( "path/filepath" "strings" "testing" - "time" - "github.com/fortytw2/leaktest" "github.com/gohugoio/hugo/htesting" qt "github.com/frankban/quicktest" @@ -318,7 +316,7 @@ Some content. // https://github.com/gohugoio/hugo/issues/5375 func TestSiteBuildTimeout(t *testing.T) { if !htesting.IsCI() { - defer leaktest.CheckTimeout(t, 10*time.Second)() + //defer leaktest.CheckTimeout(t, 10*time.Second)() } b := newTestSitesBuilder(t) diff --git a/hugolib/image_test.go b/hugolib/image_test.go index 0dacf2a33..4726f5b49 100644 --- a/hugolib/image_test.go +++ b/hugolib/image_test.go @@ -21,11 +21,11 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/htesting" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" ) // We have many tests for the different resize operations etc. in the resource package, @@ -38,7 +38,7 @@ func TestImageOps(t *testing.T) { defer clean() newBuilder := func(timeout interface{}) *sitesBuilder { - v := viper.New() + v := config.New() v.Set("workingDir", workDir) v.Set("baseURL", "https://example.org") v.Set("timeout", timeout) diff --git a/hugolib/js_test.go b/hugolib/js_test.go index cd0883fd5..75dc0e7de 100644 --- a/hugolib/js_test.go +++ b/hugolib/js_test.go @@ -21,11 +21,10 @@ import ( "testing" "github.com/gohugoio/hugo/common/hexec" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/htesting" - "github.com/spf13/viper" - qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/hugofs" @@ -88,7 +87,7 @@ document.body.textContent = greeter(user);` c.Assert(err, qt.IsNil) defer clean() - v := viper.New() + v := config.New() v.Set("workingDir", workDir) v.Set("disableKinds", []string{"taxonomy", "term", "page"}) b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger()) @@ -162,7 +161,7 @@ func TestJSBuild(t *testing.T) { c.Assert(err, qt.IsNil) defer clean() - config := fmt.Sprintf(` + tomlConfig := fmt.Sprintf(` baseURL = "https://example.org" workingDir = %q @@ -177,8 +176,8 @@ path="github.com/gohugoio/hugoTestProjectJSModImports" `, workDir) b := newTestSitesBuilder(t) - b.Fs = hugofs.NewDefault(viper.New()) - b.WithWorkingDir(workDir).WithConfigFile("toml", config).WithLogger(loggers.NewInfoLogger()) + b.Fs = hugofs.NewDefault(config.New()) + b.WithWorkingDir(workDir).WithConfigFile("toml", tomlConfig).WithLogger(loggers.NewInfoLogger()) b.WithSourceFile("go.mod", `module github.com/gohugoio/tests/testHugoModules go 1.15 diff --git a/hugolib/minify_publisher_test.go b/hugolib/minify_publisher_test.go index 66e674ade..ef460efa2 100644 --- a/hugolib/minify_publisher_test.go +++ b/hugolib/minify_publisher_test.go @@ -16,13 +16,13 @@ package hugolib import ( "testing" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/config" ) func TestMinifyPublisher(t *testing.T) { t.Parallel() - v := viper.New() + v := config.New() v.Set("minify", true) v.Set("baseURL", "https://example.org/") diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index 3df997452..759fadd2d 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -336,7 +336,7 @@ func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, fron if frontmatter != nil { // Needed for case insensitive fetching of params values - maps.ToLower(frontmatter) + maps.PrepareParams(frontmatter) if p.bucket != nil { // Check for any cascade define on itself. if cv, found := frontmatter["cascade"]; found { diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 9d23aaa5c..5bc3db22f 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -37,7 +37,6 @@ import ( "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" "github.com/spf13/afero" - "github.com/spf13/viper" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/deps" @@ -786,7 +785,7 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { c := qt.New(t) // We need to use the OS fs for this. - cfg := viper.New() + cfg := config.New() fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} @@ -1066,7 +1065,7 @@ func TestChompBOM(t *testing.T) { func TestPageWithEmoji(t *testing.T) { for _, enableEmoji := range []bool{true, false} { - v := viper.New() + v := config.New() v.Set("enableEmoji", enableEmoji) b := newTestSitesBuilder(t).WithViper(v) diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index 7d775871a..b63d663e3 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -23,6 +23,8 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/helpers" @@ -35,7 +37,6 @@ import ( "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/deps" - "github.com/spf13/viper" qt "github.com/frankban/quicktest" ) @@ -352,12 +353,11 @@ func TestMultilingualDisableDefaultLanguage(t *testing.T) { c := qt.New(t) _, cfg := newTestBundleSourcesMultilingual(t) - cfg.Set("disableLanguages", []string{"en"}) - - err := loadDefaultSettingsFor(cfg) + l := configLoader{cfg: cfg} + err := l.applyConfigDefaults() c.Assert(err, qt.IsNil) - err = loadLanguageSettings(cfg, nil) + err = l.loadLanguageSettings(nil) c.Assert(err, qt.Not(qt.IsNil)) c.Assert(err.Error(), qt.Contains, "cannot disable default language") } @@ -397,7 +397,7 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { c := qt.New(t) // We need to use the OS fs for this. - cfg := viper.New() + cfg := config.New() fs := hugofs.NewFrom(hugofs.Os, cfg) workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugosym") @@ -696,7 +696,7 @@ Single content. b.AssertFileContent("public/section-not-bundle/single/index.html", "Section Single", "|

Single content.

") } -func newTestBundleSources(t testing.TB) (*hugofs.Fs, *viper.Viper) { +func newTestBundleSources(t testing.TB) (*hugofs.Fs, config.Provider) { cfg, fs := newTestCfgBasic() c := qt.New(t) @@ -863,7 +863,7 @@ Content for 은행. return fs, cfg } -func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) { +func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, config.Provider) { cfg, fs := newTestCfgBasic() workDir := "/work" @@ -1319,7 +1319,7 @@ func TestPageBundlerHome(t *testing.T) { workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-bundler-home") c.Assert(err, qt.IsNil) - cfg := viper.New() + cfg := config.New() cfg.Set("workingDir", workDir) fs := hugofs.NewFrom(hugofs.Os, cfg) diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index c7a42acdc..45221af71 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -130,7 +130,7 @@ func (c *pagesCollector) isCascadingEdit(dir contentDirKey) (bool, string) { section = s - maps.ToLower(pf.FrontMatter) + maps.PrepareParams(pf.FrontMatter) cascade1, ok := pf.FrontMatter["cascade"] hasCascade := n.p.bucket.cascade != nil && len(n.p.bucket.cascade) > 0 if !ok { diff --git a/hugolib/paths/paths_test.go b/hugolib/paths/paths_test.go index 59dbf0e00..d3ead4d17 100644 --- a/hugolib/paths/paths_test.go +++ b/hugolib/paths/paths_test.go @@ -16,17 +16,16 @@ package paths import ( "testing" - "github.com/gohugoio/hugo/langs" - qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/langs" ) func TestNewPaths(t *testing.T) { c := qt.New(t) - v := viper.New() + v := config.New() fs := hugofs.NewMem(v) v.Set("languages", map[string]interface{}{ diff --git a/hugolib/resource_chain_babel_test.go b/hugolib/resource_chain_babel_test.go index 61b2ef6d5..9e5c9c4a5 100644 --- a/hugolib/resource_chain_babel_test.go +++ b/hugolib/resource_chain_babel_test.go @@ -19,14 +19,14 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/common/hexec" jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/htesting" - "github.com/spf13/viper" - qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/hugofs" @@ -91,7 +91,7 @@ class Car2 { var logBuf bytes.Buffer logger := loggers.NewBasicLoggerForWriter(jww.LevelInfo, &logBuf) - v := viper.New() + v := config.New() v.Set("workingDir", workDir) v.Set("disableKinds", []string{"taxonomy", "term", "page"}) b := newTestSitesBuilder(t).WithLogger(logger) diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 9ea1d8529..a367237ab 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -20,6 +20,8 @@ import ( "math/rand" "os" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/resources/resource_transformers/tocss/dartsass" "path/filepath" @@ -35,8 +37,6 @@ import ( "github.com/gohugoio/hugo/htesting" - "github.com/spf13/viper" - qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/hugofs" @@ -65,7 +65,7 @@ func TestSCSSWithIncludePaths(t *testing.T) { c.Assert(err, qt.IsNil) defer clean() - v := viper.New() + v := config.New() v.Set("workingDir", workDir) b := newTestSitesBuilder(c).WithLogger(loggers.NewErrorLogger()) // Need to use OS fs for this. @@ -130,7 +130,7 @@ func TestSCSSWithRegularCSSImport(t *testing.T) { c.Assert(err, qt.IsNil) defer clean() - v := viper.New() + v := config.New() v.Set("workingDir", workDir) b := newTestSitesBuilder(c).WithLogger(loggers.NewErrorLogger()) // Need to use OS fs for this. @@ -230,7 +230,7 @@ func TestSCSSWithThemeOverrides(t *testing.T) { theme := "mytheme" themesDir := filepath.Join(workDir, "themes") themeDirs := filepath.Join(themesDir, theme) - v := viper.New() + v := config.New() v.Set("workingDir", workDir) v.Set("theme", theme) b := newTestSitesBuilder(c).WithLogger(loggers.NewErrorLogger()) @@ -345,7 +345,7 @@ func TestSCSSWithIncludePathsSass(t *testing.T) { c.Assert(err, qt.IsNil) defer clean1() - v := viper.New() + v := config.New() v.Set("workingDir", workDir) v.Set("theme", "mytheme") b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) @@ -974,7 +974,7 @@ h1 { var logBuf bytes.Buffer - newTestBuilder := func(v *viper.Viper) *sitesBuilder { + newTestBuilder := func(v config.Provider) *sitesBuilder { v.Set("workingDir", workDir) v.Set("disableKinds", []string{"taxonomy", "term", "page"}) logger := loggers.NewBasicLoggerForWriter(jww.LevelInfo, &logBuf) @@ -997,7 +997,7 @@ Styles Content: Len: {{ len $styles.Content }}| return b } - b := newTestBuilder(viper.New()) + b := newTestBuilder(config.New()) cssDir := filepath.Join(workDir, "assets", "css", "components") b.Assert(os.MkdirAll(cssDir, 0777), qt.IsNil) @@ -1049,7 +1049,7 @@ Styles Content: Len: 770878| build := func(s string, shouldFail bool) error { b.Assert(os.RemoveAll(filepath.Join(workDir, "public")), qt.IsNil) - v := viper.New() + v := config.New() v.Set("build", map[string]interface{}{ "useResourceCacheWhen": s, }) diff --git a/hugolib/robotstxt_test.go b/hugolib/robotstxt_test.go index 6bc39e97c..2035c235f 100644 --- a/hugolib/robotstxt_test.go +++ b/hugolib/robotstxt_test.go @@ -16,7 +16,7 @@ package hugolib import ( "testing" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/config" ) const robotTxtTemplate = `User-agent: Googlebot @@ -28,7 +28,7 @@ const robotTxtTemplate = `User-agent: Googlebot func TestRobotsTXTOutput(t *testing.T) { t.Parallel() - cfg := viper.New() + cfg := config.New() cfg.Set("baseURL", "http://auth/bub/") cfg.Set("enableRobotsTXT", true) diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 51187a003..7eb0d01de 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -20,11 +20,10 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/markup/asciidocext" "github.com/gohugoio/hugo/markup/rst" - "github.com/spf13/viper" - "github.com/gohugoio/hugo/parser/pageparser" "github.com/gohugoio/hugo/resources/page" @@ -1214,7 +1213,7 @@ title: "Hugo Rocks!" func TestShortcodeEmoji(t *testing.T) { t.Parallel() - v := viper.New() + v := config.New() v.Set("enableEmoji", true) builder := newTestSitesBuilder(t).WithViper(v) @@ -1279,7 +1278,7 @@ func TestShortcodeRef(t *testing.T) { t.Run(fmt.Sprintf("plainIDAnchors=%t", plainIDAnchors), func(t *testing.T) { t.Parallel() - v := viper.New() + v := config.New() v.Set("baseURL", "https://example.org") v.Set("blackfriday", map[string]interface{}{ "plainIDAnchors": plainIDAnchors, diff --git a/hugolib/site.go b/hugolib/site.go index 12714892d..9921dcc97 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -77,7 +77,6 @@ import ( "github.com/spf13/afero" "github.com/spf13/cast" - "github.com/spf13/viper" ) // Site contains all the information relevant for constructing a static @@ -501,9 +500,9 @@ But this also means that your site configuration may not do what you expect. If var relatedContentConfig related.Config if cfg.Language.IsSet("related") { - relatedContentConfig, err = related.DecodeConfig(cfg.Language.Get("related")) + relatedContentConfig, err = related.DecodeConfig(cfg.Language.GetParams("related")) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to decode related config") } } else { relatedContentConfig = related.DefaultConfig @@ -574,7 +573,8 @@ func NewSite(cfg deps.DepsCfg) (*Site, error) { return nil, err } - if err = applyDeps(cfg, s); err != nil { + var l configLoader + if err = l.applyDeps(cfg, s); err != nil { return nil, err } @@ -586,11 +586,11 @@ func NewSite(cfg deps.DepsCfg) (*Site, error) { // Note: This is mainly used in single site tests. // TODO(bep) test refactor -- remove func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateManager) error) (*Site, error) { - v := viper.New() - if err := loadDefaultSettingsFor(v); err != nil { + l := configLoader{cfg: config.New()} + if err := l.applyConfigDefaults(); err != nil { return nil, err } - return newSiteForLang(langs.NewDefaultLanguage(v), withTemplate...) + return newSiteForLang(langs.NewDefaultLanguage(l.cfg), withTemplate...) } // NewEnglishSite creates a new site in English language. @@ -598,11 +598,11 @@ func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateManager) error) ( // Note: This is mainly used in single site tests. // TODO(bep) test refactor -- remove func NewEnglishSite(withTemplate ...func(templ tpl.TemplateManager) error) (*Site, error) { - v := viper.New() - if err := loadDefaultSettingsFor(v); err != nil { + l := configLoader{cfg: config.New()} + if err := l.applyConfigDefaults(); err != nil { return nil, err } - return newSiteForLang(langs.NewLanguage("en", v), withTemplate...) + return newSiteForLang(langs.NewLanguage("en", l.cfg), withTemplate...) } // newSiteForLang creates a new site in the given language. @@ -1314,7 +1314,7 @@ func (s *Site) initializeSiteInfo() error { return vvv } default: - m := cast.ToStringMapBool(v) + m := maps.ToStringMapBool(v) uglyURLs = func(p page.Page) bool { return m[p.Section()] } diff --git a/hugolib/site_output_test.go b/hugolib/site_output_test.go index 1961dd06f..f3455f369 100644 --- a/hugolib/site_output_test.go +++ b/hugolib/site_output_test.go @@ -19,13 +19,13 @@ import ( "testing" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/resources/page" "github.com/spf13/afero" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/output" - "github.com/spf13/viper" ) func TestSiteWithPageOutputs(t *testing.T) { @@ -333,7 +333,7 @@ func TestCreateSiteOutputFormats(t *testing.T) { page.KindSection: []string{"JSON"}, } - cfg := viper.New() + cfg := config.New() cfg.Set("outputs", outputsConfig) outputs, err := createSiteOutputFormats(output.DefaultFormats, cfg.GetStringMap("outputs"), false) @@ -358,7 +358,7 @@ func TestCreateSiteOutputFormats(t *testing.T) { // Issue #4528 t.Run("Mixed case", func(t *testing.T) { c := qt.New(t) - cfg := viper.New() + cfg := config.New() outputsConfig := map[string]interface{}{ // Note that we in Hugo 0.53.0 renamed this Kind to "taxonomy", @@ -380,7 +380,7 @@ func TestCreateSiteOutputFormatsInvalidConfig(t *testing.T) { page.KindHome: []string{"FOO", "JSON"}, } - cfg := viper.New() + cfg := config.New() cfg.Set("outputs", outputsConfig) _, err := createSiteOutputFormats(output.DefaultFormats, cfg.GetStringMap("outputs"), false) @@ -394,7 +394,7 @@ func TestCreateSiteOutputFormatsEmptyConfig(t *testing.T) { page.KindHome: []string{}, } - cfg := viper.New() + cfg := config.New() cfg.Set("outputs", outputsConfig) outputs, err := createSiteOutputFormats(output.DefaultFormats, cfg.GetStringMap("outputs"), false) @@ -409,7 +409,7 @@ func TestCreateSiteOutputFormatsCustomFormats(t *testing.T) { page.KindHome: []string{}, } - cfg := viper.New() + cfg := config.New() cfg.Set("outputs", outputsConfig) var ( diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 365679a32..e25991164 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -23,10 +23,9 @@ import ( "testing" "github.com/gobuffalo/flect" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/publisher" - "github.com/spf13/viper" - qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/resources/page" @@ -363,7 +362,7 @@ func TestMainSections(t *testing.T) { c := qt.New(t) for _, paramSet := range []bool{false, true} { c.Run(fmt.Sprintf("param-%t", paramSet), func(c *qt.C) { - v := viper.New() + v := config.New() if paramSet { v.Set("params", map[string]interface{}{ "mainSections": []string{"a1", "a2"}, diff --git a/hugolib/template_test.go b/hugolib/template_test.go index f487cec67..abb6d32f9 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -19,20 +19,19 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/identity" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/tpl" - - "github.com/spf13/viper" ) func TestTemplateLookupOrder(t *testing.T) { var ( fs *hugofs.Fs - cfg *viper.Viper + cfg config.Provider th testHelper ) diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 09988f972..451022e5c 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -30,6 +30,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/resources/page" @@ -39,7 +40,6 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/tpl" - "github.com/spf13/viper" "github.com/gohugoio/hugo/resources/resource" @@ -83,7 +83,7 @@ type sitesBuilder struct { // Default toml con