Create a struct with all of Hugo's config options

Primary motivation is documentation, but it will also hopefully simplify the code.

Also,

* Lower case the default output format names; this is in line with the custom ones (map keys) and how
it's treated all the places. This avoids doing `stringds.EqualFold` everywhere.

Closes #10896
Closes #10620

38 files changed, 3295 insertions, 5347 deletions

diff --git a/commands/commandeer.go b/commands/commandeer.go
index 45385d509..ed578e9bf 100644
--- a/commands/commandeer.go
+++ b/commands/commandeer.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
+// Copyright 2023 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,513 +14,593 @@
 package commands
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"os"
+	"os/signal"
 	"path/filepath"
-	"regexp"
 	"sync"
+	"sync/atomic"
+	"syscall"
 	"time"
 
-	hconfig "github.com/gohugoio/hugo/config"
+	jww "github.com/spf13/jwalterweatherman"
 
-	"golang.org/x/sync/semaphore"
+	"github.com/bep/clock"
+	"github.com/bep/lazycache"
+	"github.com/bep/overlayfs"
+	"github.com/bep/simplecobra"
 
-	"github.com/gohugoio/hugo/common/herrors"
+	"github.com/gohugoio/hugo/common/hstrings"
 	"github.com/gohugoio/hugo/common/htime"
-	"github.com/gohugoio/hugo/common/hugo"
-	"github.com/gohugoio/hugo/common/paths"
-
-	"github.com/spf13/cast"
-	jww "github.com/spf13/jwalterweatherman"
-
 	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/common/paths"
 	"github.com/gohugoio/hugo/config"
-
-	"github.com/spf13/cobra"
-
-	"github.com/gohugoio/hugo/hugolib"
-	"github.com/spf13/afero"
-
-	"github.com/bep/clock"
-	"github.com/bep/debounce"
-	"github.com/bep/overlayfs"
-	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/config/allconfig"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/hugolib"
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
 )
 
-type commandeerHugoState struct {
-	*deps.DepsCfg
-	hugoSites *hugolib.HugoSites
-	fsCreate  sync.Once
-	created   chan struct{}
-}
-
-type commandeer struct {
-	*commandeerHugoState
-
-	logger       loggers.Logger
-	serverConfig *config.Server
-
-	buildLock func() (unlock func(), err error)
-
-	// Loading state
-	mustHaveConfigFile bool
-	failOnInitErr      bool
-	running            bool
-
-	// Currently only set when in "fast render mode". But it seems to
-	// be fast enough that we could maybe just add it for all server modes.
-	changeDetector *fileChangeDetector
-
-	// We need to reuse these on server rebuilds.
-	publishDirFs       afero.Fs
-	publishDirStaticFs afero.Fs
-	publishDirServerFs afero.Fs
-
-	h    *hugoBuilderCommon
-	ftch flagsToConfigHandler
-
-	visitedURLs *types.EvictingStringQueue
-
-	cfgInit func(c *commandeer) error
-
-	// We watch these for changes.
-	configFiles []string
-
-	// Used in cases where we get flooded with events in server mode.
-	debounce func(f func())
-
-	serverPorts []serverPortListener
-
-	languages          langs.Languages
-	doLiveReload       bool
-	renderStaticToDisk bool
-	fastRenderMode     bool
-	showErrorInBrowser bool
-	wasError           bool
-
-	configured bool
-	paused     bool
-
-	fullRebuildSem *semaphore.Weighted
+var (
+	errHelp = errors.New("help requested")
+)
 
-	// Any error from the last build.
-	buildErr error
+// Execute executes a command.
+func Execute(args []string) error {
+	x, err := newExec()
+	if err != nil {
+		return err
+	}
+	args = mapLegacyArgs(args)
+	cd, err := x.Execute(context.Background(), args)
+	if err != nil {
+		if err == errHelp {
+			cd.CobraCommand.Help()
+			fmt.Println()
+			return nil
+		}
+		if simplecobra.IsCommandError(err) {
+			// Print the help, but also return the error to fail the command.
+			cd.CobraCommand.Help()
+			fmt.Println()
+		}
+	}
+	return err
 }
 
-type serverPortListener struct {
-	p  int
-	ln net.Listener
+type commonConfig struct {
+	mu      sync.Mutex
+	configs *allconfig.Configs
+	cfg     config.Provider
+	fs      *hugofs.Fs
 }
 
-func newCommandeerHugoState() *commandeerHugoState {
-	return &commandeerHugoState{
-		created: make(chan struct{}),
-	}
+func (c *commonConfig) getFs() *hugofs.Fs {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.fs
 }
 
-func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
-	<-c.created
-	return c.hugoSites
+// This is the root command.
+type rootCommand struct {
+	Printf  func(format string, v ...interface{})
+	Println func(a ...interface{})
+	Out     io.Writer
+
+	logger loggers.Logger
+
+	// The main cache busting key for the caches below.
+	configVersionID atomic.Int32
+
+	// Some, but not all commands need access to these.
+	// Some needs more than one, so keep them in a small cache.
+	commonConfigs *lazycache.Cache[int32, *commonConfig]
+	hugoSites     *lazycache.Cache[int32, *hugolib.HugoSites]
+
+	commands []simplecobra.Commander
+
+	// Flags
+	source          string
+	baseURL         string
+	buildWatch      bool
+	forceSyncStatic bool
+	panicOnWarning  bool
+	environment     string
+	poll            string
+	gc              bool
+
+	// Profile flags (for debugging of performance problems)
+	cpuprofile   string
+	memprofile   string
+	mutexprofile string
+	traceprofile string
+	printm       bool
+
+	// TODO(bep) var vs string
+	logging        bool
+	verbose        bool
+	verboseLog     bool
+	debug          bool
+	quiet          bool
+	renderToMemory bool
+
+	cfgFile string
+	cfgDir  string
+	logFile string
 }
 
-func (c *commandeerHugoState) hugoTry() *hugolib.HugoSites {
-	select {
-	case <-c.created:
-		return c.hugoSites
-	case <-time.After(time.Millisecond * 100):
-		return nil
+func (r *rootCommand) Build(cd *simplecobra.Commandeer, bcfg hugolib.BuildCfg, cfg config.Provider) (*hugolib.HugoSites, error) {
+	h, err := r.Hugo(cfg)
+	if err != nil {
+		return nil, err
+	}
+	if err := h.Build(bcfg); err != nil {
+		return nil, err
 	}
+
+	return h, nil
 }
 
-func (c *commandeer) errCount() int {
-	return int(c.logger.LogCounters().ErrorCounter.Count())
+func (r *rootCommand) Commands() []simplecobra.Commander {
+	return r.commands
 }
 
-func (c *commandeer) getErrorWithContext() any {
-	errCount := c.errCount()
+func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commonConfig, error) {
+	cc, _, err := r.commonConfigs.GetOrCreate(key, func(key int32) (*commonConfig, error) {
+		fs := oldConf.fs
+		configs, err := allconfig.LoadConfig(
+			allconfig.ConfigSourceDescriptor{
+				Flags:       oldConf.cfg,
+				Fs:          fs.Source,
+				Filename:    r.cfgFile,
+				ConfigDir:   r.cfgDir,
+				Environment: r.environment,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
 
-	if errCount == 0 {
-		return nil
-	}
+		if !configs.Base.C.Clock.IsZero() {
+			// TODO(bep) find a better place for this.
+			htime.Clock = clock.Start(configs.Base.C.Clock)
+		}
+
+		return &commonConfig{
+			configs: configs,
+			cfg:     oldConf.cfg,
+			fs:      fs,
+		}, nil
 
-	m := make(map[string]any)
+	})
 
-	//xwm["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors())))
-	m["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors())))
-	m["Version"] = hugo.BuildVersionString()
-	ferrors := herrors.UnwrapFileErrorsWithErrorContext(c.buildErr)
-	m["Files"] = ferrors
+	return cc, err
 
-	return m
 }
 
-func (c *commandeer) Set(key string, value any) {
-	if c.configured {
-		panic("commandeer cannot be changed")
+func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commonConfig, error) {
+	if cfg == nil {
+		panic("cfg must be set")
 	}
-	c.Cfg.Set(key, value)
-}
+	cc, _, err := r.commonConfigs.GetOrCreate(key, func(key int32) (*commonConfig, error) {
+		var dir string
+		if r.source != "" {
+			dir, _ = filepath.Abs(r.source)
+		} else {
+			dir, _ = os.Getwd()
+		}
 
-func (c *commandeer) initFs(fs *hugofs.Fs) error {
-	c.publishDirFs = fs.PublishDir
-	c.publishDirStaticFs = fs.PublishDirStatic
-	c.publishDirServerFs = fs.PublishDirServer
-	c.DepsCfg.Fs = fs
+		if cfg == nil {
+			cfg = config.New()
+		}
+		if !cfg.IsSet("publishDir") {
+			cfg.Set("publishDir", "public")
+		}
+		if !cfg.IsSet("renderToDisk") {
+			cfg.Set("renderToDisk", true)
+		}
+		if !cfg.IsSet("workingDir") {
+			cfg.Set("workingDir", dir)
+		}
+		cfg.Set("publishDirStatic", cfg.Get("publishDir"))
+		cfg.Set("publishDirDynamic", cfg.Get("publishDir"))
 
-	return nil
-}
+		renderStaticToDisk := cfg.GetBool("renderStaticToDisk")
 
-func (c *commandeer) initClock(loc *time.Location) error {
-	bt := c.Cfg.GetString("clock")
-	if bt == "" {
-		return nil
-	}
+		sourceFs := hugofs.Os
+		var desinationFs afero.Fs
+		if cfg.GetBool("renderToDisk") {
+			desinationFs = hugofs.Os
+		} else {
+			desinationFs = afero.NewMemMapFs()
+			if renderStaticToDisk {
+				// Hybrid, render dynamic content to Root.
+				cfg.Set("publishDirDynamic", "/")
+			} else {
+				// Rendering to memoryFS, publish to Root regardless of publishDir.
+				cfg.Set("publishDirDynamic", "/")
+				cfg.Set("publishDirStatic", "/")
+			}
+		}
 
-	t, err := cast.StringToDateInDefaultLocation(bt, loc)
-	if err != nil {
-		return fmt.Errorf(`failed to parse "clock" flag: %s`, err)
-	}
+		fs := hugofs.NewFromSourceAndDestination(sourceFs, desinationFs, cfg)
+
+		if renderStaticToDisk {
+			dynamicFs := fs.PublishDir
+			publishDirStatic := cfg.GetString("publishDirStatic")
+			workingDir := cfg.GetString("workingDir")
+			absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)
+			staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic)
+
+			// Serve from both the static and dynamic fs,
+			// the first will take priority.
+			// THis is a read-only filesystem,
+			// we do all the writes to
+			// fs.Destination and fs.DestinationStatic.
+			fs.PublishDirServer = overlayfs.New(
+				overlayfs.Options{
+					Fss: []afero.Fs{
+						dynamicFs,
+						staticFs,
+					},
+				},
+			)
+			fs.PublishDirStatic = staticFs
 
-	htime.Clock = clock.Start(t)
-	return nil
-}
+		}
 
-func newCommandeer(mustHaveConfigFile, failOnInitErr, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, cfgInit func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
-	var rebuildDebouncer func(f func())
-	if running {
-		// The time value used is tested with mass content replacements in a fairly big Hugo site.
-		// It is better to wait for some seconds in those cases rather than get flooded
-		// with rebuilds.
-		rebuildDebouncer = debounce.New(4 * time.Second)
-	}
+		configs, err := allconfig.LoadConfig(
+			allconfig.ConfigSourceDescriptor{
+				Flags:       cfg,
+				Fs:          fs.Source,
+				Filename:    r.cfgFile,
+				ConfigDir:   r.cfgDir,
+				Environment: r.environment,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
 
-	out := io.Discard
-	if !h.quiet {
-		out = os.Stdout
-	}
+		base := configs.Base
 
-	c := &commandeer{
-		h:                   h,
-		ftch:                f,
-		commandeerHugoState: newCommandeerHugoState(),
-		cfgInit:             cfgInit,
-		visitedURLs:         types.NewEvictingStringQueue(10),
-		debounce:            rebuildDebouncer,
-		fullRebuildSem:      semaphore.NewWeighted(1),
-
-		// Init state
-		mustHaveConfigFile: mustHaveConfigFile,
-		failOnInitErr:      failOnInitErr,
-		running:            running,
-
-		// This will be replaced later, but we need something to log to before the configuration is read.
-		logger: loggers.NewLogger(jww.LevelWarn, jww.LevelError, out, io.Discard, running),
-	}
+		if !base.C.Clock.IsZero() {
+			// TODO(bep) find a better place for this.
+			htime.Clock = clock.Start(configs.Base.C.Clock)
+		}
 
-	return c, c.loadConfig()
-}
+		if base.LogPathWarnings {
+			// Note that we only care about the "dynamic creates" here,
+			// so skip the static fs.
+			fs.PublishDir = hugofs.NewCreateCountingFs(fs.PublishDir)
+		}
+
+		commonConfig := &commonConfig{
+			configs: configs,
+			cfg:     cfg,
+			fs:      fs,
+		}
+
+		return commonConfig, nil
+	})
 
-type fileChangeDetector struct {
-	sync.Mutex
-	current map[string]string
-	prev    map[string]string
+	return cc, err
 
-	irrelevantRe *regexp.Regexp
 }
 
-func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
-	f.Lock()
-	defer f.Unlock()
-	f.current[name] = md5sum
+func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) {
+	h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) {
+		conf.mu.Lock()
+		defer conf.mu.Unlock()
+		depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, Logger: r.logger}
+		return hugolib.NewHugoSites(depsCfg)
+	})
+	return h, err
 }
 
-func (f *fileChangeDetector) changed() []string {
-	if f == nil {
-		return nil
-	}
-	f.Lock()
-	defer f.Unlock()
-	var c []string
-	for k, v := range f.current {
-		vv, found := f.prev[k]
-		if !found || v != vv {
-			c = append(c, k)
+func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) {
+	h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) {
+		conf, err := r.ConfigFromProvider(key, cfg)
+		if err != nil {
+			return nil, err
 		}
-	}
-
-	return f.filterIrrelevant(c)
+		depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, Logger: r.logger}
+		return hugolib.NewHugoSites(depsCfg)
+	})
+	return h, err
 }
 
-func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
-	var filtered []string
-	for _, v := range in {
-		if !f.irrelevantRe.MatchString(v) {
-			filtered = append(filtered, v)
-		}
-	}
-	return filtered
+func (r *rootCommand) Name() string {
+	return "hugo"
 }
 
-func (f *fileChangeDetector) PrepareNew() {
-	if f == nil {
-		return
+func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
+	if !r.buildWatch {
+		defer r.timeTrack(time.Now(), "Total")
 	}
 
-	f.Lock()
-	defer f.Unlock()
+	b := newHugoBuilder(r, nil)
 
-	if f.current == nil {
-		f.current = make(map[string]string)
-		f.prev = make(map[string]string)
-		return
+	if err := b.loadConfig(cd, true); err != nil {
+		return err
 	}
 
-	f.prev = make(map[string]string)
-	for k, v := range f.current {
-		f.prev[k] = v
+	err := func() error {
+		if r.buildWatch {
+			defer r.timeTrack(time.Now(), "Built")
+		}
+		err := b.build()
+		if err != nil {
+			r.Println("Error:", err.Error())
+		}
+		return err
+	}()
+
+	if err != nil {
+		return err
 	}
-	f.current = make(map[string]string)
-}
 
-func (c *commandeer) loadConfig() error {
-	if c.DepsCfg == nil {
-		c.DepsCfg = &deps.DepsCfg{}
+	if !r.buildWatch {
+		// Done.
+		return nil
 	}
 
-	if c.logger != nil {
-		// Truncate the error log if this is a reload.
-		c.logger.Reset()
+	watchDirs, err := b.getDirList()
+	if err != nil {
+		return err
 	}
 
-	cfg := c.DepsCfg
-	c.configured = false
-	cfg.Running = c.running
-	loggers.PanicOnWarning.Store(c.h.panicOnWarning)
+	watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs)
 
-	var dir string
-	if c.h.source != "" {
-		dir, _ = filepath.Abs(c.h.source)
-	} else {
-		dir, _ = os.Getwd()
+	for _, group := range watchGroups {
+		r.Printf("Watching for changes in %s\n", group)
 	}
-
-	var sourceFs afero.Fs = hugofs.Os
-	if c.DepsCfg.Fs != nil {
-		sourceFs = c.DepsCfg.Fs.Source
+	watcher, err := b.newWatcher(r.poll, watchDirs...)
+	if err != nil {
+		return err
 	}
 
-	environment := c.h.getEnvironment(c.running)
+	defer watcher.Close()
 
-	doWithConfig := func(cfg config.Provider) error {
-		if c.ftch != nil {
-			c.ftch.flagsToConfig(cfg)
-		}
+	r.Println("Press Ctrl+C to stop")
 
-		cfg.Set("workingDir", dir)
-		cfg.Set("environment", environment)
-		return nil
-	}
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 
-	cfgSetAndInit := func(cfg config.Provider) error {
-		c.Cfg = cfg
-		if c.cfgInit == nil {
-			return nil
-		}
-		err := c.cfgInit(c)
-		return err
-	}
+	<-sigs
 
-	configPath := c.h.source
-	if configPath == "" {
-		configPath = dir
-	}
-	config, configFiles, err := hugolib.LoadConfig(
-		hugolib.ConfigSourceDescriptor{
-			Fs:           sourceFs,
-			Logger:       c.logger,
-			Path:         configPath,
-			WorkingDir:   dir,
-			Filename:     c.h.cfgFile,
-			AbsConfigDir: c.h.getConfigDir(dir),
-			Environment:  environment,
-		},
-		cfgSetAndInit,
-		doWithConfig)
+	return nil
+}
 
-	if err != nil {
-		// We should improve the error handling here,
-		// but with hugo mod init and similar there is a chicken and egg situation
-		// with modules already configured in config.toml, so ignore those errors.
-		if c.mustHaveConfigFile || (c.failOnInitErr && !moduleNotFoundRe.MatchString(err.Error())) {
-			return err
-		} else {
-			// Just make it a warning.
-			c.logger.Warnln(err)
+func (r *rootCommand) Init(cd, runner *simplecobra.Commandeer) error {
+	r.Out = os.Stdout
+	if r.quiet {
+		r.Out = io.Discard
+	}
+	r.Printf = func(format string, v ...interface{}) {
+		if !r.quiet {
+			fmt.Fprintf(r.Out, format, v...)
 		}
-	} else if c.mustHaveConfigFile && len(configFiles) == 0 {
-		return hugolib.ErrNoConfigFile
 	}
-
-	c.configFiles = configFiles
-
-	var ok bool
-	loc := time.Local
-	c.languages, ok = c.Cfg.Get("languagesSorted").(langs.Languages)
-	if ok {
-		loc = langs.GetLocation(c.languages[0])
+	r.Println = func(a ...interface{}) {
+		if !r.quiet {
+			fmt.Fprintln(r.Out, a...)
+		}
 	}
-
-	err = c.initClock(loc)
+	_, running := runner.Command.(*serverCommand)
+	var err error
+	r.logger, err = r.createLogger(running)
 	if err != nil {
 		return err
 	}
 
-	// Set some commonly used flags
-	c.doLiveReload = c.running && !c.Cfg.GetBool("disableLiveReload")
-	c.fastRenderMode = c.running && !c.Cfg.GetBool("disableFastRender")
-	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")
+	loggers.PanicOnWarning.Store(r.panicOnWarning)
+	r.commonConfigs = lazycache.New[int32, *commonConfig](lazycache.Options{MaxEntries: 5})
+	r.hugoSites = lazycache.New[int32, *hugolib.HugoSites](lazycache.Options{MaxEntries: 5})
 
-	// This is potentially double work, but we need to do this one more time now
-	// that all the languages have been configured.
-	if c.cfgInit != nil {
-		if err := c.cfgInit(c); err != nil {
-			return err
+	return nil
+}
+
+func (r *rootCommand) createLogger(running bool) (loggers.Logger, error) {
+	var (
+		logHandle       = io.Discard
+		logThreshold    = jww.LevelWarn
+		outHandle