diff options
Diffstat (limited to 'commands')
38 files changed, 3295 insertions, 5347 deletions
diff --git a/commands/commandeer.go b/commands/commandeer.go index 45385d509..ed578e9bf 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,513 +14,593 @@ package commands import ( + "context" "errors" "fmt" "io" - "net" "os" + "os/signal" "path/filepath" - "regexp" "sync" + "sync/atomic" + "syscall" "time" - hconfig "github.com/gohugoio/hugo/config" + jww "github.com/spf13/jwalterweatherman" - "golang.org/x/sync/semaphore" + "github.com/bep/clock" + "github.com/bep/lazycache" + "github.com/bep/overlayfs" + "github.com/bep/simplecobra" - "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hstrings" "github.com/gohugoio/hugo/common/htime" - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/common/paths" - - "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" - "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/config" - - "github.com/spf13/cobra" - - "github.com/gohugoio/hugo/hugolib" - "github.com/spf13/afero" - - "github.com/bep/clock" - "github.com/bep/debounce" - "github.com/bep/overlayfs" - "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/config/allconfig" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/hugolib" + "github.com/spf13/afero" + "github.com/spf13/cobra" ) -type commandeerHugoState struct { - *deps.DepsCfg - hugoSites *hugolib.HugoSites - fsCreate sync.Once - created chan struct{} -} - -type commandeer struct { - *commandeerHugoState - - logger loggers.Logger - serverConfig *config.Server - - buildLock func() (unlock func(), err error) - - // Loading state - mustHaveConfigFile bool - failOnInitErr bool - running bool - - // Currently only set when in "fast render mode". But it seems to - // be fast enough that we could maybe just add it for all server modes. - changeDetector *fileChangeDetector - - // We need to reuse these on server rebuilds. - publishDirFs afero.Fs - publishDirStaticFs afero.Fs - publishDirServerFs afero.Fs - - h *hugoBuilderCommon - ftch flagsToConfigHandler - - visitedURLs *types.EvictingStringQueue - - cfgInit func(c *commandeer) error - - // We watch these for changes. - configFiles []string - - // Used in cases where we get flooded with events in server mode. - debounce func(f func()) - - serverPorts []serverPortListener - - languages langs.Languages - doLiveReload bool - renderStaticToDisk bool - fastRenderMode bool - showErrorInBrowser bool - wasError bool - - configured bool - paused bool - - fullRebuildSem *semaphore.Weighted +var ( + errHelp = errors.New("help requested") +) - // Any error from the last build. - buildErr error +// Execute executes a command. +func Execute(args []string) error { + x, err := newExec() + if err != nil { + return err + } + args = mapLegacyArgs(args) + cd, err := x.Execute(context.Background(), args) + if err != nil { + if err == errHelp { + cd.CobraCommand.Help() + fmt.Println() + return nil + } + if simplecobra.IsCommandError(err) { + // Print the help, but also return the error to fail the command. + cd.CobraCommand.Help() + fmt.Println() + } + } + return err } -type serverPortListener struct { - p int - ln net.Listener +type commonConfig struct { + mu sync.Mutex + configs *allconfig.Configs + cfg config.Provider + fs *hugofs.Fs } -func newCommandeerHugoState() *commandeerHugoState { - return &commandeerHugoState{ - created: make(chan struct{}), - } +func (c *commonConfig) getFs() *hugofs.Fs { + c.mu.Lock() + defer c.mu.Unlock() + return c.fs } -func (c *commandeerHugoState) hugo() *hugolib.HugoSites { - <-c.created - return c.hugoSites +// This is the root command. +type rootCommand struct { + Printf func(format string, v ...interface{}) + Println func(a ...interface{}) + Out io.Writer + + logger loggers.Logger + + // The main cache busting key for the caches below. + configVersionID atomic.Int32 + + // Some, but not all commands need access to these. + // Some needs more than one, so keep them in a small cache. + commonConfigs *lazycache.Cache[int32, *commonConfig] + hugoSites *lazycache.Cache[int32, *hugolib.HugoSites] + + commands []simplecobra.Commander + + // Flags + source string + baseURL string + buildWatch bool + forceSyncStatic bool + panicOnWarning bool + environment string + poll string + gc bool + + // Profile flags (for debugging of performance problems) + cpuprofile string + memprofile string + mutexprofile string + traceprofile string + printm bool + + // TODO(bep) var vs string + logging bool + verbose bool + verboseLog bool + debug bool + quiet bool + renderToMemory bool + + cfgFile string + cfgDir string + logFile string } -func (c *commandeerHugoState) hugoTry() *hugolib.HugoSites { - select { - case <-c.created: - return c.hugoSites - case <-time.After(time.Millisecond * 100): - return nil +func (r *rootCommand) Build(cd *simplecobra.Commandeer, bcfg hugolib.BuildCfg, cfg config.Provider) (*hugolib.HugoSites, error) { + h, err := r.Hugo(cfg) + if err != nil { + return nil, err + } + if err := h.Build(bcfg); err != nil { + return nil, err } + + return h, nil } -func (c *commandeer) errCount() int { - return int(c.logger.LogCounters().ErrorCounter.Count()) +func (r *rootCommand) Commands() []simplecobra.Commander { + return r.commands } -func (c *commandeer) getErrorWithContext() any { - errCount := c.errCount() +func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commonConfig, error) { + cc, _, err := r.commonConfigs.GetOrCreate(key, func(key int32) (*commonConfig, error) { + fs := oldConf.fs + configs, err := allconfig.LoadConfig( + allconfig.ConfigSourceDescriptor{ + Flags: oldConf.cfg, + Fs: fs.Source, + Filename: r.cfgFile, + ConfigDir: r.cfgDir, + Environment: r.environment, + }, + ) + if err != nil { + return nil, err + } - if errCount == 0 { - return nil - } + if !configs.Base.C.Clock.IsZero() { + // TODO(bep) find a better place for this. + htime.Clock = clock.Start(configs.Base.C.Clock) + } + + return &commonConfig{ + configs: configs, + cfg: oldConf.cfg, + fs: fs, + }, nil - m := make(map[string]any) + }) - //xwm["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors()))) - m["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors()))) - m["Version"] = hugo.BuildVersionString() - ferrors := herrors.UnwrapFileErrorsWithErrorContext(c.buildErr) - m["Files"] = ferrors + return cc, err - return m } -func (c *commandeer) Set(key string, value any) { - if c.configured { - panic("commandeer cannot be changed") +func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commonConfig, error) { + if cfg == nil { + panic("cfg must be set") } - c.Cfg.Set(key, value) -} + cc, _, err := r.commonConfigs.GetOrCreate(key, func(key int32) (*commonConfig, error) { + var dir string + if r.source != "" { + dir, _ = filepath.Abs(r.source) + } else { + dir, _ = os.Getwd() + } -func (c *commandeer) initFs(fs *hugofs.Fs) error { - c.publishDirFs = fs.PublishDir - c.publishDirStaticFs = fs.PublishDirStatic - c.publishDirServerFs = fs.PublishDirServer - c.DepsCfg.Fs = fs + if cfg == nil { + cfg = config.New() + } + if !cfg.IsSet("publishDir") { + cfg.Set("publishDir", "public") + } + if !cfg.IsSet("renderToDisk") { + cfg.Set("renderToDisk", true) + } + if !cfg.IsSet("workingDir") { + cfg.Set("workingDir", dir) + } + cfg.Set("publishDirStatic", cfg.Get("publishDir")) + cfg.Set("publishDirDynamic", cfg.Get("publishDir")) - return nil -} + renderStaticToDisk := cfg.GetBool("renderStaticToDisk") -func (c *commandeer) initClock(loc *time.Location) error { - bt := c.Cfg.GetString("clock") - if bt == "" { - return nil - } + sourceFs := hugofs.Os + var desinationFs afero.Fs + if cfg.GetBool("renderToDisk") { + desinationFs = hugofs.Os + } else { + desinationFs = afero.NewMemMapFs() + if renderStaticToDisk { + // Hybrid, render dynamic content to Root. + cfg.Set("publishDirDynamic", "/") + } else { + // Rendering to memoryFS, publish to Root regardless of publishDir. + cfg.Set("publishDirDynamic", "/") + cfg.Set("publishDirStatic", "/") + } + } - t, err := cast.StringToDateInDefaultLocation(bt, loc) - if err != nil { - return fmt.Errorf(`failed to parse "clock" flag: %s`, err) - } + fs := hugofs.NewFromSourceAndDestination(sourceFs, desinationFs, cfg) + + if renderStaticToDisk { + dynamicFs := fs.PublishDir + publishDirStatic := cfg.GetString("publishDirStatic") + workingDir := cfg.GetString("workingDir") + absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic) + staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic) + + // Serve from both the static and dynamic fs, + // the first will take priority. + // THis is a read-only filesystem, + // we do all the writes to + // fs.Destination and fs.DestinationStatic. + fs.PublishDirServer = overlayfs.New( + overlayfs.Options{ + Fss: []afero.Fs{ + dynamicFs, + staticFs, + }, + }, + ) + fs.PublishDirStatic = staticFs - htime.Clock = clock.Start(t) - return nil -} + } -func newCommandeer(mustHaveConfigFile, failOnInitErr, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, cfgInit func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) { - var rebuildDebouncer func(f func()) - if running { - // The time value used is tested with mass content replacements in a fairly big Hugo site. - // It is better to wait for some seconds in those cases rather than get flooded - // with rebuilds. - rebuildDebouncer = debounce.New(4 * time.Second) - } + configs, err := allconfig.LoadConfig( + allconfig.ConfigSourceDescriptor{ + Flags: cfg, + Fs: fs.Source, + Filename: r.cfgFile, + ConfigDir: r.cfgDir, + Environment: r.environment, + }, + ) + if err != nil { + return nil, err + } - out := io.Discard - if !h.quiet { - out = os.Stdout - } + base := configs.Base - c := &commandeer{ - h: h, - ftch: f, - commandeerHugoState: newCommandeerHugoState(), - cfgInit: cfgInit, - visitedURLs: types.NewEvictingStringQueue(10), - debounce: rebuildDebouncer, - fullRebuildSem: semaphore.NewWeighted(1), - - // Init state - mustHaveConfigFile: mustHaveConfigFile, - failOnInitErr: failOnInitErr, - running: running, - - // This will be replaced later, but we need something to log to before the configuration is read. - logger: loggers.NewLogger(jww.LevelWarn, jww.LevelError, out, io.Discard, running), - } + if !base.C.Clock.IsZero() { + // TODO(bep) find a better place for this. + htime.Clock = clock.Start(configs.Base.C.Clock) + } - return c, c.loadConfig() -} + if base.LogPathWarnings { + // Note that we only care about the "dynamic creates" here, + // so skip the static fs. + fs.PublishDir = hugofs.NewCreateCountingFs(fs.PublishDir) + } + + commonConfig := &commonConfig{ + configs: configs, + cfg: cfg, + fs: fs, + } + + return commonConfig, nil + }) -type fileChangeDetector struct { - sync.Mutex - current map[string]string - prev map[string]string + return cc, err - irrelevantRe *regexp.Regexp } -func (f *fileChangeDetector) OnFileClose(name, md5sum string) { - f.Lock() - defer f.Unlock() - f.current[name] = md5sum +func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) { + h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) { + conf.mu.Lock() + defer conf.mu.Unlock() + depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, Logger: r.logger} + return hugolib.NewHugoSites(depsCfg) + }) + return h, err } -func (f *fileChangeDetector) changed() []string { - if f == nil { - return nil - } - f.Lock() - defer f.Unlock() - var c []string - for k, v := range f.current { - vv, found := f.prev[k] - if !found || v != vv { - c = append(c, k) +func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) { + h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) { + conf, err := r.ConfigFromProvider(key, cfg) + if err != nil { + return nil, err } - } - - return f.filterIrrelevant(c) + depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, Logger: r.logger} + return hugolib.NewHugoSites(depsCfg) + }) + return h, err } -func (f *fileChangeDetector) filterIrrelevant(in []string) []string { - var filtered []string - for _, v := range in { - if !f.irrelevantRe.MatchString(v) { - filtered = append(filtered, v) - } - } - return filtered +func (r *rootCommand) Name() string { + return "hugo" } -func (f *fileChangeDetector) PrepareNew() { - if f == nil { - return +func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + if !r.buildWatch { + defer r.timeTrack(time.Now(), "Total") } - f.Lock() - defer f.Unlock() + b := newHugoBuilder(r, nil) - if f.current == nil { - f.current = make(map[string]string) - f.prev = make(map[string]string) - return + if err := b.loadConfig(cd, true); err != nil { + return err } - f.prev = make(map[string]string) - for k, v := range f.current { - f.prev[k] = v + err := func() error { + if r.buildWatch { + defer r.timeTrack(time.Now(), "Built") + } + err := b.build() + if err != nil { + r.Println("Error:", err.Error()) + } + return err + }() + + if err != nil { + return err } - f.current = make(map[string]string) -} -func (c *commandeer) loadConfig() error { - if c.DepsCfg == nil { - c.DepsCfg = &deps.DepsCfg{} + if !r.buildWatch { + // Done. + return nil } - if c.logger != nil { - // Truncate the error log if this is a reload. - c.logger.Reset() + watchDirs, err := b.getDirList() + if err != nil { + return err } - cfg := c.DepsCfg - c.configured = false - cfg.Running = c.running - loggers.PanicOnWarning.Store(c.h.panicOnWarning) + watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs) - var dir string - if c.h.source != "" { - dir, _ = filepath.Abs(c.h.source) - } else { - dir, _ = os.Getwd() + for _, group := range watchGroups { + r.Printf("Watching for changes in %s\n", group) } - - var sourceFs afero.Fs = hugofs.Os - if c.DepsCfg.Fs != nil { - sourceFs = c.DepsCfg.Fs.Source + watcher, err := b.newWatcher(r.poll, watchDirs...) + if err != nil { + return err } - environment := c.h.getEnvironment(c.running) + defer watcher.Close() - doWithConfig := func(cfg config.Provider) error { - if c.ftch != nil { - c.ftch.flagsToConfig(cfg) - } + r.Println("Press Ctrl+C to stop") - cfg.Set("workingDir", dir) - cfg.Set("environment", environment) - return nil - } + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - cfgSetAndInit := func(cfg config.Provider) error { - c.Cfg = cfg - if c.cfgInit == nil { - return nil - } - err := c.cfgInit(c) - return err - } + <-sigs - configPath := c.h.source - if configPath == "" { - configPath = dir - } - config, configFiles, err := hugolib.LoadConfig( - hugolib.ConfigSourceDescriptor{ - Fs: sourceFs, - Logger: c.logger, - Path: configPath, - WorkingDir: dir, - Filename: c.h.cfgFile, - AbsConfigDir: c.h.getConfigDir(dir), - Environment: environment, - }, - cfgSetAndInit, - doWithConfig) + return nil +} - if err != nil { - // We should improve the error handling here, - // but with hugo mod init and similar there is a chicken and egg situation - // with modules already configured in config.toml, so ignore those errors. - if c.mustHaveConfigFile || (c.failOnInitErr && !moduleNotFoundRe.MatchString(err.Error())) { - return err - } else { - // Just make it a warning. - c.logger.Warnln(err) +func (r *rootCommand) Init(cd, runner *simplecobra.Commandeer) error { + r.Out = os.Stdout + if r.quiet { + r.Out = io.Discard + } + r.Printf = func(format string, v ...interface{}) { + if !r.quiet { + fmt.Fprintf(r.Out, format, v...) } - } else if c.mustHaveConfigFile && len(configFiles) == 0 { - return hugolib.ErrNoConfigFile } - - c.configFiles = configFiles - - var ok bool - loc := time.Local - c.languages, ok = c.Cfg.Get("languagesSorted").(langs.Languages) - if ok { - loc = langs.GetLocation(c.languages[0]) + r.Println = func(a ...interface{}) { + if !r.quiet { + fmt.Fprintln(r.Out, a...) + } } - - err = c.initClock(loc) + _, running := runner.Command.(*serverCommand) + var err error + r.logger, err = r.createLogger(running) if err != nil { return err } - // Set some commonly used flags - c.doLiveReload = c.running && !c.Cfg.GetBool("disableLiveReload") - c.fastRenderMode = c.running && !c.Cfg.GetBool("disableFastRender") - c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError") + loggers.PanicOnWarning.Store(r.panicOnWarning) + r.commonConfigs = lazycache.New[int32, *commonConfig](lazycache.Options{MaxEntries: 5}) + r.hugoSites = lazycache.New[int32, *hugolib.HugoSites](lazycache.Options{MaxEntries: 5}) - // This is potentially double work, but we need to do this one more time now - // that all the languages have been configured. - if c.cfgInit != nil { - if err := c.cfgInit(c); err != nil { - return err + return nil +} + +func (r *rootCommand) createLogger(running bool) (loggers.Logger, error) { + var ( + logHandle = io.Discard + logThreshold = jww.LevelWarn + outHandle = r.Out + stdoutThreshold = jww.LevelWarn |