Rework the Destination filesystem to make --renderStaticToDisk work

See #9626

75 files changed, 650 insertions, 565 deletions

diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go
index 1ff3b8112..1ed020ef1 100644
--- a/cache/filecache/filecache_config_test.go
+++ b/cache/filecache/filecache_config_test.go
@@ -184,7 +184,7 @@ dir = "/"
 }
 
 func newTestConfig() config.Provider {
-	cfg := config.New()
+	cfg := config.NewWithTestDefaults()
 	cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject"))
 	cfg.Set("contentDir", "content")
 	cfg.Set("dataDir", "data")
diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go
index 6a051a264..47b5a7fcf 100644
--- a/cache/filecache/filecache_test.go
+++ b/cache/filecache/filecache_test.go
@@ -342,6 +342,7 @@ func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec
 	cfg, err := config.FromConfigString(configStr, "toml")
 	c.Assert(err, qt.IsNil)
 	initConfig(fs, cfg)
+	config.SetBaseTestDefaults(cfg)
 	p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, nil)
 	c.Assert(err, qt.IsNil)
 	return p
diff --git a/commands/commandeer.go b/commands/commandeer.go
index ced149e7a..1162a4b70 100644
--- a/commands/commandeer.go
+++ b/commands/commandeer.go
@@ -30,6 +30,7 @@ import (
 
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/common/paths"
 
 	jww "github.com/spf13/jwalterweatherman"
 
@@ -42,6 +43,7 @@ import (
 	"github.com/spf13/afero"
 
 	"github.com/bep/debounce"
+	"github.com/bep/overlayfs"
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
@@ -73,8 +75,10 @@ type commandeer struct {
 	// be fast enough that we could maybe just add it for all server modes.
 	changeDetector *fileChangeDetector
 
-	// We need to reuse this on server rebuilds.
-	destinationFs afero.Fs
+	// We need to reuse these on server rebuilds.
+	// These 2 will be different if --renderStaticToDisk is set.
+	publishDirFs       afero.Fs
+	publishDirServerFs afero.Fs
 
 	h    *hugoBuilderCommon
 	ftch flagsToConfigHandler
@@ -162,7 +166,8 @@ func (c *commandeer) Set(key string, value any) {
 }
 
 func (c *commandeer) initFs(fs *hugofs.Fs) error {
-	c.destinationFs = fs.Destination
+	c.publishDirFs = fs.PublishDir
+	c.publishDirServerFs = fs.PublishDirServer
 	c.DepsCfg.Fs = fs
 
 	return nil
@@ -378,28 +383,63 @@ func (c *commandeer) loadConfig() error {
 	createMemFs := config.GetBool("renderToMemory")
 	c.renderStaticToDisk = config.GetBool("renderStaticToDisk")
 
-	if createMemFs && !c.renderStaticToDisk {
+	if createMemFs {
 		// Rendering to memoryFS, publish to Root regardless of publishDir.
 		config.Set("publishDir", "/")
+		config.Set("publishDirStatic", "/")
+	} else if c.renderStaticToDisk {
+		// Hybrid, render dynamic content to Root.
+		config.Set("publishDirStatic", config.Get("publishDir"))
+		config.Set("publishDir", "/")
+
 	}
 
 	c.fsCreate.Do(func() {
 		fs := hugofs.NewFrom(sourceFs, config)
 
-		if c.destinationFs != nil {
+		if c.publishDirFs != nil {
 			// Need to reuse the destination on server rebuilds.
-			fs.Destination = c.destinationFs
-		} else if createMemFs && c.renderStaticToDisk {
-			// Writes the dynamic output on memory,
-			// while serve others directly from publishDir
+			fs.PublishDir = c.publishDirFs
+			fs.PublishDirServer = c.publishDirServerFs
+		} else {
 			publishDir := config.GetString("publishDir")
-			writableFs := afero.NewBasePathFs(afero.NewMemMapFs(), publishDir)
-			publicFs := afero.NewOsFs()
-			fs.Destination = afero.NewCopyOnWriteFs(afero.NewReadOnlyFs(publicFs), writableFs)
-			fs.DestinationStatic = publicFs
-		} else if createMemFs {
-			// Hugo writes the output to memory instead of the disk.
-			fs.Destination = new(afero.MemMapFs)
+			publishDirStatic := config.GetString("publishDirStatic")
+			workingDir := config.GetString("workingDir")
+			absPublishDir := paths.AbsPathify(workingDir, publishDir)
+			absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)
+
+			if c.renderStaticToDisk {
+				// Writes the dynamic output oton memory,
+				// while serve others directly from /public on disk.
+				dynamicFs := afero.NewMemMapFs()
+				staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic)
+
+				// Serve from both the static and dynamic fs,
+				// the first will take priority.
+				// THis is a read-only filesystem,
+				// we do all the writes to
+				// fs.Destination and fs.DestinationStatic.
+				fs.PublishDirServer = overlayfs.New(
+					overlayfs.Options{
+						Fss: []afero.Fs{
+							dynamicFs,
+							staticFs,
+						},
+					},
+				)
+				fs.PublishDir = dynamicFs
+				fs.PublishDirStatic = staticFs
+			} else if createMemFs {
+				// Hugo writes the output to memory instead of the disk.
+				fs.PublishDir = new(afero.MemMapFs)
+				fs.PublishDirServer = fs.PublishDir
+				fs.PublishDirStatic = fs.PublishDir
+			} else {
+				// Write everything to disk.
+				fs.PublishDir = afero.NewBasePathFs(afero.NewOsFs(), absPublishDir)
+				fs.PublishDirServer = fs.PublishDir
+				fs.PublishDirStatic = fs.PublishDir
+			}
 		}
 
 		if c.fastRenderMode {
@@ -413,15 +453,15 @@ func (c *commandeer) loadConfig() error {
 			}
 
 			changeDetector.PrepareNew()
-			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
-			fs.DestinationStatic = hugofs.NewHashingFs(fs.DestinationStatic, changeDetector)
+			fs.PublishDir = hugofs.NewHashingFs(fs.PublishDir, changeDetector)
+			fs.PublishDirStatic = hugofs.NewHashingFs(fs.PublishDirStatic, changeDetector)
 			c.changeDetector = changeDetector
 		}
 
 		if c.Cfg.GetBool("logPathWarnings") {
 			// Note that we only care about the "dynamic creates" here,
 			// so skip the static fs.
-			fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
+			fs.PublishDir = hugofs.NewCreateCountingFs(fs.PublishDir)
 		}
 
 		// To debug hard-to-find path issues.
diff --git a/commands/commands.go b/commands/commands.go
index 01f076d1a..99b0866e5 100644
--- a/commands/commands.go
+++ b/commands/commands.go
@@ -18,10 +18,9 @@ import (
 	"os"
 	"time"
 
-	"github.com/gohugoio/hugo/hugolib/paths"
-
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/common/loggers"
+	hpaths "github.com/gohugoio/hugo/common/paths"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/cobra"
@@ -243,14 +242,14 @@ func (cc *hugoBuilderCommon) timeTrack(start time.Time, name string) {
 
 func (cc *hugoBuilderCommon) getConfigDir(baseDir string) string {
 	if cc.cfgDir != "" {
-		return paths.AbsPathify(baseDir, cc.cfgDir)
+		return hpaths.AbsPathify(baseDir, cc.cfgDir)
 	}
 
 	if v, found := os.LookupEnv("HUGO_CONFIGDIR"); found {
-		return paths.AbsPathify(baseDir, v)
+		return hpaths.AbsPathify(baseDir, v)
 	}
 
-	return paths.AbsPathify(baseDir, "config")
+	return hpaths.AbsPathify(baseDir, "config")
 }
 
 func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string {
diff --git a/commands/commands_test.go b/commands/commands_test.go
index 43c7f8520..e3ec7bd99 100644
--- a/commands/commands_test.go
+++ b/commands/commands_test.go
@@ -22,8 +22,6 @@ import (
 
 	"github.com/gohugoio/hugo/config"
 
-	"github.com/gohugoio/hugo/htesting"
-
 	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/hugofs"
@@ -38,15 +36,13 @@ import (
 func TestExecute(t *testing.T) {
 	c := qt.New(t)
 
-	createSite := func(c *qt.C) (string, func()) {
-		dir, clean, err := createSimpleTestSite(t, testSiteConfig{})
-		c.Assert(err, qt.IsNil)
-		return dir, clean
+	createSite := func(c *qt.C) string {
+		dir := createSimpleTestSite(t, testSiteConfig{})
+		return dir
 	}
 
 	c.Run("hugo", func(c *qt.C) {
-		dir, clean := createSite(c)
-		defer clean()
+		dir := createSite(c)
 		resp := Execute([]string{"-s=" + dir})
 		c.Assert(resp.Err, qt.IsNil)
 		result := resp.Result
@@ -56,8 +52,7 @@ func TestExecute(t *testing.T) {
 	})
 
 	c.Run("hugo, set environment", func(c *qt.C) {
-		dir, clean := createSite(c)
-		defer clean()
+		dir := createSite(c)
 		resp := Execute([]string{"-s=" + dir, "-e=staging"})
 		c.Assert(resp.Err, qt.IsNil)
 		result := resp.Result
@@ -65,9 +60,8 @@ func TestExecute(t *testing.T) {
 	})
 
 	c.Run("convert toJSON", func(c *qt.C) {
-		dir, clean := createSite(c)
+		dir := createSite(c)
 		output := filepath.Join(dir, "myjson")
-		defer clean()
 		resp := Execute([]string{"convert", "toJSON", "-s=" + dir, "-e=staging", "-o=" + output})
 		c.Assert(resp.Err, qt.IsNil)
 		converted := readFileFrom(c, filepath.Join(output, "content", "p1.md"))
@@ -75,8 +69,7 @@ func TestExecute(t *testing.T) {
 	})
 
 	c.Run("config, set environment", func(c *qt.C) {
-		dir, clean := createSite(c)
-		defer clean()
+		dir := createSite(c)
 		out, err := captureStdout(func() error {
 			resp := Execute([]string{"config", "-s=" + dir, "-e=staging"})
 			return resp.Err
@@ -86,16 +79,14 @@ func TestExecute(t *testing.T) {
 	})
 
 	c.Run("deploy, environment set", func(c *qt.C) {
-		dir, clean := createSite(c)
-		defer clean()
+		dir := createSite(c)
 		resp := Execute([]string{"deploy", "-s=" + dir, "-e=staging", "--target=mydeployment", "--dryRun"})
 		c.Assert(resp.Err, qt.Not(qt.IsNil))
 		c.Assert(resp.Err.Error(), qt.Contains, `no driver registered for "hugocloud"`)
 	})
 
 	c.Run("list", func(c *qt.C) {
-		dir, clean := createSite(c)
-		defer clean()
+		dir := createSite(c)
 		out, err := captureStdout(func() error {
 			resp := Execute([]string{"list", "all", "-s=" + dir, "-e=staging"})
 			return resp.Err
@@ -105,8 +96,7 @@ func TestExecute(t *testing.T) {
 	})
 
 	c.Run("new theme", func(c *qt.C) {
-		dir, clean := createSite(c)
-		defer clean()
+		dir := createSite(c)
 		themesDir := filepath.Join(dir, "mythemes")
 		resp := Execute([]string{"new", "theme", "mytheme", "-s=" + dir, "-e=staging", "--themesDir=" + themesDir})
 		c.Assert(resp.Err, qt.IsNil)
@@ -115,8 +105,7 @@ func TestExecute(t *testing.T) {
 	})
 
 	c.Run("new site", func(c *qt.C) {
-		dir, clean := createSite(c)
-		defer clean()
+		dir := createSite(c)
 		siteDir := filepath.Join(dir, "mysite")
 		resp := Execute([]string{"new", "site", siteDir, "-e=staging"})
 		c.Assert(resp.Err, qt.IsNil)
@@ -167,7 +156,7 @@ func TestFlags(t *testing.T) {
 			name: "ignoreVendorPaths",
 			args: []string{"server", "--ignoreVendorPaths=github.com/**"},
 			check: func(c *qt.C, cmd *serverCmd) {
-				cfg := config.New()
+				cfg := config.NewWithTestDefaults()
 				cmd.flagsToConfig(cfg)
 				c.Assert(cfg.Get("ignoreVendorPaths"), qt.Equals, "github.com/**")
 			},
@@ -208,7 +197,7 @@ func TestFlags(t *testing.T) {
 				c.Assert(sc.serverPort, qt.Equals, 1366)
 				c.Assert(sc.environment, qt.Equals, "testing")
 
-				cfg := config.New()
+				cfg := config.NewWithTestDefaults()
 				sc.flagsToConfig(cfg)
 				c.Assert(cfg.GetString("publi