diff options
33 files changed, 1027 insertions, 142 deletions
diff --git a/cache/dynacache/dynacache.go b/cache/dynacache/dynacache.go index 6190dd234..5007e27ba 100644 --- a/cache/dynacache/dynacache.go +++ b/cache/dynacache/dynacache.go @@ -38,6 +38,11 @@ import ( const minMaxSize = 10 +type KeyIdentity struct { + Key any + Identity identity.Identity +} + // New creates a new cache. func New(opts Options) *Cache { if opts.CheckInterval == 0 { @@ -64,14 +69,14 @@ func New(opts Options) *Cache { infol := opts.Log.InfoCommand("dynacache") - evictedIdentities := collections.NewStack[identity.Identity]() + evictedIdentities := collections.NewStack[KeyIdentity]() onEvict := func(k, v any) { if !opts.Watching { return } identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool { - evictedIdentities.Push(id) + evictedIdentities.Push(KeyIdentity{Key: k, Identity: id}) return false }) resource.MarkStale(v) @@ -124,7 +129,7 @@ type Cache struct { partitions map[string]PartitionManager onEvict func(k, v any) - evictedIdentities *collections.Stack[identity.Identity] + evictedIdentities *collections.Stack[KeyIdentity] opts Options infol logg.LevelLogger @@ -135,10 +140,15 @@ type Cache struct { } // DrainEvictedIdentities drains the evicted identities from the cache. -func (c *Cache) DrainEvictedIdentities() []identity.Identity { +func (c *Cache) DrainEvictedIdentities() []KeyIdentity { return c.evictedIdentities.Drain() } +// DrainEvictedIdentitiesMatching drains the evicted identities from the cache that match the given predicate. +func (c *Cache) DrainEvictedIdentitiesMatching(predicate func(KeyIdentity) bool) []KeyIdentity { + return c.evictedIdentities.DrainMatching(predicate) +} + // ClearMatching clears all partition for which the predicate returns true. func (c *Cache) ClearMatching(predicatePartition func(k string, p PartitionManager) bool, predicateValue func(k, v any) bool) { if predicatePartition == nil { diff --git a/cache/httpcache/httpcache.go b/cache/httpcache/httpcache.go index ff360001f..98f7fedd4 100644 --- a/cache/httpcache/httpcache.go +++ b/cache/httpcache/httpcache.go @@ -83,7 +83,6 @@ func (c *Config) Compile() (ConfigCompiled, error) { } // PollConfig holds the configuration for polling remote resources to detect changes in watch mode. -// TODO1 make sure this enabled only in watch mode. type PollConfig struct { // What remote resources to apply this configuration to. For GlobMatcher diff --git a/common/collections/stack.go b/common/collections/stack.go index 0f1581626..96d32fe4b 100644 --- a/common/collections/stack.go +++ b/common/collections/stack.go @@ -65,3 +65,16 @@ func (s *Stack[T]) Drain() []T { s.items = nil return items } + +func (s *Stack[T]) DrainMatching(predicate func(T) bool) []T { + s.mu.Lock() + defer s.mu.Unlock() + var items []T + for i := len(s.items) - 1; i >= 0; i-- { + if predicate(s.items[i]) { + items = append(items, s.items[i]) + s.items = append(s.items[:i], s.items[i+1:]...) + } + } + return items +} diff --git a/common/herrors/errors.go b/common/herrors/errors.go index 7c389c1ae..e7f91462e 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -68,6 +68,20 @@ func (e *TimeoutError) Is(target error) bool { return ok } +// errMessage wraps an error with a message. +type errMessage struct { + msg string + err error +} + +func (e *errMessage) Error() string { + return e.msg +} + +func (e *errMessage) Unwrap() error { + return e.err +} + // IsFeatureNotAvailableError returns true if the given error is or contains a FeatureNotAvailableError. func IsFeatureNotAvailableError(err error) bool { return errors.Is(err, &FeatureNotAvailableError{}) @@ -121,19 +135,38 @@ func IsNotExist(err error) bool { var nilPointerErrRe = regexp.MustCompile(`at <(.*)>: error calling (.*?): runtime error: invalid memory address or nil pointer dereference`) -func ImproveIfNilPointer(inErr error) (outErr error) { +const deferredPrefix = "__hdeferred/" + +var deferredStringToRemove = regexp.MustCompile(`executing "__hdeferred/.*" `) + +// ImproveRenderErr improves the error message for rendering errors. +func ImproveRenderErr(inErr error) (outErr error) { outErr = inErr + msg := improveIfNilPointerMsg(inErr) + if msg != "" { + outErr = &errMessage{msg: msg, err: outErr} + } + if strings.Contains(inErr.Error(), deferredPrefix) { + msg := deferredStringToRemove.ReplaceAllString(inErr.Error(), "executing ") + outErr = &errMessage{msg: msg, err: outErr} + } + return +} + +func improveIfNilPointerMsg(inErr error) string { m := nilPointerErrRe.FindStringSubmatch(inErr.Error()) if len(m) == 0 { - return + return "" } call := m[1] field := m[2] parts := strings.Split(call, ".") + if len(parts) < 2 { + return "" + } receiverName := parts[len(parts)-2] receiver := strings.Join(parts[:len(parts)-1], ".") s := fmt.Sprintf("– %s is nil; wrap it in if or with: {{ with %s }}{{ .%s }}{{ end }}", receiverName, receiver, field) - outErr = errors.New(nilPointerErrRe.ReplaceAllString(inErr.Error(), s)) - return + return nilPointerErrRe.ReplaceAllString(inErr.Error(), s) } diff --git a/common/hugio/hasBytesWriter.go b/common/hugio/hasBytesWriter.go index 5148c82f9..d2bcd1bb4 100644 --- a/common/hugio/hasBytesWriter.go +++ b/common/hugio/hasBytesWriter.go @@ -17,24 +17,35 @@ import ( "bytes" ) -// HasBytesWriter is a writer that will set Match to true if the given pattern -// is found in the stream. +// HasBytesWriter is a writer will match against a slice of patterns. type HasBytesWriter struct { - Match bool - Pattern []byte + Patterns []*HasBytesPattern i int done bool buff []byte } +type HasBytesPattern struct { + Match bool + Pattern []byte +} + +func (h *HasBytesWriter) patternLen() int { + l := 0 + for _, p := range h.Patterns { + l += len(p.Pattern) + } + return l +} + func (h *HasBytesWriter) Write(p []byte) (n int, err error) { if h.done { return len(p), nil } if len(h.buff) == 0 { - h.buff = make([]byte, len(h.Pattern)*2) + h.buff = make([]byte, h.patternLen()*2) } for i := range p { @@ -46,11 +57,23 @@ func (h *HasBytesWriter) Write(p []byte) (n int, err error) { h.i = len(h.buff) / 2 } - if bytes.Contains(h.buff, h.Pattern) { - h.Match = true - h.done = true - return len(p), nil + for _, pp := range h.Patterns { + if bytes.Contains(h.buff, pp.Pattern) { + pp.Match = true + done := true + for _, ppp := range h.Patterns { + if !ppp.Match { + done = false + break + } + } + if done { + h.done = true + } + return len(p), nil + } } + } return len(p), nil diff --git a/common/hugio/hasBytesWriter_test.go b/common/hugio/hasBytesWriter_test.go index af53fa5dd..49487ab0b 100644 --- a/common/hugio/hasBytesWriter_test.go +++ b/common/hugio/hasBytesWriter_test.go @@ -34,8 +34,11 @@ func TestHasBytesWriter(t *testing.T) { var b bytes.Buffer h := &HasBytesWriter{ - Pattern: []byte("__foo"), + Patterns: []*HasBytesPattern{ + {Pattern: []byte("__foo")}, + }, } + return h, io.MultiWriter(&b, h) } @@ -46,19 +49,19 @@ func TestHasBytesWriter(t *testing.T) { for i := 0; i < 22; i++ { h, w := neww() fmt.Fprintf(w, rndStr()+"abc __foobar"+rndStr()) - c.Assert(h.Match, qt.Equals, true) + c.Assert(h.Patterns[0].Match, qt.Equals, true) h, w = neww() fmt.Fprintf(w, rndStr()+"abc __f") fmt.Fprintf(w, "oo bar"+rndStr()) - c.Assert(h.Match, qt.Equals, true) + c.Assert(h.Patterns[0].Match, qt.Equals, true) h, w = neww() fmt.Fprintf(w, rndStr()+"abc __moo bar") - c.Assert(h.Match, qt.Equals, false) + c.Assert(h.Patterns[0].Match, qt.Equals, false) } h, w := neww() fmt.Fprintf(w, "__foo") - c.Assert(h.Match, qt.Equals, true) + c.Assert(h.Patterns[0].Match, qt.Equals, true) } diff --git a/common/maps/cache.go b/common/maps/cache.go index 3723d318e..7cd7410c2 100644 --- a/common/maps/cache.go +++ b/common/maps/cache.go @@ -74,6 +74,26 @@ func (c *Cache[K, T]) ForEeach(f func(K, T)) { } } +func (c *Cache[K, T]) Drain() map[K]T { + c.Lock() + m := c.m + c.m = make(map[K]T) + c.Unlock() + return m +} + +func (c *Cache[K, T]) Len() int { + c.RLock() + defer c.RUnlock() + return len(c.m) +} + +func (c *Cache[K, T]) Reset() { + c.Lock() + c.m = make(map[K]T) + c.Unlock() +} + // SliceCache is a simple thread safe cache backed by a map. type SliceCache[T any] struct { m map[string][]T diff --git a/common/paths/path.go b/common/paths/path.go index 906270cae..de91d6a2f 100644 --- a/common/paths/path.go +++ b/common/paths/path.go @@ -237,12 +237,17 @@ func prettifyPath(in string, b filepathPathBridge) string { return b.Join(b.Dir(in), name, "index"+ext) } -// CommonDir returns the common directory of the given paths. -func CommonDir(path1, path2 string) string { +// CommonDirPath returns the common directory of the given paths. +func CommonDirPath(path1, path2 string) string { if path1 == "" || path2 == "" { return "" } + hadLeadingSlash := strings.HasPrefix(path1, "/") || strings.HasPrefix(path2, "/") + + path1 = TrimLeading(path1) + path2 = TrimLeading(path2) + p1 := strings.Split(path1, "/") p2 := strings.Split(path2, "/") @@ -256,7 +261,13 @@ func CommonDir(path1, path2 string) string { } } - return strings.Join(common, "/") + s := strings.Join(common, "/") + + if hadLeadingSlash && s != "" { + s = "/" + s + } + + return s } // Sanitize sanitizes string to be used in Hugo's file paths and URLs, allowing only @@ -384,12 +395,27 @@ func PathEscape(pth string) string { // ToSlashTrimLeading is just a filepath.ToSlash with an added / prefix trimmer. func ToSlashTrimLeading(s string) string { - return strings.TrimPrefix(filepath.ToSlash(s), "/") + return TrimLeading(filepath.ToSlash(s)) +} + +// TrimLeading trims the leading slash from the given string. +func TrimLeading(s string) string { + return strings.TrimPrefix(s, "/") } // ToSlashTrimTrailing is just a filepath.ToSlash with an added / suffix trimmer. func ToSlashTrimTrailing(s string) string { - return strings.TrimSuffix(filepath.ToSlash(s), "/") + return TrimTrailing(filepath.ToSlash(s)) +} + +// TrimTrailing trims the trailing slash from the given string. +func TrimTrailing(s string) string { + return strings.TrimSuffix(s, "/") +} + +// ToSlashTrim trims any leading and trailing slashes from the given string and converts it to a forward slash separated path. +func ToSlashTrim(s string) string { + return strings.Trim(filepath.ToSlash(s), "/") } // ToSlashPreserveLeading converts the path given to a forward slash separated path @@ -397,3 +423,8 @@ func ToSlashTrimTrailing(s string) string { func ToSlashPreserveLeading(s string) string { return "/" + strings.Trim(filepath.ToSlash(s), "/") } + +// IsSameFilePath checks if s1 and s2 are the same file path. +func IsSameFilePath(s1, s2 string) bool { + return path.Clean(ToSlashTrim(s1)) == path.Clean(ToSlashTrim(s2)) +} diff --git a/common/paths/path_test.go b/common/paths/path_test.go index 3605bfc43..bc27df6c6 100644 --- a/common/paths/path_test.go +++ b/common/paths/path_test.go @@ -262,3 +262,52 @@ func TestFieldsSlash(t *testing.T) { c.Assert(FieldsSlash("/"), qt.DeepEquals, []string{}) c.Assert(FieldsSlash(""), qt.DeepEquals, []string{}) } + +func TestCommonDirPath(t *testing.T) { + c := qt.New(t) + + for _, this := range []struct { + a, b, expected string + }{ + {"/a/b/c", "/a/b/d", "/a/b"}, + {"/a/b/c", "a/b/d", "/a/b"}, + {"a/b/c", "/a/b/d", "/a/b"}, + {"a/b/c", "a/b/d", "a/b"}, + {"/a/b/c", "/a/b/c", "/a/b/c"}, + {"/a/b/c", "/a/b/c/d", "/a/b/c"}, + {"/a/b/c", "/a/b", "/a/b"}, + {"/a/b/c", "/a", "/a"}, + {"/a/b/c", "/d/e/f", ""}, + } { + c.Assert(CommonDirPath(this.a, this.b), qt.Equals, this.expected, qt.Commentf("a: %s b: %s", this.a, this.b)) + } +} + +func TestIsSameFilePath(t *testing.T) { + c := qt.New(t) + + for _, this := range []struct { + a, b string + expected bool + }{ + {"/a/b/c", "/a/b/c", true}, + {"/a/b/c", "/a/b/c/", true}, + {"/a/b/c", "/a/b/d", false}, + {"/a/b/c", "/a/b", false}, + {"/a/b/c", "/a/b/c/d", false}, + {"/a/b/c", "/a/b/cd", false}, + {"/a/b/c", "/a/b/cc", false}, + {"/a/b/c", "/a/b/c/", true}, + {"/a/b/c", "/a/b/c//", true}, + {"/a/b/c", "/a/b/c/.", true}, + {"/a/b/c", "/a/b/c/./", true}, + {"/a/b/c", "/a/b/c/./.", true}, + {"/a/b/c", "/a/b/c/././", true}, + {"/a/b/c", "/a/b/c/././.", true}, + {"/a/b/c", "/a/b/c/./././", true}, + {"/a/b/c", "/a/b/c/./././.", true}, + {"/a/b/c", "/a/b/c/././././", true}, + } { + c.Assert(IsSameFilePath(filepath.FromSlash(this.a), filepath.FromSlash(this.b)), qt.Equals, this.expected, qt.Commentf("a: %s b: %s", this.a, this.b)) + } +} diff --git a/config/allconfig/load.go b/config/allconfig/load.go index edf8295bf..117b8e89c 100644 --- a/config/allconfig/load.go +++ b/config/allconfig/load.go @@ -458,6 +458,7 @@ func (l *configLoader) loadModules(configs *Configs) (modules.ModulesConfig, *mo conf := configs.Base workingDir := bcfg.WorkingDir themesDir := bcfg.ThemesDir + publishDir := bcfg.PublishDir cfg := configs.LoadingInfo.Cfg @@ -492,6 +493,7 @@ func (l *configLoader) loadModules(configs *Configs) (modules.ModulesConfig, *mo HookBeforeFinalize: hook, WorkingDir: workingDir, ThemesDir: themesDir, + PublishDir: publishDir, Environment: l.Environment, CacheDir: conf.Caches.CacheDirModules(), ModuleConfig: conf.Module, diff --git a/deps/deps.go b/deps/deps.go index 678f8a2fc..4805af1aa 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -15,6 +15,7 @@ import ( "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/allconfig" @@ -135,6 +136,15 @@ func (d *Deps) Init() error { if d.BuildState == nil { d.BuildState = &BuildState{} } + if d.BuildState.DeferredExecutions == nil { + if d.BuildState.DeferredExecutionsGroupedByRenderingContext == nil { + d.BuildState.DeferredExecutionsGroupedByRenderingContext = make(map[tpl.RenderingContext]*DeferredExecutions) + } + d.BuildState.DeferredExecutions = &DeferredExecutions{ + Executions: maps.NewCache[string, *tpl.DeferredExecution](), + FilenamesWithPostPrefix: maps.NewCache[string, bool](), + } + } if d.BuildStartListeners == nil { d.BuildStartListeners = &Listeners{} @@ -161,20 +171,29 @@ func (d *Deps) Init() error { } if d.PathSpec == nil { - hashBytesReceiverFunc := func(name string, match bool) { - if !match { - return + hashBytesReceiverFunc := func(name string, match []byte) { + s := string(match) + switch s { + case postpub.PostProcessPrefix: + d.BuildState.AddFilenameWithPostPrefix(name) + case tpl.HugoDeferredTemplatePrefix: + d.BuildState.DeferredExecutions.FilenamesWithPostPrefix.Set(name, true) } - d.BuildState.AddFilenameWithPostPrefix(name) } // Skip binary files. mediaTypes := d.Conf.GetConfigSection("mediaTypes").(media.Types) - hashBytesSHouldCheck := func(name string) bool { + hashBytesShouldCheck := func(name string) bool { ext := strings.TrimPrefix(filepath.Ext(name), ".") return mediaTypes.IsTextSuffix(ext) } - d.Fs.PublishDir = hugofs.NewHasBytesReceiver(d.Fs.PublishDir, hashBytesSHouldCheck, hashBytesReceiverFunc, []byte(postpub.PostProcessPrefix)) + d.Fs.PublishDir = hugofs.NewHasBytesReceiver( + d.Fs.PublishDir, + hashBytesShouldCheck, + hashBytesReceiverFunc, + []byte(tpl.HugoDeferredTemplatePrefix), + []byte(postpub.PostProcessPrefix)) + pathSpec, err := helpers.NewPathSpec(d.Fs, d.Conf, d.Log) if err != nil { return err @@ -371,10 +390,37 @@ type BuildState struct { // A set of filenames in /public that // contains a post-processing prefix. filenamesWithPostPrefix map[string]bool + + DeferredExecutions *DeferredExecutions + + // Deferred executions grouped by rendering context. + DeferredExecutionsGroupedByRenderingContext map[tpl.RenderingContext]*DeferredExecutions +} + +type DeferredExecutions struct { + // A set of filenames in /public that + // contains a post-processing prefix. + FilenamesWithPostPrefix *maps.Cache[string, bool] + + // Maps a placeholder to a deferred execution. + Executions *maps.Cache[string, *tpl.DeferredExecution] } var _ identity.SignalRebuilder = (*BuildState)(nil) +// StartStageRender will be called before a stage is rendered. +func (b *BuildState) StartStageRender(stage tpl.RenderingContext) { +} + +// StopStageRender will be called after a stage is rendered. +func (b *BuildState) StopStageRender(stage tpl.RenderingContext) { + b.DeferredExecutionsGroupedByRenderingContext[stage] = b.DeferredExecutions + b.DeferredExecutions = &DeferredExecutions{ + Executions: maps.NewCache[string, *tpl.DeferredExecution](), + FilenamesWithPostPrefix: maps.NewCache[string, bool](), + } +} + func (b *BuildState) SignalRebuild(ids ...identity.Identity) { b.OnSignalRebuild(ids...) } diff --git a/hugofs/hasbytes_fs.go b/hugofs/hasbytes_fs.go index 238fbc9c4..ac9e881ef 100644 --- a/hugofs/hasbytes_fs.go +++ b/hugofs/hasbytes_fs.go @@ -28,12 +28,12 @@ var ( type hasBytesFs struct { afero.Fs shouldCheck func(name string) bool - hasBytesCallback func(name string, match bool) - pattern []byte + hasBytesCallback func(name string, match []byte) + patterns [][]byte } -func NewHasBytesReceiver(delegate afero.Fs, shouldCheck func(name string) bool, hasBytesCallback func(name string, match bool), pattern []byte) afero.Fs { - return &hasBytesFs{Fs: delegate, shouldCheck: shouldCheck, hasBytesCallback: hasBytesCallback, pattern: pattern} +func NewHasBytesReceiver(delegate afero.Fs, shouldCheck func(name string) bool, hasBytesCallback func(name string, match []byte), patterns ...[]byte) afero.Fs { + return &hasBytesFs{Fs: delegate, shouldCheck: shouldCheck, hasBytesCallback: hasBytesCallback, patterns: patterns} } func (fs *hasBytesFs) UnwrapFilesystem() afero.Fs { @@ -60,10 +60,15 @@ func (fs *hasBytesFs) wrapFile(f afero.File) afero.File { if !fs.shouldCheck(f.Name()) { return f } + patterns := make([]*hugio.HasBytesPattern, len(fs.patterns)) + for i, p := range fs.patterns { + patterns[i] = &hugio.HasBytesPattern{Pattern: p} + } + return &hasBytesFile{ File: f, hbw: &hugio.HasBytesWriter{ - Pattern: fs.pattern, + Patterns: patterns, }, hasBytesCallback: fs.hasBytesCallback, } @@ -74,7 +79,7 @@ func (fs *hasBytesFs) Name() string { } type hasBytesFile struct { - hasBytesCallback func(name string, match bool) + hasBytesCallback func(name string, match []byte) hbw *hugio.HasBytesWriter afero.File } @@ -88,6 +93,10 @@ func (h *hasBytesFile) Write(p []byte) (n int, err error) { } func (h *hasBytesFile) Close() error { - h.hasBytesCallback(h.Name(), h.hbw.Match) + for _, p := range h.hbw.Patterns { + if p.Match { + h.hasBytesCallback(h.Name(), p.Pattern) + } + } return h.File.Close() } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index c91403c79..2ecd88e9e 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -323,6 +323,7 @@ type ComponentPath struct { Component string Path string Lang string + Watch bool } func (c ComponentPath) ComponentPathJoined() string { @@ -376,6 +377,7 @@ func (fs *RootMappingFs) ReverseLookupComponent(component, filename string) ([]C Component: first.FromBase, Path: paths.ToSlashTrimLeading(filename), Lang: first.Meta.Lang, + Watch: first.Meta.Watch, }) } diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index f9709df15..0a9063e23 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -33,6 +33,7 @@ import ( "github.com/gohugoio/hugo/common/rungroup" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/identity" @@ -1002,7 +1003,7 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) { } const indentStr = " " p := n.(*pageState) - s := strings.TrimPrefix(keyPage, paths.CommonDir(prevKey, keyPage)) + s := strings.TrimPrefix(keyPage, paths.CommonDirPath(prevKey, keyPage)) lenIndent := len(keyPage) - len(s) fmt.Fprint(w, |