diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2024-05-17 17:06:47 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2024-06-04 16:07:39 +0200 |
commit | 447108fed2842e264897659856e9fd9cdc32ca23 (patch) | |
tree | 53687693e04496919dd84266cc1edc16746101b0 | |
parent | c71e24af5172e230baa5f7dfa2078721cda38df4 (diff) |
Add a HTTP cache for remote resources.
Fixes #12502
Closes #11891
32 files changed, 1149 insertions, 235 deletions
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 093d2941c..01c466ca6 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import ( "sync" "time" + "github.com/gohugoio/httpcache" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/hugofs" @@ -182,6 +183,15 @@ func (c *Cache) ReadOrCreate(id string, return } +// NamedLock locks the given id. The lock is released when the returned function is called. +func (c *Cache) NamedLock(id string) func() { + id = cleanID(id) + c.nlocker.Lock(id) + return func() { + c.nlocker.Unlock(id) + } +} + // GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will // be invoked and the result cached. // This method is protected by a named lock using the given id as identifier. @@ -218,7 +228,23 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It var buff bytes.Buffer return info, hugio.ToReadCloser(&buff), - afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff)) + c.writeReader(id, io.TeeReader(r, &buff)) +} + +func (c *Cache) writeReader(id string, r io.Reader) error { + dir := filepath.Dir(id) + if dir != "" { + _ = c.Fs.MkdirAll(dir, 0o777) + } + f, err := c.Fs.Create(id) + if err != nil { + return err + } + defer f.Close() + + _, _ = io.Copy(f, r) + + return nil } // GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice. @@ -253,9 +279,10 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item return info, b, nil } - if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil { + if err := c.writeReader(id, bytes.NewReader(b)); err != nil { return info, nil, err } + return info, b, nil } @@ -305,16 +332,8 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser { return nil } - if c.maxAge > 0 { - fi, err := c.Fs.Stat(id) - if err != nil { - return nil - } - - if c.isExpired(fi.ModTime()) { - c.Fs.Remove(id) - return nil - } + if removed, err := c.removeIfExpired(id); err != nil || removed { + return nil } f, err := c.Fs.Open(id) @@ -325,6 +344,49 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser { return f } +func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) { + if c.maxAge == 0 { + // No caching. + return nil, false + } + + f, err := c.Fs.Open(id) + if err != nil { + return nil, false + } + defer f.Close() + + b, err := io.ReadAll(f) + if err != nil { + return nil, false + } + + removed, err := c.removeIfExpired(id) + if err != nil { + return nil, false + } + + return b, removed +} + +func (c *Cache) removeIfExpired(id string) (bool, error) { + if c.maxAge <= 0 { + return false, nil + } + + fi, err := c.Fs.Stat(id) + if err != nil { + return false, err + } + + if c.isExpired(fi.ModTime()) { + c.Fs.Remove(id) + return true, nil + } + + return false, nil +} + func (c *Cache) isExpired(modTime time.Time) bool { if c.maxAge < 0 { return false @@ -398,3 +460,37 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) { func cleanID(name string) string { return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator) } + +// AsHTTPCache returns an httpcache.Cache implementation for this file cache. +// Note that none of the methods are protected by named locks, so you need to make sure +// to do that in your own code. +func (c *Cache) AsHTTPCache() httpcache.Cache { + return &httpCache{c: c} +} + +type httpCache struct { + c *Cache +} + +func (h *httpCache) Get(id string) (resp []byte, ok bool) { + id = cleanID(id) + b, removed := h.c.getBytesAndRemoveIfExpired(id) + + return b, !removed +} + +func (h *httpCache) Set(id string, resp []byte) { + if h.c.maxAge == 0 { + return + } + + id = cleanID(id) + + if err := h.c.writeReader(id, bytes.NewReader(resp)); err != nil { + panic(err) + } +} + +func (h *httpCache) Delete(key string) { + h.c.Fs.Remove(key) +} diff --git a/cache/httpcache/httpcache.go b/cache/httpcache/httpcache.go new file mode 100644 index 000000000..ff360001f --- /dev/null +++ b/cache/httpcache/httpcache.go @@ -0,0 +1,208 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpcache + +import ( + "encoding/json" + "time" + + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/config" + "github.com/mitchellh/mapstructure" +) + +// DefaultConfig holds the default configuration for the HTTP cache. +var DefaultConfig = Config{ + Cache: Cache{ + For: GlobMatcher{ + Excludes: []string{"**"}, + }, + }, + Polls: []PollConfig{ + { + For: GlobMatcher{ + Includes: []string{"**"}, + }, + Disable: true, + }, + }, +} + +// Config holds the configuration for the HTTP cache. +type Config struct { + // Configures the HTTP cache behaviour (RFC 9111). + // When this is not enabled for a resource, Hugo will go straight to the file cache. + Cache Cache + + // Polls holds a list of configurations for polling remote resources to detect changes in watch mode. + // This can be disabled for some resources, typically if they are known to not change. + Polls []PollConfig +} + +type Cache struct { + // Enable HTTP cache behaviour (RFC 9111) for these rsources. + For GlobMatcher +} + +func (c *Config) Compile() (ConfigCompiled, error) { + var cc ConfigCompiled + + p, err := c.Cache.For.CompilePredicate() + if err != nil { + return cc, err + } + + cc.For = p + + for _, pc := range c.Polls { + + p, err := pc.For.CompilePredicate() + if err != nil { + return cc, err + } + + cc.PollConfigs = append(cc.PollConfigs, PollConfigCompiled{ + For: p, + Config: pc, + }) + } + + return cc, nil +} + +// PollConfig holds the configuration for polling remote resources to detect changes in watch mode. +// TODO1 make sure this enabled only in watch mode. +type PollConfig struct { + // What remote resources to apply this configuration to. + For GlobMatcher + + // Disable polling for this configuration. + Disable bool + + // Low is the lower bound for the polling interval. + // This is the starting point when the resource has recently changed, + // if that resource stops changing, the polling interval will gradually increase towards High. + Low time.Duration + + // High is the upper bound for the polling interval. + // This is the interval used when the resource is stable. + High time.Duration +} + +func (c PollConfig) MarshalJSON() (b []byte, err error) { + // Marshal the durations as strings. + type Alias PollConfig + return json.Marshal(&struct { + Low string + High string + Alias + }{ + Low: c.Low.String(), + High: c.High.String(), + Alias: (Alias)(c), + }) +} + +type GlobMatcher struct { + // Excludes holds a list of glob patterns that will be excluded. + Excludes []string + + // Includes holds a list of glob patterns that will be included. + Includes []string +} + +type ConfigCompiled struct { + For predicate.P[string] + PollConfigs []PollConfigCompiled +} + +func (c *ConfigCompiled) PollConfigFor(s string) PollConfigCompiled { + for _, pc := range c.PollConfigs { + if pc.For(s) { + return pc + } + } + return PollConfigCompiled{} +} + +func (c *ConfigCompiled) IsPollingDisabled() bool { + for _, pc := range c.PollConfigs { + if !pc.Config.Disable { + return false + } + } + return true +} + +type PollConfigCompiled struct { + For predicate.P[string] + Config PollConfig +} + +func (p PollConfigCompiled) IsZero() bool { + return p.For == nil +} + +func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) { + var p predicate.P[string] + for _, include := range gm.Includes { + g, err := glob.Compile(include, '/') + if err != nil { + return nil, err + } + fn := func(s string) bool { + return g.Match(s) + } + p = p.Or(fn) + } + + for _, exclude := range gm.Excludes { + g, err := glob.Compile(exclude, '/') + if err != nil { + return nil, err + } + fn := func(s string) bool { + return !g.Match(s) + } + p = p.And(fn) + } + + return p, nil +} + +func DecodeConfig(bcfg config.BaseConfig, m map[string]any) (Config, error) { + if len(m) == 0 { + return DefaultConfig, nil + } + + var c Config + + dc := &mapstructure.DecoderConfig{ + Result: &c, + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + } + + decoder, err := mapstructure.NewDecoder(dc) + if err != nil { + return c, err + } + + if err := decoder.Decode(m); err != nil { + return c, err + } + + return c, nil +} diff --git a/cache/httpcache/httpcache_integration_test.go b/cache/httpcache/httpcache_integration_test.go new file mode 100644 index 000000000..d3337c023 --- /dev/null +++ b/cache/httpcache/httpcache_integration_test.go @@ -0,0 +1,64 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpcache_test + +import ( + "testing" + "time" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/hugolib" +) + +func TestConfigCustom(t *testing.T) { + files := ` +-- hugo.toml -- +[httpcache] +[httpcache.cache.for] +includes = ["**gohugo.io**"] +[[httpcache.polls]] +low = "5s" +high = "32s" +[httpcache.polls.for] +includes = ["**gohugo.io**"] + + +` + + b := hugolib.Test(t, files) + + httpcacheConf := b.H.Configs.Base.HTTPCache + compiled := b.H.Configs.Base.C.HTTPCache + + b.Assert(httpcacheConf.Cache.For.Includes, qt.DeepEquals, []string{"**gohugo.io**"}) + b.Assert(httpcacheConf.Cache.For.Excludes, qt.IsNil) + + pc := compiled.PollConfigFor("https://gohugo.io/foo.jpg") + b.Assert(pc.Config.Low, qt.Equals, 5*time.Second) + b.Assert(pc.Config.High, qt.Equals, 32*time.Second) + b.Assert(compiled.PollConfigFor("https://example.com/foo.jpg").IsZero(), qt.IsTrue) +} + +func TestConfigDefault(t *testing.T) { + files := ` +-- hugo.toml -- +` + b := hugolib.Test(t, files) + + compiled := b.H.Configs.Base.C.HTTPCache + + b.Assert(compiled.For("https://gohugo.io/posts.json"), qt.IsFalse) + b.Assert(compiled.For("https://gohugo.io/foo.jpg"), qt.IsFalse) + b.Assert(compiled.PollConfigFor("https://gohugo.io/foo.jpg").Config.Disable, qt.IsTrue) +} diff --git a/cache/httpcache/httpcache_test.go b/cache/httpcache/httpcache_test.go new file mode 100644 index 000000000..e3659f97b --- /dev/null +++ b/cache/httpcache/httpcache_test.go @@ -0,0 +1,42 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpcache + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestGlobMatcher(t *testing.T) { + c := qt.New(t) + + g := GlobMatcher{ + Includes: []string{"**/*.jpg", "**.png", "**/bar/**"}, + Excludes: []string{"**/foo.jpg", "**.css"}, + } + + p, err := g.CompilePredicate() + c.Assert(err, qt.IsNil) + + c.Assert(p("foo.jpg"), qt.IsFalse) + c.Assert(p("foo.png"), qt.IsTrue) + c.Assert(p("foo/bar.jpg"), qt.IsTrue) + c.Assert(p("foo/bar.png"), qt.IsTrue) + c.Assert(p("foo/bar/foo.jpg"), qt.IsFalse) + c.Assert(p("foo/bar/foo.css"), qt.IsFalse) + c.Assert(p("foo.css"), qt.IsFalse) + c.Assert(p("foo/bar/foo.css"), qt.IsFalse) + c.Assert(p("foo/bar/foo.xml"), qt.IsTrue) +} diff --git a/commands/commandeer.go b/commands/commandeer.go index 59fe32f74..f18a95bb9 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -48,6 +48,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/resources/kinds" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -103,6 +104,9 @@ type rootCommand struct { commonConfigs *lazycache.Cache[int32, *commonConfig] hugoSites *lazycache.Cache[int32, *hugolib.HugoSites] + // changesFromBuild received from Hugo in watch mode. + changesFromBuild chan []identity.Identity + commands []simplecobra.Commander // Flags @@ -304,7 +308,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) { h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) { - depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()} + depsCfg := r.newDepsConfig(conf) return hugolib.NewHugoSites(depsCfg) }) return h, err @@ -316,12 +320,16 @@ func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) { if err != nil { return nil, err } - depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()} + depsCfg := r.newDepsConfig(conf) return hugolib.NewHugoSites(depsCfg) }) return h, err } +func (r *rootCommand) newDepsConfig(conf *commonConfig) deps.DepsCfg { + return deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level(), ChangesFromBuild: r.changesFromBuild} +} + func (r *rootCommand) Name() string { return "hugo" } @@ -408,6 +416,8 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error { return err } + r.changesFromBuild = make(chan []identity.Identity, 10) + r.commonConfigs = lazycache.New(lazycache.Options[int32, *commonConfig]{MaxEntries: 5}) // We don't want to keep stale HugoSites in memory longer than needed. r.hugoSites = lazycache.New(lazycache.Options[int32, *hugolib.HugoSites]{ diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index 32b7e1de8..99bd8a04a 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -43,6 +43,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib" "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/livereload" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/watcher" @@ -343,6 +344,24 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa go func() { for { select { + case changes := <-c.r.changesFromBuild: + unlock, err := h.LockBuild() + if err != nil { + c.r.logger.Errorln("Failed to acquire a build lock: %s", err) + return + } + c.changeDetector.PrepareNew() + err = c.rebuildSitesForChanges(changes) + if err != nil { + c.r.logger.Errorln("Error while watching:", err) + } + if c.s != nil && c.s.doLiveReload { + if c.changeDetector == nil || len(c.changeDetector.changed()) > 0 { + livereload.ForceRefresh() + } + } + unlock() + case evs := <-watcher.Events: unlock, err := h.LockBuild() if err != nil { @@ -1019,6 +1038,19 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error { return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...) } +func (c *hugoBuilder) rebuildSitesForChanges(ids []identity.Identity) error { + c.errState.setBuildErr(nil) + h, err := c.hugo() + if err != nil { + return err + } + whatChanged := &hugolib.WhatChanged{} + whatChanged.Add(ids...) + err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}) + c.errState.setBuildErr(err) + return err +} + func (c *hugoBuilder) reloadConfig() error { c.r.Reset() c.r.configVersionID.Add(1) diff --git a/common/maps/maps.go b/common/maps/maps.go index 2686baad6..f9171ebf2 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -112,17 +112,17 @@ func ToSliceStringMap(in any) ([]map[string]any, error) { } // LookupEqualFold finds key in m with case insensitive equality checks. -func LookupEqualFold[T any | string](m map[string]T, key string) (T, bool) { +func LookupEqualFold[T any | string](m map[string]T, key string) (T, string, bool) { if v, found := m[key]; found { - return v, true + return v, key, true } for k, v := range m { if strings.EqualFold(k, key) { - return v, true + return v, k, true } } var s T - return s, false + return s, "", false } // MergeShallow merges src into dst, but only if the key does not already exist in dst. diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go index 098098388..b4f9c5a3d 100644 --- a/common/maps/maps_test.go +++ b/common/maps/maps_test.go @@ -180,16 +180,18 @@ func TestLookupEqualFold(t *testing.T) { "B": "bv", } - v, found := LookupEqualFold(m1, "b") + v, k, found := LookupEqualFold(m1, "b") c.Assert(found, qt.IsTrue) c.Assert(v, qt.Equals, "bv") + c.Assert(k, qt.Equals, "B") m2 := map[string]string{ "a": "av", "B": "bv", } - v, found = LookupEqualFold(m2, "b") + v, k, found = LookupEqualFold(m2, "b") c.Assert(found, qt.IsTrue) + c.Assert(k, qt.Equals, "B") c.Assert(v, qt.Equals, "bv") } diff --git a/common/predicate/predicate.go b/common/predicate/predicate.go index f9cb1bb2b..f71536474 100644 --- a/common/predicate/predicate.go +++ b/common/predicate/predicate.go @@ -24,6 +24,9 @@ func (p P[T]) And(ps ...P[T]) P[T] { return false |