diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-11-08 10:24:13 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-11-13 14:19:42 +0100 |
commit | f7aeaa61291dd75f92901bcbeecc7fce07a28dec (patch) | |
tree | 409c03e259a38fce2beeab46655fd5108c84cd5c /hugolib | |
parent | 7d78a2afd3c4a6c4af77a4ddcbd2a82f15986048 (diff) |
Add a consolidated file cache
This commits reworks how file caching is performed in Hugo. Now there is only one way, and it can be configured.
This is the default configuration:
```toml
[caches]
[caches.getjson]
dir = ":cacheDir"
maxAge = -1
[caches.getcsv]
dir = ":cacheDir"
maxAge = -1
[caches.images]
dir = ":resourceDir/_gen"
maxAge = -1
[caches.assets]
dir = ":resourceDir/_gen"
maxAge = -1
```
You can override any of these cache setting in your own `config.toml`.
The placeholders explained:
`:cacheDir`: This is the value of the `cacheDir` config option if set (can also be set via OS env variable `HUGO_CACHEDIR`). It will fall back to `/opt/build/cache/hugo_cache/` on Netlify, or a `hugo_cache` directory below the OS temp dir for the others.
`:resourceDir`: This is the value of the `resourceDir` config option.
`maxAge` is the time in seconds before a cache entry will be evicted, -1 means forever and 0 effectively turns that particular cache off.
This means that if you run your builds on Netlify, all caches configured with `:cacheDir` will be saved and restored on the next build. For other CI vendors, please read their documentation. For an CircleCI example, see https://github.com/bep/hugo-sass-test/blob/6c3960a8f4b90e8938228688bc49bdcdd6b2d99e/.circleci/config.yml
Fixes #5404
Diffstat (limited to 'hugolib')
-rw-r--r-- | hugolib/filesystems/basefs.go | 8 | ||||
-rw-r--r-- | hugolib/filesystems/basefs_test.go | 3 | ||||
-rw-r--r-- | hugolib/prune_resources.go | 33 |
3 files changed, 9 insertions, 35 deletions
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index ee1c870d9..8fc1861b8 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -81,7 +81,6 @@ type SourceFilesystems struct { Layouts *SourceFilesystem Archetypes *SourceFilesystem Assets *SourceFilesystem - Resources *SourceFilesystem // This is a unified read-only view of the project's and themes' workdir. Work *SourceFilesystem @@ -376,13 +375,6 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { } b.result.Assets = sfs - sfs, err = b.createFs(true, false, "resourceDir", "resources") - if err != nil { - return nil, err - } - - b.result.Resources = sfs - sfs, err = b.createFs(false, true, "", "") if err != nil { return nil, err diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index ec6ccb30c..8e9b7235c 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -108,7 +108,6 @@ theme = ["atheme"] checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes checkFileCount(bfs.Archetypes.Fs, "", assert, 10) // 8 + 2 themes checkFileCount(bfs.Assets.Fs, "", assert, 9) - checkFileCount(bfs.Resources.Fs, "", assert, 10) checkFileCount(bfs.Work.Fs, "", assert, 78) assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames) @@ -229,8 +228,6 @@ func TestRealDirs(t *testing.T) { assert.Equal(filepath.Join(root, "myassets/scss"), realDirs[0]) assert.Equal(filepath.Join(themesDir, "mytheme/assets/scss"), realDirs[len(realDirs)-1]) - checkFileCount(bfs.Resources.Fs, "", assert, 3) - assert.NotNil(bfs.themeFs) fi, b, err := bfs.themeFs.(afero.Lstater).LstatIfPossible(filepath.Join("resources", "t1.txt")) assert.NoError(err) diff --git a/hugolib/prune_resources.go b/hugolib/prune_resources.go index 28802c6f2..63623d75d 100644 --- a/hugolib/prune_resources.go +++ b/hugolib/prune_resources.go @@ -14,7 +14,6 @@ package hugolib import ( - "fmt" "io" "os" "strings" @@ -27,21 +26,12 @@ import ( // GC requires a build first. func (h *HugoSites) GC() (int, error) { s := h.Sites[0] - fs := h.PathSpec.BaseFs.Resources.Fs + assetsCacheFs := h.Deps.FileCaches.AssetsCache().Fs + imageCacheFs := h.Deps.FileCaches.ImageCache().Fs - imageCacheDir := s.ResourceSpec.GenImagePath - if len(imageCacheDir) < 10 { - panic("invalid image cache") - } - assetsCacheDir := s.ResourceSpec.GenAssetsPath - if len(assetsCacheDir) < 10 { - panic("invalid assets cache") - } - - isImageInUse := func(filename string) bool { - key := strings.TrimPrefix(filename, imageCacheDir) + isImageInUse := func(name string) bool { for _, site := range h.Sites { - if site.ResourceSpec.IsInImageCache(key) { + if site.ResourceSpec.IsInImageCache(name) { return true } } @@ -49,10 +39,9 @@ func (h *HugoSites) GC() (int, error) { return false } - isAssetInUse := func(filename string) bool { - key := strings.TrimPrefix(filename, assetsCacheDir) + isAssetInUse := func(name string) bool { // These assets are stored in tuplets with an added extension to the key. - key = strings.TrimSuffix(key, helpers.Ext(key)) + key := strings.TrimSuffix(name, helpers.Ext(name)) for _, site := range h.Sites { if site.ResourceSpec.ResourceCache.Contains(key) { return true @@ -62,17 +51,13 @@ func (h *HugoSites) GC() (int, error) { return false } - walker := func(dirname string, inUse func(filename string) bool) (int, error) { + walker := func(fs afero.Fs, dirname string, inUse func(filename string) bool) (int, error) { counter := 0 err := afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error { if info == nil { return nil } - if !strings.HasPrefix(path, dirname) { - return fmt.Errorf("Invalid state, walk outside of resource dir: %q", path) - } - if info.IsDir() { f, err := fs.Open(path) if err != nil { @@ -103,8 +88,8 @@ func (h *HugoSites) GC() (int, error) { return counter, err } - imageCounter, err1 := walker(imageCacheDir, isImageInUse) - assetsCounter, err2 := walker(assetsCacheDir, isAssetInUse) + imageCounter, err1 := walker(imageCacheFs, "", isImageInUse) + assetsCounter, err2 := walker(assetsCacheFs, "", isAssetInUse) totalCount := imageCounter + assetsCounter if err1 != nil { |