diff options
Diffstat (limited to 'cache/filecache')
| -rw-r--r-- | cache/filecache/filecache.go | 384 | ||||
| -rw-r--r-- | cache/filecache/filecache_config.go | 248 | ||||
| -rw-r--r-- | cache/filecache/filecache_config_test.go | 198 | ||||
| -rw-r--r-- | cache/filecache/filecache_pruner.go | 127 | ||||
| -rw-r--r-- | cache/filecache/filecache_pruner_test.go | 110 | ||||
| -rw-r--r-- | cache/filecache/filecache_test.go | 349 |
6 files changed, 1416 insertions, 0 deletions
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go new file mode 100644 index 000000000..63d939ef6 --- /dev/null +++ b/cache/filecache/filecache.go @@ -0,0 +1,384 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/helpers" + + "github.com/BurntSushi/locker" + "github.com/spf13/afero" +) + +// ErrFatal can be used to signal an unrecoverable error. +var ErrFatal = errors.New("fatal filecache error") + +const ( + filecacheRootDirname = "filecache" +) + +// Cache caches a set of files in a directory. This is usually a file on +// disk, but since this is backed by an Afero file system, it can be anything. +type Cache struct { + Fs afero.Fs + + // Max age for items in this cache. Negative duration means forever, + // 0 is effectively turning this cache off. + maxAge time.Duration + + // When set, we just remove this entire root directory on expiration. + pruneAllRootDir string + + nlocker *lockTracker +} + +type lockTracker struct { + seenMu sync.RWMutex + seen map[string]struct{} + + *locker.Locker +} + +// Lock tracks the ids in use. We use this information to do garbage collection +// after a Hugo build. +func (l *lockTracker) Lock(id string) { + l.seenMu.RLock() + if _, seen := l.seen[id]; !seen { + l.seenMu.RUnlock() + l.seenMu.Lock() + l.seen[id] = struct{}{} + l.seenMu.Unlock() + } else { + l.seenMu.RUnlock() + } + + l.Locker.Lock(id) +} + +// ItemInfo contains info about a cached file. +type ItemInfo struct { + // This is the file's name relative to the cache's filesystem. + Name string +} + +// NewCache creates a new file cache with the given filesystem and max age. +func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache { + return &Cache{ + Fs: fs, + nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})}, + maxAge: maxAge, + pruneAllRootDir: pruneAllRootDir, + } +} + +// lockedFile is a file with a lock that is released on Close. +type lockedFile struct { + afero.File + unlock func() +} + +func (l *lockedFile) Close() error { + defer l.unlock() + return l.File.Close() +} + +// WriteCloser returns a transactional writer into the cache. +// It's important that it's closed when done. +func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) { + id = cleanID(id) + c.nlocker.Lock(id) + + info := ItemInfo{Name: id} + + f, err := helpers.OpenFileForWriting(c.Fs, id) + if err != nil { + c.nlocker.Unlock(id) + return info, nil, err + } + + return info, &lockedFile{ + File: f, + unlock: func() { c.nlocker.Unlock(id) }, + }, nil +} + +// ReadOrCreate tries to lookup the file in cache. +// If found, it is passed to read and then closed. +// If not found a new file is created and passed to create, which should close +// it when done. +func (c *Cache) ReadOrCreate(id string, + read func(info ItemInfo, r io.ReadSeeker) error, + create func(info ItemInfo, w io.WriteCloser) error) (info ItemInfo, err error) { + id = cleanID(id) + + c.nlocker.Lock(id) + defer c.nlocker.Unlock(id) + + info = ItemInfo{Name: id} + + if r := c.getOrRemove(id); r != nil { + err = read(info, r) + defer r.Close() + if err == nil || err == ErrFatal { + // See https://github.com/gohugoio/hugo/issues/6401 + // To recover from file corruption we handle read errors + // as the cache item was not found. + // Any file permission issue will also fail in the next step. + return + } + } + + f, err := helpers.OpenFileForWriting(c.Fs, id) + if err != nil { + return + } + + err = create(info, f) + + return +} + +// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will +// be invoked and the result cached. +// This method is protected by a named lock using the given id as identifier. +func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) { + id = cleanID(id) + + c.nlocker.Lock(id) + defer c.nlocker.Unlock(id) + + info := ItemInfo{Name: id} + + if r := c.getOrRemove(id); r != nil { + return info, r, nil + } + + var ( + r io.ReadCloser + err error + ) + + r, err = create() + if err != nil { + return info, nil, err + } + + if c.maxAge == 0 { + // No caching. + return info, hugio.ToReadCloser(r), nil + } + + var buff bytes.Buffer + return info, + hugio.ToReadCloser(&buff), + afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff)) +} + +// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice. +func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (ItemInfo, []byte, error) { + id = cleanID(id) + + c.nlocker.Lock(id) + defer c.nlocker.Unlock(id) + + info := ItemInfo{Name: id} + + if r := c.getOrRemove(id); r != nil { + defer r.Close() + b, err := ioutil.ReadAll(r) + return info, b, err + } + + var ( + b []byte + err error + ) + + b, err = create() + if err != nil { + return info, nil, err + } + + if c.maxAge == 0 { + return info, b, nil + } + + if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil { + return info, nil, err + } + return info, b, nil +} + +// GetBytes gets the file content with the given id from the cache, nil if none found. +func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) { + id = cleanID(id) + + c.nlocker.Lock(id) + defer c.nlocker.Unlock(id) + + info := ItemInfo{Name: id} + + if r := c.getOrRemove(id); r != nil { + defer r.Close() + b, err := ioutil.ReadAll(r) + return info, b, err + } + + return info, nil, nil +} + +// Get gets the file with the given id from the cahce, nil if none found. +func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) { + id = cleanID(id) + + c.nlocker.Lock(id) + defer c.nlocker.Unlock(id) + + info := ItemInfo{Name: id} + + r := c.getOrRemove(id) + + return info, r, nil +} + +// getOrRemove gets the file with the given id. If it's expired, it will +// be removed. +func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser { + if c.maxAge == 0 { + // No caching. + return nil + } + + if c.maxAge > 0 { + fi, err := c.Fs.Stat(id) + if err != nil { + return nil + } + + if c.isExpired(fi.ModTime()) { + c.Fs.Remove(id) + return nil + } + } + + f, err := c.Fs.Open(id) + if err != nil { + return nil + } + + return f +} + +func (c *Cache) isExpired(modTime time.Time) bool { + if c.maxAge < 0 { + return false + } + + // Note the use of time.Since here. + // We cannot use Hugo's global Clock for this. + return c.maxAge == 0 || time.Since(modTime) > c.maxAge +} + +// For testing +func (c *Cache) getString(id string) string { + id = cleanID(id) + + c.nlocker.Lock(id) + defer c.nlocker.Unlock(id) + + f, err := c.Fs.Open(id) + if err != nil { + return "" + } + defer f.Close() + + b, _ := ioutil.ReadAll(f) + return string(b) +} + +// Caches is a named set of caches. +type Caches map[string]*Cache + +// Get gets a named cache, nil if none found. +func (f Caches) Get(name string) *Cache { + return f[strings.ToLower(name)] +} + +// NewCaches creates a new set of file caches from the given +// configuration. +func NewCaches(p *helpers.PathSpec) (Caches, error) { + var dcfg Configs + if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok { + dcfg = c + } else { + var err error + dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg) + if err != nil { + return nil, err + } + } + + fs := p.Fs.Source + + m := make(Caches) + for k, v := range dcfg { + var cfs afero.Fs + + if v.isResourceDir { + cfs = p.BaseFs.ResourcesCache + } else { + cfs = fs + } + + if cfs == nil { + // TODO(bep) we still have some places that do not initialize the + // full dependencies of a site, e.g. the import Jekyll command. + // That command does not need these caches, so let us just continue + // for now. + continue + } + + baseDir := v.Dir + + if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) { + return nil, err + } + + bfs := afero.NewBasePathFs(cfs, baseDir) + + var pruneAllRootDir string + if k == cacheKeyModules { + pruneAllRootDir = "pkg" + } + + m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir) + } + + return m, nil +} + +func cleanID(name string) string { + return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator) +} diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go new file mode 100644 index 000000000..a82133ab7 --- /dev/null +++ b/cache/filecache/filecache_config.go @@ -0,0 +1,248 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "fmt" + "path" + "path/filepath" + "strings" + "time" + + "github.com/gohugoio/hugo/common/maps" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/helpers" + + "errors" + + "github.com/mitchellh/mapstructure" + "github.com/spf13/afero" +) + +const ( + cachesConfigKey = "caches" + + resourcesGenDir = ":resourceDir/_gen" + cacheDirProject = ":cacheDir/:project" +) + +var defaultCacheConfig = Config{ + MaxAge: -1, // Never expire + Dir: cacheDirProject, +} + +const ( + cacheKeyGetJSON = "getjson" + cacheKeyGetCSV = "getcsv" + cacheKeyImages = "images" + cacheKeyAssets = "assets" + cacheKeyModules = "modules" + cacheKeyGetResource = "getresource" +) + +type Configs map[string]Config + +func (c Configs) CacheDirModules() string { + return c[cacheKeyModules].Dir +} + +var defaultCacheConfigs = Configs{ + cacheKeyModules: { + MaxAge: -1, + Dir: ":cacheDir/modules", + }, + cacheKeyGetJSON: defaultCacheConfig, + cacheKeyGetCSV: defaultCacheConfig, + cacheKeyImages: { + MaxAge: -1, + Dir: resourcesGenDir, + }, + cacheKeyAssets: { + MaxAge: -1, + Dir: resourcesGenDir, + }, + cacheKeyGetResource: Config{ + MaxAge: -1, // Never expire + Dir: cacheDirProject, + }, +} + +type Config struct { + // Max age of cache entries in this cache. Any items older than this will + // be removed and not returned from the cache. + // a negative value means forever, 0 means cache is disabled. + MaxAge time.Duration + + // The directory where files are stored. + Dir string + + // Will resources/_gen will get its own composite filesystem that + // also checks any theme. + isResourceDir bool +} + +// GetJSONCache gets the file cache for getJSON. +func (f Caches) GetJSONCache() *Cache { + return f[cacheKeyGetJSON] +} + +// GetCSVCache gets the file cache for getCSV. +func (f Caches) GetCSVCache() *Cache { + return f[cacheKeyGetCSV] +} + +// ImageCache gets the file cache for processed images. +func (f Caches) ImageCache() *Cache { + return f[cacheKeyImages] +} + +// ModulesCache gets the file cache for Hugo Modules. +func (f Caches) ModulesCache() *Cache { + return f[cacheKeyModules] +} + +// AssetsCache gets the file cache for assets (processed resources, SCSS etc.). +func (f Caches) AssetsCache() *Cache { + return f[cacheKeyAssets] +} + +// GetResourceCache gets the file cache for remote resources. +func (f Caches) GetResourceCache() *Cache { + return f[cacheKeyGetResource] +} + +func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { + c := make(Configs) + valid := make(map[string]bool) + // Add defaults + for k, v := range defaultCacheConfigs { + c[k] = v + valid[k] = true + } + + m := cfg.GetStringMap(cachesConfigKey) + + _, isOsFs := fs.(*afero.OsFs) + + for k, v := range m { + if _, ok := v.(maps.Params); !ok { + continue + } + cc := defaultCacheConfig + + dc := &mapstructure.DecoderConfig{ + Result: &cc, + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + } + + decoder, err := mapstructure.NewDecoder(dc) + if err != nil { + return c, err + } + + if err := decoder.Decode(v); err != nil { + return nil, fmt.Errorf("failed to decode filecache config: %w", err) + } + + if cc.Dir == "" { + return c, errors.New("must provide cache Dir") + } + + name := strings.ToLower(k) + if !valid[name] { + return nil, fmt.Errorf("%q is not a valid cache name", name) + } + + c[name] = cc + } + + // This is a very old flag in Hugo, but we need to respect it. + disabled := cfg.GetBool("ignoreCache") + + for k, v := range c { + dir := filepath.ToSlash(filepath.Clean(v.Dir)) + hadSlash := strings.HasPrefix(dir, "/") + parts := strings.Split(dir, "/") + + for i, part := range parts { + if strings.HasPrefix(part, ":") { + resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part) + if err != nil { + return c, err + } + if isResource { + v.isResourceDir = true + } + parts[i] = resolved + } + } + + dir = path.Join(parts...) + if hadSlash { + dir = "/" + dir + } + v.Dir = filepath.Clean(filepath.FromSlash(dir)) + + if !v.isResourceDir { + if isOsFs && !filepath.IsAbs(v.Dir) { + return c, fmt.Errorf("%q must resolve to an absolute directory", v.Dir) + } + + // Avoid cache in root, e.g. / (Unix) or c:\ (Windows) + if len(strings.TrimPrefix(v.Dir, filepath.VolumeName(v.Dir))) == 1 { + return c, fmt.Errorf("%q is a root folder and not allowed as cache dir", v.Dir) + } + } + + if !strings.HasPrefix(v.Dir, "_gen") { + // We do cache eviction (file removes) and since the user can set + // his/hers own cache directory, we really want to make sure + // we do not delete any files that do not belong to this cache. + // We do add the cache name as the root, but this is an extra safe + // guard. We skip the files inside /resources/_gen/ because + // that would be breaking. + v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k) + } else { + v.Dir = filepath.Join(v.Dir, k) + } + + if disabled { + v.MaxAge = 0 + } + + c[k] = v + } + + return c, nil +} + +// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ... +func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) { + workingDir := cfg.GetString("workingDir") + + switch strings.ToLower(placeholder) { + case ":resourcedir": + return "", true, nil + case ":cachedir": + d, err := helpers.GetCacheDir(fs, cfg) + return d, false, err + case ":project": + return filepath.Base(workingDir), false, nil + } + + return "", false, fmt.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder) +} diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go new file mode 100644 index 000000000..1ed020ef1 --- /dev/null +++ b/cache/filecache/filecache_config_test.go @@ -0,0 +1,198 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/config" + + qt "github.com/frankban/quicktest" +) + +func TestDecodeConfig(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + configStr := ` +resourceDir = "myresources" +contentDir = "content" +dataDir = "data" +i18nDir = "i18n" +layoutDir = "layouts" +assetDir = "assets" +archetypeDir = "archetypes" + +[caches] +[caches.getJSON] +maxAge = "10m" +dir = "/path/to/c1" +[caches.getCSV] +maxAge = "11h" +dir = "/path/to/c2" +[caches.images] +dir = "/path/to/c3" +[caches.getResource] +dir = "/path/to/c4" +` + + cfg, err := config.FromConfigString(configStr, "toml") + c.Assert(err, qt.IsNil) + fs := afero.NewMemMapFs() + decoded, err := DecodeConfig(fs, cfg) + c.Assert(err, qt.IsNil) + + c.Assert(len(decoded), qt.Equals, 6) + + c2 := decoded["getcsv"] + c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s") + c.Assert(c2.Dir, qt.Equals, filepath.FromSlash("/path/to/c2/filecache/getcsv")) + + c3 := decoded["images"] + c.Assert(c3.MaxAge, qt.Equals, time.Duration(-1)) + c.Assert(c3.Dir, qt.Equals, filepath.FromSlash("/path/to/c3/filecache/images")) + + c4 := decoded["getresource"] + c.Assert(c4.MaxAge, qt.Equals, time.Duration(-1)) + c.Assert(c4.Dir, qt.Equals, filepath.FromSlash("/path/to/c4/filecache/getresource")) +} + +func TestDecodeConfigIgnoreCache(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + configStr := ` +resourceDir = "myresources" +contentDir = "content" +dataDir = "data" +i18nDir = "i18n" +layoutDir = "layouts" +assetDir = "assets" +archeTypedir = "archetypes" + +ignoreCache = true +[caches] +[caches.getJSON] +maxAge = 1234 +dir = "/path/to/c1" +[caches.getCSV] +maxAge = 3456 +dir = "/path/to/c2" +[caches.images] +dir = "/path/to/c3" +[caches.getResource] +dir = "/path/to/c4" +` + + cfg, err := config.FromConfigString(configStr, "toml") + c.Assert(err, qt.IsNil) + fs := afero.NewMemMapFs() + decoded, err := DecodeConfig(fs, cfg) + c.Assert(err, qt.IsNil) + + c.Assert(len(decoded), qt.Equals, 6) + + for _, v := range decoded { + c.Assert(v.MaxAge, qt.Equals, time.Duration(0)) + } +} + +func TestDecodeConfigDefault(t *testing.T) { + c := qt.New(t) + cfg := newTestConfig() + + if runtime.GOOS == "windows" { + cfg.Set("resourceDir", "c:\\cache\\resources") + cfg.Set("cacheDir", "c:\\cache\\thecache") + + } else { + cfg.Set("resourceDir", "/cache/resources") + cfg.Set("cacheDir", "/cache/thecache") + } + + fs := afero.NewMemMapFs() + + decoded, err := DecodeConfig(fs, cfg) + + c.Assert(err, qt.IsNil) + + c.Assert(len(decoded), qt.Equals, 6) + + imgConfig := decoded[cacheKeyImages] + jsonConfig := decoded[cacheKeyGetJSON] + + if runtime.GOOS == "windows" { + c.Assert(imgConfig.Dir, qt.Equals, filepath.FromSlash("_gen/images")) + } else { + c.Assert(imgConfig.Dir, qt.Equals, "_gen/images") + c.Assert(jsonConfig.Dir, qt.Equals, "/cache/thecache/hugoproject/filecache/getjson") + } + + c.Assert(imgConfig.isResourceDir, qt.Equals, true) + c.Assert(jsonConfig.isResourceDir, qt.Equals, false) +} + +func TestDecodeConfigInvalidDir(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + configStr := ` +resourceDir = "myresources" +contentDir = "content" +dataDir = "data" +i18nDir = "i18n" +layoutDir = "layouts" +assetDir = "assets" +archeTypedir = "archetypes" + +[caches] +[caches.getJSON] +maxAge = "10m" +dir = "/" + +` + if runtime.GOOS == "windows" { + configStr = strings.Replace(configStr, "/", "c:\\\\", 1) + } + + cfg, err := config.FromConfigString(configStr, "toml") + c.Assert(err, qt.IsNil) + fs := afero.NewMemMapFs() + + _, err = DecodeConfig(fs, cfg) + c.Assert(err, qt.Not(qt.IsNil)) +} + +func newTestConfig() config.Provider { + cfg := config.NewWithTestDefaults() + cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject")) + cfg.Set("contentDir", "content") + cfg.Set("dataDir", "data") + cfg.Set("resourceDir", "resources") + cfg.Set("i18nDir", "i18n") + cfg.Set("layoutDir", "layouts") + cfg.Set("archetypeDir", "archetypes") + cfg.Set("assetDir", "assets") + + return cfg +} diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go new file mode 100644 index 000000000..5734af199 --- /dev/null +++ b/cache/filecache/filecache_pruner.go @@ -0,0 +1,127 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "fmt" + "io" + "os" + + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/hugofs" + + "github.com/spf13/afero" +) + +// Prune removes expired and unused items from this cache. +// The last one requires a full build so the cache usage can be tracked. +// Note that we operate directly on the filesystem here, so this is not +// thread safe. +func (c Caches) Prune() (int, error) { + counter := 0 + for k, cache := range c { + + count, err := cache.Prune(false) + + counter += count + + if err != nil { + if herrors.IsNotExist(err) { + continue + } + return counter, fmt.Errorf("failed to prune cache %q: %w", k, err) + } + + } + + return counter, nil +} + +// Prune removes expired and unused items from this cache. +// If force is set, everything will be removed not considering expiry time. +func (c *Cache) Prune(force bool) (int, error) { + if c.pruneAllRootDir != "" { + return c.pruneRootDir(force) + } + + counter := 0 + + err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error { + if info == nil { + return nil + } + + name = cleanID(name) + + if info.IsDir() { + f, err := c.Fs.Open(name) + if err != nil { + // This cache dir may not exist. + return nil + } + defer f.Close() + _, err = f.Readdirnames(1) + if err == io.EOF { + // Empty dir. + err = c.Fs.Remove(name) + } + + if err != nil && !herrors.IsNotExist(err) { + return err + } + + return nil + } + + shouldRemove := force || c.isExpired(info.ModTime()) + + if !shouldRemove && len(c.nlocker.seen) > 0 { + // Remove it if it's not been touched/used in the last build. + _, seen := c.nlocker.seen[name] + shouldRemove = !seen + } + + if shouldRemove { + err := c.Fs.Remove(name) + if err == nil { + counter++ + } + + if err != nil && !herrors.IsNotExist(err) { + return err + } + + } + + return nil + }) + + return counter, err +} + +func (c *Cache) pruneRootDir(force bool) (int, error) { + info, err := c.Fs.Stat(c.pruneAllRootDir) + if err != nil { + if herrors.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + if !force && !c.isExpired(info.ModTime()) { + return 0, nil + } + + return hugofs.MakeReadableAndRemoveAllModulePkgDir(c.Fs, c.pruneAllRootDir) +} diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go new file mode 100644 index 000000000..46e1317ce --- /dev/null +++ b/cache/filecache/filecache_pruner_test.go @@ -0,0 +1,110 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "fmt" + "testing" + "time" + + "github.com/spf13/afero" + + qt "github.com/frankban/quicktest" +) + +func TestPrune(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + configStr := ` +resourceDir = "myresources" +contentDir = "content" +dataDir = "data" +i18nDir = "i18n" +layoutDir = "layouts" +assetDir = "assets" +archeTypedir = "archetypes" + +[caches] +[caches.getjson] +maxAge = "200ms" +dir = "/cache/c" +[caches.getcsv] +maxAge = "200ms" +dir = "/cache/d" +[caches.assets] +maxAge = "200ms" +dir = ":resourceDir/_gen" +[caches.images] |