diff options
Diffstat (limited to 'cache')
-rw-r--r-- | cache/filecache/filecache.go | 49 | ||||
-rw-r--r-- | cache/filecache/filecache_config.go | 56 | ||||
-rw-r--r-- | cache/filecache/filecache_config_test.go | 45 | ||||
-rw-r--r-- | cache/filecache/filecache_pruner.go | 117 | ||||
-rw-r--r-- | cache/filecache/filecache_pruner_test.go | 11 | ||||
-rw-r--r-- | cache/filecache/filecache_test.go | 64 |
6 files changed, 223 insertions, 119 deletions
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 6ad417117..bf004c8f7 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -44,6 +44,9 @@ type Cache struct { // 0 is effectively turning this cache off. maxAge time.Duration + // When set, we just remove this entire root directory on expiration. + pruneAllRootDir string + nlocker *lockTracker } @@ -77,11 +80,12 @@ type ItemInfo struct { } // NewCache creates a new file cache with the given filesystem and max age. -func NewCache(fs afero.Fs, maxAge time.Duration) *Cache { +func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache { return &Cache{ - Fs: fs, - nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})}, - maxAge: maxAge, + Fs: fs, + nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})}, + maxAge: maxAge, + pruneAllRootDir: pruneAllRootDir, } } @@ -307,9 +311,15 @@ func (f Caches) Get(name string) *Cache { // NewCaches creates a new set of file caches from the given // configuration. func NewCaches(p *helpers.PathSpec) (Caches, error) { - dcfg, err := decodeConfig(p) - if err != nil { - return nil, err + var dcfg Configs + if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok { + dcfg = c + } else { + var err error + dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg) + if err != nil { + return nil, err + } } fs := p.Fs.Source @@ -319,30 +329,25 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) { var cfs afero.Fs if v.isResourceDir { - cfs = p.BaseFs.Resources.Fs + cfs = p.BaseFs.ResourcesCache } else { cfs = fs } - var baseDir string - if !strings.HasPrefix(v.Dir, "_gen") { - // We do cache eviction (file removes) and since the user can set - // his/hers own cache directory, we really want to make sure - // we do not delete any files that do not belong to this cache. - // We do add the cache name as the root, but this is an extra safe - // guard. We skip the files inside /resources/_gen/ because - // that would be breaking. - baseDir = filepath.Join(v.Dir, filecacheRootDirname, k) - } else { - baseDir = filepath.Join(v.Dir, k) - } - if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) { + baseDir := v.Dir + + if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) { return nil, err } bfs := afero.NewBasePathFs(cfs, baseDir) - m[k] = NewCache(bfs, v.MaxAge) + var pruneAllRootDir string + if k == cacheKeyModules { + pruneAllRootDir = "pkg" + } + + m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir) } return m, nil diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go index a6a0252b2..0c6b569c1 100644 --- a/cache/filecache/filecache_config.go +++ b/cache/filecache/filecache_config.go @@ -19,6 +19,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" "github.com/mitchellh/mapstructure" @@ -32,7 +34,7 @@ const ( resourcesGenDir = ":resourceDir/_gen" ) -var defaultCacheConfig = cacheConfig{ +var defaultCacheConfig = Config{ MaxAge: -1, // Never expire Dir: ":cacheDir/:project", } @@ -42,9 +44,20 @@ const ( cacheKeyGetCSV = "getcsv" cacheKeyImages = "images" cacheKeyAssets = "assets" + cacheKeyModules = "modules" ) -var defaultCacheConfigs = map[string]cacheConfig{ +type Configs map[string]Config + +func (c Configs) CacheDirModules() string { + return c[cacheKeyModules].Dir +} + +var defaultCacheConfigs = Configs{ + cacheKeyModules: { + MaxAge: -1, + Dir: ":cacheDir/modules", + }, cacheKeyGetJSON: defaultCacheConfig, cacheKeyGetCSV: defaultCacheConfig, cacheKeyImages: { @@ -57,9 +70,7 @@ var defaultCacheConfigs = map[string]cacheConfig{ }, } -type cachesConfig map[string]cacheConfig - -type cacheConfig struct { +type Config struct { // Max age of cache entries in this cache. Any items older than this will // be removed and not returned from the cache. // a negative value means forever, 0 means cache is disabled. @@ -88,13 +99,18 @@ func (f Caches) ImageCache() *Cache { return f[cacheKeyImages] } +// ModulesCache gets the file cache for Hugo Modules. +func (f Caches) ModulesCache() *Cache { + return f[cacheKeyModules] +} + // AssetsCache gets the file cache for assets (processed resources, SCSS etc.). func (f Caches) AssetsCache() *Cache { return f[cacheKeyAssets] } -func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { - c := make(cachesConfig) +func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { + c := make(Configs) valid := make(map[string]bool) // Add defaults for k, v := range defaultCacheConfigs { @@ -102,11 +118,9 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { valid[k] = true } - cfg := p.Cfg - m := cfg.GetStringMap(cachesConfigKey) - _, isOsFs := p.Fs.Source.(*afero.OsFs) + _, isOsFs := fs.(*afero.OsFs) for k, v := range m { cc := defaultCacheConfig @@ -148,7 +162,7 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { for i, part := range parts { if strings.HasPrefix(part, ":") { - resolved, isResource, err := resolveDirPlaceholder(p, part) + resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part) if err != nil { return c, err } @@ -176,6 +190,18 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { } } + if !strings.HasPrefix(v.Dir, "_gen") { + // We do cache eviction (file removes) and since the user can set + // his/hers own cache directory, we really want to make sure + // we do not delete any files that do not belong to this cache. + // We do add the cache name as the root, but this is an extra safe + // guard. We skip the files inside /resources/_gen/ because + // that would be breaking. + v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k) + } else { + v.Dir = filepath.Join(v.Dir, k) + } + if disabled { v.MaxAge = 0 } @@ -187,15 +213,17 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { } // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ... -func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) { +func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) { + workingDir := cfg.GetString("workingDir") + switch strings.ToLower(placeholder) { case ":resourcedir": return "", true, nil case ":cachedir": - d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg) + d, err := helpers.GetCacheDir(fs, cfg) return d, false, err case ":project": - return filepath.Base(p.WorkingDir), false, nil + return filepath.Base(workingDir), false, nil } return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder) diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go index b0f5d2dc0..f2f75344b 100644 --- a/cache/filecache/filecache_config_test.go +++ b/cache/filecache/filecache_config_test.go @@ -20,10 +20,9 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/helpers" + "github.com/spf13/afero" "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -57,22 +56,19 @@ dir = "/path/to/c3" cfg, err := config.FromConfigString(configStr, "toml") assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) + fs := afero.NewMemMapFs() + decoded, err := DecodeConfig(fs, cfg) assert.NoError(err) - decoded, err := decodeConfig(p) - assert.NoError(err) - - assert.Equal(4, len(decoded)) + assert.Equal(5, len(decoded)) c2 := decoded["getcsv"] assert.Equal("11h0m0s", c2.MaxAge.String()) - assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir) + assert.Equal(filepath.FromSlash("/path/to/c2/filecache/getcsv"), c2.Dir) c3 := decoded["images"] assert.Equal(time.Duration(-1), c3.MaxAge) - assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir) + assert.Equal(filepath.FromSlash("/path/to/c3/filecache/images"), c3.Dir) } @@ -105,14 +101,11 @@ dir = "/path/to/c3" cfg, err := config.FromConfigString(configStr, "toml") assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - decoded, err := decodeConfig(p) + fs := afero.NewMemMapFs() + decoded, err := DecodeConfig(fs, cfg) assert.NoError(err) - assert.Equal(4, len(decoded)) + assert.Equal(5, len(decoded)) for _, v := range decoded { assert.Equal(time.Duration(0), v.MaxAge) @@ -133,24 +126,22 @@ func TestDecodeConfigDefault(t *testing.T) { cfg.Set("cacheDir", "/cache/thecache") } - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + fs := afero.NewMemMapFs() - decoded, err := decodeConfig(p) + decoded, err := DecodeConfig(fs, cfg) assert.NoError(err) - assert.Equal(4, len(decoded)) + assert.Equal(5, len(decoded)) imgConfig := decoded[cacheKeyImages] jsonConfig := decoded[cacheKeyGetJSON] if runtime.GOOS == "windows" { - assert.Equal("_gen", imgConfig.Dir) + assert.Equal(filepath.FromSlash("_gen/images"), imgConfig.Dir) } else { - assert.Equal("_gen", imgConfig.Dir) - assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir) + assert.Equal("_gen/images", imgConfig.Dir) + assert.Equal("/cache/thecache/hugoproject/filecache/getjson", jsonConfig.Dir) } assert.True(imgConfig.isResourceDir) @@ -183,11 +174,9 @@ dir = "/" cfg, err := config.FromConfigString(configStr, "toml") assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + fs := afero.NewMemMapFs() - _, err = decodeConfig(p) + _, err = DecodeConfig(fs, cfg) assert.Error(err) } diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go index 322eabf92..c6fd4497e 100644 --- a/cache/filecache/filecache_pruner.go +++ b/cache/filecache/filecache_pruner.go @@ -28,53 +28,100 @@ import ( func (c Caches) Prune() (int, error) { counter := 0 for k, cache := range c { - err := afero.Walk(cache.Fs, "", func(name string, info os.FileInfo, err error) error { - if info == nil { - return nil - } - name = cleanID(name) - - if info.IsDir() { - f, err := cache.Fs.Open(name) - if err != nil { - // This cache dir may not exist. - return nil - } - defer f.Close() - _, err = f.Readdirnames(1) - if err == io.EOF { - // Empty dir. - return cache.Fs.Remove(name) - } + count, err := cache.Prune(false) + + if err != nil { + return counter, errors.Wrapf(err, "failed to prune cache %q", k) + } + + counter += count + + } + + return counter, nil +} + +// Prune removes expired and unused items from this cache. +// If force is set, everything will be removed not considering expiry time. +func (c *Cache) Prune(force bool) (int, error) { + if c.pruneAllRootDir != "" { + return c.pruneRootDir(force) + } + + counter := 0 + + err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error { + if info == nil { + return nil + } + + name = cleanID(name) + if info.IsDir() { + f, err := c.Fs.Open(name) + if err != nil { + // This cache dir may not exist. return nil } + defer f.Close() + _, err = f.Readdirnames(1) + if err == io.EOF { + // Empty dir. + return c.Fs.Remove(name) + } + + return nil + } - shouldRemove := cache.isExpired(info.ModTime()) + shouldRemove := force || c.isExpired(info.ModTime()) - if !shouldRemove && len(cache.nlocker.seen) > 0 { - // Remove it if it's not been touched/used in the last build. - _, seen := cache.nlocker.seen[name] - shouldRemove = !seen - } + if !shouldRemove && len(c.nlocker.seen) > 0 { + // Remove it if it's not been touched/used in the last build. + _, seen := c.nlocker.seen[name] + shouldRemove = !seen + } - if shouldRemove { - err := cache.Fs.Remove(name) - if err == nil { - counter++ - } - return err + if shouldRemove { + err := c.Fs.Remove(name) + if err == nil { + counter++ } + return err + } - return nil - }) + return nil + }) - if err != nil { - return counter, errors.Wrapf(err, "failed to prune cache %q", k) + return counter, err +} + +func (c *Cache) pruneRootDir(force bool) (int, error) { + + info, err := c.Fs.Stat(c.pruneAllRootDir) + if err != nil { + if os.IsNotExist(err) { + return 0, nil } + return 0, err + } + if !force && !c.isExpired(info.ModTime()) { + return 0, nil } - return counter, nil + counter := 0 + // Module cache has 0555 directories; make them writable in order to remove content. + afero.Walk(c.Fs, c.pruneAllRootDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() { + counter++ + c.Fs.Chmod(path, 0777) + } + return nil + }) + return 1, c.Fs.RemoveAll(c.pruneAllRootDir) + } diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go index e62a6315a..72c6781ac 100644 --- a/cache/filecache/filecache_pruner_test.go +++ b/cache/filecache/filecache_pruner_test.go @@ -18,9 +18,7 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -54,14 +52,9 @@ maxAge = "200ms" dir = ":resourceDir/_gen" ` - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} { msg := fmt.Sprintf("cache: %s", name) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + p := newPathsSpec(t, afero.NewMemMapFs(), configStr) caches, err := NewCaches(p) assert.NoError(err) cache := caches[name] diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index 5ac2e9beb..a03c3116a 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -25,6 +25,9 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" @@ -83,12 +86,7 @@ dir = ":cacheDir/c" configStr = replacer.Replace(configStr) configStr = strings.Replace(configStr, "\\", winPathSep, -1) - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - - fs := hugofs.NewFrom(osfs, cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + p := newPathsSpec(t, osfs, configStr) caches, err := NewCaches(p) assert.NoError(err) @@ -207,11 +205,7 @@ dir = "/cache/c" ` - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + p := newPathsSpec(t, afero.NewMemMapFs(), configStr) caches, err := NewCaches(p) assert.NoError(err) @@ -255,3 +249,51 @@ func TestCleanID(t *testing.T) { assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("/a/b//c.txt"))) assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("a/b//c.txt"))) } + +func initConfig(fs afero.Fs, cfg config.Provider) error { + if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil { + return err + } + + modConfig, err := modules.DecodeConfig(cfg) + if err != nil { + return err + } + + workingDir := cfg.GetString("workingDir") + themesDir := cfg.GetString("themesDir") + if !filepath.IsAbs(themesDir) { + themesDir = filepath.Join(workingDir, themesDir) + } + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: fs, + WorkingDir: workingDir, + ThemesDir: themesDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + moduleConfig, err := modulesClient.Collect() + if err != nil { + return err + } + + if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil { + return err + } + + cfg.Set("allModules", moduleConfig.ActiveModules) + + return nil +} + +func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec { + assert := require.New(t) + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + initConfig(fs, cfg) + p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg) + assert.NoError(err) + return p + +} |