diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-11-08 10:24:13 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-11-13 14:19:42 +0100 |
commit | f7aeaa61291dd75f92901bcbeecc7fce07a28dec (patch) | |
tree | 409c03e259a38fce2beeab46655fd5108c84cd5c /resource | |
parent | 7d78a2afd3c4a6c4af77a4ddcbd2a82f15986048 (diff) |
Add a consolidated file cache
This commits reworks how file caching is performed in Hugo. Now there is only one way, and it can be configured.
This is the default configuration:
```toml
[caches]
[caches.getjson]
dir = ":cacheDir"
maxAge = -1
[caches.getcsv]
dir = ":cacheDir"
maxAge = -1
[caches.images]
dir = ":resourceDir/_gen"
maxAge = -1
[caches.assets]
dir = ":resourceDir/_gen"
maxAge = -1
```
You can override any of these cache setting in your own `config.toml`.
The placeholders explained:
`:cacheDir`: This is the value of the `cacheDir` config option if set (can also be set via OS env variable `HUGO_CACHEDIR`). It will fall back to `/opt/build/cache/hugo_cache/` on Netlify, or a `hugo_cache` directory below the OS temp dir for the others.
`:resourceDir`: This is the value of the `resourceDir` config option.
`maxAge` is the time in seconds before a cache entry will be evicted, -1 means forever and 0 effectively turns that particular cache off.
This means that if you run your builds on Netlify, all caches configured with `:cacheDir` will be saved and restored on the next build. For other CI vendors, please read their documentation. For an CircleCI example, see https://github.com/bep/hugo-sass-test/blob/6c3960a8f4b90e8938228688bc49bdcdd6b2d99e/.circleci/config.yml
Fixes #5404
Diffstat (limited to 'resource')
-rw-r--r-- | resource/bundler/bundler.go | 2 | ||||
-rw-r--r-- | resource/create/create.go | 2 | ||||
-rw-r--r-- | resource/image.go | 101 | ||||
-rw-r--r-- | resource/image_cache.go | 88 | ||||
-rw-r--r-- | resource/image_test.go | 19 | ||||
-rw-r--r-- | resource/resource.go | 26 | ||||
-rw-r--r-- | resource/resource_cache.go | 61 | ||||
-rw-r--r-- | resource/testhelpers_test.go | 13 | ||||
-rw-r--r-- | resource/transform.go | 19 |
9 files changed, 153 insertions, 178 deletions
diff --git a/resource/bundler/bundler.go b/resource/bundler/bundler.go index 39ce9b521..70b8ee536 100644 --- a/resource/bundler/bundler.go +++ b/resource/bundler/bundler.go @@ -105,7 +105,7 @@ func (c *Client) Concat(targetPath string, resources resource.Resources) (resour } composite, err := c.rs.NewForFs( - c.rs.BaseFs.Resources.Fs, + c.rs.FileCaches.AssetsCache().Fs, resource.ResourceSourceDescriptor{ LazyPublish: true, OpenReadSeekCloser: concatr, diff --git a/resource/create/create.go b/resource/create/create.go index 0f909a36d..db23930e4 100644 --- a/resource/create/create.go +++ b/resource/create/create.go @@ -51,7 +51,7 @@ func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { return c.rs.NewForFs( - c.rs.BaseFs.Resources.Fs, + c.rs.FileCaches.AssetsCache().Fs, resource.ResourceSourceDescriptor{ LazyPublish: true, OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { diff --git a/resource/image.go b/resource/image.go index fc79cbef5..0fdabf788 100644 --- a/resource/image.go +++ b/resource/image.go @@ -21,6 +21,7 @@ import ( "image/draw" "image/jpeg" "io" + "io/ioutil" "os" "strconv" "strings" @@ -125,9 +126,6 @@ type Image struct { copyToDestinationInit sync.Once - // Lock used when creating alternate versions of this image. - createMu sync.Mutex - imaging *Imaging format imaging.Format @@ -245,7 +243,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c } } - return i.spec.imageCache.getOrCreate(i, conf, func(resourceCacheFilename string) (*Image, error) { + return i.spec.imageCache.getOrCreate(i, conf, func() (*Image, image.Image, error) { imageProcSem <- true defer func() { <-imageProcSem @@ -260,7 +258,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c src, err := i.decodeSource() if err != nil { - return nil, &os.PathError{Op: errOp, Path: errPath, Err: err} + return nil, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} } if conf.Rotate != 0 { @@ -270,7 +268,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c converted, err := f(src, conf) if err != nil { - return ci, &os.PathError{Op: errOp, Path: errPath, Err: err} + return ci, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} } if i.format == imaging.PNG { @@ -286,7 +284,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} ci.configLoaded = true - return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.targetFilenames()...) + return ci, converted, nil }) } @@ -462,79 +460,37 @@ func (i *Image) decodeSource() (image.Image, error) { return img, err } -func (i *Image) copyToDestination(src string) error { - var res error - i.copyToDestinationInit.Do(func() { - targetFilenames := i.targetFilenames() - var changedFilenames []string - - // Fast path: - // This is a processed version of the original. - // If it exists on destination with the same filename and file size, it is - // the same file, so no need to transfer it again. - for _, targetFilename := range targetFilenames { - if fi, err := i.spec.BaseFs.PublishFs.Stat(targetFilename); err == nil && fi.Size() == i.osFileInfo.Size() { - continue - } - changedFilenames = append(changedFilenames, targetFilename) - } - - if len(changedFilenames) == 0 { - return - } - - in, err := i.sourceFs().Open(src) - if err != nil { - res = err - return - } - defer in.Close() - - out, err := helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, changedFilenames...) - - if err != nil { - res = err - return +func (i *Image) openDestinationsForWriting() (io.WriteCloser, error) { + targetFilenames := i.targetFilenames() + var changedFilenames []string + + // Fast path: + // This is a processed version of the original. + // If it exists on destination with the same filename and file size, it is + // the same file, so no need to transfer it again. + for _, targetFilename := range targetFilenames { + if fi, err := i.spec.BaseFs.PublishFs.Stat(targetFilename); err == nil && fi.Size() == i.osFileInfo.Size() { + continue } - defer out.Close() - - _, err = io.Copy(out, in) - if err != nil { - res = err - return - } - }) - - if res != nil { - return fmt.Errorf("failed to copy image to destination: %s", res) + changedFilenames = append(changedFilenames, targetFilename) } - return nil -} -func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename string, targetFilenames ...string) error { + if len(changedFilenames) == 0 { + return struct { + io.Writer + io.Closer + }{ + ioutil.Discard, + ioutil.NopCloser(nil), + }, nil - file1, err := helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, targetFilenames...) - if err != nil { - return err } - defer file1.Close() - - var w io.Writer + return helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, changedFilenames...) - if resourceCacheFilename != "" { - // Also save it to the image resource cache for later reuse. - file2, err := helpers.OpenFileForWriting(i.spec.BaseFs.Resources.Fs, resourceCacheFilename) - if err != nil { - return err - } - - w = io.MultiWriter(file1, file2) - defer file2.Close() - } else { - w = file1 - } +} +func (i *Image) encodeTo(conf imageConfig, img image.Image, w io.Writer) error { switch i.format { case imaging.JPEG: @@ -557,7 +513,6 @@ func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resource default: return imaging.Encode(w, img, i.format) } - } func (i *Image) clone() *Image { diff --git a/resource/image_cache.go b/resource/image_cache.go index 470c24c99..22c86ea32 100644 --- a/resource/image_cache.go +++ b/resource/image_cache.go @@ -15,19 +15,25 @@ package resource import ( "fmt" + "image" + "io" "path/filepath" "strings" "sync" "time" + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" ) type imageCache struct { - cacheDir string pathSpec *helpers.PathSpec - mu sync.RWMutex + fileCache *filecache.Cache + + mu sync.RWMutex store map[string]*Image } @@ -66,7 +72,7 @@ func (c *imageCache) clear() { } func (c *imageCache) getOrCreate( - parent *Image, conf imageConfig, create func(resourceCacheFilename string) (*Image, error)) (*Image, error) { + parent *Image, conf imageConfig, createImage func() (*Image, image.Image, error)) (*Image, error) { relTarget := parent.relTargetPathFromConfig(conf) key := parent.relTargetPathForRel(relTarget.path(), false, false) @@ -80,58 +86,76 @@ func (c *imageCache) getOrCreate( return img, nil } - // Now look in the file cache. - // Multiple Go routines can invoke same operation on the same image, so - // we need to make sure this is serialized per source image. - parent.createMu.Lock() - defer parent.createMu.Unlock() + // These funcs are protected by a named lock. + // read clones the parent to its new name and copies + // the content to the destinations. + read := func(info filecache.ItemInfo, r io.Reader) error { + img = parent.clone() + img.relTargetDirFile.file = relTarget.file + img.sourceFilename = info.Name + + w, err := img.openDestinationsForWriting() + if err != nil { + return err + } + + defer w.Close() + _, err = io.Copy(w, r) + return err + } + + // create creates the image and encodes it to w (cache) and to its destinations. + create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + var conv image.Image + img, conv, err = createImage() + if err != nil { + w.Close() + return + } + img.relTargetDirFile.file = relTarget.file + img.sourceFilename = info.Name + + destinations, err := img.openDestinationsForWriting() + if err != nil { + w.Close() + return err + } + + mw := hugio.NewMultiWriteCloser(w, destinations) + defer mw.Close() - cacheFilename := filepath.Join(c.cacheDir, key) + return img.encodeTo(conf, conv, mw) + } + + // Now look in the file cache. // The definition of this counter is not that we have processed that amount // (e.g. resized etc.), it can be fetched from file cache, // but the count of processed image variations for this site. c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.Resources.Fs) + _, err := c.fileCache.ReadOrCreate(key, read, create) if err != nil { return nil, err } - if exists { - img = parent.clone() - } else { - img, err = create(cacheFilename) - if err != nil { - return nil, err - } - } - img.relTargetDirFile.file = relTarget.file - img.sourceFilename = cacheFilename - // We have to look in the resources file system for this. - img.overriddenSourceFs = img.spec.BaseFs.Resources.Fs + // The file is now stored in this cache. + img.overriddenSourceFs = c.fileCache.Fs c.mu.Lock() if img2, found := c.store[key]; found { c.mu.Unlock() return img2, nil } - c.store[key] = img - c.mu.Unlock() - if !exists { - // File already written to destination - return img, nil - } - - return img, img.copyToDestination(cacheFilename) + return img, nil } -func newImageCache(ps *helpers.PathSpec, cacheDir string) *imageCache { - return &imageCache{pathSpec: ps, store: make(map[string]*Image), cacheDir: cacheDir} +func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { + return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*Image)} } func timeTrack(start time.Time, name string) { diff --git a/resource/image_test.go b/resource/image_test.go index a9aacbdce..07eab5d56 100644 --- a/resource/image_test.go +++ b/resource/image_test.go @@ -62,6 +62,7 @@ func TestImageTransformBasic(t *testing.T) { assert := require.New(t) image := fetchSunset(assert) + fileCache := image.spec.FileCaches.ImageCache().Fs assert.Equal("/a/sunset.jpg", image.RelPermalink()) assert.Equal("image", image.ResourceType()) @@ -76,19 +77,20 @@ func TestImageTransformBasic(t *testing.T) { assert.NoError(err) assert.Equal(320, resized0x.Width()) assert.Equal(200, resized0x.Height()) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized0x.RelPermalink(), 320, 200) + + assertFileCache(assert, fileCache, resized0x.RelPermalink(), 320, 200) resizedx0, err := image.Resize("200x") assert.NoError(err) assert.Equal(200, resizedx0.Width()) assert.Equal(125, resizedx0.Height()) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedx0.RelPermalink(), 200, 125) + assertFileCache(assert, fileCache, resizedx0.RelPermalink(), 200, 125) resizedAndRotated, err := image.Resize("x200 r90") assert.NoError(err) assert.Equal(125, resizedAndRotated.Width()) assert.Equal(200, resizedAndRotated.Height()) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAndRotated.RelPermalink(), 125, 200) + assertFileCache(assert, fileCache, resizedAndRotated.RelPermalink(), 125, 200) assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink()) assert.Equal(300, resized.Width()) @@ -113,21 +115,21 @@ func TestImageTransformBasic(t *testing.T) { assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink()) assert.Equal(200, filled.Width()) assert.Equal(100, filled.Height()) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filled.RelPermalink(), 200, 100) + assertFileCache(assert, fileCache, filled.RelPermalink(), 200, 100) smart, err := image.Fill("200x100 smart") assert.NoError(err) assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink()) assert.Equal(200, smart.Width()) assert.Equal(100, smart.Height()) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, smart.RelPermalink(), 200, 100) + assertFileCache(assert, fileCache, smart.RelPermalink(), 200, 100) // Check cache filledAgain, err := image.Fill("200x100 bottomLeft") assert.NoError(err) assert.True(filled == filledAgain) assert.True(filled.sourceFilename == filledAgain.sourceFilename) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filledAgain.RelPermalink(), 200, 100) + assertFileCache(assert, fileCache, filledAgain.RelPermalink(), 200, 100) } @@ -286,6 +288,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert := require.New(t) image := fetchImage(assert, "sub/gohugoio2.png") + fileCache := image.spec.FileCaches.ImageCache().Fs assert.Equal(imaging.PNG, image.format) assert.Equal("/a/sub/gohugoio2.png", image.RelPermalink()) @@ -297,7 +300,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink()) assert.Equal(101, resized.Width()) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized.RelPermalink(), 101, 101) + assertFileCache(assert, fileCache, resized.RelPermalink(), 101, 101) publishedImageFilename := filepath.Clean(resized.RelPermalink()) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename)) @@ -309,7 +312,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.NoError(err) assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink()) assert.Equal(101, resizedAgain.Width()) - assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAgain.RelPermalink(), 101, 101) + assertFileCache(assert, fileCache, resizedAgain.RelPermalink(), 101, 101) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) } diff --git a/resource/resource.go b/resource/resource.go index be3ebdb8b..350519913 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -28,6 +28,7 @@ import ( "github.com/gohugoio/hugo/tpl" "github.com/pkg/errors" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/loggers" @@ -280,12 +281,15 @@ type Spec struct { imageCache *imageCache ResourceCache *ResourceCache - - GenImagePath string - GenAssetsPath string + FileCaches filecache.Caches } -func NewSpec(s *helpers.PathSpec, logger *loggers.Logger, outputFormats output.Formats, mimeTypes media.Types) (*Spec, error) { +func NewSpec( + s *helpers.PathSpec, + fileCaches filecache.Caches, + logger *loggers.Logger, + outputFormats output.Formats, + mimeTypes media.Types) (*Spec, error) { imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) if err != nil { @@ -296,24 +300,16 @@ func NewSpec(s *helpers.PathSpec, logger *loggers.Logger, outputFormats output.F logger = loggers.NewErrorLogger() } - genImagePath := filepath.FromSlash("_gen/images") - // The transformed assets (CSS etc.) - genAssetsPath := filepath.FromSlash("_gen/assets") - rs := &Spec{PathSpec: s, Logger: logger, - GenImagePath: genImagePath, - GenAssetsPath: genAssetsPath, imaging: &imaging, MediaTypes: mimeTypes, OutputFormats: outputFormats, + FileCaches: fileCaches, imageCache: newImageCache( + fileCaches.ImageCache(), + s, - // We're going to write a cache pruning routine later, so make it extremely - // unlikely that the user shoots him or herself in the foot - // and this is set to a value that represents data he/she - // cares about. This should be set in stone once released. - genImagePath, )} rs.ResourceCache = newResourceCache(rs) diff --git a/resource/resource_cache.go b/resource/resource_cache.go index e0b86ec9e..e7c6ab6d0 100644 --- a/resource/resource_cache.go +++ b/resource/resource_cache.go @@ -15,14 +15,13 @@ package resource import ( "encoding/json" - "io/ioutil" + "io" "path" "path/filepath" "strings" "sync" - "github.com/gohugoio/hugo/helpers" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/cache/filecache" "github.com/BurntSushi/locker" ) @@ -35,8 +34,10 @@ const ( type ResourceCache struct { rs *Spec - cache map[string]Resource sync.RWMutex + cache map[string]Resource + + fileCache *filecache.Cache // Provides named resource locks. nlocker *locker.Locker @@ -56,9 +57,10 @@ func ResourceKeyPartition(filename string) string { func newResourceCache(rs *Spec) *ResourceCache { return &ResourceCache{ - rs: rs, - cache: make(map[string]Resource), - nlocker: locker.NewLocker(), + rs: rs, + fileCache: rs.FileCaches.AssetsCache(), + cache: make(map[string]Resource), + nlocker: locker.NewLocker(), } } @@ -118,65 +120,56 @@ func (c *ResourceCache) GetOrCreate(partition, key string, f func() (Resource, e } func (c *ResourceCache) getFilenames(key string) (string, string) { - filenameBase := filepath.Join(c.rs.GenAssetsPath, key) - filenameMeta := filenameBase + ".json" - filenameContent := filenameBase + ".content" + filenameMeta := key + ".json" + filenameContent := key + ".content" return filenameMeta, filenameContent } -func (c *ResourceCache) getFromFile(key string) (afero.File, transformedResourceMetadata, bool) { +func (c *ResourceCache) getFromFile(key string) (filecache.ItemInfo, io.ReadCloser, transformedResourceMetadata, bool) { c.RLock() defer c.RUnlock() var meta transformedResourceMetadata filenameMeta, filenameContent := c.getFilenames(key) - fMeta, err := c.rs.Resources.Fs.Open(filenameMeta) - if err != nil { - return nil, meta, false - } - defer fMeta.Close() - jsonContent, err := ioutil.ReadAll(fMeta) - if err != nil { - return nil, meta, false + _, jsonContent, _ := c.fileCache.GetBytes(filenameMeta) + if jsonContent == nil { + return filecache.ItemInfo{}, nil, meta, false } if err := json.Unmarshal(jsonContent, &meta); err != nil { - return nil, meta, false + return filecache.ItemInfo{}, nil, meta, false } - fContent, err := c.rs.Resources.Fs.Open(filenameContent) - if err != nil { - return nil, meta, false - } + fi, rc, _ := c.fileCache.Get(filenameContent) + + return fi, rc, meta, rc != nil - return fContent, meta, true } // writeMeta writes the metadata to file and returns a writer for the content part. -func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) (afero.File, error) { +func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) (filecache.ItemInfo, io.WriteCloser, error) { filenameMeta, filenameContent := c.getFilenames(key) raw, err := json.Marshal(meta) if err != nil { - return nil, err + return filecache.ItemInfo{}, nil, err } - fm, err := c.openResourceFileForWriting(filenameMeta) + _, fm, err := c.fileCache.WriteCloser(filenameMeta) if err != nil { - return nil, err + return filecache.ItemInfo{}, nil, err } + defer fm.Close() if _, err := fm.Write(raw); err != nil { - return nil, err + return filecache.ItemInfo{}, nil, err } - return c.openResourceFileForWriting(filenameContent) + fi, fc, err := c.fileCache.WriteCloser(filenameContent) -} + return fi, fc, err -func (c *ResourceCache) openResourceFileForWriting(filename string) (afero.File, error) { - return helpers.OpenFileForWriting(c.rs.Resources.Fs, filename) } func (c *ResourceCache) set(key string, r Resource) { diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go index 2a5d2b3cd..05a02a6fc 100644 --- a/resource/testhelpers_test.go +++ b/resource/testhelpers_test.go @@ -13,6 +13,7 @@ import ( "runtime" "strings" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/media" @@ -49,10 +50,12 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * fs := hugofs.NewMem(cfg) s, err := helpers.NewPathSpec(fs, cfg) + assert.NoError(err) + filecaches, err := filecache.NewCachesFromPaths(s.Paths) assert.NoError(err) - spec, err := NewSpec(s, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, output.DefaultFormats, media.DefaultTypes) assert.NoError(err) return spec } @@ -70,7 +73,7 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { } cfg.Set("workingDir", workDir) - cfg.Set("resourceDir", filepath.Join(workDir, "res")) + cfg.Set("resourceDir", "resources") cfg.Set("contentDir", "content") cfg.Set("dataDir", "data") cfg.Set("i18nDir", "i18n") @@ -83,10 +86,12 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { fs.Destination = &afero.MemMapFs{} s, err := helpers.NewPathSpec(fs, cfg) + assert.NoError(err) + filecaches, err := filecache.NewCachesFromPaths(s.Paths) assert.NoError(err) - spec, err := NewSpec(s, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, output.DefaultFormats, media.DefaultTypes) assert.NoError(err) return spec @@ -144,7 +149,7 @@ func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, w } func assertFileCache(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { - assertImageFile(assert, fs, filepath.Join("_gen/images", filename), width, height) + assertImageFile(assert, fs, filepath.Clean(filename), width, height) } func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) { diff --git a/resource/transform.go b/resource/transform.go index a4b1ca27c..0b5772dde 100644 --- a/resource/transform.go +++ b/resource/transform.go @@ -24,7 +24,6 @@ import ( "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" "github.com/mitchellh/hashstructure" - "github.com/spf13/afero" "fmt" "io" @@ -210,12 +209,12 @@ func (r *transformedResource) transferTransformedValues(another *transformedReso } func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser { - f, meta, found := r.cache.getFromFile(key) + fi, f, meta, found := r.cache.getFromFile(key) if !found { return nil } r.transformedResourceMetadata = meta - r.sourceFilename = f.Name() + r.sourceFilename = fi.Name return f } @@ -263,7 +262,7 @@ func (r *transformedResource) initContent() error { var err error r.contentInit.Do(func() { var b []byte - b, err := afero.ReadFile(r.cache.rs.Resources.Fs, r.sourceFilename) + _, b, err = r.cache.fileCache.GetBytes(r.sourceFilename) if err != nil { return } @@ -434,16 +433,15 @@ func (r *transformedResource) transform(setContent bool) (err error) { } defer publicw.Close() - publishwriters := []io.Writer{publicw} + publishwriters := []io.WriteCloser{publicw} if transformedContentr == nil { // Also write it to the cache - metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata) + fi, metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata) if err != nil { return err } - r.sourceFilename = metaw.Name() - defer metaw.Close() + r.sourceFilename = fi.Name publishwriters = append(publishwriters, metaw) @@ -460,11 +458,12 @@ func (r *transformedResource) transform(setContent bool) (err error) { if setContent { contentmemw = bp.GetBuffer() defer bp.PutBuffer(contentmemw) - publishwriters = append(publishwriters, contentmemw) + publishwriters = append(publishwriters, hugio.ToWriteCloser(contentmemw)) } - publishw := io.MultiWriter(publishwriters...) + publishw := hugio.NewMultiWriteCloser(publishwriters...) _, r.transformErr = io.Copy(publishw, transformedContentr) + publishw.Close() if setContent { r.contentInit.Do(func() { |