From f9978ed16476ca6d233a89669c62c798cdf9db9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 18 Aug 2019 11:21:27 +0200 Subject: Image resource refactor This commit pulls most of the image related logic into its own package, to make it easier to reason about and extend. This is also a rewrite of the transformation logic used in Hugo Pipes, mostly to allow constructs like the one below: {{ ($myimg | fingerprint ).Width }} Fixes #5903 Fixes #6234 Fixes #6266 --- resources/image.go | 506 ++----------- resources/image_cache.go | 63 +- resources/image_test.go | 189 ++--- resources/images/config.go | 276 +++++++ resources/images/config_test.go | 125 ++++ resources/images/image.go | 170 +++++ resources/images/smartcrop.go | 75 ++ resources/internal/key.go | 61 ++ resources/internal/key_test.go | 36 + resources/resource.go | 827 +++++++++------------ resources/resource/resourcetypes.go | 20 +- resources/resource_cache.go | 2 +- resources/resource_metadata.go | 20 +- resources/resource_metadata_test.go | 2 +- resources/resource_spec.go | 304 ++++++++ resources/resource_test.go | 25 +- .../resource_transformers/htesting/testhelpers.go | 80 ++ .../resource_transformers/integrity/integrity.go | 25 +- .../integrity/integrity_test.go | 24 + resources/resource_transformers/minifier/minify.go | 18 +- .../resource_transformers/minifier/minify_test.go | 43 ++ resources/resource_transformers/postcss/postcss.go | 12 +- .../templates/execute_as_template.go | 22 +- .../resource_transformers/tocss/scss/client.go | 12 +- resources/smartcrop.go | 77 -- resources/testhelpers_test.go | 44 +- resources/transform.go | 598 ++++++++------- resources/transform_test.go | 428 ++++++++++- 28 files changed, 2560 insertions(+), 1524 deletions(-) create mode 100644 resources/images/config.go create mode 100644 resources/images/config_test.go create mode 100644 resources/images/image.go create mode 100644 resources/images/smartcrop.go create mode 100644 resources/internal/key.go create mode 100644 resources/internal/key_test.go create mode 100644 resources/resource_spec.go create mode 100644 resources/resource_transformers/htesting/testhelpers.go create mode 100644 resources/resource_transformers/minifier/minify_test.go delete mode 100644 resources/smartcrop.go (limited to 'resources') diff --git a/resources/image.go b/resources/image.go index f1aae2996..e1a816942 100644 --- a/resources/image.go +++ b/resources/image.go @@ -14,198 +14,98 @@ package resources import ( - "errors" "fmt" "image" "image/color" "image/draw" - "image/jpeg" - "io" + _ "image/gif" + _ "image/png" "os" - "strconv" "strings" - "sync" "github.com/gohugoio/hugo/resources/resource" _errors "github.com/pkg/errors" "github.com/disintegration/imaging" - "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" - "github.com/mitchellh/mapstructure" + "github.com/gohugoio/hugo/resources/images" // Blind import for image.Decode - _ "image/gif" - _ "image/png" // Blind import for image.Decode _ "golang.org/x/image/webp" ) var ( - _ resource.Resource = (*Image)(nil) - _ resource.Source = (*Image)(nil) - _ resource.Cloner = (*Image)(nil) + _ resource.Image = (*imageResource)(nil) + _ resource.Source = (*imageResource)(nil) + _ resource.Cloner = (*imageResource)(nil) ) -// Imaging contains default image processing configuration. This will be fetched -// from site (or language) config. -type Imaging struct { - // Default image quality setting (1-100). Only used for JPEG images. - Quality int - - // Resample filter used. See https://github.com/disintegration/imaging - ResampleFilter string +// ImageResource represents an image resource. +type imageResource struct { + *images.Image - // The anchor used in Fill. Default is "smart", i.e. Smart Crop. - Anchor string + baseResource } -const ( - defaultJPEGQuality = 75 - defaultResampleFilter = "box" -) - -var ( - imageFormats = map[string]imaging.Format{ - ".jpg": imaging.JPEG, - ".jpeg": imaging.JPEG, - ".png": imaging.PNG, - ".tif": imaging.TIFF, - ".tiff": imaging.TIFF, - ".bmp": imaging.BMP, - ".gif": imaging.GIF, +func (i *imageResource) Clone() resource.Resource { + gr := i.baseResource.Clone().(baseResource) + return &imageResource{ + Image: i.WithSpec(gr), + baseResource: gr, } - - // Add or increment if changes to an image format's processing requires - // re-generation. - imageFormatsVersions = map[imaging.Format]int{ - imaging.PNG: 2, // Floyd Steinberg dithering - } - - // Increment to mark all processed images as stale. Only use when absolutely needed. - // See the finer grained smartCropVersionNumber and imageFormatsVersions. - mainImageVersionNumber = 0 -) - -var anchorPositions = map[string]imaging.Anchor{ - strings.ToLower("Center"): imaging.Center, - strings.ToLower("TopLeft"): imaging.TopLeft, - strings.ToLower("Top"): imaging.Top, - strings.ToLower("TopRight"): imaging.TopRight, - strings.ToLower("Left"): imaging.Left, - strings.ToLower("Right"): imaging.Right, - strings.ToLower("BottomLeft"): imaging.BottomLeft, - strings.ToLower("Bottom"): imaging.Bottom, - strings.ToLower("BottomRight"): imaging.BottomRight, } -var imageFilters = map[string]imaging.ResampleFilter{ - strings.ToLower("NearestNeighbor"): imaging.NearestNeighbor, - strings.ToLower("Box"): imaging.Box, - strings.ToLower("Linear"): imaging.Linear, - strings.ToLower("Hermite"): imaging.Hermite, - strings.ToLower("MitchellNetravali"): imaging.MitchellNetravali, - strings.ToLower("CatmullRom"): imaging.CatmullRom, - strings.ToLower("BSpline"): imaging.BSpline, - strings.ToLower("Gaussian"): imaging.Gaussian, - strings.ToLower("Lanczos"): imaging.Lanczos, - strings.ToLower("Hann"): imaging.Hann, - strings.ToLower("Hamming"): imaging.Hamming, - strings.ToLower("Blackman"): imaging.Blackman, - strings.ToLower("Bartlett"): imaging.Bartlett, - strings.ToLower("Welch"): imaging.Welch, - strings.ToLower("Cosine"): imaging.Cosine, -} - -// Image represents an image resource. -type Image struct { - config image.Config - configInit sync.Once - configLoaded bool - - imaging *Imaging - - format imaging.Format - - *genericResource -} +func (i *imageResource) cloneWithUpdates(u *transformationUpdate) (baseResource, error) { + base, err := i.baseResource.cloneWithUpdates(u) + if err != nil { + return nil, err + } -// Width returns i's width. -func (i *Image) Width() int { - i.initConfig() - return i.config.Width -} + var img *images.Image -// Height returns i's height. -func (i *Image) Height() int { - i.initConfig() - return i.config.Height -} + if u.isContenChanged() { + img = i.WithSpec(base) + } else { + img = i.Image + } -// WithNewBase implements the Cloner interface. -func (i *Image) WithNewBase(base string) resource.Resource { - return &Image{ - imaging: i.imaging, - format: i.format, - genericResource: i.genericResource.WithNewBase(base).(*genericResource)} + return &imageResource{ + Image: img, + baseResource: base, + }, nil } // Resize resizes the image to the specified width and height using the specified resampling // filter and returns the transformed image. If one of width or height is 0, the image aspect // ratio is preserved. -func (i *Image) Resize(spec string) (*Image, error) { - return i.doWithImageConfig("resize", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - return imaging.Resize(src, conf.Width, conf.Height, conf.Filter), nil +func (i *imageResource) Resize(spec string) (resource.Image, error) { + return i.doWithImageConfig("resize", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Resize(src, conf) }) } // Fit scales down the image using the specified resample filter to fit the specified // maximum width and height. -func (i *Image) Fit(spec string) (*Image, error) { - return i.doWithImageConfig("fit", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - return imaging.Fit(src, conf.Width, conf.Height, conf.Filter), nil +func (i *imageResource) Fit(spec string) (resource.Image, error) { + return i.doWithImageConfig("fit", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Fit(src, conf) }) } // Fill scales the image to the smallest possible size that will cover the specified dimensions, // crops the resized image to the specified dimensions using the given anchor point. // Space delimited config: 200x300 TopLeft -func (i *Image) Fill(spec string) (*Image, error) { - return i.doWithImageConfig("fill", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - if conf.AnchorStr == smartCropIdentifier { - return smartCrop(src, conf.Width, conf.Height, conf.Anchor, conf.Filter) - } - return imaging.Fill(src, conf.Width, conf.Height, conf.Anchor, conf.Filter), nil +func (i *imageResource) Fill(spec string) (resource.Image, error) { + return i.doWithImageConfig("fill", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Fill(src, conf) }) } -// Holds configuration to create a new image from an existing one, resize etc. -type imageConfig struct { - Action string - - // Quality ranges from 1 to 100 inclusive, higher is better. - // This is only relevant for JPEG images. - // Default is 75. - Quality int - - // Rotate rotates an image by the given angle counter-clockwise. - // The rotation will be performed first. - Rotate int - - Width int - Height int - - Filter imaging.ResampleFilter - FilterStr string - - Anchor imaging.Anchor - AnchorStr string -} - -func (i *Image) isJPEG() bool { - name := strings.ToLower(i.relTargetDirFile.file) +func (i *imageResource) isJPEG() bool { + name := strings.ToLower(i.getResourcePaths().relTargetDirFile.file) return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg") } @@ -218,42 +118,20 @@ const imageProcWorkers = 1 var imageProcSem = make(chan bool, imageProcWorkers) -func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, conf imageConfig) (image.Image, error)) (*Image, error) { - conf, err := parseImageConfig(spec) +func (i *imageResource) doWithImageConfig(action, spec string, f func(src image.Image, conf images.ImageConfig) (image.Image, error)) (resource.Image, error) { + conf, err := i.decodeImageConfig(action, spec) if err != nil { return nil, err } - conf.Action = action - - if conf.Quality <= 0 && i.isJPEG() { - // We need a quality setting for all JPEGs - conf.Quality = i.imaging.Quality - } - - if conf.FilterStr == "" { - conf.FilterStr = i.imaging.ResampleFilter - conf.Filter = imageFilters[conf.FilterStr] - } - - if conf.AnchorStr == "" { - conf.AnchorStr = i.imaging.Anchor - if !strings.EqualFold(conf.AnchorStr, smartCropIdentifier) { - conf.Anchor = anchorPositions[conf.AnchorStr] - } - } - return i.spec.imageCache.getOrCreate(i, conf, func() (*Image, image.Image, error) { + return i.getSpec().imageCache.getOrCreate(i, conf, func() (*imageResource, image.Image, error) { imageProcSem <- true defer func() { <-imageProcSem }() - ci := i.clone() - errOp := action - errPath := i.sourceFilename - - ci.setBasePath(conf) + errPath := i.getSourceFilename() src, err := i.decodeSource() if err != nil { @@ -267,10 +145,10 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c converted, err := f(src, conf) if err != nil { - return ci, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} + return nil, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} } - if i.format == imaging.PNG { + if i.Format == imaging.PNG { // Apply the colour palette from the source if paletted, ok := src.(*image.Paletted); ok { tmp := image.NewPaletted(converted.Bounds(), paletted.Palette) @@ -279,177 +157,30 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c } } - b := converted.Bounds() - ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} - ci.configLoaded = true + ci := i.clone(converted) + ci.setBasePath(conf) return ci, converted, nil }) - -} - -func (i imageConfig) key(format imaging.Format) string { - k := strconv.Itoa(i.Width) + "x" + strconv.Itoa(i.Height) - if i.Action != "" { - k += "_" + i.Action - } - if i.Quality > 0 { - k += "_q" + strconv.Itoa(i.Quality) - } - if i.Rotate != 0 { - k += "_r" + strconv.Itoa(i.Rotate) - } - anchor := i.AnchorStr - if anchor == smartCropIdentifier { - anchor = anchor + strconv.Itoa(smartCropVersionNumber) - } - - k += "_" + i.FilterStr - - if strings.EqualFold(i.Action, "fill") { - k += "_" + anchor - } - - if v, ok := imageFormatsVersions[format]; ok { - k += "_" + strconv.Itoa(v) - } - - if mainImageVersionNumber > 0 { - k += "_" + strconv.Itoa(mainImageVersionNumber) - } - - return k -} - -func newImageConfig(width, height, quality, rotate int, filter, anchor string) imageConfig { - var c imageConfig - - c.Width = width - c.Height = height - c.Quality = quality - c.Rotate = rotate - - if filter != "" { - filter = strings.ToLower(filter) - if v, ok := imageFilters[filter]; ok { - c.Filter = v - c.FilterStr = filter - } - } - - if anchor != "" { - anchor = strings.ToLower(anchor) - if v, ok := anchorPositions[anchor]; ok { - c.Anchor = v - c.AnchorStr = anchor - } - } - - return c } -func parseImageConfig(config string) (imageConfig, error) { - var ( - c imageConfig - err error - ) - - if config == "" { - return c, errors.New("image config cannot be empty") - } - - parts := strings.Fields(config) - for _, part := range parts { - part = strings.ToLower(part) - - if part == smartCropIdentifier { - c.AnchorStr = smartCropIdentifier - } else if pos, ok := anchorPositions[part]; ok { - c.Anchor = pos - c.AnchorStr = part - } else if filter, ok := imageFilters[part]; ok { - c.Filter = filter - c.FilterStr = part - } else if part[0] == 'q' { - c.Quality, err = strconv.Atoi(part[1:]) - if err != nil { - return c, err - } - if c.Quality < 1 || c.Quality > 100 { - return c, errors.New("quality ranges from 1 to 100 inclusive") - } - } else if part[0] == 'r' { - c.Rotate, err = strconv.Atoi(part[1:]) - if err != nil { - return c, err - } - } else if strings.Contains(part, "x") { - widthHeight := strings.Split(part, "x") - if len(widthHeight) <= 2 { - first := widthHeight[0] - if first != "" { - c.Width, err = strconv.Atoi(first) - if err != nil { - return c, err - } - } - - if len(widthHeight) == 2 { - second := widthHeight[1] - if second != "" { - c.Height, err = strconv.Atoi(second) - if err != nil { - return c, err - } - } - } - } else { - return c, errors.New("invalid image dimensions") - } - - } - } - - if c.Width == 0 && c.Height == 0 { - return c, errors.New("must provide Width or Height") +func (i *imageResource) decodeImageConfig(action, spec string) (images.ImageConfig, error) { + conf, err := images.DecodeImageConfig(action, spec, i.Proc.Cfg) + if err != nil { + return conf, err } - return c, nil -} - -func (i *Image) initConfig() error { - var err error - i.configInit.Do(func() { - if i.configLoaded { - return - } - - var ( - f hugio.ReadSeekCloser - config image.Config - ) - - f, err = i.ReadSeekCloser() - if err != nil { - return - } - defer f.Close() - - config, _, err = image.DecodeConfig(f) - if err != nil { - return - } - i.config = config - }) + iconf := i.Proc.Cfg - if err != nil { - return _errors.Wrap(err, "failed to load image config") + if conf.Quality <= 0 && i.isJPEG() { + // We need a quality setting for all JPEGs + conf.Quality = iconf.Quality } - return nil + return conf, nil } -func (i *Image) decodeSource() (image.Image, error) { +func (i *imageResource) decodeSource() (image.Image, error) { f, err := i.ReadSeekCloser() if err != nil { return nil, _errors.Wrap(err, "failed to open image for decode") @@ -459,80 +190,39 @@ func (i *Image) decodeSource() (image.Image, error) { return img, err } -// returns an opened file or nil if nothing to write. -func (i *Image) openDestinationsForWriting() (io.WriteCloser, error) { - targetFilenames := i.targetFilenames() - var changedFilenames []string - - // Fast path: - // This is a processed version of the original; - // check if it already existis at the destination. - for _, targetFilename := range targetFilenames { - if _, err := i.spec.BaseFs.PublishFs.Stat(targetFilename); err == nil { - continue - } - changedFilenames = append(changedFilenames, targetFilename) - } - - if len(changedFilenames) == 0 { - return nil, nil - } - - return helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, changedFilenames...) - -} - -func (i *Image) encodeTo(conf imageConfig, img image.Image, w io.Writer) error { - switch i.format { - case imaging.JPEG: - - var rgba *image.RGBA - quality := conf.Quality +func (i *imageResource) clone(img image.Image) *imageResource { + spec := i.baseResource.Clone().(baseResource) - if nrgba, ok := img.(*image.NRGBA); ok { - if nrgba.Opaque() { - rgba = &image.RGBA{ - Pix: nrgba.Pix, - Stride: nrgba.Stride, - Rect: nrgba.Rect, - } - } - } - if rgba != nil { - return jpeg.Encode(w, rgba, &jpeg.Options{Quality: quality}) - } - return jpeg.Encode(w, img, &jpeg.Options{Quality: quality}) - default: - return imaging.Encode(w, img, i.format) + var image *images.Image + if img != nil { + image = i.WithImage(img) + } else { + image = i.WithSpec(spec) } -} -func (i *Image) clone() *Image { - g := *i.genericResource - g.resourceContent = &resourceContent{} - if g.publishOnce != nil { - g.publishOnce = &publishOnce{logger: g.publishOnce.logger} + return &imageResource{ + Image: image, + baseResource: spec, } - - return &Image{ - imaging: i.imaging, - format: i.format, - genericResource: &g} } -func (i *Image) setBasePath(conf imageConfig) { - i.relTargetDirFile = i.relTargetPathFromConfig(conf) +func (i *imageResource) setBasePath(conf images.ImageConfig) { + i.getResourcePaths().relTargetDirFile = i.relTargetPathFromConfig(conf) } -func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { - p1, p2 := helpers.FileAndExt(i.relTargetDirFile.file) +func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile { + p1, p2 := helpers.FileAndExt(i.getResourcePaths().relTargetDirFile.file) + if conf.Action == "trace" { + p2 = ".svg" + } - idStr := fmt.Sprintf("_hu%s_%d", i.hash, i.osFileInfo.Size()) + h, _ := i.hash() + idStr := fmt.Sprintf("_hu%s_%d", h, i.size()) // Do not change for no good reason. const md5Threshold = 100 - key := conf.key(i.format) + key := conf.Key(i.Format) // It is useful to have the key in clear text, but when nesting transforms, it // can easily be too long to read, and maybe even too long @@ -554,43 +244,7 @@ func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { } return dirFile{ - dir: i.relTargetDirFile.dir, + dir: i.getResourcePaths().relTargetDirFile.dir, file: fmt.Sprintf("%s%s_%s%s", p1, idStr, key, p2), } - -} - -func decodeImaging(m map[string]interface{}) (Imaging, error) { - var i Imaging - if err := mapstructure.WeakDecode(m, &i); err != nil { - return i, err - } - - if i.Quality == 0 { - i.Quality = defaultJPEGQuality - } else if i.Quality < 0 || i.Quality > 100 { - return i, errors.New("JPEG quality must be a number between 1 and 100") - } - - if i.Anchor == "" || strings.EqualFold(i.Anchor, smartCropIdentifier) { - i.Anchor = smartCropIdentifier - } else { - i.Anchor = strings.ToLower(i.Anchor) - if _, found := anchorPositions[i.Anchor]; !found { - return i, errors.New("invalid anchor value in imaging config") - } - } - - if i.ResampleFilter == "" { - i.ResampleFilter = defaultResampleFilter - } else { - filter := strings.ToLower(i.ResampleFilter) - _, found := imageFilters[filter] - if !found { - return i, fmt.Errorf("%q is not a valid resample filter", filter) - } - i.ResampleFilter = filter - } - - return i, nil } diff --git a/resources/image_cache.go b/resources/image_cache.go index 3324e442e..3a9e3c2c5 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -20,7 +20,7 @@ import ( "strings" "sync" - "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/resources/images" "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" @@ -32,7 +32,7 @@ type imageCache struct { fileCache *filecache.Cache mu sync.RWMutex - store map[string]*Image + store map[string]*resourceAdapter } func (c *imageCache) isInCache(key string) bool { @@ -66,33 +66,34 @@ func (c *imageCache) normalizeKey(key string) string { func (c *imageCache) clear() { c.mu.Lock() defer c.mu.Unlock() - c.store = make(map[string]*Image) + c.store = make(map[string]*resourceAdapter) } func (c *imageCache) getOrCreate( - parent *Image, conf imageConfig, createImage func() (*Image, image.Image, error)) (*Image, error) { - + parent *imageResource, conf images.ImageConfig, + createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) { relTarget := parent.relTargetPathFromConfig(conf) key := parent.relTargetPathForRel(relTarget.path(), false, false, false) // First check the in-memory store, then the disk. c.mu.RLock() - img, found := c.store[key] + cachedImage, found := c.store[key] c.mu.RUnlock() if found { - return img, nil + return cachedImage, nil } + var img *imageResource + // These funcs are protected by a named lock. // read clones the parent to its new name and copies // the content to the destinations. read := func(info filecache.ItemInfo, r io.Reader) error { - img = parent.clone() - img.relTargetDirFile.file = relTarget.file - img.sourceFilename = info.Name - // Make sure it's always loaded by sourceFilename. - img.openReadSeekerCloser = nil + img = parent.clone(nil) + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) w, err := img.openDestinationsForWriting() if err != nil { @@ -109,29 +110,20 @@ func (c *imageCache) getOrCreate( return err } - // create creates the image and encodes it to w (cache) and to its destinations. + // create creates the image and encodes it to the cache (w). create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + defer w.Close() + var conv image.Image img, conv, err = createImage() if err != nil { - w.Close() return } - img.relTargetDirFile.file = relTarget.file - img.sourceFilename = info.Name + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - destinations, err := img.openDestinationsForWriting() - if err != nil { - w.Close() - return err - } - - if destinations != nil { - w = hugio.NewMultiWriteCloser(w, destinations) - } - defer w.Close() - - return img.encodeTo(conf, conv, w) + return img.EncodeTo(conf, conv, w) } // Now look in the file cache. @@ -147,20 +139,21 @@ func (c *imageCache) getOrCreate( } // The file is now stored in this cache. - img.sourceFs = c.fileCache.Fs + img.setSourceFs(c.fileCache.Fs) c.mu.Lock() - if img2, found := c.store[key]; found { + if cachedImage, found = c.store[key]; found { c.mu.Unlock() - return img2, nil + return cachedImage, nil } - c.store[key] = img - c.mu.Unlock() - return img, nil + imgAdapter := newResourceAdapter(parent.getSpec(), true, img) + c.store[key] = imgAdapter + c.mu.Unlock() + return imgAdapter, nil } func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { - return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*Image)} + return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)} } diff --git a/resources/image_test.go b/resources/image_test.go index 96a66d999..31169444d 100644 --- a/resources/image_test.go +++ b/resources/image_test.go @@ -18,121 +18,101 @@ import ( "math/rand" "path/filepath" "strconv" + "sync" "testing" - "github.com/gohugoio/hugo/htesting/hqt" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources/resource" - "github.com/disintegration/imaging" + "github.com/google/go-cmp/cmp" - "sync" + "github.com/gohugoio/hugo/htesting/hqt" qt "github.com/frankban/quicktest" ) -func TestParseImageConfig(t *testing.T) { - for i, this := range []struct { - in string - expect interface{} - }{ - {"300x400", newImageConfig(300, 400, 0, 0, "", "")}, - {"100x200 bottomRight", newImageConfig(100, 200, 0, 0, "", "BottomRight")}, - {"10x20 topleft Lanczos", newImageConfig(10, 20, 0, 0, "Lanczos", "topleft")}, - {"linear left 10x r180", newImageConfig(10, 0, 0, 180, "linear", "left")}, - {"x20 riGht Cosine q95", newImageConfig(0, 20, 95, 0, "cosine", "right")}, - - {"", false}, - {"foo", false}, - } { - result, err := parseImageConfig(this.in) - if b, ok := this.expect.(bool); ok && !b { - if err == nil { - t.Errorf("[%d] parseImageConfig didn't return an expected error", i) - } - } else { - if err != nil { - t.Fatalf("[%d] err: %s", i, err) - } - if fmt.Sprint(result) != fmt.Sprint(this.expect) { - t.Fatalf("[%d] got\n%v\n but expected\n%v", i, result, this.expect) - } - } - } -} +var eq = qt.CmpEquals( + cmp.Comparer(func(p1, p2 *resourceAdapter) bool { + return p1.resourceAdapterInner == p2.resourceAdapterInner + }), + cmp.Comparer(func(p1, p2 *genericResource) bool { return p1 == p2 }), + cmp.Comparer(func(m1, m2 media.Type) bool { + return m1.Type() == m2.Type() + }), +) func TestImageTransformBasic(t *testing.T) { - c := qt.New(t) image := fetchSunset(c) - fileCache := image.spec.FileCaches.ImageCache().Fs + + fileCache := image.(specProvider).getSpec().FileCaches.ImageCache().Fs + + assertWidthHeight := func(img resource.Image, w, h int) { + c.Helper() + c.Assert(img, qt.Not(qt.IsNil)) + c.Assert(img.Width(), qt.Equals, w) + c.Assert(img.Height(), qt.Equals, h) + } c.Assert(image.RelPermalink(), qt.Equals, "/a/sunset.jpg") c.Assert(image.ResourceType(), qt.Equals, "image") + assertWidthHeight(image, 900, 562) resized, err := image.Resize("300x200") c.Assert(err, qt.IsNil) c.Assert(image != resized, qt.Equals, true) - c.Assert(image.genericResource != resized.genericResource, qt.Equals, true) - c.Assert(image.sourceFilename != resized.sourceFilename, qt.Equals, true) + c.Assert(image, qt.Not(eq), resized) + assertWidthHeight(resized, 300, 200) + assertWidthHeight(image, 900, 562) resized0x, err := image.Resize("x200") c.Assert(err, qt.IsNil) - c.Assert(resized0x.Width(), qt.Equals, 320) - c.Assert(resized0x.Height(), qt.Equals, 200) - + assertWidthHeight(resized0x, 320, 200) assertFileCache(c, fileCache, resized0x.RelPermalink(), 320, 200) resizedx0, err := image.Resize("200x") c.Assert(err, qt.IsNil) - c.Assert(resizedx0.Width(), qt.Equals, 200) - c.Assert(resizedx0.Height(), qt.Equals, 125) + assertWidthHeight(resizedx0, 200, 125) assertFileCache(c, fileCache, resizedx0.RelPermalink(), 200, 125) resizedAndRotated, err := image.Resize("x200 r90") c.Assert(err, qt.IsNil) - c.Assert(resizedAndRotated.Width(), qt.Equals, 125) - c.Assert(resizedAndRotated.Height(), qt.Equals, 200) + assertWidthHeight(resizedAndRotated, 125, 200) assertFileCache(c, fileCache, resizedAndRotated.RelPermalink(), 125, 200) + assertWidthHeight(resized, 300, 200) c.Assert(resized.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg") - c.Assert(resized.Width(), qt.Equals, 300) - c.Assert(resized.Height(), qt.Equals, 200) fitted, err := resized.Fit("50x50") c.Assert(err, qt.IsNil) c.Assert(fitted.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_625708021e2bb281c9f1002f88e4753f.jpg") - c.Assert(fitted.Width(), qt.Equals, 50) - c.Assert(fitted.Height(), qt.Equals, 33) + assertWidthHeight(fitted, 50, 33) // Check the MD5 key threshold fittedAgain, _ := fitted.Fit("10x20") fittedAgain, err = fittedAgain.Fit("10x20") c.Assert(err, qt.IsNil) c.Assert(fittedAgain.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_3f65ba24dc2b7fba0f56d7f104519157.jpg") - c.Assert(fittedAgain.Width(), qt.Equals, 10) - c.Assert(fittedAgain.Height(), qt.Equals, 6) + assertWidthHeight(fittedAgain, 10, 6) filled, err := image.Fill("200x100 bottomLeft") c.Assert(err, qt.IsNil) c.Assert(filled.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg") - c.Assert(filled.Width(), qt.Equals, 200) - c.Assert(filled.Height(), qt.Equals, 100) + assertWidthHeight(filled, 200, 100) assertFileCache(c, fileCache, filled.RelPermalink(), 200, 100) smart, err := image.Fill("200x100 smart") c.Assert(err, qt.IsNil) - c.Assert(smart.RelPermalink(), qt.Equals, fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber)) - c.Assert(smart.Width(), qt.Equals, 200) - c.Assert(smart.Height(), qt.Equals, 100) + c.Assert(smart.RelPermalink(), qt.Equals, fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", 1)) + assertWidthHeight(smart, 200, 100) assertFileCache(c, fileCache, smart.RelPermalink(), 200, 100) // Check cache filledAgain, err := image.Fill("200x100 bottomLeft") c.Assert(err, qt.IsNil) - c.Assert(filled == filledAgain, qt.Equals, true) - c.Assert(filled.sourceFilename == filledAgain.sourceFilename, qt.Equals, true) + c.Assert(filled, eq, filledAgain) assertFileCache(c, fileCache, filledAgain.RelPermalink(), 200, 100) - } // https://github.com/gohugoio/hugo/issues/4261 @@ -158,6 +138,7 @@ func TestImageTransformLongFilename(t *testing.T) { func TestImageTransformUppercaseExt(t *testing.T) { c := qt.New(t) image := fetchImage(c, "sunrise.JPG") + resized, err := image.Resize("200x") c.Assert(err, qt.IsNil) c.Assert(resized, qt.Not(qt.IsNil)) @@ -173,17 +154,16 @@ func TestImagePermalinkPublishOrder(t *testing.T) { } t.Run(name, func(t *testing.T) { - c := qt.New(t) spec := newTestResourceOsFs(c) - check1 := func(img *Image) { + check1 := func(img resource.Image) { resizedLink := "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_100x50_resize_q75_box.jpg" c.Assert(img.RelPermalink(), qt.Equals, resizedLink) assertImageFile(c, spec.PublishFs, resizedLink, 100, 50) } - check2 := func(img *Image) { + check2 := func(img resource.Image) { c.Assert(img.RelPermalink(), qt.Equals, "/a/sunset.jpg") assertImageFile(c, spec.PublishFs, "a/sunset.jpg", 900, 562) } @@ -198,18 +178,16 @@ func TestImagePermalinkPublishOrder(t *testing.T) { resized, err := orignal.Resize("100x50") c.Assert(err, qt.IsNil) - check1(resized) + check1(resized.(resource.Image)) if !checkOriginalFirst { check2(orignal) } }) } - } func TestImageTransformConcurrent(t *testing.T) { - var wg sync.WaitGroup c := qt.New(t) @@ -239,12 +217,7 @@ func TestImageTransformConcurrent(t *testing.T) { t.Error(err) } - _, err = r2.decodeSource() - if err != nil { - t.Error("Err decode:", err) - } - - img = r1 + img = r2 } } }(i + 20) @@ -253,58 +226,12 @@ func TestImageTransformConcurrent(t *testing.T) { wg.Wait() } -func TestDecodeImaging(t *testing.T) { - c := qt.New(t) - m := map[string]interface{}{ - "quality": 42, - "resampleFilter": "NearestNeighbor", - "anchor": "topLeft", - } - - imaging, err := decodeImaging(m) - - c.Assert(err, qt.IsNil) - c.Assert(imaging.Quality, qt.Equals, 42) - c.Assert(imaging.ResampleFilter, qt.Equals, "nearestneighbor") - c.Assert(imaging.Anchor, qt.Equals, "topleft") - - m = map[string]interface{}{} - - imaging, err = decodeImaging(m) - c.Assert(err, qt.IsNil) - c.Assert(imaging.Quality, qt.Equals, defaultJPEGQuality) - c.Assert(imaging.ResampleFilter, qt.Equals, "box") - c.Assert(imaging.Anchor, qt.Equals, "smart") - - _, err = decodeImaging(map[string]interface{}{ - "quality": 123, - }) - c.Assert(err, qt.Not(qt.IsNil)) - - _, err = decodeImaging(map[string]interface{}{ - "resampleFilter": "asdf", - }) - c.Assert(err, qt.Not(qt.IsNil)) - - _, err = decodeImaging(map[string]interface{}{ - "anchor": "asdf", - }) - c.Assert(err, qt.Not(qt.IsNil)) - - imaging, err = decodeImaging(map[string]interface{}{ - "anchor": "Smart", - }) - c.Assert(err, qt.IsNil) - c.Assert(imaging.Anchor, qt.Equals, "smart") - -} - func TestImageWithMetadata(t *testing.T) { c := qt.New(t) image := fetchSunset(c) - var meta = []map[string]interface{}{ + meta := []map[string]interface{}{ { "title": "My Sunset", "name": "Sunset #:counter", @@ -318,71 +245,69 @@ func TestImageWithMetadata(t *testing.T) { resized, err := image.Resize("200x") c.Assert(err, qt.IsNil) c.Assert(resized.Name(), qt.Equals, "Sunset #1") - } func TestImageResize8BitPNG(t *testing.T) { - c := qt.New(t) image := fetchImage(c, "gohugoio.png") - c.Assert(image.format, qt.Equals, imaging.PNG) + c.Assert(image.MediaType().Type(), qt.Equals, "image/png") c.Assert(image.RelPermalink(), qt.Equals, "/a/gohugoio.png") c.Assert(image.ResourceType(), qt.Equals, "image") resized, err := image.Resize("800x") c.Assert(err, qt.IsNil) - c.Assert(resized.format, qt.Equals, imaging.PNG) + c.Assert(resized.MediaType().Type(), qt.Equals, "image/png") c.Assert(resized.RelPermalink(), qt.Equals, "/a/gohugoio_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_800x0_resize_linear_2.png") c.Assert(resized.Width(), qt.Equals, 800) - } func TestImageResizeInSubPath(t *testing.T) { - c := qt.New(t) image := fetchImage(c, "sub/gohugoio2.png") - fileCache := image.spec.FileCaches.ImageCache().Fs + fileCache := image.(specProvider).getSpec().FileCaches.ImageCache().Fs - c.Assert(image.format, qt.Equals, imaging.PNG) + c.Assert(image.MediaType(), eq, media.PNGType) c.Assert(image.RelPermalink(), qt.Equals, "/a/sub/gohugoio2.png") c.Assert(image.ResourceType(), qt.Equals, "image") resized, err := image.Resize("101x101") c.Assert(err, qt.IsNil) - c.Assert(resized.format, qt.Equals, imaging.PNG) + c.Assert(resized.MediaType().Type(), qt.Equals, "image/png") c.Assert(resized.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png") c.Assert(resized.Width(), qt.Equals, 101) assertFileCache(c, fileCache, resized.RelPermalink(), 101, 101) publishedImageFilename := filepath.Clean(resized.RelPermalink()) - assertImageFile(c, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) - c.Assert(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) + + spec := image.(specProvider).getSpec() + + assertImageFile(c, spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) + c.Assert(spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) // Cleare mem cache to simulate reading from the file cache. - resized.spec.imageCache.clear() + spec.imageCache.clear() resizedAgain, err := image.Resize("101x101") c.Assert(err, qt.IsNil) c.Assert(resizedAgain.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png") c.Assert(resizedAgain.Width(), qt.Equals, 101) assertFileCache(c, fileCache, resizedAgain.RelPermalink(), 101, 101) - assertImageFile(c, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) - + assertImageFile(c, image.(specProvider).getSpec().BaseFs.PublishFs, publishedImageFilename, 101, 101) } func TestSVGImage(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) svg := fetchResourceForSpec(spec, c, "circle.svg") c.Assert(svg, qt.Not(qt.IsNil)) } func TestSVGImageContent(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) svg := fetchResourceForSpec(spec, c, "circle.svg") c.Assert(svg, qt.Not(qt.IsNil)) diff --git a/resources/images/config.go b/resources/images/config.go new file mode 100644 index 000000000..c4605c9cf --- /dev/null +++ b/resources/images/config.go @@ -0,0 +1,276 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package images + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/disintegration/imaging" + "github.com/mitchellh/mapstructure" +) + +const ( + defaultJPEGQuality = 75 + defaultResampleFilter = "box" +) + +var ( + imageFormats = map[string]imaging.Format{ + ".jpg": imaging.JPEG, + ".jpeg": imaging.JPEG, + ".png": imaging.PNG, + ".tif": imaging.TIFF, + ".tiff": imaging.TIFF, + ".bmp": imaging.BMP, + ".gif": imaging.GIF, + } + + // Add or increment if changes to an image format's processing requires + // re-generation. + imageFormatsVersions = map[imaging.Format]int{ + imaging.PNG: 2, // Floyd Steinberg dithering + } + + // Increment to mark all processed images as stale. Only use when absolutely needed. + // See the finer grained smartCropVersionNumber and imageFormatsVersions. + mainImageVersionNumber = 0 + + // Increment to mark all traced SVGs as stale. + traceVersionNumber = 0 +) + +var anchorPositions = map[string]imaging.Anchor{ + strings.ToLower("Center"): imaging.Center, + strings.ToLower("TopLeft"): imaging.TopLeft, + strings.ToLower("Top"): imaging.Top, + strings.ToLower("TopRight"): imaging.TopRight, + strings.ToLower("Left"): imaging.Left, + strings.ToLower("Right"): imaging.Right, + strings.ToLower("BottomLeft"): imaging.BottomLeft, + strings.ToLower("Bottom"): imaging.Bottom, + strings.ToLower("BottomRight"): imaging.BottomRight, +} + +var imageFilters = map[string]imaging.ResampleFilter{ + strings.ToLower("NearestNeighbor"): imaging.NearestNeighbor, + strings.ToLower("Box"): imaging.Box, + strings.ToLower("Linear"): imaging.Linear, + strings.ToLower("Hermite"): imaging.Hermite, + strings.ToLower("MitchellNetravali"): imaging.MitchellNetravali, + strings.ToLower("CatmullRom"): imaging.CatmullRom, + strings.ToLower("BSpline"): imaging.BSpline, + strings.ToLower("Gaussian"): imaging.Gaussian, + strings.ToLower("Lanczos"): imaging.Lanczos, + strings.ToLower("Hann"): imaging.Hann, + strings.ToLower("Hamming"): imaging.Hamming, + strings.ToLower("Blackman"): imaging.Blackman, + strings.ToLower("Bartlett"): imaging.Bartlett, + strings.ToLower("Welch"): imaging.Welch, + strings.ToLower("Cosine"): imaging.Cosine, +} + +func ImageFormatFromExt(ext string) (imaging.Format, bool) { + f, found := imageFormats[ext] + return f, found +} + +func DecodeConfig(m map[string]interface{}) (Imaging, error) { + var i Imaging + if err := mapstructure.WeakDecode(m, &i); err != nil { + return i, err + } + + if i.Quality == 0 { + i.Quality = defaultJPEGQuality + } else if i.Quality < 0 || i.Quality > 100 { + return i, errors.New("JPEG quality must be a number between 1 and 100") + } + + if i.Anchor == "" || strings.EqualFold(i.Anchor, SmartCropIdentifier) { + i.Anchor = SmartCropIdentifier + } else { + i.Anchor = strings.ToLower(i.Anchor) + if _, found := anchorPositions[i.Anchor]; !found { + return i, errors.New("invalid anchor value in imaging config") + } + } + + if i.ResampleFilter == "" { + i.ResampleFilter = defaultResampleFilter + } else { + filter := strings.ToLower(i.ResampleFilter) + _, found := imageFilters[filter] + if !found { + return i, fmt.Errorf("%q is not a valid resample filter", filter) + } + i.ResampleFilter = filter + } + + return i, nil +} + +func DecodeImageConfig(action, config string, defaults Imaging) (ImageConfig, error) { + var ( + c ImageConfig + err error + ) + + c.Action = action + + if config == "" { + return c, errors.New("image config cannot be empty") + } + + parts := strings.Fields(config) + for _, part := range parts { + part = strings.ToLower(part) + + if part == SmartCropIdentifier { + c.AnchorStr = SmartCropIdentifier + } else if pos, ok := anchorPositions[part]; ok { + c.Anchor = pos + c.AnchorStr = part + } else if filter, ok := imageFilters[part]; ok { + c.Filter = filter + c.FilterStr = part + } else if part[0] == 'q' { + c.Quality, err = strconv.Atoi(part[1:]) + if err != nil { + return c, err + } + if c.Quality < 1 || c.Quality > 100 { + return c, errors.New("quality ranges from 1 to 100 inclusive") + } + } else if part[0] == 'r' { + c.Rotate, err = strconv.Atoi(part[1:]) + if err != nil { + return c, err + } + } else if strings.Contains(part, "x") { + widthHeight := strings.Split(part, "x") + if len(widthHeight) <= 2 { + first := widthHeight[0] + if first != "" { + c.Width, err = strconv.Atoi(first) + if err != nil { + return c, err + } + } + + if len(widthHeight) == 2 { + second := widthHeight[1] + if second != "" { + c.Height, err = strconv.Atoi(second) + if err != nil { + return c, err + } + } + } + } else { + return c, errors.New("invalid image dimensions") + } + + } + } + + if c.Width == 0 && c.Height == 0 { + return c, errors.New("must provide Width or Height") + } + + if c.FilterStr == "" { + c.FilterStr = defaults.ResampleFilter + c.Filter = imageFilters[c.FilterStr] + } + + if c.AnchorStr == "" { + c.AnchorStr = defaults.Anchor + if !strings.EqualFold(c.AnchorStr, SmartCropIdentifier) { + c.Anchor = anchorPositions[c.AnchorStr] + } + } + + return c, nil +} + +// ImageConfig holds configuration to create a new image from an existing one, resize etc. +type ImageConfig struct { + Action string + + // Quality ranges from 1 to 100 inclusive, higher is better. + // This is only relevant for JPEG images. + // Default is 75. + Quality int + + // Rotate rotates an image by the given angle counter-clockwise. + // The rotation will be performed first. + Rotate int + + Width int + Height int + + Filter imaging.ResampleFilter + FilterStr string + + Anchor imaging.Anchor + AnchorStr string +} + +func (i ImageConfig) Key(format imaging.Format) string { + k := strconv.Itoa(i.Width) + "x" + strconv.Itoa(i.Height) + if i.Action != "" { + k += "_" + i.Action + } + if i.Quality > 0 { + k += "_q" + strconv.Itoa(i.Quality) + } + if i.Rotate != 0 { + k += "_r" + strconv.Itoa(i.Rotate) + } + anchor := i.AnchorStr + if anchor == SmartCropIdentifier { + anchor = anchor + strconv.Itoa(smartCropVersionNumber) + } + + k += "_" + i.FilterStr + + if strings.EqualFold(i.Action, "fill") { + k += "_" + anchor + } + + if v, ok := imageFormatsVersions[format]; ok { + k += "_" + strconv.Itoa(v) + } + + if mainImageVersionNumber > 0 { + k += "_" + strconv.Itoa(mainImageVersionNumber) + } + + return k +} + +// Imaging contains default image processing configuration. This will be fetched +// from site (or language) config. +type Imaging struct { + // Default image quality setting (1-100). Only used for JPEG images. + Quality int + + // Resample filter used. See https://github.com/disintegration/imaging + ResampleFilter string + + // The anchor used in Fill. Default is "smart", i.e. Smart Crop. + Anchor string +} diff --git a/resources/images/config_test.go b/resources/images/config_test.go new file mode 100644 index 000000000..91f4b663a --- /dev/null +++ b/resources/images/config_test.go @@ -0,0 +1,125 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package images + +import ( + "fmt" + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestDecodeConfig(t *testing.T) { + c := qt.New(t) + m := map[string]interface{}{ + "quality": 42, + "resampleFilter": "NearestNeighbor", + "anchor": "topLeft", + } + + imaging, err := DecodeConfig(m) + + c.Assert(err, qt.IsNil) + c.Assert(imaging.Quality, qt.Equals, 42) + c.Assert(imaging.ResampleFilter, qt.Equals, "nearestneighbor") + c.Assert(imaging.Anchor, qt.Equals, "topleft") + + m = map[string]interface{}{} + + imaging, err = DecodeConfig(m) + c.Assert(err, qt.IsNil) + c.Assert(imaging.Quality, qt.Equals, defaultJPEGQuality) + c.Assert(imaging.ResampleFilter, qt.Equals, "box") + c.Assert(imaging.Anchor, qt.Equals, "smart") + + _, err = DecodeConfig(map[string]interface{}{ + "quality": 123, + }) + c.Assert(err, qt.Not(qt.IsNil)) + + _, err = DecodeConfig(map[string]interface{}{ + "resampleFilter": "asdf", + }) + c.Assert(err, qt.Not(qt.IsNil)) + + _, err = DecodeConfig(map[string]interface{}{ + "anchor": "asdf", + }) + c.Assert(err, qt.Not(qt.IsNil)) + + imaging, err = DecodeConfig(map[string]interface{}{ + "anchor": "Smart", + }) + c.Assert(err, qt.IsNil) + c.Assert(imaging.Anchor, qt.Equals, "smart") +} + +func TestDecodeImageConfig(t *testing.T) { + for i, this := range []struct { + in string + expect interface{} + }{ + {"300x400", newImageConfig(300, 400, 0, 0, "", "")}, + {"100x200 bottomRight", newImageConfig(100, 200, 0, 0, "", "BottomRight")}, + {"10x20 topleft Lanczos", newImageConfig(10, 20, 0, 0, "Lanczos", "topleft")}, + {"linear left 10x r180", newImageConfig(10, 0, 0, 180, "linear", "left")}, + {"x20 riGht Cosine q95", newImageConfig(0, 20, 95, 0, "cosine", "right")}, + + {"", false}, + {"foo", false}, + } { + + result, err := DecodeImageConfig("resize", this.in, Imaging{}) + if b, ok := this.expect.(bool); ok && !b { + if err == nil { + t.Errorf("[%d] parseImageConfig didn't return an expected error", i) + } + } else { + if err != nil { + t.Fatalf("[%d] err: %s", i, err) + } + if fmt.Sprint(result) != fmt.Sprint(this.expect) { + t.Fatalf("[%d] got\n%v\n but expected\n%v", i, result, this.expect) + } + } + } +} + +func newImageConfig(width, height, quality, rotate int, filter, anchor string) ImageConfig { + var c ImageConfig + c.Action = "resize" + c.Width = width + c.Height = height + c.Quality = quality + c.Rotate = rotate + + if filter != "" { + filter = strings.ToLower(filter) + if v, ok := imageFilters[filter]; ok { + c.Filter = v + c.FilterStr = filter + } + } + + if anchor != "" { + anchor = strings.ToLower(anchor) + if v, ok := anchorPositions[anchor]; ok { + c.Anchor = v + c.AnchorStr = anchor + } + } + + return c +} diff --git a/resources/images/image.go b/resources/images/image.go new file mode 100644 index 000000000..b39e84972 --- /dev/null +++ b/resources/images/image.go @@ -0,0 +1,170 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package images + +import ( + "image" + "image/jpeg" + "io" + "sync" + + "github.com/disintegration/imaging" + "github.com/gohugoio/hugo/common/hugio" + "github.com/pkg/errors" +) + +func NewImage(f imaging.Format, proc *ImageProcessor, img image.Image, s Spec) *Image { + if img != nil { + return &Image{ + Format: f, + Proc: proc, + Spec: s, + imageConfig: &imageConfig{ + config: imageConfigFromImage(img), + configLoaded: true, + }, + } + } + return &Image{Format: f, Proc: proc, Spec: s, imageConfig: &imageConfig{}} +} + +type Image struct { + Format imaging.Format + + Proc *ImageProcessor + + Spec Spec + + *imageConfig +} + +func (i *Image) EncodeTo(conf ImageConfig, img image.Image, w io.Writer) error { + switch i.Format { + case imaging.JPEG: + + var rgba *image.RGBA + quality := conf.Quality + + if nrgba, ok := img.(*image.NRGBA); ok { + if nrgba.Opaque() { + rgba = &image.RGBA{ + Pix: nrgba.Pix, + Stride: nrgba.Stride, + Rect: nrgba.Rect, + } + } + } + if rgba != nil { + return jpeg.Encode(w, rgba, &jpeg.Options{Quality: quality}) + } + return jpeg.Encode(w, img, &jpeg.Options{Quality: quality}) + default: + return imaging.Encode(w, img, i.Format) + } +} + +// Height returns i's height. +func (i *Image) Height() int { + i.initConfig() + return i.config.Height +} + +// Width returns i's width. +func (i *Image) Width() int { + i.initConfig() + return i.config.Width +} + +func (i Image) WithImage(img image.Image) *Image { + i.Spec = nil + i.imageConfig = &imageConfig{ + config: imageConfigFromImage(img), + configLoaded: true, + } + + return &i +} + +func (i Image) WithSpec(s Spec) *Image { + i.Spec = s + i.imageConfig = &imageConfig{} + return &i +} + +func (i *Image) initConfig() error { + var err error + i.configInit.Do(func() { + if i.configLoaded { + return + } + + var ( + f hugio.ReadSeekCloser + config image.Config + ) + + f, err = i.Spec.ReadSeekCloser() + if err != nil { + return + } + defer f.Close() + + config, _, err = image.DecodeConfig(f) + if err != nil { + return + } + i.config = config + }) + + if err != nil { + return errors.Wrap(err, "failed to load image config") + } + + return nil +} + +type ImageProcessor struct { + Cfg Imaging +} + +func (p *ImageProcessor) Fill(src image.Image, conf ImageConfig) (image.Image, error) { + if conf.AnchorStr == SmartCropIdentifier { + return smartCrop(src, conf.Width, conf.Height, conf.Anchor, conf.Filter) + } + return imaging.Fill(src, conf.Width, conf.Height, conf.Anchor, conf.Filter), nil +} + +func (p *ImageProcessor) Fit(src image.Image, conf ImageConfig) (image.Image, error) { + return imaging.Fit(src, conf.Width, conf.Height, conf.Filter), nil +} + +func (p *ImageProcessor) Resize(src image.Image, conf ImageConfig) (image.Image, error) { + return imaging.Resize(src, conf.Width, conf.Height, conf.Filter), nil +} + +type Spec interface { + // Loads the image source. + ReadSeekCloser() (hugio.ReadSeekCloser, error) +} + +type imageConfig struct { + config image.Config + configInit sync.Once + configLoaded bool +} + +func imageConfigFromImage(img image.Image) image.Config { + b := img.Bounds() + return image.Config{Width: b.Max.X, Height: b.Max.Y} +} diff --git a/resources/images/smartcrop.go b/resources/images/smartcrop.go new file mode 100644 index 000000000..0b35b8280 --- /dev/null +++ b/resources/images/smartcrop.go @@ -0,0 +1,75 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package images + +import ( + "image" + + "github.com/disintegration/imaging" + "github.com/muesli/smartcrop" +) + +const ( + // Do not change. + // TODO(bep) image unexport + SmartCropIdentifier = "smart" + + // This is just a increment, starting on 1. If Smart Crop improves its cropping, we + // need a way to trigger a re-generation of the crops in the wild, so increment this. + smartCropVersionNumber = 1 +) + +func newSmartCropAnalyzer(filter imaging.ResampleFilter) smartcrop.Analyzer { + return smartcrop.NewAnalyzer(imagingResizer{filter: filter}) +} + +// Needed by smartcrop +type imagingResizer struct { + filter imaging.ResampleFilter +} + +func (r imagingResizer) Resize(img image.Image, width, height uint) image.Image { + return imaging.Resize(img, int(width), int(height), r.filter) +} + +func smartCrop(img image.Image, width, height int, anchor imaging.Anchor, filter imaging.ResampleFilter) (*image.NRGBA, error) { + if width <= 0 || height <= 0 { + return &image.NRGBA{}, nil + } + + srcBounds := img.Bounds() + srcW := srcBounds.Dx() + srcH := srcBounds.Dy() + + if srcW <= 0 || srcH <= 0 { + return &image.NRGBA{}, nil + } + + if srcW == width && srcH == height { + return imaging.Clone(img), nil + } + + smart := newSmartCropAnalyzer(filter) + + rect, err := smart.FindBestCrop(img, width, height) + if err != nil { + return nil, err + } + + b := img.Bounds().Intersect(rect) + + cropped := imaging.Crop(img, b) + + return imaging.Resize(cropped, width, height, filter), nil +} diff --git a/resources/internal/key.go b/resources/internal/key.go new file mode 100644 index 000000000..3dce8b350 --- /dev/null +++ b/resources/internal/key.go @@ -0,0 +1,61 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "strconv" + + bp "github.com/gohugoio/hugo/bufferpool" + + "github.com/mitchellh/hashstructure" +) + +// ResourceTransformationKey are provided by the different transformation implementations. +// It identifies the transformation (name) and its configuration (elements). +// We combine this in a chain with the rest of the transformations +// with the target filename and a content hash of the origin to use as cache key. +type ResourceTransformationKey struct { + Name string + elements []interface{} +} + +// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation +// name and elements. We will create a 64 bit FNV hash from the elements, which when combined +// with the other key elements should be unique for all practical applications. +func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey { + return ResourceTransformationKey{Name: name, elements: elements} +} + +// Value returns the Key as a string. +// Do not change this without good reasons. +func (k ResourceTransformationKey) Value() string { + if len(k.elements) == 0 { + return k.Name + } + + sb := bp.GetBuffer() + defer bp.PutBuffer(sb) + + sb.WriteString(k.Name) + for _, element := range k.elements { + hash, err := hashstructure.Hash(element, nil) + if err != nil { + panic(err) + } + sb.WriteString("_") + sb.WriteString(strconv.FormatUint(hash, 10)) + } + + return sb.String() +} diff --git a/resources/internal/key_test.go b/resources/internal/key_test.go new file mode 100644 index 000000000..9b6a23d87 --- /dev/null +++ b/resources/internal/key_test.go @@ -0,0 +1,36 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +type testStruct struct { + Name string + V1 int64 + V2 int32 + V3 int + V4 uint64 +} + +func TestResourceTransformationKey(t *testing.T) { + // We really need this key to be portable across OSes. + key := NewResourceTransformationKey("testing", + testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)}) + c := qt.New(t) + c.Assert("testing_518996646957295636", qt.Equals, key.Value()) +} diff --git a/resources/resource.go b/resources/resource.go index 92bcbd0fc..3859e6044 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -17,30 +17,23 @@ import ( "fmt" "io" "io/ioutil" - "mime" "os" "path" "path/filepath" - "strings" "sync" "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/source" - "github.com/gohugoio/hugo/output" - "github.com/gohugoio/hugo/tpl" "github.com/pkg/errors" - "github.com/gohugoio/hugo/cache/filecache" - "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hugio" - "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" "github.com/spf13/afero" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/source" ) var ( @@ -51,80 +44,10 @@ var ( _ resource.Cloner = (*genericResource)(nil) _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) _ permalinker = (*genericResource)(nil) - _ collections.Slicer = (*genericResource)(nil) _ resource.Identifier = (*genericResource)(nil) + _ fileInfo = (*genericResource)(nil) ) -var noData = make(map[string]interface{}) - -type permalinker interface { - relPermalinkFor(target string) string - permalinkFor(target string) string - relTargetPathsFor(target string) []string - relTargetPaths() []string - TargetPath() string -} - -type Spec struct { - *helpers.PathSpec - - MediaTypes media.Types - OutputFormats output.Formats - - Logger *loggers.Logger - - TextTemplates tpl.TemplateParseFinder - - Permalinks page.PermalinkExpander - - // Holds default filter settings etc. - imaging *Imaging - - imageCache *imageCache - ResourceCache *ResourceCache - FileCaches filecache.Caches -} - -func NewSpec( - s *helpers.PathSpec, - fileCaches filecache.Caches, - logger *loggers.Logger, - outputFormats output.Formats, - mimeTypes media.Types) (*Spec, error) { - - imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) - if err != nil { - return nil, err - } - - if logger == nil { - logger = loggers.NewErrorLogger() - } - - permalinks, err := page.NewPermalinkExpander(s) - if err != nil { - return nil, err - } - - rs := &Spec{PathSpec: s, - Logger: logger, - imaging: &imaging, - MediaTypes: mimeTypes, - OutputFormats: outputFormats, - Permalinks: permalinks, - FileCaches: fileCaches, - imageCache: newImageCache( - fileCaches.ImageCache(), - - s, - )} - - rs.ResourceCache = newResourceCache(rs) - - return rs, nil - -} - type ResourceSourceDescriptor struct { // TargetPaths is a callback to fetch paths's relative to its owner. TargetPaths func() page.TargetPaths @@ -161,136 +84,77 @@ func (r ResourceSourceDescriptor) Filename() string { return r.SourceFilename } -func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { - return r.newResourceFor(fd) +type ResourceTransformer interface { + resource.Resource + Transformer } -func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) { - if fd.OpenReadSeekCloser == nil { - if fd.SourceFile != nil && fd.SourceFilename != "" { - return nil, errors.New("both SourceFile and AbsSourceFilename provided") - } else if fd.SourceFile == nil && fd.SourceFilename == "" { - return nil, errors.New("either SourceFile or AbsSourceFilename must be provided") - } - } - - if fd.RelTargetFilename == "" { - fd.RelTargetFilename = fd.Filename() - } - - if len(fd.TargetBasePaths) == 0 { - // If not set, we publish the same resource to all hosts. - fd.TargetBasePaths = r.MultihostTargetBasePaths - } - - return r.newResource(fd.Fs, fd) +type Transformer interface { + Transform(...ResourceTransformation) (ResourceTransformer, error) } -func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { - fi := fd.FileInfo - var sourceFilename string - - if fd.OpenReadSeekCloser != nil { - } else if fd.SourceFilename != "" { - var err error - fi, err = sourceFs.Stat(fd.SourceFilename) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - sourceFilename = fd.SourceFilename - } else { - sourceFilename = fd.SourceFile.Filename() - } - - if fd.RelTargetFilename == "" { - fd.RelTargetFilename = sourceFilename - } +type baseResourceResource interface { + resource.Cloner + resource.ContentProvider + resource.Resource + resource.Identifier +} - ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename)) - mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) - // TODO(bep) we need to handle these ambigous types better, but in this context - // we most likely want the application/xml type. - if mimeType.Suffix() == "xml" && mimeType.SubType == "rss" { - mimeType, found = r.MediaTypes.GetByType("application/xml") - } +type baseResourceInternal interface { + resource.Source - if !found { - // A fallback. Note that mime.TypeByExtension is slow by Hugo standards, - // so we should configure media types to avoid this lookup for most - // situations. - mimeStr := mime.TypeByExtension(ext) - if mimeStr != "" { - mimeType, _ = media.FromStringAndExt(mimeStr, ext) - } - } + fileInfo + metaAssigner + targetPather - gr := r.newGenericResourceWithBase( - sourceFs, - fd.LazyPublish, - fd.OpenReadSeekCloser, - fd.TargetBasePaths, - fd.TargetPaths, - fi, - sourceFilename, - fd.RelTargetFilename, - mimeType) - - if mimeType.MainType == "image" { - imgFormat, ok := imageFormats[ext] - if !ok { - // This allows SVG etc. to be used as resources. They will not have the methods of the Image, but - // that would not (currently) have worked. - return gr, nil - } + ReadSeekCloser() (hugio.ReadSeekCloser, error) - if err := gr.initHash(); err != nil { - return nil, err - } + // Internal + cloneWithUpdates(*transformationUpdate) (baseResource, error) + tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser - return &Image{ - format: imgFormat, - imaging: r.imaging, - genericResource: gr}, nil - } - return gr, nil + specProvider + getResourcePaths() *resourcePathDescriptor + getTargetFilenames() []string + openDestinationsForWriting() (io.WriteCloser, error) + openPublishFileForWriting(relTarget