diff options
34 files changed, 2593 insertions, 1475 deletions
diff --git a/common/herrors/errors.go b/common/herrors/errors.go index e484ecb80..ff8eab116 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -52,6 +52,7 @@ func FprintStackTrace(w io.Writer, err error) { // defer herrors.Recover() func Recover(args ...interface{}) { if r := recover(); r != nil { + fmt.Println("ERR:", r) args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n") fmt.Println(args...) } diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go index dc303b2e5..660c76a44 100644 --- a/htesting/test_helpers.go +++ b/htesting/test_helpers.go @@ -14,8 +14,10 @@ package htesting import ( + "math/rand" "runtime" "strings" + "time" "github.com/spf13/afero" ) @@ -37,3 +39,20 @@ func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) { } return tempDir, func() { fs.RemoveAll(tempDir) }, nil } + +// BailOut panics with a stack trace after the given duration. Useful for +// hanging tests. +func BailOut(after time.Duration) { + time.AfterFunc(after, func() { + buf := make([]byte, 1<<16) + runtime.Stack(buf, true) + panic(string(buf)) + }) + +} + +var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) + +func RandIntn(n int) int { + return rnd.Intn(n) +} diff --git a/hugolib/assets/images/sunset.jpg b/hugolib/assets/images/sunset.jpg Binary files differnew file mode 100644 index 000000000..7d7307bed --- /dev/null +++ b/hugolib/assets/images/sunset.jpg diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index 4b3eef512..1f7addb28 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -42,8 +42,7 @@ import ( ) func TestPageBundlerSiteRegular(t *testing.T) { - t.Parallel() - + c := qt.New(t) baseBaseURL := "https://example.com" for _, baseURLPath := range []string{"", "/hugo"} { @@ -55,15 +54,14 @@ func TestPageBundlerSiteRegular(t *testing.T) { } ugly := ugly canonify := canonify - t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId), - func(t *testing.T) { - t.Parallel() + c.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId), + func(c *qt.C) { + c.Parallel() baseURL := baseBaseURL + baseURLPath relURLBase := baseURLPath if canonify { relURLBase = "" } - c := qt.New(t) fs, cfg := newTestBundleSources(t) cfg.Set("baseURL", baseURL) cfg.Set("canonifyURLs", canonify) diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 84c871e4d..2b32587eb 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "io" "os" "path/filepath" "testing" @@ -167,6 +168,64 @@ T1: {{ $r.Content }} } +func TestResourceChainBasic(t *testing.T) { + t.Parallel() + + b := newTestSitesBuilder(t) + b.WithTemplatesAdded("index.html", ` +{{ $hello := "<h1> Hello World! </h1>" | resources.FromString "hello.html" | fingerprint "sha512" | minify | fingerprint }} + +HELLO: {{ $hello.Name }}|{{ $hello.RelPermalink }}|{{ $hello.Content | safeHTML }} + +{{ $img := resources.Get "images/sunset.jpg" }} +{{ $fit := $img.Fit "200x200" }} +{{ $fit2 := $fit.Fit "100x200" }} +{{ $img = $img | fingerprint }} +SUNSET: {{ $img.Name }}|{{ $img.RelPermalink }}|{{ $img.Width }}|{{ len $img.Content }} +FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }} +`) + + fs := b.Fs.Source + + imageDir := filepath.Join("assets", "images") + b.Assert(os.MkdirAll(imageDir, 0777), qt.IsNil) + src, err := os.Open("testdata/sunset.jpg") + b.Assert(err, qt.IsNil) + out, err := fs.Create(filepath.Join(imageDir, "sunset.jpg")) + b.Assert(err, qt.IsNil) + _, err = io.Copy(out, src) + b.Assert(err, qt.IsNil) + out.Close() + + b.Running() + + for i := 0; i < 2; i++ { + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", + ` +SUNSET: images/sunset.jpg|/images/sunset.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587 +FIT: images/sunset.jpg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x200_fit_q75_box.jpg|200 + +`) + + b.EditFiles("page1.md", ` +--- +title: "Page 1 edit" +summary: "Edited summary" +--- + +Edited content. + +`) + + b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil) + b.H.ResourceSpec.ClearCaches() + + } +} + func TestResourceChain(t *testing.T) { t.Parallel() @@ -353,9 +412,11 @@ Publish 2: {{ $cssPublish2.Permalink }} "Publish 1: body{color:blue} /external1.min.css", "Publish 2: http://example.com/external2.min.css", ) - c.Assert(b.CheckExists("public/external2.min.css"), qt.Equals, true) - c.Assert(b.CheckExists("public/external1.min.css"), qt.Equals, true) - c.Assert(b.CheckExists("public/inline.min.css"), qt.Equals, false) + b.Assert(b.CheckExists("public/external2.css"), qt.Equals, false) + b.Assert(b.CheckExists("public/external1.css"), qt.Equals, false) + b.Assert(b.CheckExists("public/external2.min.css"), qt.Equals, true) + b.Assert(b.CheckExists("public/external1.min.css"), qt.Equals, true) + b.Assert(b.CheckExists("public/inline.min.css"), qt.Equals, false) }}, {"unmarshal", func() bool { return true }, func(b *sitesBuilder) { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index e7d3b99fb..f1c19366d 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -536,6 +536,7 @@ func (s *sitesBuilder) changeEvents() []fsnotify.Event { } func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder { + s.Helper() defer func() { s.changedFiles = nil }() diff --git a/resources/image.go b/resources/image.go index f1aae2996..e1a816942 100644 --- a/resources/image.go +++ b/resources/image.go @@ -14,198 +14,98 @@ package resources import ( - "errors" "fmt" "image" "image/color" "image/draw" - "image/jpeg" - "io" + _ "image/gif" + _ "image/png" "os" - "strconv" "strings" - "sync" "github.com/gohugoio/hugo/resources/resource" _errors "github.com/pkg/errors" "github.com/disintegration/imaging" - "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" - "github.com/mitchellh/mapstructure" + "github.com/gohugoio/hugo/resources/images" // Blind import for image.Decode - _ "image/gif" - _ "image/png" // Blind import for image.Decode _ "golang.org/x/image/webp" ) var ( - _ resource.Resource = (*Image)(nil) - _ resource.Source = (*Image)(nil) - _ resource.Cloner = (*Image)(nil) + _ resource.Image = (*imageResource)(nil) + _ resource.Source = (*imageResource)(nil) + _ resource.Cloner = (*imageResource)(nil) ) -// Imaging contains default image processing configuration. This will be fetched -// from site (or language) config. -type Imaging struct { - // Default image quality setting (1-100). Only used for JPEG images. - Quality int - - // Resample filter used. See https://github.com/disintegration/imaging - ResampleFilter string +// ImageResource represents an image resource. +type imageResource struct { + *images.Image - // The anchor used in Fill. Default is "smart", i.e. Smart Crop. - Anchor string + baseResource } -const ( - defaultJPEGQuality = 75 - defaultResampleFilter = "box" -) - -var ( - imageFormats = map[string]imaging.Format{ - ".jpg": imaging.JPEG, - ".jpeg": imaging.JPEG, - ".png": imaging.PNG, - ".tif": imaging.TIFF, - ".tiff": imaging.TIFF, - ".bmp": imaging.BMP, - ".gif": imaging.GIF, +func (i *imageResource) Clone() resource.Resource { + gr := i.baseResource.Clone().(baseResource) + return &imageResource{ + Image: i.WithSpec(gr), + baseResource: gr, } - - // Add or increment if changes to an image format's processing requires - // re-generation. - imageFormatsVersions = map[imaging.Format]int{ - imaging.PNG: 2, // Floyd Steinberg dithering - } - - // Increment to mark all processed images as stale. Only use when absolutely needed. - // See the finer grained smartCropVersionNumber and imageFormatsVersions. - mainImageVersionNumber = 0 -) - -var anchorPositions = map[string]imaging.Anchor{ - strings.ToLower("Center"): imaging.Center, - strings.ToLower("TopLeft"): imaging.TopLeft, - strings.ToLower("Top"): imaging.Top, - strings.ToLower("TopRight"): imaging.TopRight, - strings.ToLower("Left"): imaging.Left, - strings.ToLower("Right"): imaging.Right, - strings.ToLower("BottomLeft"): imaging.BottomLeft, - strings.ToLower("Bottom"): imaging.Bottom, - strings.ToLower("BottomRight"): imaging.BottomRight, } -var imageFilters = map[string]imaging.ResampleFilter{ - strings.ToLower("NearestNeighbor"): imaging.NearestNeighbor, - strings.ToLower("Box"): imaging.Box, - strings.ToLower("Linear"): imaging.Linear, - strings.ToLower("Hermite"): imaging.Hermite, - strings.ToLower("MitchellNetravali"): imaging.MitchellNetravali, - strings.ToLower("CatmullRom"): imaging.CatmullRom, - strings.ToLower("BSpline"): imaging.BSpline, - strings.ToLower("Gaussian"): imaging.Gaussian, - strings.ToLower("Lanczos"): imaging.Lanczos, - strings.ToLower("Hann"): imaging.Hann, - strings.ToLower("Hamming"): imaging.Hamming, - strings.ToLower("Blackman"): imaging.Blackman, - strings.ToLower("Bartlett"): imaging.Bartlett, - strings.ToLower("Welch"): imaging.Welch, - strings.ToLower("Cosine"): imaging.Cosine, -} - -// Image represents an image resource. -type Image struct { - config image.Config - configInit sync.Once - configLoaded bool - - imaging *Imaging - - format imaging.Format - - *genericResource -} +func (i *imageResource) cloneWithUpdates(u *transformationUpdate) (baseResource, error) { + base, err := i.baseResource.cloneWithUpdates(u) + if err != nil { + return nil, err + } -// Width returns i's width. -func (i *Image) Width() int { - i.initConfig() - return i.config.Width -} + var img *images.Image -// Height returns i's height. -func (i *Image) Height() int { - i.initConfig() - return i.config.Height -} + if u.isContenChanged() { + img = i.WithSpec(base) + } else { + img = i.Image + } -// WithNewBase implements the Cloner interface. -func (i *Image) WithNewBase(base string) resource.Resource { - return &Image{ - imaging: i.imaging, - format: i.format, - genericResource: i.genericResource.WithNewBase(base).(*genericResource)} + return &imageResource{ + Image: img, + baseResource: base, + }, nil } // Resize resizes the image to the specified width and height using the specified resampling // filter and returns the transformed image. If one of width or height is 0, the image aspect // ratio is preserved. -func (i *Image) Resize(spec string) (*Image, error) { - return i.doWithImageConfig("resize", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - return imaging.Resize(src, conf.Width, conf.Height, conf.Filter), nil +func (i *imageResource) Resize(spec string) (resource.Image, error) { + return i.doWithImageConfig("resize", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Resize(src, conf) }) } // Fit scales down the image using the specified resample filter to fit the specified // maximum width and height. -func (i *Image) Fit(spec string) (*Image, error) { - return i.doWithImageConfig("fit", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - return imaging.Fit(src, conf.Width, conf.Height, conf.Filter), nil +func (i *imageResource) Fit(spec string) (resource.Image, error) { + return i.doWithImageConfig("fit", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Fit(src, conf) }) } // Fill scales the image to the smallest possible size that will cover the specified dimensions, // crops the resized image to the specified dimensions using the given anchor point. // Space delimited config: 200x300 TopLeft -func (i *Image) Fill(spec string) (*Image, error) { - return i.doWithImageConfig("fill", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - if conf.AnchorStr == smartCropIdentifier { - return smartCrop(src, conf.Width, conf.Height, conf.Anchor, conf.Filter) - } - return imaging.Fill(src, conf.Width, conf.Height, conf.Anchor, conf.Filter), nil +func (i *imageResource) Fill(spec string) (resource.Image, error) { + return i.doWithImageConfig("fill", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Fill(src, conf) }) } -// Holds configuration to create a new image from an existing one, resize etc. -type imageConfig struct { - Action string - - // Quality ranges from 1 to 100 inclusive, higher is better. - // This is only relevant for JPEG images. - // Default is 75. - Quality int - - // Rotate rotates an image by the given angle counter-clockwise. - // The rotation will be performed first. - Rotate int - - Width int - Height int - - Filter imaging.ResampleFilter - FilterStr string - - Anchor imaging.Anchor - AnchorStr string -} - -func (i *Image) isJPEG() bool { - name := strings.ToLower(i.relTargetDirFile.file) +func (i *imageResource) isJPEG() bool { + name := strings.ToLower(i.getResourcePaths().relTargetDirFile.file) return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg") } @@ -218,42 +118,20 @@ const imageProcWorkers = 1 var imageProcSem = make(chan bool, imageProcWorkers) -func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, conf imageConfig) (image.Image, error)) (*Image, error) { - conf, err := parseImageConfig(spec) +func (i *imageResource) doWithImageConfig(action, spec string, f func(src image.Image, conf images.ImageConfig) (image.Image, error)) (resource.Image, error) { + conf, err := i.decodeImageConfig(action, spec) if err != nil { return nil, err } - conf.Action = action - - if conf.Quality <= 0 && i.isJPEG() { - // We need a quality setting for all JPEGs - conf.Quality = i.imaging.Quality - } - - if conf.FilterStr == "" { - conf.FilterStr = i.imaging.ResampleFilter - conf.Filter = imageFilters[conf.FilterStr] - } - - if conf.AnchorStr == "" { - conf.AnchorStr = i.imaging.Anchor - if !strings.EqualFold(conf.AnchorStr, smartCropIdentifier) { - conf.Anchor = anchorPositions[conf.AnchorStr] - } - } - return i.spec.imageCache.getOrCreate(i, conf, func() (*Image, image.Image, error) { + return i.getSpec().imageCache.getOrCreate(i, conf, func() (*imageResource, image.Image, error) { imageProcSem <- true defer func() { <-imageProcSem }() - ci := i.clone() - errOp := action - errPath := i.sourceFilename - - ci.setBasePath(conf) + errPath := i.getSourceFilename() src, err := i.decodeSource() if err != nil { @@ -267,10 +145,10 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c converted, err := f(src, conf) if err != nil { - return ci, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} + return nil, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} } - if i.format == imaging.PNG { + if i.Format == imaging.PNG { // Apply the colour palette from the source if paletted, ok := src.(*image.Paletted); ok { tmp := image.NewPaletted(converted.Bounds(), paletted.Palette) @@ -279,177 +157,30 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c } } - b := converted.Bounds() - ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} - ci.configLoaded = true + ci := i.clone(converted) + ci.setBasePath(conf) return ci, converted, nil }) - -} - -func (i imageConfig) key(format imaging.Format) string { - k := strconv.Itoa(i.Width) + "x" + strconv.Itoa(i.Height) - if i.Action != "" { - k += "_" + i.Action - } - if i.Quality > 0 { - k += "_q" + strconv.Itoa(i.Quality) - } - if i.Rotate != 0 { - k += "_r" + strconv.Itoa(i.Rotate) - } - anchor := i.AnchorStr - if anchor == smartCropIdentifier { - anchor = anchor + strconv.Itoa(smartCropVersionNumber) - } - - k += "_" + i.FilterStr - - if strings.EqualFold(i.Action, "fill") { - k += "_" + anchor - } - - if v, ok := imageFormatsVersions[format]; ok { - k += "_" + strconv.Itoa(v) - } - - if mainImageVersionNumber > 0 { - k += "_" + strconv.Itoa(mainImageVersionNumber) - } - - return k -} - -func newImageConfig(width, height, quality, rotate int, filter, anchor string) imageConfig { - var c imageConfig - - c.Width = width - c.Height = height - c.Quality = quality - c.Rotate = rotate - - if filter != "" { - filter = strings.ToLower(filter) - if v, ok := imageFilters[filter]; ok { - c.Filter = v - c.FilterStr = filter - } - } - - if anchor != "" { - anchor = strings.ToLower(anchor) - if v, ok := anchorPositions[anchor]; ok { - c.Anchor = v - c.AnchorStr = anchor - } - } - - return c } -func parseImageConfig(config string) (imageConfig, error) { - var ( - c imageConfig - err error - ) - - if config == "" { - return c, errors.New("image config cannot be empty") - } - - parts := strings.Fields(config) - for _, part := range parts { - part = strings.ToLower(part) - - if part == smartCropIdentifier { - c.AnchorStr = smartCropIdentifier - } else if pos, ok := anchorPositions[part]; ok { - c.Anchor = pos - c.AnchorStr = part - } else if filter, ok := imageFilters[part]; ok { - c.Filter = filter - c.FilterStr = part - } else if part[0] == 'q' { - c.Quality, err = strconv.Atoi(part[1:]) - if err != nil { - return c, err - } - if c.Quality < 1 || c.Quality > 100 { - return c, errors.New("quality ranges from 1 to 100 inclusive") - } - } else if part[0] == 'r' { - c.Rotate, err = strconv.Atoi(part[1:]) - if err != nil { - return c, err - } - } else if strings.Contains(part, "x") { - widthHeight := strings.Split(part, "x") - if len(widthHeight) <= 2 { - first := widthHeight[0] - if first != "" { - c.Width, err = strconv.Atoi(first) - if err != nil { - return c, err - } - } - - if len(widthHeight) == 2 { - second := widthHeight[1] - if second != "" { - c.Height, err = strconv.Atoi(second) - if err != nil { - return c, err - } - } - } - } else { - return c, errors.New("invalid image dimensions") - } - - } - } - - if c.Width == 0 && c.Height == 0 { - return c, errors.New("must provide Width or Height") +func (i *imageResource) decodeImageConfig(action, spec string) (images.ImageConfig, error) { + conf, err := images.DecodeImageConfig(action, spec, i.Proc.Cfg) + if err != nil { + return conf, err } - return c, nil -} - -func (i *Image) initConfig() error { - var err error - i.configInit.Do(func() { - if i.configLoaded { - return - } - - var ( - f hugio.ReadSeekCloser - config image.Config - ) - - f, err = i.ReadSeekCloser() - if err != nil { - return - } - defer f.Close() - - config, _, err = image.DecodeConfig(f) - if err != nil { - return - } - i.config = config - }) + iconf := i.Proc.Cfg - if err != nil { - return _errors.Wrap(err, "failed to load image config") + if conf.Quality <= 0 && i.isJPEG() { + // We need a quality setting for all JPEGs + conf.Quality = iconf.Quality } - return nil + return conf, nil } -func (i *Image) decodeSource() (image.Image, error) { +func (i *imageResource) decodeSource() (image.Image, error) { f, err := i.ReadSeekCloser() if err != nil { return nil, _errors.Wrap(err, "failed to open image for decode") @@ -459,80 +190,39 @@ func (i *Image) decodeSource() (image.Image, error) { return img, err } -// returns an opened file or nil if nothing to write. -func (i *Image) openDestinationsForWriting() (io.WriteCloser, error) { - targetFilenames := i.targetFilenames() - var changedFilenames []string - - // Fast path: - // This is a processed version of the original; - // check if it already existis at the destination. - for _, targetFilename := range targetFilenames { - if _, err := i.spec.BaseFs.PublishFs.Stat(targetFilename); err == nil { - continue |
