From 2f721f8ec69c52202815cd1b543ca4bf535c0901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Tue, 25 Feb 2020 21:40:02 +0100 Subject: Add basic "post resource publish support" Fixes #7146 --- deps/deps.go | 26 +-- deps/deps_test.go | 10 ++ hugolib/filesystems/basefs.go | 2 +- hugolib/hugo_sites_build.go | 101 ++++++++++++ hugolib/page__common.go | 3 +- hugolib/page__new.go | 3 +- hugolib/resource_chain_test.go | 78 ++++++++- identity/identity.go | 16 ++ resources/post_publish.go | 51 ++++++ resources/postpub/fields.go | 59 +++++++ resources/postpub/fields_test.go | 45 ++++++ resources/postpub/postpub.go | 177 +++++++++++++++++++++ resources/resource/resourcetypes.go | 25 ++- resources/resource_spec.go | 36 +++-- .../resource_transformers/htesting/testhelpers.go | 2 +- resources/testhelpers_test.go | 4 +- resources/transform.go | 11 +- tpl/resources/resources.go | 6 + 18 files changed, 619 insertions(+), 36 deletions(-) create mode 100644 resources/post_publish.go create mode 100644 resources/postpub/fields.go create mode 100644 resources/postpub/fields_test.go create mode 100644 resources/postpub/postpub.go diff --git a/deps/deps.go b/deps/deps.go index e482b2df7..82a16ba59 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -2,6 +2,7 @@ package deps import ( "sync" + "sync/atomic" "time" "github.com/pkg/errors" @@ -92,8 +93,9 @@ type Deps struct { // BuildStartListeners will be notified before a build starts. BuildStartListeners *Listeners - // Atomic flags set during a build. - BuildFlags *BuildFlags + // Atomic values set during a build. + // This is common/global for all sites. + BuildState *BuildState *globalErrHandler } @@ -236,8 +238,9 @@ func New(cfg DepsCfg) (*Deps, error) { } errorHandler := &globalErrHandler{} + buildState := &BuildState{} - resourceSpec, err := resources.NewSpec(ps, fileCaches, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) + resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -275,7 +278,7 @@ func New(cfg DepsCfg) (*Deps, error) { Site: cfg.Site, FileCaches: fileCaches, BuildStartListeners: &Listeners{}, - BuildFlags: &BuildFlags{}, + BuildState: buildState, Timeout: time.Duration(timeoutms) * time.Millisecond, globalErrHandler: errorHandler, } @@ -308,7 +311,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er // The resource cache is global so reuse. // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) + d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } @@ -376,10 +379,15 @@ type DepsCfg struct { Running bool } -// BuildFlags are flags that may be turned on during a build. -type BuildFlags struct { +// BuildState are flags that may be turned on during a build. +type BuildState struct { + counter uint64 } -func NewBuildFlags() BuildFlags { - return BuildFlags{} +func (b *BuildState) Incr() int { + return int(atomic.AddUint64(&b.counter, uint64(1))) +} + +func NewBuildState() BuildState { + return BuildState{} } diff --git a/deps/deps_test.go b/deps/deps_test.go index a7450a41c..5c58ed7a3 100644 --- a/deps/deps_test.go +++ b/deps/deps_test.go @@ -15,8 +15,18 @@ package deps import ( "testing" + + qt "github.com/frankban/quicktest" ) func TestBuildFlags(t *testing.T) { + c := qt.New(t) + var bf BuildState + bf.Incr() + bf.Incr() + bf.Incr() + + c.Assert(bf.Incr(), qt.Equals, 4) + } diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 47d6d11f5..57a95a037 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -345,7 +345,7 @@ func NewBase(p *paths.Paths, logger *loggers.Logger, options ...func(*BaseFs) er logger = loggers.NewWarningLogger() } - publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) + publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)) b := &BaseFs{ PublishFs: publishFs, diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 15eca4bb3..6a65605fc 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -17,7 +17,17 @@ import ( "bytes" "context" "fmt" + "os" "runtime/trace" + "strings" + + "github.com/gohugoio/hugo/common/para" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/resources/postpub" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/output" @@ -138,6 +148,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { } } + if err := h.postProcess(); err != nil { + h.SendError(err) + } + if h.Metrics != nil { var b bytes.Buffer h.Metrics.WriteMetrics(&b) @@ -321,3 +335,90 @@ func (h *HugoSites) render(config *BuildCfg) error { return nil } + +func (h *HugoSites) postProcess() error { + var toPostProcess []resource.OriginProvider + for _, s := range h.Sites { + for _, v := range s.ResourceSpec.PostProcessResources { + toPostProcess = append(toPostProcess, v) + } + } + + if len(toPostProcess) == 0 { + return nil + } + + workers := para.New(config.GetNumWorkerMultiplier()) + g, _ := workers.Start(context.Background()) + + handleFile := func(filename string) error { + + content, err := afero.ReadFile(h.BaseFs.PublishFs, filename) + if err != nil { + return err + } + + k := 0 + changed := false + + for { + l := bytes.Index(content[k:], []byte(postpub.PostProcessPrefix)) + if l == -1 { + break + } + m := bytes.Index(content[k+l:], []byte(postpub.PostProcessSuffix)) + len(postpub.PostProcessSuffix) + + low, high := k+l, k+l+m + + field := content[low:high] + + forward := l + m + + for i, r := range toPostProcess { + if r == nil { + panic(fmt.Sprintf("resource %d to post process is nil", i+1)) + } + v, ok := r.GetFieldString(string(field)) + if ok { + content = append(content[:low], append([]byte(v), content[high:]...)...) + changed = true + forward = len(v) + break + } + } + + k += forward + } + + if changed { + return afero.WriteFile(h.BaseFs.PublishFs, filename, content, 0666) + } + + return nil + + } + + _ = afero.Walk(h.BaseFs.PublishFs, "", func(path string, info os.FileInfo, err error) error { + if info == nil || info.IsDir() { + return nil + } + + if !strings.HasSuffix(path, "html") { + return nil + } + + g.Run(func() error { + return handleFile(path) + }) + + return nil + }) + + // Prepare for a new build. + for _, s := range h.Sites { + s.ResourceSpec.PostProcessResources = make(map[string]postpub.PostPublishedResource) + } + + return g.Wait() + +} diff --git a/hugolib/page__common.go b/hugolib/page__common.go index be6bb090b..d1c7ba866 100644 --- a/hugolib/page__common.go +++ b/hugolib/page__common.go @@ -86,7 +86,8 @@ type pageCommon struct { resource.ResourceDataProvider resource.ResourceMetaProvider resource.ResourceParamsProvider - resource.ResourceTypesProvider + resource.ResourceTypeProvider + resource.MediaTypeProvider resource.TranslationKeyProvider compare.Eqer diff --git a/hugolib/page__new.go b/hugolib/page__new.go index 938c13d7c..9ec089f27 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -49,7 +49,8 @@ func newPageBase(metaProvider *pageMeta) (*pageState, error) { PageMetaProvider: metaProvider, RelatedKeywordsProvider: metaProvider, OutputFormatsProvider: page.NopPage, - ResourceTypesProvider: pageTypesProvider, + ResourceTypeProvider: pageTypesProvider, + MediaTypeProvider: pageTypesProvider, RefProvider: page.NopPage, ShortcodeInfoProvider: page.NopPage, LanguageProvider: s, diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 8bca6c7b5..0d0c9203a 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -14,13 +14,16 @@ package hugolib import ( + "fmt" "io" + "math/rand" "os" "os/exec" "path/filepath" "runtime" "strings" "testing" + "time" "github.com/gohugoio/hugo/common/herrors" @@ -352,6 +355,80 @@ Edited content. } } +func TestResourceChainPostProcess(t *testing.T) { + t.Parallel() + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + + b := newTestSitesBuilder(t) + b.WithContent("page1.md", "---\ntitle: Page1\n---") + b.WithContent("page2.md", "---\ntitle: Page2\n---") + + b.WithTemplates( + "_default/single.html", `{{ $hello := "

Hello World!

" | resources.FromString "hello.html" | minify | fingerprint "md5" | resources.PostProcess }} +HELLO: {{ $hello.RelPermalink }} +`, + "index.html", `Start. +{{ $hello := "

Hello World!

" | resources.FromString "hello.html" | minify | fingerprint "md5" | resources.PostProcess }} + +HELLO: {{ $hello.RelPermalink }}|Integrity: {{ $hello.Data.Integrity }}|MediaType: {{ $hello.MediaType.Type }} +HELLO2: Name: {{ $hello.Name }}|Content: {{ $hello.Content }}|Title: {{ $hello.Title }}|ResourceType: {{ $hello.ResourceType }} + +`+strings.Repeat("a b", rnd.Intn(10)+1)+` + + +End.`) + + b.Running() + b.Build(BuildCfg{}) + b.AssertFileContent("public/index.html", + `Start. +HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html|Integrity: md5-otHLJPJLMip9rVIEFMUj6Q==|MediaType: text/html +HELLO2: Name: hello.html|Content:

Hello World!

|Title: hello.html|ResourceType: html +End.`) + + b.AssertFileContent("public/page1/index.html", `HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html`) + b.AssertFileContent("public/page2/index.html", `HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html`) + +} + +func BenchmarkResourceChainPostProcess(b *testing.B) { + + for i := 0; i < b.N; i++ { + b.StopTimer() + s := newTestSitesBuilder(b) + for i := 0; i < 300; i++ { + s.WithContent(fmt.Sprintf("page%d.md", i+1), "---\ntitle: Page\n---") + } + s.WithTemplates("_default/single.html", `Start. +Some text. + + +{{ $hello1 := "

Hello World 2!

" | resources.FromString "hello.html" | minify | fingerprint "md5" | resources.PostProcess }} +{{ $hello2 := "

Hello World 2!

" | resources.FromString (printf "%s.html" .Path) | minify | fingerprint "md5" | resources.PostProcess }} + +Some more text. + +HELLO: {{ $hello1.RelPermalink }}|Integrity: {{ $hello1.Data.Integrity }}|MediaType: {{ $hello1.MediaType.Type }} + +Some more text. + +HELLO2: Name: {{ $hello2.Name }}|Content: {{ $hello2.Content }}|Title: {{ $hello2.Title }}|ResourceType: {{ $hello2.ResourceType }} + +Some more text. + +HELLO2_2: Name: {{ $hello2.Name }}|Content: {{ $hello2.Content }}|Title: {{ $hello2.Title }}|ResourceType: {{ $hello2.ResourceType }} + +End. +`) + + b.StartTimer() + s.Build(BuildCfg{}) + + } + +} + func TestResourceChains(t *testing.T) { t.Parallel() @@ -769,7 +846,6 @@ func TestResourceChainPostCSS(t *testing.T) { } if runtime.GOOS == "windows" { - // TODO(bep) t.Skip("skip npm test on Windows") } diff --git a/identity/identity.go b/identity/identity.go index 7e03120b4..ac3558d16 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -4,6 +4,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" ) // NewIdentityManager creates a new Manager starting at id. @@ -139,3 +140,18 @@ func (im *identityManager) Search(id Identity) Provider { defer im.Unlock() return im.ids.search(0, id.GetIdentity()) } + +// Incrementer increments and returns the value. +// Typically used for IDs. +type Incrementer interface { + Incr() int +} + +// IncrementByOne implements Incrementer adding 1 every time Incr is called. +type IncrementByOne struct { + counter uint64 +} + +func (c *IncrementByOne) Incr() int { + return int(atomic.AddUint64(&c.counter, uint64(1))) +} diff --git a/resources/post_publish.go b/resources/post_publish.go new file mode 100644 index 000000000..b2adfa5ce --- /dev/null +++ b/resources/post_publish.go @@ -0,0 +1,51 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "github.com/gohugoio/hugo/resources/postpub" + "github.com/gohugoio/hugo/resources/resource" +) + +type transformationKeyer interface { + TransformationKey() string +} + +// PostProcess wraps the given Resource for later processing. +func (spec *Spec) PostProcess(r resource.Resource) (postpub.PostPublishedResource, error) { + key := r.(transformationKeyer).TransformationKey() + spec.postProcessMu.RLock() + result, found := spec.PostProcessResources[key] + spec.postProcessMu.RUnlock() + if found { + return result, nil + } + + spec.postProcessMu.Lock() + defer spec.postProcessMu.Unlock() + + // Double check + result, found = spec.PostProcessResources[key] + if found { + return result, nil + } + + result = postpub.NewPostPublishResource(spec.incr.Incr(), r) + if result == nil { + panic("got nil result") + } + spec.PostProcessResources[key] = result + + return result, nil +} diff --git a/resources/postpub/fields.go b/resources/postpub/fields.go new file mode 100644 index 000000000..f1cfe6092 --- /dev/null +++ b/resources/postpub/fields.go @@ -0,0 +1,59 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postpub + +import ( + "reflect" +) + +const ( + FieldNotSupported = "__field_not_supported" +) + +func structToMapWithPlaceholders(root string, in interface{}, createPlaceholder func(s string) string) map[string]interface{} { + m := structToMap(in) + insertFieldPlaceholders(root, m, createPlaceholder) + return m +} + +func structToMap(s interface{}) map[string]interface{} { + m := make(map[string]interface{}) + t := reflect.TypeOf(s) + + for i := 0; i < t.NumMethod(); i++ { + method := t.Method(i) + if method.PkgPath != "" { + continue + } + if method.Type.NumIn() == 1 { + m[method.Name] = "" + } + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if field.PkgPath != "" { + continue + } + m[field.Name] = "" + } + return m +} + +// insert placeholder for the templates. Do it very shallow for now. +func insertFieldPlaceholders(root string, m map[string]interface{}, createPlaceholder func(s string) string) { + for k, _ := range m { + m[k] = createPlaceholder(root + "." + k) + } +} diff --git a/resources/postpub/fields_test.go b/resources/postpub/fields_test.go new file mode 100644 index 000000000..fa0c9190a --- /dev/null +++ b/resources/postpub/fields_test.go @@ -0,0 +1,45 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postpub + +import ( + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/gohugoio/hugo/media" +) + +func TestCreatePlaceholders(t *testing.T) { + c := qt.New(t) + + m := structToMap(media.CSSType) + + insertFieldPlaceholders("foo", m, func(s string) string { + return "pre_" + s + "_post" + }) + + c.Assert(m, qt.DeepEquals, map[string]interface{}{ + "FullSuffix": "pre_foo.FullSuffix_post", + "Type": "pre_foo.Type_post", + "MainType": "pre_foo.MainType_post", + "Delimiter": "pre_foo.Delimiter_post", + "MarshalJSON": "pre_foo.MarshalJSON_post", + "String": "pre_foo.String_post", + "Suffix": "pre_foo.Suffix_post", + "SubType": "pre_foo.SubType_post", + "Suffixes": "pre_foo.Suffixes_post", + }) + +} diff --git a/resources/postpub/postpub.go b/resources/postpub/postpub.go new file mode 100644 index 000000000..3a1dd2f85 --- /dev/null +++ b/resources/postpub/postpub.go @@ -0,0 +1,177 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postpub + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources/resource" +) + +type PostPublishedResource interface { + resource.ResourceTypeProvider + resource.ResourceLinksProvider + resource.ResourceMetaProvider + resource.ResourceParamsProvider + resource.ResourceDataProvider + resource.OriginProvider + + MediaType() map[string]interface{} +} + +const ( + PostProcessPrefix = "__h_pp_l1" + PostProcessSuffix = "__e" +) + +func NewPostPublishResource(id int, r resource.Resource) PostPublishedResource { + return &PostPublishResource{ + prefix: PostProcessPrefix + "_" + strconv.Itoa(id) + "_", + delegate: r, + } +} + +// postPublishResource holds a Resource to be transformed post publishing. +type PostPublishResource struct { + prefix string + delegate resource.Resource +} + +func (r *PostPublishResource) field(name string) string { + return r.prefix + name + PostProcessSuffix +} + +func (r *PostPublishResource) Permalink() string { + return r.field("Permalink") +} + +func (r *PostPublishResource) RelPermalink() string { + return r.field("RelPermalink") +} + +func (r *PostPublishResource) Origin() resource.Resource { + return r.delegate +} + +func (r *PostPublishResource) GetFieldString(pattern string) (string, bool) { + if r == nil { + panic("resource is nil") + } + prefixIdx := strings.Index(pattern, r.prefix) + if prefixIdx == -1 { + // Not a method on this resource. + return "", false + } + + fieldAccessor := pattern[prefixIdx+len(r.prefix) : strings.Index(pattern, PostProcessSuffix)] + + d := r.delegate + switch { + case fieldAccessor == "RelPermalink": + return d.RelPermalink(), true + case fieldAccessor == "Permalink": + return d.Permalink(), true + case fieldAccessor == "Name": + return d.Name(), true + case fieldAccessor == "Title": + return d.Title(), true + case fieldAccessor == "ResourceType": + return d.ResourceType(), true + case fieldAccessor == "Content": + content, err := d.(resource.ContentProvider).Content() + if err != nil { + return "", true + } + return cast.ToString(content), true + case strings.HasPrefix(fieldAccessor, "MediaType"): + return r.fieldToString(d.MediaType(), fieldAccessor), true + case fieldAccessor == "Data.Integrity": + return cast.ToString((d.Data().(map[string]interface{})["Integrity"])), true + default: + panic(fmt.Sprintf("unknown field accessor %q", fieldAccessor)) + } + +} + +func (r *PostPublishResource) fieldToString(receiver interface{}, path string) string { + fieldname := strings.Split(path, ".")[1] + + receiverv := reflect.ValueOf(receiver) + switch receiverv.Kind() { + case reflect.Map: + v := receiverv.MapIndex(reflect.ValueOf(fieldname)) + return cast.ToString(v.Interface()) + default: + v := receiverv.FieldByName(fieldname) + if !v.IsValid() { + method := receiverv.MethodByName(fieldname) + if method.IsValid() { + vals := method.Call(nil) + if len(vals) > 0 { + v = vals[0] + } + + } + } + + if v.IsValid() { + return cast.ToString(v.Interface()) + } + return "" + } +} + +func (r *PostPublishResource) Data() interface{} { + m := map[string]interface{}{ + "Integrity": "", + } + insertFieldPlaceholders("Data", m, r.field) + return m +} + +func (r *PostPublishResource) MediaType() map[string]interface{} { + m := structToMapWithPlaceholders("MediaType", media.Type{}, r.field) + return m +} + +func (r *PostPublishResource) ResourceType() string { + return r.field("ResourceType") +} + +func (r *PostPublishResource) Name() string { + return r.field("Name") +} + +func (r *PostPublishResource) Title() string { + return r.field("Title") +} + +func (r *PostPublishResource) Params() maps.Params { + panic(r.fieldNotSupported("Params")) +} + +func (r *PostPublishResource) Content() (interface{}, error) { + return r.field("Content"), nil +} + +func (r *PostPublishResource) fieldNotSupported(name string) string { + return fmt.Sprintf("method .%s is currently not supported in post-publish transformations.", name) +} diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index b525d7d55..62431c06c 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -28,9 +28,17 @@ type Cloner interface { Clone() Resource } +// OriginProvider provides the original Resource if this is wrapped. +// This is an internal Hugo interface and not meant for use in the templates. +type OriginProvider interface { + Origin() Resource + GetFieldString(pattern string) (string, bool) +} + // Resource represents a linkable resource, i.e. a content page, image etc. type Resource interface { - ResourceTypesProvider + ResourceTypeProvider + MediaTypeProvider ResourceLinksProvider ResourceMetaProvider ResourceParamsProvider @@ -53,16 +61,23 @@ type ImageOps interface { Exif() (*exif.Exif, error) } -type ResourceTypesProvider interface { - // MediaType is this resource's MIME type. - MediaType() media.Type - +type ResourceTypeProvider interface { // ResourceType is the resource type. For most file types, this is the main // part of the MIME type, e.g. "image", "application", "text" etc. // For content pages, this value is "page". ResourceType() string } +type ResourceTypesProvider interface { + ResourceTypeProvider + MediaTypeProvider +} + +type MediaTypeProvider interface { + // MediaType is this resource's MIME type. + MediaType() media.Type +} + type ResourceLinksProvider interface { // Permalink represents the absolute link to this resource. Permalink() string diff --git a/resources/resource_spec.go b/resources/resource_spec.go index d094998a4..81eed2f02 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -21,14 +21,16 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/config" - - "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/resources/postpub" "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/loggers" @@ -44,6 +46,7 @@ import ( func NewSpec( s *helpers.PathSpec, fileCaches filecache.Caches, + incr identity.Incrementer, logger *loggers.Logger, errorHandler herrors.ErrorSender, outputFormats output.Formats, @@ -59,6 +62,10 @@ func NewSpec( return nil, err } + if incr == nil { + incr = &identity.IncrementByOne{} + } + if logger == nil { logger = loggers.NewErrorLogger() } @@ -68,15 +75,18 @@ func NewSpec( return nil, err } - rs := &Spec{PathSpec: s, - Logger: logger, - ErrorSender: errorHandler, - imaging: imaging, - MediaTypes: mimeTypes, - OutputFormats: outputFormats, - Permalinks: permalinks, - BuildConfig: config.DecodeBuild(s.Cfg), - FileCaches: fileCaches, + rs := &Spec{ + PathSpec: s, + Logger: logger, + ErrorSender: errorHandler, + imaging: imaging, + incr: incr, + MediaTypes: mimeTypes, + OutputFormats: outputFormats, + Permalinks: permalinks, + BuildConfig: config.DecodeBuild(s.Cfg), + FileCaches: fileCaches, + PostProcessResources: make(map[string]postpub.PostPublishedResource), imageCache: newImageCache( fileCaches.ImageCache(), @@ -106,9 +116,13 @@ type Spec struct { // Holds default filter settings etc. imaging *images.ImageProcessor + incr identity.Incrementer imageCache *imageCache ResourceCache *ResourceCache FileCaches filecache.Caches + + postProcessMu sync.RWMutex + PostProcessResources map[string]postpub.PostPublishedResource } func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 752f571f7..8eacf7da4 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -51,7 +51,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 87652a00f..0462f7ecd 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -90,7 +90,7 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -129,7 +129,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir diff --git a/resources/transform.go b/resources/transform.go index e88307afe..6cb257817 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -296,9 +296,7 @@ func (r *resourceAdapter) publish() { } -func (r *resourceAdapter) transform(publish, setContent bool) error { - cache := r.spec.ResourceCache - +func (r *resourceAdapter) TransformationKey() string { // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. var key string @@ -307,8 +305,13 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { } base := ResourceCacheKey(r.target.Key()) + return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key) +} + +func (r *resourceAdapter) transform(publish, setContent bool) error { + cache := r.spec.ResourceCache - key = cache.cleanKey(base) + "_" + helpers.MD5String(key) + key := r.TransformationKey() cached, found := cache.get(key) diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index a1055632c..90fb58b4b 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -19,6 +19,8 @@ import ( "fmt" "path/filepath" + "github.com/gohugoio/hugo/resources/postpub" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/resources" @@ -273,6 +275,10 @@ func (ns *Namespace) PostCSS(args ...interface{}) (resource.Resource, error) { return ns.postcssClient.Process(r, options) } +func (ns *Namespace) PostProcess(r resource.Resource) (postpub.PostPublishedResource, error) { + return ns.deps.ResourceSpec.PostProcess(r) +} + // We allow string or a map as the first argument in some cases. func (ns *Namespace) resolveIfFirstArgIsString(args []interface{}) (resources.ResourceTransformer, string, bool) { if len(args) != 2 { -- cgit v1.2.3