diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2020-02-25 21:40:02 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2020-04-07 21:59:20 +0200 |
commit | 2f721f8ec69c52202815cd1b543ca4bf535c0901 (patch) | |
tree | cae7d1ee9ce867a4bffc70e94513f09e2aebe162 /resources | |
parent | 8568928aa8e82a6bd7de4555c3703d8835fbd25b (diff) |
Add basic "post resource publish support"
Fixes #7146
Diffstat (limited to 'resources')
-rw-r--r-- | resources/post_publish.go | 51 | ||||
-rw-r--r-- | resources/postpub/fields.go | 59 | ||||
-rw-r--r-- | resources/postpub/fields_test.go | 45 | ||||
-rw-r--r-- | resources/postpub/postpub.go | 177 | ||||
-rw-r--r-- | resources/resource/resourcetypes.go | 25 | ||||
-rw-r--r-- | resources/resource_spec.go | 36 | ||||
-rw-r--r-- | resources/resource_transformers/htesting/testhelpers.go | 2 | ||||
-rw-r--r-- | resources/testhelpers_test.go | 4 | ||||
-rw-r--r-- | resources/transform.go | 11 |
9 files changed, 387 insertions, 23 deletions
diff --git a/resources/post_publish.go b/resources/post_publish.go new file mode 100644 index 000000000..b2adfa5ce --- /dev/null +++ b/resources/post_publish.go @@ -0,0 +1,51 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "github.com/gohugoio/hugo/resources/postpub" + "github.com/gohugoio/hugo/resources/resource" +) + +type transformationKeyer interface { + TransformationKey() string +} + +// PostProcess wraps the given Resource for later processing. +func (spec *Spec) PostProcess(r resource.Resource) (postpub.PostPublishedResource, error) { + key := r.(transformationKeyer).TransformationKey() + spec.postProcessMu.RLock() + result, found := spec.PostProcessResources[key] + spec.postProcessMu.RUnlock() + if found { + return result, nil + } + + spec.postProcessMu.Lock() + defer spec.postProcessMu.Unlock() + + // Double check + result, found = spec.PostProcessResources[key] + if found { + return result, nil + } + + result = postpub.NewPostPublishResource(spec.incr.Incr(), r) + if result == nil { + panic("got nil result") + } + spec.PostProcessResources[key] = result + + return result, nil +} diff --git a/resources/postpub/fields.go b/resources/postpub/fields.go new file mode 100644 index 000000000..f1cfe6092 --- /dev/null +++ b/resources/postpub/fields.go @@ -0,0 +1,59 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postpub + +import ( + "reflect" +) + +const ( + FieldNotSupported = "__field_not_supported" +) + +func structToMapWithPlaceholders(root string, in interface{}, createPlaceholder func(s string) string) map[string]interface{} { + m := structToMap(in) + insertFieldPlaceholders(root, m, createPlaceholder) + return m +} + +func structToMap(s interface{}) map[string]interface{} { + m := make(map[string]interface{}) + t := reflect.TypeOf(s) + + for i := 0; i < t.NumMethod(); i++ { + method := t.Method(i) + if method.PkgPath != "" { + continue + } + if method.Type.NumIn() == 1 { + m[method.Name] = "" + } + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if field.PkgPath != "" { + continue + } + m[field.Name] = "" + } + return m +} + +// insert placeholder for the templates. Do it very shallow for now. +func insertFieldPlaceholders(root string, m map[string]interface{}, createPlaceholder func(s string) string) { + for k, _ := range m { + m[k] = createPlaceholder(root + "." + k) + } +} diff --git a/resources/postpub/fields_test.go b/resources/postpub/fields_test.go new file mode 100644 index 000000000..fa0c9190a --- /dev/null +++ b/resources/postpub/fields_test.go @@ -0,0 +1,45 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postpub + +import ( + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/gohugoio/hugo/media" +) + +func TestCreatePlaceholders(t *testing.T) { + c := qt.New(t) + + m := structToMap(media.CSSType) + + insertFieldPlaceholders("foo", m, func(s string) string { + return "pre_" + s + "_post" + }) + + c.Assert(m, qt.DeepEquals, map[string]interface{}{ + "FullSuffix": "pre_foo.FullSuffix_post", + "Type": "pre_foo.Type_post", + "MainType": "pre_foo.MainType_post", + "Delimiter": "pre_foo.Delimiter_post", + "MarshalJSON": "pre_foo.MarshalJSON_post", + "String": "pre_foo.String_post", + "Suffix": "pre_foo.Suffix_post", + "SubType": "pre_foo.SubType_post", + "Suffixes": "pre_foo.Suffixes_post", + }) + +} diff --git a/resources/postpub/postpub.go b/resources/postpub/postpub.go new file mode 100644 index 000000000..3a1dd2f85 --- /dev/null +++ b/resources/postpub/postpub.go @@ -0,0 +1,177 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postpub + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources/resource" +) + +type PostPublishedResource interface { + resource.ResourceTypeProvider + resource.ResourceLinksProvider + resource.ResourceMetaProvider + resource.ResourceParamsProvider + resource.ResourceDataProvider + resource.OriginProvider + + MediaType() map[string]interface{} +} + +const ( + PostProcessPrefix = "__h_pp_l1" + PostProcessSuffix = "__e" +) + +func NewPostPublishResource(id int, r resource.Resource) PostPublishedResource { + return &PostPublishResource{ + prefix: PostProcessPrefix + "_" + strconv.Itoa(id) + "_", + delegate: r, + } +} + +// postPublishResource holds a Resource to be transformed post publishing. +type PostPublishResource struct { + prefix string + delegate resource.Resource +} + +func (r *PostPublishResource) field(name string) string { + return r.prefix + name + PostProcessSuffix +} + +func (r *PostPublishResource) Permalink() string { + return r.field("Permalink") +} + +func (r *PostPublishResource) RelPermalink() string { + return r.field("RelPermalink") +} + +func (r *PostPublishResource) Origin() resource.Resource { + return r.delegate +} + +func (r *PostPublishResource) GetFieldString(pattern string) (string, bool) { + if r == nil { + panic("resource is nil") + } + prefixIdx := strings.Index(pattern, r.prefix) + if prefixIdx == -1 { + // Not a method on this resource. + return "", false + } + + fieldAccessor := pattern[prefixIdx+len(r.prefix) : strings.Index(pattern, PostProcessSuffix)] + + d := r.delegate + switch { + case fieldAccessor == "RelPermalink": + return d.RelPermalink(), true + case fieldAccessor == "Permalink": + return d.Permalink(), true + case fieldAccessor == "Name": + return d.Name(), true + case fieldAccessor == "Title": + return d.Title(), true + case fieldAccessor == "ResourceType": + return d.ResourceType(), true + case fieldAccessor == "Content": + content, err := d.(resource.ContentProvider).Content() + if err != nil { + return "", true + } + return cast.ToString(content), true + case strings.HasPrefix(fieldAccessor, "MediaType"): + return r.fieldToString(d.MediaType(), fieldAccessor), true + case fieldAccessor == "Data.Integrity": + return cast.ToString((d.Data().(map[string]interface{})["Integrity"])), true + default: + panic(fmt.Sprintf("unknown field accessor %q", fieldAccessor)) + } + +} + +func (r *PostPublishResource) fieldToString(receiver interface{}, path string) string { + fieldname := strings.Split(path, ".")[1] + + receiverv := reflect.ValueOf(receiver) + switch receiverv.Kind() { + case reflect.Map: + v := receiverv.MapIndex(reflect.ValueOf(fieldname)) + return cast.ToString(v.Interface()) + default: + v := receiverv.FieldByName(fieldname) + if !v.IsValid() { + method := receiverv.MethodByName(fieldname) + if method.IsValid() { + vals := method.Call(nil) + if len(vals) > 0 { + v = vals[0] + } + + } + } + + if v.IsValid() { + return cast.ToString(v.Interface()) + } + return "" + } +} + +func (r *PostPublishResource) Data() interface{} { + m := map[string]interface{}{ + "Integrity": "", + } + insertFieldPlaceholders("Data", m, r.field) + return m +} + +func (r *PostPublishResource) MediaType() map[string]interface{} { + m := structToMapWithPlaceholders("MediaType", media.Type{}, r.field) + return m +} + +func (r *PostPublishResource) ResourceType() string { + return r.field("ResourceType") +} + +func (r *PostPublishResource) Name() string { + return r.field("Name") +} + +func (r *PostPublishResource) Title() string { + return r.field("Title") +} + +func (r *PostPublishResource) Params() maps.Params { + panic(r.fieldNotSupported("Params")) +} + +func (r *PostPublishResource) Content() (interface{}, error) { + return r.field("Content"), nil +} + +func (r *PostPublishResource) fieldNotSupported(name string) string { + return fmt.Sprintf("method .%s is currently not supported in post-publish transformations.", name) +} diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index b525d7d55..62431c06c 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -28,9 +28,17 @@ type Cloner interface { Clone() Resource } +// OriginProvider provides the original Resource if this is wrapped. +// This is an internal Hugo interface and not meant for use in the templates. +type OriginProvider interface { + Origin() Resource + GetFieldString(pattern string) (string, bool) +} + // Resource represents a linkable resource, i.e. a content page, image etc. type Resource interface { - ResourceTypesProvider + ResourceTypeProvider + MediaTypeProvider ResourceLinksProvider ResourceMetaProvider ResourceParamsProvider @@ -53,16 +61,23 @@ type ImageOps interface { Exif() (*exif.Exif, error) } -type ResourceTypesProvider interface { - // MediaType is this resource's MIME type. - MediaType() media.Type - +type ResourceTypeProvider interface { // ResourceType is the resource type. For most file types, this is the main // part of the MIME type, e.g. "image", "application", "text" etc. // For content pages, this value is "page". ResourceType() string } +type ResourceTypesProvider interface { + ResourceTypeProvider + MediaTypeProvider +} + +type MediaTypeProvider interface { + // MediaType is this resource's MIME type. + MediaType() media.Type +} + type ResourceLinksProvider interface { // Permalink represents the absolute link to this resource. Permalink() string diff --git a/resources/resource_spec.go b/resources/resource_spec.go index d094998a4..81eed2f02 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -21,14 +21,16 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/config" - - "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/resources/postpub" "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/loggers" @@ -44,6 +46,7 @@ import ( func NewSpec( s *helpers.PathSpec, fileCaches filecache.Caches, + incr identity.Incrementer, logger *loggers.Logger, errorHandler herrors.ErrorSender, outputFormats output.Formats, @@ -59,6 +62,10 @@ func NewSpec( return nil, err } + if incr == nil { + incr = &identity.IncrementByOne{} + } + if logger == nil { logger = loggers.NewErrorLogger() } @@ -68,15 +75,18 @@ func NewSpec( return nil, err } - rs := &Spec{PathSpec: s, - Logger: logger, - ErrorSender: errorHandler, - imaging: imaging, - MediaTypes: mimeTypes, - OutputFormats: outputFormats, - Permalinks: permalinks, - BuildConfig: config.DecodeBuild(s.Cfg), - FileCaches: fileCaches, + rs := &Spec{ + PathSpec: s, + Logger: logger, + ErrorSender: errorHandler, + imaging: imaging, + incr: incr, + MediaTypes: mimeTypes, + OutputFormats: outputFormats, + Permalinks: permalinks, + BuildConfig: config.DecodeBuild(s.Cfg), + FileCaches: fileCaches, + PostProcessResources: make(map[string]postpub.PostPublishedResource), imageCache: newImageCache( fileCaches.ImageCache(), @@ -106,9 +116,13 @@ type Spec struct { // Holds default filter settings etc. imaging *images.ImageProcessor + incr identity.Incrementer imageCache *imageCache ResourceCache *ResourceCache FileCaches filecache.Caches + + postProcessMu sync.RWMutex + PostProcessResources map[string]postpub.PostPublishedResource } func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 752f571f7..8eacf7da4 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -51,7 +51,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 87652a00f..0462f7ecd 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -90,7 +90,7 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -129,7 +129,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir diff --git a/resources/transform.go b/resources/transform.go index e88307afe..6cb257817 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -296,9 +296,7 @@ func (r *resourceAdapter) publish() { } -func (r *resourceAdapter) transform(publish, setContent bool) error { - cache := r.spec.ResourceCache - +func (r *resourceAdapter) TransformationKey() string { // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. var key string @@ -307,8 +305,13 @@ func (r *resourceAdapter) transform(publish, setContent bool) error { } base := ResourceCacheKey(r.target.Key()) + return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key) +} + +func (r *resourceAdapter) transform(publish, setContent bool) error { + cache := r.spec.ResourceCache - key = cache.cleanKey(base) + "_" + helpers.MD5String(key) + key := r.TransformationKey() cached, found := cache.get(key) |