From 822dc627a1cfdf1f97882f27761675ac6ace7669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 21 Dec 2018 16:21:13 +0100 Subject: tpl/transform: Add transform.Unmarshal func Fixes #5428 --- tpl/transform/init.go | 8 ++ tpl/transform/remarshal.go | 30 ++----- tpl/transform/remarshal_test.go | 32 ------- tpl/transform/transform.go | 14 ++- tpl/transform/transform_test.go | 7 +- tpl/transform/unmarshal.go | 98 +++++++++++++++++++++ tpl/transform/unmarshal_test.go | 185 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 312 insertions(+), 62 deletions(-) create mode 100644 tpl/transform/unmarshal.go create mode 100644 tpl/transform/unmarshal_test.go (limited to 'tpl/transform') diff --git a/tpl/transform/init.go b/tpl/transform/init.go index 86951c253..62cb0a9c3 100644 --- a/tpl/transform/init.go +++ b/tpl/transform/init.go @@ -95,6 +95,14 @@ func init() { }, ) + ns.AddMethodMapping(ctx.Unmarshal, + []string{"unmarshal"}, + [][2]string{ + {`{{ "hello = \"Hello World\"" | transform.Unmarshal }}`, "map[hello:Hello World]"}, + {`{{ "hello = \"Hello World\"" | resources.FromString "data/greetings.toml" | transform.Unmarshal }}`, "map[hello:Hello World]"}, + }, + ) + return ns } diff --git a/tpl/transform/remarshal.go b/tpl/transform/remarshal.go index fd0742b7f..144964f0a 100644 --- a/tpl/transform/remarshal.go +++ b/tpl/transform/remarshal.go @@ -2,9 +2,10 @@ package transform import ( "bytes" - "errors" "strings" + "github.com/pkg/errors" + "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/cast" @@ -34,9 +35,9 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error) return "", err } - fromFormat, err := detectFormat(from) - if err != nil { - return "", err + fromFormat := metadecoders.FormatFromContentString(from) + if fromFormat == "" { + return "", errors.New("failed to detect format from content") } meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat) @@ -56,24 +57,3 @@ func toFormatMark(format string) (metadecoders.Format, error) { return "", errors.New("failed to detect target data serialization format") } - -func detectFormat(data string) (metadecoders.Format, error) { - jsonIdx := strings.Index(data, "{") - yamlIdx := strings.Index(data, ":") - tomlIdx := strings.Index(data, "=") - - if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) { - return metadecoders.JSON, nil - } - - if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) { - return metadecoders.YAML, nil - } - - if tomlIdx != -1 { - return metadecoders.TOML, nil - } - - return "", errors.New("failed to detect data serialization format") - -} diff --git a/tpl/transform/remarshal_test.go b/tpl/transform/remarshal_test.go index 1416afff3..07414ccb4 100644 --- a/tpl/transform/remarshal_test.go +++ b/tpl/transform/remarshal_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -171,34 +170,3 @@ func TestTestRemarshalError(t *testing.T) { assert.Error(err) } - -func TestRemarshalDetectFormat(t *testing.T) { - t.Parallel() - assert := require.New(t) - - for i, test := range []struct { - data string - expect interface{} - }{ - {`foo = "bar"`, metadecoders.TOML}, - {` foo = "bar"`, metadecoders.TOML}, - {`foo="bar"`, metadecoders.TOML}, - {`foo: "bar"`, metadecoders.YAML}, - {`foo:"bar"`, metadecoders.YAML}, - {`{ "foo": "bar"`, metadecoders.JSON}, - {`asdfasdf`, false}, - {``, false}, - } { - errMsg := fmt.Sprintf("[%d] %s", i, test.data) - - result, err := detectFormat(test.data) - - if b, ok := test.expect.(bool); ok && !b { - assert.Error(err, errMsg) - continue - } - - assert.NoError(err, errMsg) - assert.Equal(test.expect, result) - } -} diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index 777e31c3e..42e36eb0f 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -19,6 +19,8 @@ import ( "html" "html/template" + "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/spf13/cast" @@ -26,14 +28,22 @@ import ( // New returns a new instance of the transform-namespaced template functions. func New(deps *deps.Deps) *Namespace { + cache := namedmemcache.New() + deps.BuildStartListeners.Add( + func() { + cache.Clear() + }) + return &Namespace{ - deps: deps, + cache: cache, + deps: deps, } } // Namespace provides template functions for the "transform" namespace. type Namespace struct { - deps *deps.Deps + cache *namedmemcache.Cache + deps *deps.Deps } // Emojify returns a copy of s with all emoji codes replaced with actual emojis. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index 34de4a6fd..a09ec6fbd 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -34,7 +34,6 @@ func TestEmojify(t *testing.T) { t.Parallel() v := viper.New() - v.Set("contentDir", "content") ns := New(newDeps(v)) for i, test := range []struct { @@ -215,7 +214,6 @@ func TestPlainify(t *testing.T) { t.Parallel() v := viper.New() - v.Set("contentDir", "content") ns := New(newDeps(v)) for i, test := range []struct { @@ -241,8 +239,11 @@ func TestPlainify(t *testing.T) { } func newDeps(cfg config.Provider) *deps.Deps { + cfg.Set("contentDir", "content") + cfg.Set("i18nDir", "i18n") + l := langs.NewLanguage("en", cfg) - l.Set("i18nDir", "i18n") + cs, err := helpers.NewContentSpec(l) if err != nil { panic(err) diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go new file mode 100644 index 000000000..bf7db8920 --- /dev/null +++ b/tpl/transform/unmarshal.go @@ -0,0 +1,98 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transform + +import ( + "io/ioutil" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/resource" + "github.com/pkg/errors" + + "github.com/spf13/cast" +) + +// Unmarshal unmarshals the data given, which can be either a string +// or a Resource. Supported formats are JSON, TOML and YAML. +func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) { + + // All the relevant Resource types implements ReadSeekCloserResource, + // which should be the most effective way to get the content. + if r, ok := data.(resource.ReadSeekCloserResource); ok { + var key string + var reader hugio.ReadSeekCloser + + if k, ok := r.(resource.Identifier); ok { + key = k.Key() + } + + if key == "" { + reader, err := r.ReadSeekCloser() + if err != nil { + return nil, err + } + defer reader.Close() + + key, err = helpers.MD5FromReader(reader) + if err != nil { + return nil, err + } + + reader.Seek(0, 0) + } + + return ns.cache.GetOrCreate(key, func() (interface{}, error) { + f := metadecoders.FormatFromMediaType(r.MediaType()) + if f == "" { + return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + } + + if reader == nil { + var err error + reader, err = r.ReadSeekCloser() + if err != nil { + return nil, err + } + defer reader.Close() + } + + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + return metadecoders.Unmarshal(b, f) + }) + + } + + dataStr, err := cast.ToStringE(data) + if err != nil { + return nil, errors.Errorf("type %T not supported", data) + } + + key := helpers.MD5String(dataStr) + + return ns.cache.GetOrCreate(key, func() (interface{}, error) { + f := metadecoders.FormatFromContentString(dataStr) + if f == "" { + return nil, errors.New("unknown format") + } + + return metadecoders.Unmarshal([]byte(dataStr), f) + }) +} diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go new file mode 100644 index 000000000..77e14edad --- /dev/null +++ b/tpl/transform/unmarshal_test.go @@ -0,0 +1,185 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transform + +import ( + "fmt" + "math/rand" + "strings" + "testing" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/resource" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +const ( + testJSON = ` + +{ + "ROOT_KEY": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } +} + + ` +) + +var _ resource.ReadSeekCloserResource = (*testContentResource)(nil) + +type testContentResource struct { + content string + mime media.Type + + key string +} + +func (t testContentResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloserFromString(t.content), nil +} + +func (t testContentResource) MediaType() media.Type { + return t.mime +} + +func (t testContentResource) Key() string { + return t.key +} + +func TestUnmarshal(t *testing.T) { + + v := viper.New() + ns := New(newDeps(v)) + assert := require.New(t) + + assertSlogan := func(m map[string]interface{}) { + assert.Equal("Hugo Rocks!", m["slogan"]) + } + + for i, test := range []struct { + data interface{} + expect interface{} + }{ + {`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) { + assertSlogan(m) + }}, + // errors + {"thisisnotavaliddataformat", false}, + {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false}, + {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false}, + {"thisisnotavaliddataformat", false}, + {`{ notjson }`, false}, + {tstNoStringer{}, false}, + } { + errMsg := fmt.Sprintf("[%d]", i) + + result, err := ns.Unmarshal(test.data) + + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, errMsg) + } else if fn, ok := test.expect.(func(m map[string]interface{})); ok { + assert.NoError(err, errMsg) + m, ok := result.(map[string]interface{}) + assert.True(ok, errMsg) + fn(m) + } else { + assert.NoError(err, errMsg) + assert.Equal(test.expect, result, errMsg) + } + + } +} + +func BenchmarkUnmarshalString(b *testing.B) { + v := viper.New() + ns := New(newDeps(v)) + + const numJsons = 100 + + var jsons [numJsons]string + for i := 0; i < numJsons; i++ { + jsons[i] = strings.Replace(testJSON, "ROOT_KEY", fmt.Sprintf("root%d", i), 1) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)]) + if err != nil { + b.Fatal(err) + } + if result == nil { + b.Fatal("no result") + } + } +} + +func BenchmarkUnmarshalResource(b *testing.B) { + v := viper.New() + ns := New(newDeps(v)) + + const numJsons = 100 + + var jsons [numJsons]testContentResource + for i := 0; i < numJsons; i++ { + key := fmt.Sprintf("root%d", i) + jsons[i] = testContentResource{key: key, content: strings.Replace(testJSON, "ROOT_KEY", key, 1), mime: media.JSONType} + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)]) + if err != nil { + b.Fatal(err) + } + if result == nil { + b.Fatal("no result") + } + } +} -- cgit v1.2.3