From a5744697971d296eb973e04e4259fe9e516b908f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 23 Dec 2018 10:40:32 +0100 Subject: Add CSV support to transform.Unmarshal Fixes #5555 --- tpl/transform/remarshal.go | 4 +- tpl/transform/unmarshal.go | 93 ++++++++++++++++++++++++++++++++++++++--- tpl/transform/unmarshal_test.go | 71 ++++++++++++++++++++++++------- 3 files changed, 146 insertions(+), 22 deletions(-) (limited to 'tpl/transform') diff --git a/tpl/transform/remarshal.go b/tpl/transform/remarshal.go index 144964f0a..62d826b43 100644 --- a/tpl/transform/remarshal.go +++ b/tpl/transform/remarshal.go @@ -35,12 +35,12 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error) return "", err } - fromFormat := metadecoders.FormatFromContentString(from) + fromFormat := metadecoders.Default.FormatFromContentString(from) if fromFormat == "" { return "", errors.New("failed to detect format from content") } - meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat) + meta, err := metadecoders.Default.UnmarshalToMap([]byte(from), fromFormat) var result bytes.Buffer if err := parser.InterfaceToConfig(meta, mark, &result); err != nil { diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index bf7db8920..d83cafd3a 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -15,8 +15,10 @@ package transform import ( "io/ioutil" + "strings" "github.com/gohugoio/hugo/common/hugio" + "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/parser/metadecoders" @@ -27,8 +29,33 @@ import ( ) // Unmarshal unmarshals the data given, which can be either a string -// or a Resource. Supported formats are JSON, TOML and YAML. -func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) { +// or a Resource. Supported formats are JSON, TOML, YAML, and CSV. +// You can optional provide an Options object as the first argument. +func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { + if len(args) < 1 || len(args) > 2 { + return nil, errors.New("unmarshal takes 1 or 2 arguments") + } + + var data interface{} + var decoder = metadecoders.Default + + if len(args) == 1 { + data = args[0] + } else { + m, ok := args[0].(map[string]interface{}) + if !ok { + return nil, errors.New("first argument must be a map") + } + + var err error + + data = args[1] + decoder, err = decodeDecoder(m) + if err != nil { + return nil, errors.WithMessage(err, "failed to decode options") + } + + } // All the relevant Resource types implements ReadSeekCloserResource, // which should be the most effective way to get the content. @@ -75,7 +102,7 @@ func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) { return nil, err } - return metadecoders.Unmarshal(b, f) + return decoder.Unmarshal(b, f) }) } @@ -88,11 +115,67 @@ func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) { key := helpers.MD5String(dataStr) return ns.cache.GetOrCreate(key, func() (interface{}, error) { - f := metadecoders.FormatFromContentString(dataStr) + f := decoder.FormatFromContentString(dataStr) if f == "" { return nil, errors.New("unknown format") } - return metadecoders.Unmarshal([]byte(dataStr), f) + return decoder.Unmarshal([]byte(dataStr), f) }) } + +func decodeDecoder(m map[string]interface{}) (metadecoders.Decoder, error) { + opts := metadecoders.Default + + if m == nil { + return opts, nil + } + + // mapstructure does not support string to rune conversion, so do that manually. + // See https://github.com/mitchellh/mapstructure/issues/151 + for k, v := range m { + if strings.EqualFold(k, "Comma") { + r, err := stringToRune(v) + if err != nil { + return opts, err + } + opts.Comma = r + delete(m, k) + + } else if strings.EqualFold(k, "Comment") { + r, err := stringToRune(v) + if err != nil { + return opts, err + } + opts.Comment = r + delete(m, k) + } + } + + err := mapstructure.WeakDecode(m, &opts) + + return opts, err +} + +func stringToRune(v interface{}) (rune, error) { + s, err := cast.ToStringE(v) + if err != nil { + return 0, err + } + + if len(s) == 0 { + return 0, nil + } + + var r rune + + for i, rr := range s { + if i == 0 { + r = rr + } else { + return 0, errors.Errorf("invalid character: %q", v) + } + } + + return r, nil +} diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 77e14edad..00424c693 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -89,38 +89,74 @@ func TestUnmarshal(t *testing.T) { } for i, test := range []struct { - data interface{} - expect interface{} + data interface{} + options interface{} + expect interface{} }{ - {`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) { + {`{ "slogan": "Hugo Rocks!" }`, nil, func(m map[string]interface{}) { assertSlogan(m) }}, - {`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) { + {`slogan: "Hugo Rocks!"`, nil, func(m map[string]interface{}) { assertSlogan(m) }}, - {`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) { + {`slogan = "Hugo Rocks!"`, nil, func(m map[string]interface{}) { assertSlogan(m) }}, - {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) { + {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, nil, func(m map[string]interface{}) { assertSlogan(m) }}, - {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) { + {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, nil, func(m map[string]interface{}) { assertSlogan(m) }}, - {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) { + {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, nil, func(m map[string]interface{}) { assertSlogan(m) }}, + {testContentResource{content: `1997,Ford,E350,"ac, abs, moon",3000.00 +1999,Chevy,"Venture ""Extended Edition""","",4900.00`, mime: media.CSVType}, nil, func(r [][]string) { + assert.Equal(2, len(r)) + first := r[0] + assert.Equal(5, len(first)) + assert.Equal("Ford", first[1]) + }}, + {testContentResource{content: `a;b;c`, mime: media.CSVType}, map[string]interface{}{"comma": ";"}, func(r [][]string) { + assert.Equal(r, [][]string{[]string{"a", "b", "c"}}) + + }}, + {"a,b,c", nil, func(r [][]string) { + assert.Equal(r, [][]string{[]string{"a", "b", "c"}}) + + }}, + {"a;b;c", map[string]interface{}{"comma": ";"}, func(r [][]string) { + assert.Equal(r, [][]string{[]string{"a", "b", "c"}}) + + }}, + {testContentResource{content: ` +% This is a comment +a;b;c`, mime: media.CSVType}, map[string]interface{}{"CommA": ";", "Comment": "%"}, func(r [][]string) { + assert.Equal(r, [][]string{[]string{"a", "b", "c"}}) + + }}, // errors - {"thisisnotavaliddataformat", false}, - {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false}, - {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false}, - {"thisisnotavaliddataformat", false}, - {`{ notjson }`, false}, - {tstNoStringer{}, false}, + {"thisisnotavaliddataformat", nil, false}, + {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, nil, false}, + {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, nil, false}, + {"thisisnotavaliddataformat", nil, false}, + {`{ notjson }`, nil, false}, + {tstNoStringer{}, nil, false}, } { errMsg := fmt.Sprintf("[%d]", i) - result, err := ns.Unmarshal(test.data) + ns.cache.Clear() + + var args []interface{} + + if test.options != nil { + args = []interface{}{test.options, test.data} + } else { + args = []interface{}{test.data} + } + + result, err := ns.Unmarshal(args...) if b, ok := test.expect.(bool); ok && !b { assert.Error(err, errMsg) @@ -129,6 +165,11 @@ func TestUnmarshal(t *testing.T) { m, ok := result.(map[string]interface{}) assert.True(ok, errMsg) fn(m) + } else if fn, ok := test.expect.(func(r [][]string)); ok { + assert.NoError(err, errMsg) + r, ok := result.([][]string) + assert.True(ok, errMsg) + fn(r) } else { assert.NoError(err, errMsg) assert.Equal(test.expect, result, errMsg) -- cgit v1.2.3