From 129c27ee6e9fed98dbfebeaa272fd52757b475b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 20 Oct 2018 11:16:18 +0200 Subject: parser/metadecoders: Consolidate the metadata decoders See #5324 --- parser/frontmatter.go | 232 +++++++++---------------- parser/frontmatter_test.go | 266 ----------------------------- parser/metadecoders/decoder.go | 146 +++++++++++----- parser/metadecoders/decoder_test.go | 207 ++++++++++++++++++++++ parser/metadecoders/format.go | 66 +++++++ parser/metadecoders/format_test.go | 57 +++++++ parser/metadecoders/json.go | 31 ---- parser/metadecoders/yaml.go | 84 --------- parser/page.go | 9 +- parser/page_test.go | 129 -------------- parser/pageparser/item.go | 2 +- parser/pageparser/pagelexer.go | 34 ++-- parser/pageparser/pageparser.go | 10 +- parser/pageparser/pageparser_intro_test.go | 2 - 14 files changed, 533 insertions(+), 742 deletions(-) create mode 100644 parser/metadecoders/decoder_test.go create mode 100644 parser/metadecoders/format.go create mode 100644 parser/metadecoders/format_test.go delete mode 100644 parser/metadecoders/json.go delete mode 100644 parser/metadecoders/yaml.go (limited to 'parser') diff --git a/parser/frontmatter.go b/parser/frontmatter.go index 284d3f955..a42db0cca 100644 --- a/parser/frontmatter.go +++ b/parser/frontmatter.go @@ -19,16 +19,12 @@ import ( "bytes" "encoding/json" "errors" - "fmt" "io" "strings" - "github.com/gohugoio/hugo/helpers" - - "github.com/spf13/cast" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/BurntSushi/toml" - "github.com/chaseadamsio/goorgeous" "gopkg.in/yaml.v2" ) @@ -79,6 +75,82 @@ func InterfaceToConfig(in interface{}, mark rune, w io.Writer) error { } } +func InterfaceToConfig2(in interface{}, format metadecoders.Format, w io.Writer) error { + if in == nil { + return errors.New("input was nil") + } + + switch format { + case metadecoders.YAML: + b, err := yaml.Marshal(in) + if err != nil { + return err + } + + _, err = w.Write(b) + return err + + case metadecoders.TOML: + return toml.NewEncoder(w).Encode(in) + case metadecoders.JSON: + b, err := json.MarshalIndent(in, "", " ") + if err != nil { + return err + } + + _, err = w.Write(b) + if err != nil { + return err + } + + _, err = w.Write([]byte{'\n'}) + return err + + default: + return errors.New("Unsupported Format provided") + } +} + +func InterfaceToFrontMatter2(in interface{}, format metadecoders.Format, w io.Writer) error { + if in == nil { + return errors.New("input was nil") + } + + switch format { + case metadecoders.YAML: + _, err := w.Write([]byte(YAMLDelimUnix)) + if err != nil { + return err + } + + err = InterfaceToConfig2(in, format, w) + if err != nil { + return err + } + + _, err = w.Write([]byte(YAMLDelimUnix)) + return err + + case metadecoders.TOML: + _, err := w.Write([]byte(TOMLDelimUnix)) + if err != nil { + return err + } + + err = InterfaceToConfig2(in, format, w) + + if err != nil { + return err + } + + _, err = w.Write([]byte("\n" + TOMLDelimUnix)) + return err + + default: + return InterfaceToConfig2(in, format, w) + } +} + // InterfaceToFrontMatter encodes a given input into a frontmatter // representation based upon the mark with the appropriate front matter delimiters // surrounding the output, which is written to w. @@ -155,34 +227,6 @@ func FormatSanitize(kind string) string { } } -// DetectFrontMatter detects the type of frontmatter analysing its first character. -func DetectFrontMatter(mark rune) (f *FrontmatterType) { - switch mark { - case '-': - return &FrontmatterType{HandleYAMLMetaData, []byte(YAMLDelim), []byte(YAMLDelim), false} - case '+': - return &FrontmatterType{HandleTOMLMetaData, []byte(TOMLDelim), []byte(TOMLDelim), false} - case '{': - return &FrontmatterType{HandleJSONMetaData, []byte{'{'}, []byte{'}'}, true} - case '#': - return &FrontmatterType{HandleOrgMetaData, []byte("#+"), []byte("\n"), false} - default: - return nil - } -} - -// HandleTOMLMetaData unmarshals TOML-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleTOMLMetaData(datum []byte) (map[string]interface{}, error) { - m := map[string]interface{}{} - datum = removeTOMLIdentifier(datum) - - _, err := toml.Decode(string(datum), &m) - - return m, err - -} - // removeTOMLIdentifier removes, if necessary, beginning and ending TOML // frontmatter delimiters from a byte slice. func removeTOMLIdentifier(datum []byte) []byte { @@ -200,125 +244,3 @@ func removeTOMLIdentifier(datum []byte) []byte { b = bytes.Trim(b, "\r\n") return bytes.TrimSuffix(b, []byte(TOMLDelim)) } - -// HandleYAMLMetaData unmarshals YAML-encoded datum and returns a Go interface -// representing the encoded data structure. -// TODO(bep) 2errors remove these handlers (and hopefully package) -func HandleYAMLMetaData(datum []byte) (map[string]interface{}, error) { - m := map[string]interface{}{} - err := yaml.Unmarshal(datum, &m) - - // To support boolean keys, the `yaml` package unmarshals maps to - // map[interface{}]interface{}. Here we recurse through the result - // and change all maps to map[string]interface{} like we would've - // gotten from `json`. - if err == nil { - for k, v := range m { - if vv, changed := stringifyMapKeys(v); changed { - m[k] = vv - } - } - } - - return m, err -} - -// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleYAMLData(datum []byte) (interface{}, error) { - var m interface{} - err := yaml.Unmarshal(datum, &m) - if err != nil { - return nil, err - } - - // To support boolean keys, the `yaml` package unmarshals maps to - // map[interface{}]interface{}. Here we recurse through the result - // and change all maps to map[string]interface{} like we would've - // gotten from `json`. - if mm, changed := stringifyMapKeys(m); changed { - return mm, nil - } - - return m, nil -} - -// stringifyMapKeys recurses into in and changes all instances of -// map[interface{}]interface{} to map[string]interface{}. This is useful to -// work around the impedence mismatch between JSON and YAML unmarshaling that's -// described here: https://github.com/go-yaml/yaml/issues/139 -// -// Inspired by https://github.com/stripe/stripe-mock, MIT licensed -func stringifyMapKeys(in interface{}) (interface{}, bool) { - switch in := in.(type) { - case []interface{}: - for i, v := range in { - if vv, replaced := stringifyMapKeys(v); replaced { - in[i] = vv - } - } - case map[interface{}]interface{}: - res := make(map[string]interface{}) - var ( - ok bool - err error - ) - for k, v := range in { - var ks string - - if ks, ok = k.(string); !ok { - ks, err = cast.ToStringE(k) - if err != nil { - ks = fmt.Sprintf("%v", k) - } - // TODO(bep) added in Hugo 0.37, remove some time in the future. - helpers.DistinctFeedbackLog.Printf("WARNING: YAML data/frontmatter with keys of type %T is since Hugo 0.37 converted to strings", k) - } - if vv, replaced := stringifyMapKeys(v); replaced { - res[ks] = vv - } else { - res[ks] = v - } - } - return res, true - } - - return nil, false -} - -// HandleJSONMetaData unmarshals JSON-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleJSONMetaData(datum []byte) (map[string]interface{}, error) { - m := make(map[string]interface{}) - - if datum == nil { - // Package json returns on error on nil input. - // Return an empty map to be consistent with our other supported - // formats. - return m, nil - } - - err := json.Unmarshal(datum, &m) - return m, err -} - -// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleJSONData(datum []byte) (interface{}, error) { - if datum == nil { - // Package json returns on error on nil input. - // Return an empty map to be consistent with our other supported - // formats. - return make(map[string]interface{}), nil - } - - var f interface{} - err := json.Unmarshal(datum, &f) - return f, err -} - -// HandleOrgMetaData unmarshals org-mode encoded datum and returns a Go -// interface representing the encoded data structure. -func HandleOrgMetaData(datum []byte) (map[string]interface{}, error) { - return goorgeous.OrgHeaders(datum) -} diff --git a/parser/frontmatter_test.go b/parser/frontmatter_test.go index 7281ca336..d6e6e79c3 100644 --- a/parser/frontmatter_test.go +++ b/parser/frontmatter_test.go @@ -132,116 +132,6 @@ func TestInterfaceToFrontMatter(t *testing.T) { } } -func TestHandleTOMLMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("title = \"test 1\""), map[string]interface{}{"title": "test 1"}, false}, - {[]byte("a = [1, 2, 3]"), map[string]interface{}{"a": []interface{}{int64(1), int64(2), int64(3)}}, false}, - {[]byte("b = [\n[1, 2],\n[3, 4]\n]"), map[string]interface{}{"b": []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, false}, - // errors - {[]byte("z = [\n[1, 2]\n[3, 4]\n]"), nil, true}, - } - - for i, c := range cases { - res, err := HandleTOMLMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - -func TestHandleYAMLMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("title: test 1"), map[string]interface{}{"title": "test 1"}, false}, - {[]byte("a: Easy!\nb:\n c: 2\n d: [3, 4]"), map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}, false}, - {[]byte("a:\n true: 1\n false: 2"), map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}, false}, - // errors - {[]byte("z = not toml"), nil, true}, - } - - for i, c := range cases { - res, err := HandleYAMLMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - -func TestHandleJSONMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("{\"title\": \"test 1\"}"), map[string]interface{}{"title": "test 1"}, false}, - // errors - {[]byte("{noquotes}"), nil, true}, - } - - for i, c := range cases { - res, err := HandleJSONMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - -func TestHandleOrgMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("#+title: test 1\n"), map[string]interface{}{"title": "test 1"}, false}, - } - - for i, c := range cases { - res, err := HandleOrgMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - func TestFormatToLeadRune(t *testing.T) { for i, this := range []struct { kind string @@ -264,41 +154,6 @@ func TestFormatToLeadRune(t *testing.T) { } } -func TestDetectFrontMatter(t *testing.T) { - cases := []struct { - mark rune - want *FrontmatterType - }{ - // funcs are uncomparable, so we ignore FrontmatterType.Parse in these tests - {'-', &FrontmatterType{nil, []byte(YAMLDelim), []byte(YAMLDelim), false}}, - {'+', &FrontmatterType{nil, []byte(TOMLDelim), []byte(TOMLDelim), false}}, - {'{', &FrontmatterType{nil, []byte("{"), []byte("}"), true}}, - {'#', &FrontmatterType{nil, []byte("#+"), []byte("\n"), false}}, - {'$', nil}, - } - - for _, c := range cases { - res := DetectFrontMatter(c.mark) - if res == nil { - if c.want == nil { - continue - } - - t.Fatalf("want %v, got %v", *c.want, res) - } - - if !reflect.DeepEqual(res.markstart, c.want.markstart) { - t.Errorf("markstart mismatch: want %v, got %v", c.want.markstart, res.markstart) - } - if !reflect.DeepEqual(res.markend, c.want.markend) { - t.Errorf("markend mismatch: want %v, got %v", c.want.markend, res.markend) - } - if !reflect.DeepEqual(res.includeMark, c.want.includeMark) { - t.Errorf("includeMark mismatch: want %v, got %v", c.want.includeMark, res.includeMark) - } - } -} - func TestRemoveTOMLIdentifier(t *testing.T) { cases := []struct { input string @@ -321,64 +176,6 @@ func TestRemoveTOMLIdentifier(t *testing.T) { } } -func TestStringifyYAMLMapKeys(t *testing.T) { - cases := []struct { - input interface{} - want interface{} - replaced bool - }{ - { - map[interface{}]interface{}{"a": 1, "b": 2}, - map[string]interface{}{"a": 1, "b": 2}, - true, - }, - { - map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}}, - map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}}, - true, - }, - { - map[interface{}]interface{}{true: 1, "b": false}, - map[string]interface{}{"true": 1, "b": false}, - true, - }, - { - map[interface{}]interface{}{1: "a", 2: "b"}, - map[string]interface{}{"1": "a", "2": "b"}, - true, - }, - { - map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}}, - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - true, - }, - { - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - false, - }, - { - []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}}, - []interface{}{map[string]interface{}{"1": "a", "2": "b"}}, - false, - }, - } - - for i, c := range cases { - res, replaced := stringifyMapKeys(c.input) - - if c.replaced != replaced { - t.Fatalf("[%d] Replaced mismatch: %t", i, replaced) - } - if !c.replaced { - res = c.input - } - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res) - } - } -} - func BenchmarkFrontmatterTags(b *testing.B) { for _, frontmatter := range []string{"JSON", "YAML", "YAML2", "TOML"} { @@ -388,69 +185,6 @@ func BenchmarkFrontmatterTags(b *testing.B) { } } -func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) { - maps := make([]map[interface{}]interface{}, b.N) - for i := 0; i < b.N; i++ { - maps[i] = map[interface{}]interface{}{ - "a": map[interface{}]interface{}{ - "b": 32, - "c": 43, - "d": map[interface{}]interface{}{ - "b": 32, - "c": 43, - }, - }, - "b": []interface{}{"a", "b"}, - "c": "d", - } - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - stringifyMapKeys(maps[i]) - } -} - -func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) { - m := map[string]interface{}{ - "a": map[string]interface{}{ - "b": 32, - "c": 43, - "d": map[string]interface{}{ - "b": 32, - "c": 43, - }, - }, - "b": []interface{}{"a", "b"}, - "c": "d", - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - stringifyMapKeys(m) - } -} - -func BenchmarkStringifyMapKeysIntegers(b *testing.B) { - maps := make([]map[interface{}]interface{}, b.N) - for i := 0; i < b.N; i++ { - maps[i] = map[interface{}]interface{}{ - 1: map[interface{}]interface{}{ - 4: 32, - 5: 43, - 6: map[interface{}]interface{}{ - 7: 32, - 8: 43, - }, - }, - 2: []interface{}{"a", "b"}, - 3: "d", - } - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - stringifyMapKeys(maps[i]) - } -} func doBenchmarkFrontmatter(b *testing.B, fileformat string, numTags int) { yamlTemplate := `--- name: "Tags" diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go index 280361a84..e409b7665 100644 --- a/parser/metadecoders/decoder.go +++ b/parser/metadecoders/decoder.go @@ -15,81 +15,139 @@ package metadecoders import ( "encoding/json" + "fmt" "github.com/BurntSushi/toml" "github.com/chaseadamsio/goorgeous" - "github.com/gohugoio/hugo/parser/pageparser" "github.com/pkg/errors" + "github.com/spf13/cast" yaml "gopkg.in/yaml.v2" ) -type Format string - -const ( - // These are the supported metdata formats in Hugo. Most of these are also - // supported as /data formats. - ORG Format = "org" - JSON Format = "json" - TOML Format = "toml" - YAML Format = "yaml" -) - -// FormatFromFrontMatterType will return empty if not supported. -func FormatFromFrontMatterType(typ pageparser.ItemType) Format { - switch typ { - case pageparser.TypeFrontMatterJSON: - return JSON - case pageparser.TypeFrontMatterORG: - return ORG - case pageparser.TypeFrontMatterTOML: - return TOML - case pageparser.TypeFrontMatterYAML: - return YAML - default: - return "" - } -} - // UnmarshalToMap will unmarshall data in format f into a new map. This is // what's needed for Hugo's front matter decoding. func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) { m := make(map[string]interface{}) - if data == nil { return m, nil } + err := unmarshal(data, f, &m) + + return m, err + +} + +// Unmarshal will unmarshall data in format f into an interface{}. +// This is what's needed for Hugo's /data handling. +func Unmarshal(data []byte, f Format) (interface{}, error) { + if data == nil { + return make(map[string]interface{}), nil + } + var v interface{} + err := unmarshal(data, f, &v) + + return v, err +} + +// unmarshal unmarshals data in format f into v. +func unmarshal(data []byte, f Format, v interface{}) error { + var err error switch f { case ORG: - m, err = goorgeous.OrgHeaders(data) + vv, err := goorgeous.OrgHeaders(data) + if err != nil { + return err + } + switch v.(type) { + case *map[string]interface{}: + *v.(*map[string]interface{}) = vv + default: + *v.(*interface{}) = vv + } case JSON: - err = json.Unmarshal(data, &m) + err = json.Unmarshal(data, v) case TOML: - _, err = toml.Decode(string(data), &m) + err = toml.Unmarshal(data, v) case YAML: - err = yaml.Unmarshal(data, &m) + err = yaml.Unmarshal(data, v) - // To support boolean keys, the `yaml` package unmarshals maps to + // To support boolean keys, the YAML package unmarshals maps to // map[interface{}]interface{}. Here we recurse through the result // and change all maps to map[string]interface{} like we would've // gotten from `json`. - if err == nil { - for k, v := range m { - if vv, changed := stringifyMapKeys(v); changed { - m[k] = vv - } + var ptr interface{} + switch v.(type) { + case *map[string]interface{}: + ptr = *v.(*map[string]interface{}) + case *interface{}: + ptr = *v.(*interface{}) + default: + return errors.Errorf("unknown type %T in YAML unmarshal", v) + } + + if mm, changed := stringifyMapKeys(ptr); changed { + switch v.(type) { + case *map[string]interface{}: + *v.(*map[string]interface{}) = mm.(map[string]interface{}) + case *interface{}: + *v.(*interface{}) = mm } } default: - return nil, errors.Errorf("unmarshal of format %q is not supported", f) + return errors.Errorf("unmarshal of format %q is not supported", f) } - if err != nil { - return nil, errors.Wrapf(err, "unmarshal failed for format %q", f) - } + return err - return m, nil +} + +// stringifyMapKeys recurses into in and changes all instances of +// map[interface{}]interface{} to map[string]interface{}. This is useful to +// work around the impedence mismatch between JSON and YAML unmarshaling that's +// described here: https://github.com/go-yaml/yaml/issues/139 +// +// Inspired by https://github.com/stripe/stripe-mock, MIT licensed +func stringifyMapKeys(in interface{}) (interface{}, bool) { + + switch in := in.(type) { + case []interface{}: + for i, v := range in { + if vv, replaced := stringifyMapKeys(v); replaced { + in[i] = vv + } + } + case map[string]interface{}: + for k, v := range in { + if vv, changed := stringifyMapKeys(v); changed { + in[k] = vv + } + } + case map[interface{}]interface{}: + res := make(map[string]interface{}) + var ( + ok bool + err error + ) + for k, v := range in { + var ks string + + if ks, ok = k.(string); !ok { + ks, err = cast.ToStringE(k) + if err != nil { + ks = fmt.Sprintf("%v", k) + } + } + if vv, replaced := stringifyMapKeys(v); replaced { + res[ks] = vv + } else { + res[ks] = v + } + } + return res, true + } + return nil, false } diff --git a/parser/metadecoders/decoder_test.go b/parser/metadecoders/decoder_test.go new file mode 100644 index 000000000..94cfd5a9a --- /dev/null +++ b/parser/metadecoders/decoder_test.go @@ -0,0 +1,207 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadecoders + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnmarshalToMap(t *testing.T) { + assert := require.New(t) + + expect := map[string]interface{}{"a": "b"} + + for i, test := range []struct { + data string + format Format + expect interface{} + }{ + {`a = "b"`, TOML, expect}, + {`a: "b"`, YAML, expect}, + // Make sure we get all string keys, even for YAML + {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}}, + {"a:\n true: 1\n false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}}, + {`{ "a": "b" }`, JSON, expect}, + {`#+a: b`, ORG, expect}, + // errors + {`a = b`, TOML, false}, + } { + msg := fmt.Sprintf("%d: %s", i, test.format) + m, err := UnmarshalToMap([]byte(test.data), test.format) + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, msg) + } else { + assert.NoError(err, msg) + assert.Equal(test.expect, m, msg) + } + } +} + +func TestUnmarshalToInterface(t *testing.T) { + assert := require.New(t) + + expect := map[string]interface{}{"a": "b"} + + for i, test := range []struct { + data string + format Format + expect interface{} + }{ + {`[ "Brecker", "Blake", "Redman" ]`, JSON, []interface{}{"Brecker", "Blake", "Redman"}}, + {`{ "a": "b" }`, JSON, expect}, + {`#+a: b`, ORG, expect}, + {`a = "b"`, TOML, expect}, + {`a: "b"`, YAML, expect}, + {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}}, + // errors + {`a = "`, TOML, false}, + } { + msg := fmt.Sprintf("%d: %s", i, test.format) + m, err := Unmarshal([]byte(test.data), test.format) + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, msg) + } else { + assert.NoError(err, msg) + assert.Equal(test.expect, m, msg) + } + + } + +} + +func TestStringifyYAMLMapKeys(t *testing.T) { + cases := []struct { + input interface{} + want interface{} + replaced bool + }{ + { + map[interface{}]interface{}{"a": 1, "b": 2}, + map[string]interface{}{"a": 1, "b": 2}, + true, + }, + { + map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}}, + map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}}, + true, + }, + { + map[interface{}]interface{}{true: 1, "b": false}, + map[string]interface{}{"true": 1, "b": false}, + true, + }, + { + map[interface{}]interface{}{1: "a", 2: "b"}, + map[string]interface{}{"1": "a", "2": "b"}, + true, + }, + { + map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}}, + map[string]interface{}{"a": map[string]interface{}{"b": 1}}, + true, + }, + { + map[string]interface{}{"a": map[string]interface{}{"b": 1}}, + map[string]interface{}{"a": map[string]interface{}{"b": 1}}, + false, + }, + { + []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}}, + []interface{}{map[string]interface{}{"1": "a", "2": "b"}}, + false, + }, + } + + for i, c := range cases { + res, replaced := stringifyMapKeys(c.input) + + if c.replaced != replaced { + t.Fatalf("[%d] Replaced mismatch: %t", i, replaced) + } + if !c.replaced { + res = c.input + } + if !reflect.DeepEqual(res, c.want) { + t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res) + } + } +} + +func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) { + maps := make([]map[interface{}]interface{}, b.N) + for i := 0; i < b.N; i++ { + maps[i] = map[interface{}]interface{}{ + "a": map[interface{}]interface{}{ + "b": 32, + "c": 43, + "d": map[interface{}]interface{}{ + "b": 32, + "c": 43, + }, + }, + "b": []interface{}{"a", "b"}, + "c": "d", + } + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + stringifyMapKeys(maps[i]) + } +} + +func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) { + m := map[string]interface{}{ + "a": map[string]interface{}{ + "b": 32, + "c": 43, + "d": map[string]interface{}{ + "b": 32, + "c": 43, + }, + }, + "b": []interface{}{"a", "b"}, + "c": "d", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + stringifyMapKeys(m) + } +} + +func BenchmarkStringifyMapKeysIntegers(b *testing.B) { + maps := make([]map[interface{}]interface{}, b.N) + for i := 0; i < b.N; i++ { + maps[i] = map[interface{}]interface{}{ + 1: map[interface{}]interface{}{ + 4: 32, + 5: 43, + 6: map[interface{}]interface{}{ + 7: 32, + 8: 43, + }, + }, + 2: []interface{}{"a", "b"}, + 3: "d", + } + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + stringifyMapKeys(maps[i]) + } +} diff --git a/parser/metadecoders/format.go b/parser/metadecoders/format.go new file mode 100644 index 000000000..b9f7f6919 --- /dev/null +++ b/parser/metadecoders/format.go @@ -0,0 +1,66 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadecoders + +import ( + "strings" + + "github.com/gohugoio/hugo/parser/pageparser" +) + +type Format string + +const ( + // These are the supported metdata formats in Hugo. Most of these are also + // supported as /data formats. + ORG Format = "org" + JSON Format = "json" + TOML Format = "toml" + YAML Format = "yaml" +) + +// FormatFromString turns formatStr, typically a file extension without any ".", +// into a Format. It returns an empty string for unknown formats. +func FormatFromString(formatStr string) Format { + formatStr = strings.ToLower(formatStr) + switch formatStr { + case "yaml", "yml": + return YAML + case "json": + return JSON + case "toml": + return TOML + case "org": + return ORG + } + + return "" + +} + +// FormatFromFrontMatterType will return empty if not supported. +func FormatFromFrontMatterType(typ pageparser.ItemType) Format { + switch typ { + case pageparser.TypeFrontMatterJSON: + return JSON + case pageparser.TypeFrontMatterORG: + return ORG + case pageparser.TypeFrontMatterTOML: + return TOML + case pageparser.TypeFrontMatterYAML: + return YAML + default: + return "" + } +} diff --git a/parser/metadecoders/format_test.go b/parser/metadecoders/format_test.go new file mode 100644 index 000000000..46b4e434a --- /dev/null +++ b/parser/metadecoders/format_test.go @@ -0,0 +1,57 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadecoders + +import ( + "fmt" + "testing" + + "github.com/gohugoio/hugo/parser/pageparser" + + "github.com/stretchr/testify/require" +) + +func TestFormatFromString(t *testing.T) { + assert := require.New(t) + for i, test := range []struct { + s string + expect Format + }{ + {"json", JSON}, + {"yaml", YAML}, + {"yml", YAML}, + {"toml", TOML}, + {"tOMl", TOML}, + {"org", ORG}, + {"foo", ""}, + } { + assert.Equal(test.expect, FormatFromString(test.s), fmt.Sprintf("t%d", i)) + } +} + +func TestFormatFromFrontMatterType(t *testing.T) { + assert := require.New(t) + for i, test := range []struct { + typ pageparser.ItemType + expect Format + }{ + {pageparser.TypeFrontMatterJSON, JSON}, + {pageparser.TypeFrontMatterTOML, TOML}, + {pageparser.TypeFrontMatterYAML, YAML}, + {pageparser.TypeFrontMatterORG, ORG}, + {pageparser.TypeIgnore, ""}, + } { + assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i)) + } +} diff --git a/parser/metadecoders/json.go b/parser/metadecoders/json.go deleted file mode 100644 index 21ca8a3b9..000000000 --- a/parser/metadecoders/json.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metadecoders - -import "encoding/json" - -// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleJSONData(datum []byte) (interface{}, error) { - if datum == nil { - // Package json returns on error on nil input. - // Return an empty map to be consistent with our other supported - // formats. - return make(map[string]interface{}), nil - } - - var f interface{} - err := json.Unmarshal(datum, &f) - return f, err -} diff --git a/parser/metadecoders/yaml.go b/parser/metadecoders/yaml.go deleted file mode 100644 index 21b23a9fd..000000000 --- a/parser/metadecoders/yaml.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The metadecoders package contains functions to decode metadata (e.g. page front matter) -// from different formats: TOML, YAML, JSON. -package metadecoders - -import ( - "fmt" - - "github.com/spf13/cast" - yaml "gopkg.in/yaml.v2" -) - -// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleYAMLData(datum []byte) (interface{}, error) { - var m interface{} - err := yaml.Unmarshal(datum, &m) - if err != nil { - return nil, err - } - - // To support boolean keys, the `yaml` package unmarshals maps to - // map[interface{}]interface{}. Here we recurse through the result - // and change all maps to map[string]interface{} like we would've - // gotten from `json`. - if mm, changed := stringifyMapKeys(m); changed { - return mm, nil - } - - return m, nil -} - -// stringifyMapKeys recurses into in and changes all instances of -// map[interface{}]interface{} to map[string]interface{}. This is useful to -// work around the impedence mismatch between JSON and YAML unmarshaling that's -// described here: https://github.com/go-yaml/yaml/issues/139 -// -// Inspired by https://github.com/stripe/stripe-mock, MIT licensed -func stringifyMapKeys(in interface{}) (interface{}, bool) { - switch in := in.(type) { - case []interface{}: - for i, v := range in { - if vv, replaced := stringifyMapKeys(v); replaced { - in[i] = vv - } - } - case map[interface{}]interface{}: - res := make(map[string]interface{}) - var ( - ok bool - err error - ) - for k, v := range in { - var ks string - - if ks, ok = k.(string); !ok { - ks, err = cast.ToStringE(k) - if err != nil { - ks = fmt.Sprintf("%v", k) - } - } - if vv, replaced := stringifyMapKeys(v); replaced { - res[ks] = vv - } else { - res[ks] = v - } - } - return res, true - } - - return nil, false -} diff --git a/parser/page.go b/parser/page.go index 84a59f186..68a542669 100644 --- a/parser/page.go +++ b/parser/page.go @@ -101,15 +101,8 @@ func (p *page) IsRenderable() bool { // Metadata returns the unmarshalled frontmatter data. func (p *page) Metadata() (meta map[string]interface{}, err error) { - frontmatter := p.FrontMatter() - if len(frontmatter) != 0 { - fm := DetectFrontMatter(rune(frontmatter[0])) - if fm != nil { - meta, err = fm.Parse(frontmatter) - } - } - return + return nil, nil } // ReadFrom reads the content from an io.Reader and constructs a page. diff --git a/parser/page_test.go b/parser/page_test.go index 07d7660d4..0bfe2c257 100644 --- a/parser/page_test.go +++ b/parser/page_test.go @@ -1,130 +1 @@ package parser - -import ( - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPage(t *testing.T) { - cases := []struct { - raw string - - content string - frontmatter string - renderable bool - metadata map[string]interface{} - }{ - { - testPageLeader + jsonPageFrontMatter + "\n" + testPageTrailer + jsonPageContent, - jsonPageContent, - jsonPageFrontMatter, - true, - map[string]interface{}{ - "title": "JSON Test 1", - "social": []interface{}{ - []interface{}{"a", "#"}, - []interface{}{"b", "#"}, - }, - }, - }, - { - testPageLeader + tomlPageFrontMatter + testPageTrailer + tomlPageContent, - tomlPageContent, - tomlPageFrontMatter, - true, - map[string]interface{}{ - "title": "TOML Test 1", - "social": []interface{}{ - []interface{}{"a", "#"}, - []interface{}{"b", "#"}, - }, - }, - }, - { - testPageLeader + yamlPageFrontMatter + testPageTrailer + yamlPageContent, - yamlPageContent, - yamlPageFrontMatter, - true, - map[string]interface{}{ - "title": "YAML Test 1", - "social": []interface{}{ - []interface{}{"a", "#"}, - []interface{}{"b", "#"}, - }, - }, - }, - { - testPageLeader + orgPageFrontMatter + orgPageContent, - orgPageContent, - orgPageFrontMatter, - true, - map[string]interface{}{ - "TITLE": "Org Test 1", - "categories": []string{"a", "b"}, - }, - }, - } - - for i, c := range cases { - p := pageMust(ReadFrom(strings.NewReader(c.raw))) - meta, err := p.Metadata() - - mesg := fmt.Sprintf("[%d]", i) - - require.Nil(t, err, mesg) - assert.Equal(t, c.content, string(p.Content()), mesg+" content") - assert.Equal(t, c.frontmatter, string(p.FrontMatter()), mesg+" frontmatter") - assert.Equal(t, c.renderable, p.IsRenderable(), mesg+" renderable") - assert.Equal(t, c.metadata, meta, mesg+" metadata") - } -} - -var ( - testWhitespace = "\t\t\n\n" - testPageLeader = "\ufeff" + testWhitespace + "\n" - - jsonPageContent = "# JSON Test\n" - jsonPageFrontMatter = `{ - "title": "JSON Test 1", - "social": [ - ["a", "#"], - ["b", "#"] - ] -}` - - tomlPageContent = "# TOML Test\n" - tomlPageFrontMatter = `+++ -title = "TOML Test 1" -social = [ - ["a", "#"], - ["b", "#"], -] -+++ -` - - yamlPageContent = "# YAML Test\n" - yamlPageFrontMatter = `--- -title: YAML Test 1 -social: - - - "a" - - "#" - - - "b" - - "#" ---- -` - - orgPageContent = "* Org Test\n" - orgPageFrontMatter = `#+TITLE: Org Test 1 -#+categories: a b -` - - pageHTMLComment = ` -` -) diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go index afc3b5fab..c6f6c3f38 100644 --- a/parser/pageparser/item.go +++ b/parser/pageparser/item.go @@ -20,7 +20,7 @@ import ( type Item struct { Type ItemType - pos pos + Pos Pos Val []byte } diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index a6a26016b..d3fc11bf2 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -25,7 +25,7 @@ import ( ) // position (in bytes) -type pos int +type Pos int const eof = -1 @@ -47,9 +47,9 @@ type pageLexer struct { input []byte stateStart stateFunc state stateFunc - pos pos // input position - start pos // item start position - width pos // width of last element + pos Pos // input position + start Pos // item start position + width Pos // width of last element // Set when we have parsed any summary divider summaryDividerChecked bool @@ -73,7 +73,7 @@ func (l *pageLexer) Input() []byte { // note: the input position here is normally 0 (start), but // can be set if position of first shortcode is known // TODO(bep) 2errors byte -func newPageLexer(input []byte, inputPosition pos, stateStart stateFunc) *pageLexer { +func newPageLexer(input []byte, inputPosition Pos, stateStart stateFunc) *pageLexer { lexer := &pageLexer{ input: input, pos: inputPosition, @@ -131,7 +131,7 @@ func (l *pageLexer) next() rune { } runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:]) - l.width = pos(runeWidth) + l.width = Pos(runeWidth) l.pos += l.width return runeValue } @@ -210,7 +210,7 @@ func lexMainSection(l *pageLexer) stateFunc { l3 = l.index(leftDelimSc) skip := minPositiveIndex(l1, l2, l3) if skip > 0 { - l.pos += pos(skip) + l.pos += Pos(skip) } for { @@ -234,7 +234,7 @@ func lexMainSection(l *pageLexer) stateFunc { l.emit(tText) } l.summaryDividerChecked = true - l.pos += pos(len(summaryDivider)) + l.pos += Pos(len(summaryDivider)) //l.consumeCRLF() l.emit(TypeLeadSummaryDivider) } else if l.hasPrefix(summaryDividerOrg) { @@ -242,7 +242,7 @@ func lexMainSection(l *pageLexer) stateFunc { l.emit(tText) } l.summaryDividerChecked = true - l.pos += pos(len(summaryDividerOrg)) + l.pos += Pos(len(summaryDividerOrg)) //l.consumeCRLF() l.emit(TypeSummaryDividerOrg) } @@ -291,12 +291,12 @@ LOOP: if right == -1 { return l.errorf("starting HTML comment with no end") } - l.pos += pos(right) + pos(len(htmlCOmmentEnd)) + l.pos += Pos(right) + Pos(len(htmlCOmmentEnd)) l.emit(TypeHTMLComment) } else { // Not need to look further. Hugo treats this as plain HTML, // no front matter, no shortcodes, no nothing. - l.pos = pos(len(l.input)) + l.pos = Pos(len(l.input)) l.emit(TypeHTMLDocument) } } @@ -434,7 +434,7 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string, } func lexShortcodeLeftDelim(l *pageLexer) stateFunc { - l.pos += pos(len(l.currentLeftShortcodeDelim())) + l.pos += Pos(len(l.currentLeftShortcodeDelim())) if l.hasPrefix(leftComment) { return lexShortcodeComment } @@ -451,20 +451,20 @@ func lexShortcodeComment(l *pageLexer) stateFunc { } // we emit all as text, except the comment markers l.emit(tText) - l.pos += pos(len(leftComment)) + l.pos += Pos(len(leftComment)) l.ignore() - l.pos += pos(posRightComment - len(leftComment)) + l.pos += Pos(posRightComment - len(leftComment)) l.emit(tText) - l.pos += pos(len(rightComment)) + l.pos += Pos(len(rightComment)) l.ignore() - l.pos += pos(len(l.currentRightShortcodeDelim())) + l.pos += Pos(len(l.currentRightShortcodeDelim())) l.emit(tText) return lexMainSection } func lexShortcodeRightDelim(l *pageLexer) stateFunc { l.closingState = 0 - l.pos += pos(len(l.currentRightShortcodeDelim())) + l.pos += Pos(len(l.currentRightShortcodeDelim())) l.emit(l.currentRightShortcodeDelimItem()) return lexMainSection } diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go index bc6f55dd8..0d32c0e89 100644 --- a/parser/pageparser/pageparser.go +++ b/parser/pageparser/pageparser.go @@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) { } func parseMainSection(input []byte, from int) Result { - lexer := newPageLexer(input, pos(from), lexMainSection) // TODO(bep) 2errors + lexer := newPageLexer(input, Pos(from), lexMainSection) // TODO(bep) 2errors lexer.run() return lexer } @@ -57,7 +57,7 @@ func parseMainSection(input []byte, from int) Result { // if needed. type Iterator struct { l *pageLexer - lastPos pos // position of the last item returned by nextItem + lastPos Pos // position of the last item returned by nextItem } // consumes and returns the next item @@ -69,7 +69,7 @@ func (t *Iterator) Next() Item { var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")} func (t *Iterator) current() Item { - if t.lastPos >= pos(len(t.l.items)) { + if t.lastPos >= Pos(len(t.l.items)) { return errIndexOutOfBounds } return t.l.items[t.lastPos] @@ -98,7 +98,7 @@ func (t *Iterator) Peek() Item { // PeekWalk will feed the next items in the iterator to walkFn // until it returns false. func (t *Iterator) PeekWalk(walkFn func(item Item) bool) { - for i := t.lastPos + 1; i < pos(len(t.l.items)); i++ { + for i := t.lastPos + 1; i < Pos(len(t.l.items)); i++ { item := t.l.items[i] if !walkFn(item) { break @@ -120,5 +120,5 @@ func (t *Iterator) Consume(cnt int) { // LineNumber returns the current line number. Used for logging. func (t *Iterator) LineNumber() int { - return bytes.Count(t.l.input[:t.current().pos], lf) + 1 + return bytes.Count(t.l.input[:t.current().Pos], lf) + 1 } diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go index 850254ac7..1a8c2d237 100644 --- a/parser/pageparser/pageparser_intro_test.go +++ b/parser/pageparser/pageparser_intro_test.go @@ -59,9 +59,7 @@ var frontMatterTests = []lexerTest{ {"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}}, {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}}, {"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, "\n"), tstSomeText, tstEOF}}, - {"YAML commented out front matter", "\nSome text.\n", []Item{nti(TypeHTMLComment, ""), tstSomeText, tstEOF}}, - // Note that we keep all bytes as they are, but we need to handle CRLF {"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}}, {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}}, -- cgit v1.2.3