From 12a65e76df9470d9563b91a22969ddb41b7c19aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Tue, 30 Jun 2020 16:11:05 +0200 Subject: Add openapi3.Unmarshal Fixes #7442 Fixes #7443 --- common/types/convert.go | 17 +++++-- common/types/convert_test.go | 9 ++++ go.mod | 4 +- go.sum | 6 +++ hugolib/openapi_test.go | 69 ++++++++++++++++++++++++++ parser/metadecoders/decoder.go | 24 ++++----- resources/resource/resourcetypes.go | 6 +++ tpl/openapi/openapi3/init.go | 42 ++++++++++++++++ tpl/openapi/openapi3/openapi3.go | 97 +++++++++++++++++++++++++++++++++++++ tpl/tplimpl/template_funcs.go | 1 + tpl/transform/unmarshal.go | 17 +++---- tpl/transform/unmarshal_test.go | 2 +- 12 files changed, 268 insertions(+), 26 deletions(-) create mode 100644 hugolib/openapi_test.go create mode 100644 tpl/openapi/openapi3/init.go create mode 100644 tpl/openapi/openapi3/openapi3.go diff --git a/common/types/convert.go b/common/types/convert.go index 24e01c273..137029a0e 100644 --- a/common/types/convert.go +++ b/common/types/convert.go @@ -14,6 +14,7 @@ package types import ( + "encoding/json" "html/template" "github.com/spf13/cast" @@ -59,10 +60,20 @@ func TypeToString(v interface{}) (string, bool) { // ToString converts v to a string. func ToString(v interface{}) string { + s, _ := ToStringE(v) + return s +} + +// ToStringE converts v to a string. +func ToStringE(v interface{}) (string, error) { if s, ok := TypeToString(v); ok { - return s + return s, nil } - return cast.ToString(v) - + switch s := v.(type) { + case json.RawMessage: + return string(s), nil + default: + return cast.ToStringE(v) + } } diff --git a/common/types/convert_test.go b/common/types/convert_test.go index 7f86f4c8a..8a4f04db2 100644 --- a/common/types/convert_test.go +++ b/common/types/convert_test.go @@ -14,6 +14,7 @@ package types import ( + "encoding/json" "testing" qt "github.com/frankban/quicktest" @@ -27,3 +28,11 @@ func TestToStringSlicePreserveString(t *testing.T) { c.Assert(ToStringSlicePreserveString(nil), qt.IsNil) } + +func TestToString(t *testing.T) { + c := qt.New(t) + + c.Assert(ToString([]byte("Hugo")), qt.Equals, "Hugo") + c.Assert(ToString(json.RawMessage("Hugo")), qt.Equals, "Hugo") + +} diff --git a/go.mod b/go.mod index ad6ecda86..3969f67fc 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,8 @@ require ( github.com/fortytw2/leaktest v1.3.0 github.com/frankban/quicktest v1.7.2 github.com/fsnotify/fsnotify v1.4.7 + github.com/getkin/kin-openapi v0.14.0 + github.com/ghodss/yaml v1.0.0 github.com/gobwas/glob v0.2.3 github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93 @@ -65,7 +67,7 @@ require ( google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/ini.v1 v1.51.1 // indirect - gopkg.in/yaml.v2 v2.2.7 + gopkg.in/yaml.v2 v2.3.0 ) replace github.com/markbates/inflect => github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 diff --git a/go.sum b/go.sum index 98f010ece..7549b1822 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,9 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getkin/kin-openapi v0.14.0 h1:hqwQL7kze/adt0wB+0UJR2nJm+gfUHqM0Gu4D8nByVc= +github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -352,6 +355,7 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdewolff/minify/v2 v2.6.1 h1:UJLhbs2Q/iDrqA79EEyKE48uYHeAMPVdiUzdtKsatJ8= @@ -550,6 +554,8 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hugolib/openapi_test.go b/hugolib/openapi_test.go new file mode 100644 index 000000000..82f080311 --- /dev/null +++ b/hugolib/openapi_test.go @@ -0,0 +1,69 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "strings" + "testing" +) + +func TestOpenAPI3(t *testing.T) { + const openapi3Yaml = `openapi: 3.0.0 +info: + title: Sample API + description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML. + version: 0.1.9 +servers: + - url: http://api.example.com/v1 + description: Optional server description, e.g. Main (production) server + - url: http://staging-api.example.com + description: Optional server description, e.g. Internal staging server for testing +paths: + /users: + get: + summary: Returns a list of users. + description: Optional extended description in CommonMark or HTML. + responses: + '200': # status code + description: A JSON array of user names + content: + application/json: + schema: + type: array + items: + type: string +` + + b := newTestSitesBuilder(t).Running() + b.WithSourceFile("assets/api/myapi.yaml", openapi3Yaml) + + b.WithTemplatesAdded("index.html", ` +{{ $api := resources.Get "api/myapi.yaml" | openapi3.Unmarshal }} + +API: {{ $api.Info.Title | safeHTML }} + + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `API: Sample API`) + + b.EditFiles("assets/api/myapi.yaml", strings.Replace(openapi3Yaml, "Sample API", "Hugo API", -1)) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `API: Hugo API`) + +} diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go index 1a4a57076..2624ad16f 100644 --- a/parser/metadecoders/decoder.go +++ b/parser/metadecoders/decoder.go @@ -63,7 +63,7 @@ func (d Decoder) UnmarshalToMap(data []byte, f Format) (map[string]interface{}, return m, nil } - err := d.unmarshal(data, f, &m) + err := d.UnmarshalTo(data, f, &m) return m, err } @@ -122,13 +122,13 @@ func (d Decoder) Unmarshal(data []byte, f Format) (interface{}, error) { } var v interface{} - err := d.unmarshal(data, f, &v) + err := d.UnmarshalTo(data, f, &v) return v, err } -// unmarshal unmarshals data in format f into v. -func (d Decoder) unmarshal(data []byte, f Format, v interface{}) error { +// UnmarshalTo unmarshals data in format f into v. +func (d Decoder) UnmarshalTo(data []byte, f Format, v interface{}) error { var err error @@ -156,15 +156,17 @@ func (d Decoder) unmarshal(data []byte, f Format, v interface{}) error { case *interface{}: ptr = *v.(*interface{}) default: - return errors.Errorf("unknown type %T in YAML unmarshal", v) + // Not a map. } - if mm, changed := stringifyMapKeys(ptr); changed { - switch v.(type) { - case *map[string]interface{}: - *v.(*map[string]interface{}) = mm.(map[string]interface{}) - case *interface{}: - *v.(*interface{}) = mm + if ptr != nil { + if mm, changed := stringifyMapKeys(ptr); changed { + switch v.(type) { + case *map[string]interface{}: + *v.(*map[string]interface{}) = mm.(map[string]interface{}) + case *interface{}: + *v.(*interface{}) = mm + } } } case CSV: diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 62431c06c..13ffc5ae3 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -173,6 +173,12 @@ type TranslationKeyProvider interface { TranslationKey() string } +// UnmarshableResource represents a Resource that can be unmarshaled to some other format. +type UnmarshableResource interface { + ReadSeekCloserResource + Identifier +} + type resourceTypesHolder struct { mediaType media.Type resourceType string diff --git a/tpl/openapi/openapi3/init.go b/tpl/openapi/openapi3/init.go new file mode 100644 index 000000000..1e1a4ae05 --- /dev/null +++ b/tpl/openapi/openapi3/init.go @@ -0,0 +1,42 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openapi3 + +import ( + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/tpl/internal" +) + +const name = "openapi3" + +func init() { + f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { + ctx := New(d) + + ns := &internal.TemplateFuncsNamespace{ + Name: name, + Context: func(args ...interface{}) interface{} { return ctx }, + } + + ns.AddMethodMapping(ctx.Unmarshal, + nil, + [][2]string{}, + ) + + return ns + + } + + internal.AddTemplateFuncsNamespace(f) +} diff --git a/tpl/openapi/openapi3/openapi3.go b/tpl/openapi/openapi3/openapi3.go new file mode 100644 index 000000000..7dfd2f6a7 --- /dev/null +++ b/tpl/openapi/openapi3/openapi3.go @@ -0,0 +1,97 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openapi3 + +import ( + "io/ioutil" + + gyaml "github.com/ghodss/yaml" + + "github.com/pkg/errors" + + kopenapi3 "github.com/getkin/kin-openapi/openapi3" + "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/resources/resource" +) + +// New returns a new instance of the openapi3-namespaced template functions. +func New(deps *deps.Deps) *Namespace { + // TODO1 consolidate when merging that "other branch" -- but be aware of the keys. + cache := namedmemcache.New() + deps.BuildStartListeners.Add( + func() { + cache.Clear() + }) + + return &Namespace{ + cache: cache, + deps: deps, + } +} + +// Namespace provides template functions for the "openapi3". +type Namespace struct { + cache *namedmemcache.Cache + deps *deps.Deps +} + +func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagger, error) { + + key := r.Key() + if key == "" { + return nil, errors.New("no Key set in Resource") + } + + v, err := ns.cache.GetOrCreate(key, func() (interface{}, error) { + f := metadecoders.FormatFromMediaType(r.MediaType()) + if f == "" { + return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + } + + reader, err := r.ReadSeekCloser() + if err != nil { + return nil, err + } + defer reader.Close() + + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + s := &kopenapi3.Swagger{} + switch f { + case metadecoders.YAML: + err = gyaml.Unmarshal(b, s) + default: + err = metadecoders.Default.UnmarshalTo(b, f, s) + } + if err != nil { + return nil, err + } + + err = kopenapi3.NewSwaggerLoader().ResolveRefsIn(s, nil) + + return s, err + }) + + if err != nil { + return nil, err + } + + return v.(*kopenapi3.Swagger), nil + +} diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index ccf33d8ba..9141de3f1 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -44,6 +44,7 @@ import ( _ "github.com/gohugoio/hugo/tpl/inflect" _ "github.com/gohugoio/hugo/tpl/lang" _ "github.com/gohugoio/hugo/tpl/math" + _ "github.com/gohugoio/hugo/tpl/openapi/openapi3" _ "github.com/gohugoio/hugo/tpl/os" _ "github.com/gohugoio/hugo/tpl/partials" _ "github.com/gohugoio/hugo/tpl/path" diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index da06b6aa1..b606c870a 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -17,17 +17,20 @@ import ( "io/ioutil" "strings" + "github.com/gohugoio/hugo/resources/resource" + + "github.com/gohugoio/hugo/common/types" + "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/parser/metadecoders" - "github.com/gohugoio/hugo/resources/resource" "github.com/pkg/errors" "github.com/spf13/cast" ) -// Unmarshal unmarshals the data given, which can be either a string +// Unmarshal unmarshals the data given, which can be either a string, json.RawMessage // or a Resource. Supported formats are JSON, TOML, YAML, and CSV. // You can optionally provide an options map as the first argument. func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { @@ -55,7 +58,7 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { } } - if r, ok := data.(unmarshableResource); ok { + if r, ok := data.(resource.UnmarshableResource); ok { key := r.Key() if key == "" { @@ -87,7 +90,7 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { }) } - dataStr, err := cast.ToStringE(data) + dataStr, err := types.ToStringE(data) if err != nil { return nil, errors.Errorf("type %T not supported", data) } @@ -104,12 +107,6 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { }) } -// All the relevant resources implements this interface. -type unmarshableResource interface { - resource.ReadSeekCloserResource - resource.Identifier -} - func decodeDecoder(m map[string]interface{}) (metadecoders.Decoder, error) { opts := metadecoders.Default diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 7b0caa07f..183bdefd5 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -20,11 +20,11 @@ import ( "testing" "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/media" qt "github.com/frankban/quicktest" - "github.com/gohugoio/hugo/resources/resource" "github.com/spf13/viper" ) -- cgit v1.2.3