diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2021-12-16 11:09:21 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2021-12-17 09:33:51 +0100 |
commit | 22ef5da20d1685dfe6aff3bd9364c9b1f1d0d8f8 (patch) | |
tree | ffa7339b033795291dafdd9b1bc3f23b64f55d38 /resources | |
parent | 5758c370eac6c4460cd6bb34d4475c8d347585f6 (diff) |
Add resources.GetRemote
In Hugo 0.89 we added remote support to `resources.Get`.
In hindsight that was not a great idea, as a poll from many Hugo users showed. See Issue #9285 for more details.
After this commit `resources.Get` only supports local resource lookups. If you want to support both, you need to use a construct similar to:
Also improve some option case handling.
```
{{ resource := "" }}
{{ if (urls.Parse $url).IsAbs }}
{{ $resource = resources.GetRemote $url }}
{{ else }}
{{ $resource = resources.Get $url }}
{{ end }}
```
Fixes #9285
Fixes #9296
Diffstat (limited to 'resources')
-rw-r--r-- | resources/resource_factories/create/create.go | 213 | ||||
-rw-r--r-- | resources/resource_factories/create/remote.go | 230 | ||||
-rw-r--r-- | resources/resource_factories/create/remote_test.go | 85 |
3 files changed, 315 insertions, 213 deletions
diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index f7d0efe64..f8e7e18db 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -16,15 +16,7 @@ package create import ( - "bufio" - "bytes" - "fmt" - "io" - "io/ioutil" - "mime" "net/http" - "net/http/httputil" - "net/url" "path" "path/filepath" "strings" @@ -36,13 +28,8 @@ import ( "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hugio" - "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/common/types" - "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" - - "github.com/pkg/errors" ) // Client contains methods to create Resource objects. @@ -150,203 +137,3 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro }) }) } - -// FromRemote expects one or n-parts of a URL to a resource -// If you provide multiple parts they will be joined together to the final URL. -func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) { - if err := c.validateFromRemoteArgs(uri, options); err != nil { - return nil, err - } - rURL, err := url.Parse(uri) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri) - } - - resourceID := helpers.HashString(uri, options) - - _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) { - method, reqBody, err := getMethodAndBody(options) - if err != nil { - return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri) - } - - req, err := http.NewRequest(method, uri, reqBody) - if err != nil { - return nil, errors.Wrapf(err, "failed to create request for resource %s", uri) - } - addDefaultHeaders(req) - - if _, ok := options["headers"]; ok { - headers, err := maps.ToStringMapE(options["headers"]) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri) - } - addUserProvidedHeaders(headers, req) - } - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - - if res.StatusCode != http.StatusNotFound { - if res.StatusCode < 200 || res.StatusCode > 299 { - return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode)) - } - } - - httpResponse, err := httputil.DumpResponse(res, true) - if err != nil { - return nil, err - } - - return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil - }) - if err != nil { - return nil, err - } - defer httpResponse.Close() - - res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil) - if err != nil { - return nil, err - } - - if res.StatusCode == http.StatusNotFound { - // Not found. This matches how looksup for local resources work. - return nil, nil - } - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) - } - - filename := path.Base(rURL.Path) - if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { - if _, ok := params["filename"]; ok { - filename = params["filename"] - } - } - - var extension string - if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { - extension = arr[0] - } - - // If extension was not determined by header, look for a file extention - if extension == "" { - if ext := path.Ext(filename); ext != "" { - extension = ext - } - } - - // If extension was not determined by header or file extention, try using content itself - if extension == "" { - if ct := http.DetectContentType(body); ct != "application/octet-stream" { - if ct == "image/jpeg" { - extension = ".jpg" - } else if arr, _ := mime.ExtensionsByType(ct); arr != nil { - extension = arr[0] - } - } - } - - resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension - - return c.rs.New( - resources.ResourceSourceDescriptor{ - LazyPublish: true, - OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { - return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil - }, - RelTargetFilename: filepath.Clean(resourceID), - }) - -} - -func (c *Client) validateFromRemoteArgs(uri string, options map[string]interface{}) error { - if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil { - return err - } - - if method, ok := options["method"].(string); ok { - if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(method); err != nil { - return err - } - } - return nil -} - -func addDefaultHeaders(req *http.Request, accepts ...string) { - for _, accept := range accepts { - if !hasHeaderValue(req.Header, "Accept", accept) { - req.Header.Add("Accept", accept) - } - } - if !hasHeaderKey(req.Header, "User-Agent") { - req.Header.Add("User-Agent", "Hugo Static Site Generator") - } -} - -func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) { - if headers == nil { - return - } - for key, val := range headers { - vals := types.ToStringSlicePreserveString(val) - for _, s := range vals { - req.Header.Add(key, s) - } - } -} - -func hasHeaderValue(m http.Header, key, value string) bool { - var s []string - var ok bool - - if s, ok = m[key]; !ok { - return false - } - - for _, v := range s { - if v == value { - return true - } - } - return false -} - -func hasHeaderKey(m http.Header, key string) bool { - _, ok := m[key] - return ok -} - -func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) { - if options == nil { - return "GET", nil, nil - } - - if method, ok := options["method"].(string); ok { - method = strings.ToUpper(method) - switch method { - case "GET", "DELETE", "HEAD", "OPTIONS": - return method, nil, nil - case "POST", "PUT", "PATCH": - var body []byte - if _, ok := options["body"]; ok { - switch b := options["body"].(type) { - case string: - body = []byte(b) - case []byte: - body = b - } - } - return method, bytes.NewBuffer(body), nil - } - - return "", nil, fmt.Errorf("invalid HTTP method %q", method) - } - - return "GET", nil, nil -} diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go new file mode 100644 index 000000000..53e77bc5e --- /dev/null +++ b/resources/resource_factories/create/remote.go @@ -0,0 +1,230 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package create + +import ( + "bufio" + "bytes" + "io" + "io/ioutil" + "mime" + "net/http" + "net/http/httputil" + "net/url" + "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +// FromRemote expects one or n-parts of a URL to a resource +// If you provide multiple parts they will be joined together to the final URL. +func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resource.Resource, error) { + rURL, err := url.Parse(uri) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri) + } + + resourceID := helpers.HashString(uri, optionsm) + + _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) { + options, err := decodeRemoteOptions(optionsm) + if err != nil { + return nil, errors.Wrapf(err, "failed to decode options for resource %s", uri) + } + if err := c.validateFromRemoteArgs(uri, options); err != nil { + return nil, err + } + + req, err := http.NewRequest(options.Method, uri, options.BodyReader()) + if err != nil { + return nil, errors.Wrapf(err, "failed to create request for resource %s", uri) + } + addDefaultHeaders(req) + + if options.Headers != nil { + addUserProvidedHeaders(options.Headers, req) + } + + res, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusNotFound { + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, errors.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)) + } + } + + httpResponse, err := httputil.DumpResponse(res, true) + if err != nil { + return nil, err + } + + return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil + }) + if err != nil { + return nil, err + } + defer httpResponse.Close() + + res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil) + if err != nil { + return nil, err + } + + if res.StatusCode == http.StatusNotFound { + // Not found. This matches how looksup for local resources work. + return nil, nil + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) + } + + filename := path.Base(rURL.Path) + if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { + if _, ok := params["filename"]; ok { + filename = params["filename"] + } + } + + var extension string + if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { + extension = arr[0] + } + + // If extension was not determined by header, look for a file extention + if extension == "" { + if ext := path.Ext(filename); ext != "" { + extension = ext + } + } + + // If extension was not determined by header or file extention, try using content itself + if extension == "" { + if ct := http.DetectContentType(body); ct != "application/octet-stream" { + if ct == "image/jpeg" { + extension = ".jpg" + } else if arr, _ := mime.ExtensionsByType(ct); arr != nil { + extension = arr[0] + } + } + } + + resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension + + return c.rs.New( + resources.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil + }, + RelTargetFilename: filepath.Clean(resourceID), + }) + +} + +func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error { + if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil { + return err + } + + if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(options.Method); err != nil { + return err + } + + return nil +} + +func addDefaultHeaders(req *http.Request, accepts ...string) { + for _, accept := range accepts { + if !hasHeaderValue(req.Header, "Accept", accept) { + req.Header.Add("Accept", accept) + } + } + if !hasHeaderKey(req.Header, "User-Agent") { + req.Header.Add("User-Agent", "Hugo Static Site Generator") + } +} + +func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) { + if headers == nil { + return + } + for key, val := range headers { + vals := types.ToStringSlicePreserveString(val) + for _, s := range vals { + req.Header.Add(key, s) + } + } +} + +func hasHeaderValue(m http.Header, key, value string) bool { + var s []string + var ok bool + + if s, ok = m[key]; !ok { + return false + } + + for _, v := range s { + if v == value { + return true + } + } + return false +} + +func hasHeaderKey(m http.Header, key string) bool { + _, ok := m[key] + return ok +} + +type fromRemoteOptions struct { + Method string + Headers map[string]interface{} + Body []byte +} + +func (o fromRemoteOptions) BodyReader() io.Reader { + if o.Body == nil { + return nil + } + return bytes.NewBuffer(o.Body) +} + +func decodeRemoteOptions(optionsm map[string]interface{}) (fromRemoteOptions, error) { + var options = fromRemoteOptions{ + Method: "GET", + } + + err := mapstructure.WeakDecode(optionsm, &options) + if err != nil { + return options, err + } + options.Method = strings.ToUpper(options.Method) + + return options, nil + +} diff --git a/resources/resource_factories/create/remote_test.go b/resources/resource_factories/create/remote_test.go new file mode 100644 index 000000000..42e3fc890 --- /dev/null +++ b/resources/resource_factories/create/remote_test.go @@ -0,0 +1,85 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package create + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestDecodeRemoteOptions(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + name string + args map[string]interface{} + want fromRemoteOptions + wantErr bool + }{ + { + "POST", + map[string]interface{}{ + "meThod": "PoST", + "headers": map[string]interface{}{ + "foo": "bar", + }, + }, + fromRemoteOptions{ + Method: "POST", + Headers: map[string]interface{}{ + "foo": "bar", + }, + }, + false, + }, + { + "Body", + map[string]interface{}{ + "meThod": "POST", + "body": []byte("foo"), + }, + fromRemoteOptions{ + Method: "POST", + Body: []byte("foo"), + }, + false, + }, + { + "Body, string", + map[string]interface{}{ + "meThod": "POST", + "body": "foo", + }, + fromRemoteOptions{ + Method: "POST", + Body: []byte("foo"), + }, + false, + }, + } { + c.Run(test.name, func(c *qt.C) { + got, err := decodeRemoteOptions(test.args) + isErr := qt.IsNil + if test.wantErr { + isErr = qt.IsNotNil + } + + c.Assert(err, isErr) + c.Assert(got, qt.DeepEquals, test.want) + }) + + } + +} |