From 8aa7257f652ebea70dfbb819c301800f5c25b567 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Tue, 30 Nov 2021 11:49:51 +0100 Subject: Add remote support to resources.Get Closes #5255 Supports #9044 --- resources/resource_factories/create/create.go | 181 +++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 2 deletions(-) (limited to 'resources') diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 97a3fc0c4..4832b3cbe 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -16,28 +16,47 @@ package create import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "mime" + "net/http" + "net/url" "path" "path/filepath" "strings" + "time" "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + + "github.com/pkg/errors" ) // Client contains methods to create Resource objects. // tasks to Resource objects. type Client struct { - rs *resources.Spec + rs *resources.Spec + httpClient *http.Client } // New creates a new Client with the given specification. func New(rs *resources.Spec) *Client { - return &Client{rs: rs} + return &Client{ + rs: rs, + httpClient: &http.Client{ + Timeout: 10 * time.Second, + }, + } } // Get creates a new Resource by opening the given filename in the assets filesystem. @@ -126,3 +145,161 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro }) }) } + +// FromRemote expects one or n-parts of a URL to a resource +// If you provide multiple parts they will be joined together to the final URL. +func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) { + rURL, err := url.Parse(uri) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri) + } + + resourceID := helpers.HashString(uri, options) + + return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(resourceID), func() (resource.Resource, error) { + method, reqBody, err := getMethodAndBody(options) + if err != nil { + return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri) + } + + req, err := http.NewRequest(method, uri, reqBody) + if err != nil { + return nil, errors.Wrapf(err, "failed to create request for resource %s", uri) + } + addDefaultHeaders(req) + + if _, ok := options["headers"]; ok { + headers, err := maps.ToStringMapE(options["headers"]) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri) + } + addUserProvidedHeaders(headers, req) + } + res, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode)) + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) + } + + filename := path.Base(rURL.Path) + if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { + if _, ok := params["filename"]; ok { + filename = params["filename"] + } + } + + var contentType string + if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { + contentType = arr[0] + } + + // If content type was not determined by header, look for a file extention + if contentType == "" { + if ext := path.Ext(filename); ext != "" { + contentType = ext + } + } + + // If content type was not determined by header or file extention, try using content itself + if contentType == "" { + if ct := http.DetectContentType(body); ct != "application/octet-stream" { + if arr, _ := mime.ExtensionsByType(ct); arr != nil { + contentType = arr[0] + } + } + } + + resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType + + return c.rs.New( + resources.ResourceSourceDescriptor{ + Fs: c.rs.FileCaches.AssetsCache().Fs, + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil + }, + RelTargetFilename: filepath.Clean(resourceID), + }) + }) +} + +func addDefaultHeaders(req *http.Request, accepts ...string) { + for _, accept := range accepts { + if !hasHeaderValue(req.Header, "Accept", accept) { + req.Header.Add("Accept", accept) + } + } + if !hasHeaderKey(req.Header, "User-Agent") { + req.Header.Add("User-Agent", "Hugo Static Site Generator") + } +} + +func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) { + if headers == nil { + return + } + for key, val := range headers { + vals := types.ToStringSlicePreserveString(val) + for _, s := range vals { + req.Header.Add(key, s) + } + } +} + +func hasHeaderValue(m http.Header, key, value string) bool { + var s []string + var ok bool + + if s, ok = m[key]; !ok { + return false + } + + for _, v := range s { + if v == value { + return true + } + } + return false +} + +func hasHeaderKey(m http.Header, key string) bool { + _, ok := m[key] + return ok +} + +func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) { + if options == nil { + return "GET", nil, nil + } + + if method, ok := options["method"].(string); ok { + method = strings.ToUpper(method) + switch method { + case "GET", "DELETE", "HEAD", "OPTIONS": + return method, nil, nil + case "POST", "PUT", "PATCH": + var body []byte + if _, ok := options["body"]; ok { + switch b := options["body"].(type) { + case string: + body = []byte(b) + case []byte: + body = b + } + } + return method, bytes.NewBuffer(body), nil + } + + return "", nil, fmt.Errorf("invalid HTTP method %q", method) + } + + return "GET", nil, nil +} -- cgit v1.2.3