From 66753416b5ec0f9f1be588a935d5551dfb5eebb9 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Thu, 2 Dec 2021 12:56:25 +0100 Subject: Make resources.Get use a file cache for remote resources Closes #9228 --- resources/resource_factories/create/create.go | 97 ++++++++++++++++----------- 1 file changed, 58 insertions(+), 39 deletions(-) (limited to 'resources') diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 4ad3da448..3f66642d6 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -16,12 +16,14 @@ package create import ( + "bufio" "bytes" "fmt" "io" "io/ioutil" "mime" "net/http" + "net/http/httputil" "net/url" "path" "path/filepath" @@ -32,6 +34,7 @@ import ( "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" @@ -45,8 +48,9 @@ import ( // Client contains methods to create Resource objects. // tasks to Resource objects. type Client struct { - rs *resources.Spec - httpClient *http.Client + rs *resources.Spec + httpClient *http.Client + cacheGetResource *filecache.Cache } // New creates a new Client with the given specification. @@ -56,6 +60,7 @@ func New(rs *resources.Spec) *Client { httpClient: &http.Client{ Timeout: 10 * time.Second, }, + cacheGetResource: rs.FileCaches.GetResourceCache(), } } @@ -156,10 +161,7 @@ func (c *Client) FromRemote(uri string, options map[string]interface{}) (resourc resourceID := helpers.HashString(uri, options) - // This caches to memory and will, in server mode, not be evicted unless the resourceID changes - // or the server restarts. - // There is ongoing work to improve this. - return c.rs.ResourceCache.GetOrCreate(resourceID, func() (resource.Resource, error) { + _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) { method, reqBody, err := getMethodAndBody(options) if err != nil { return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri) @@ -187,51 +189,68 @@ func (c *Client) FromRemote(uri string, options map[string]interface{}) (resourc return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode)) } - body, err := ioutil.ReadAll(res.Body) + httpResponse, err := httputil.DumpResponse(res, true) if err != nil { - return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) + return nil, err } - filename := path.Base(rURL.Path) - if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { - if _, ok := params["filename"]; ok { - filename = params["filename"] - } - } + return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil + }) + if err != nil { + return nil, err + } + defer httpResponse.Close() + + res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil) + if err != nil { + return nil, err + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) + } - var contentType string - if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { - contentType = arr[0] + filename := path.Base(rURL.Path) + if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { + if _, ok := params["filename"]; ok { + filename = params["filename"] } + } - // If content type was not determined by header, look for a file extention - if contentType == "" { - if ext := path.Ext(filename); ext != "" { - contentType = ext - } + var contentType string + if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { + contentType = arr[0] + } + + // If content type was not determined by header, look for a file extention + if contentType == "" { + if ext := path.Ext(filename); ext != "" { + contentType = ext } + } - // If content type was not determined by header or file extention, try using content itself - if contentType == "" { - if ct := http.DetectContentType(body); ct != "application/octet-stream" { - if arr, _ := mime.ExtensionsByType(ct); arr != nil { - contentType = arr[0] - } + // If content type was not determined by header or file extention, try using content itself + if contentType == "" { + if ct := http.DetectContentType(body); ct != "application/octet-stream" { + if arr, _ := mime.ExtensionsByType(ct); arr != nil { + contentType = arr[0] } } + } - resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType + resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType + + return c.rs.New( + resources.ResourceSourceDescriptor{ + Fs: c.rs.FileCaches.AssetsCache().Fs, + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil + }, + RelTargetFilename: filepath.Clean(resourceID), + }) - return c.rs.New( - resources.ResourceSourceDescriptor{ - Fs: c.rs.FileCaches.AssetsCache().Fs, - LazyPublish: true, - OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { - return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil - }, - RelTargetFilename: filepath.Clean(resourceID), - }) - }) } func addDefaultHeaders(req *http.Request, accepts ...string) { -- cgit v1.2.3