Make resources.Get use a file cache for remote resources

Closes #9228

1 files changed, 58 insertions, 39 deletions

diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go
index 4ad3da448..3f66642d6 100644
--- a/resources/resource_factories/create/create.go
+++ b/resources/resource_factories/create/create.go
@@ -16,12 +16,14 @@
 package create
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"path"
 	"path/filepath"
@@ -32,6 +34,7 @@ import (
 
 	"github.com/gohugoio/hugo/hugofs"
 
+	"github.com/gohugoio/hugo/cache/filecache"
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/types"
@@ -45,8 +48,9 @@ import (
 // Client contains methods to create Resource objects.
 // tasks to Resource objects.
 type Client struct {
-	rs         *resources.Spec
-	httpClient *http.Client
+	rs               *resources.Spec
+	httpClient       *http.Client
+	cacheGetResource *filecache.Cache
 }
 
 // New creates a new Client with the given specification.
@@ -56,6 +60,7 @@ func New(rs *resources.Spec) *Client {
 		httpClient: &http.Client{
 			Timeout: 10 * time.Second,
 		},
+		cacheGetResource: rs.FileCaches.GetResourceCache(),
 	}
 }
 
@@ -156,10 +161,7 @@ func (c *Client) FromRemote(uri string, options map[string]interface{}) (resourc
 
 	resourceID := helpers.HashString(uri, options)
 
-	// This caches to memory and will, in server mode, not be evicted unless the resourceID changes
-	// or the server restarts.
-	// There is ongoing work to improve this.
-	return c.rs.ResourceCache.GetOrCreate(resourceID, func() (resource.Resource, error) {
+	_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
 		method, reqBody, err := getMethodAndBody(options)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri)
@@ -187,51 +189,68 @@ func (c *Client) FromRemote(uri string, options map[string]interface{}) (resourc
 			return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode))
 		}
 
-		body, err := ioutil.ReadAll(res.Body)
+		httpResponse, err := httputil.DumpResponse(res, true)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
+			return nil, err
 		}
 
-		filename := path.Base(rURL.Path)
-		if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
-			if _, ok := params["filename"]; ok {
-				filename = params["filename"]
-			}
-		}
+		return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer httpResponse.Close()
+
+	res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
+	}
 
-		var contentType string
-		if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
-			contentType = arr[0]
+	filename := path.Base(rURL.Path)
+	if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
+		if _, ok := params["filename"]; ok {
+			filename = params["filename"]
 		}
+	}
 
-		// If content type was not determined by header, look for a file extention
-		if contentType == "" {
-			if ext := path.Ext(filename); ext != "" {
-				contentType = ext
-			}
+	var contentType string
+	if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
+		contentType = arr[0]
+	}
+
+	// If content type was not determined by header, look for a file extention
+	if contentType == "" {
+		if ext := path.Ext(filename); ext != "" {
+			contentType = ext
 		}
+	}
 
-		// If content type was not determined by header or file extention, try using content itself
-		if contentType == "" {
-			if ct := http.DetectContentType(body); ct != "application/octet-stream" {
-				if arr, _ := mime.ExtensionsByType(ct); arr != nil {
-					contentType = arr[0]
-				}
+	// If content type was not determined by header or file extention, try using content itself
+	if contentType == "" {
+		if ct := http.DetectContentType(body); ct != "application/octet-stream" {
+			if arr, _ := mime.ExtensionsByType(ct); arr != nil {
+				contentType = arr[0]
 			}
 		}
+	}
 
-		resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType
+	resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType
+
+	return c.rs.New(
+		resources.ResourceSourceDescriptor{
+			Fs:          c.rs.FileCaches.AssetsCache().Fs,
+			LazyPublish: true,
+			OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+				return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
+			},
+			RelTargetFilename: filepath.Clean(resourceID),
+		})
 
-		return c.rs.New(
-			resources.ResourceSourceDescriptor{
-				Fs:          c.rs.FileCaches.AssetsCache().Fs,
-				LazyPublish: true,
-				OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
-					return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
-				},
-				RelTargetFilename: filepath.Clean(resourceID),
-			})
-	})
 }
 
 func addDefaultHeaders(req *http.Request, accepts ...string) {