path: root/tpl/data/resources.go

// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package data

import (
	"bytes"
	"encoding/csv"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/spf13/afero"
	"github.com/spf13/hugo/config"
	"github.com/spf13/hugo/helpers"
	jww "github.com/spf13/jwalterweatherman"
)

var (
	remoteURLLock = &remoteLock{m: make(map[string]*sync.Mutex)}
	resSleep      = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying
	resRetries    = 1               // number of retries to load the JSON from URL or local file system
	resCacheMu    sync.RWMutex
)

type remoteLock struct {
	sync.RWMutex
	m map[string]*sync.Mutex
}

// URLLock locks an URL during download
func (l *remoteLock) URLLock(url string) {
	var (
		lock *sync.Mutex
		ok   bool
	)
	l.Lock()
	if lock, ok = l.m[url]; !ok {
		lock = &sync.Mutex{}
		l.m[url] = lock
	}
	l.Unlock()
	lock.Lock()
}

// URLUnlock unlocks an URL when the download has been finished. Use only in defer calls.
func (l *remoteLock) URLUnlock(url string) {
	l.RLock()
	defer l.RUnlock()
	if um, ok := l.m[url]; ok {
		um.Unlock()
	}
}

// getCacheFileID returns the cache ID for a string.
func getCacheFileID(cfg config.Provider, id string) string {
	return cfg.GetString("cacheDir") + url.QueryEscape(id)
}

// getCache returns the content for an ID from the file cache or an error.
// If the ID is not found, return nil,nil.
func getCache(id string, fs afero.Fs, cfg config.Provider, ignoreCache bool) ([]byte, error) {
	if ignoreCache {
		return nil, nil
	}

	resCacheMu.RLock()
	defer resCacheMu.RUnlock()

	fID := getCacheFileID(cfg, id)
	isExists, err := helpers.Exists(fID, fs)
	if err != nil {
		return nil, err
	}
	if !isExists {
		return nil, nil
	}

	return afero.ReadFile(fs, fID)
}

// writeCache writes bytes associated with an ID into the file cache.
func writeCache(id string, c []byte, fs afero.Fs, cfg config.Provider, ignoreCache bool) error {
	if ignoreCache {
		return nil
	}

	resCacheMu.Lock()
	defer resCacheMu.Unlock()

	fID := getCacheFileID(cfg, id)
	f, err := fs.Create(fID)
	if err != nil {
		return errors.New("Error: " + err.Error() + ". Failed to create file: " + fID)
	}
	defer f.Close()

	n, err := f.Write(c)
	if err != nil {
		return errors.New("Error: " + err.Error() + ". Failed to write to file: " + fID)
	}
	if n == 0 {
		return errors.New("No bytes written to file: " + fID)
	}
	return nil
}

func deleteCache(id string, fs afero.Fs, cfg config.Provider) error {
	return fs.Remove(getCacheFileID(cfg, id))
}

// getRemote loads the content of a remote file. This method is thread safe.
func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) {
	c, err := getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
	if c != nil && err == nil {
		return c, nil
	}
	if err != nil {
		return nil, err
	}

	// avoid race condition with locks, block other goroutines if the current url is processing
	remoteURLLock.URLLock(url)
	defer func() { remoteURLLock.URLUnlock(url) }()

	// avoid multiple locks due to calling getCache twice
	c, err = getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
	if c != nil && err == nil {
		return c, nil
	}
	if err != nil {
		return nil, err
	}

	jww.INFO.Printf("Downloading: %s ...", url)
	res, err := hc.Get(url)
	if err != nil {
		return nil, err
	}
	c, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return nil, err
	}
	err = writeCache(url, c, fs, cfg, cfg.GetBool("ignoreCache"))
	if err != nil {
		return nil, err
	}
	jww.INFO.Printf("... and cached to: %s", getCacheFileID(cfg, url))
	return c, nil
}

// getLocal loads the content of a local file
func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
	filename := filepath.Join(cfg.GetString("workingDir"), url)
	if e, err := helpers.Exists(filename, fs); !e {
		return nil, err
	}

	return afero.ReadFile(fs, filename)

}

// getResource loads the content of a local or remote file
func (ns *Namespace) getResource(url string) ([]byte, error) {
	if url == "" {
		return nil, nil
	}
	if strings.Contains(url, "://") {
		return getRemote(url, ns.deps.Fs.Source, ns.deps.Cfg, http.DefaultClient)
	}
	return getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg)
}

// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
// If you provide multiple parts they will be joined together to the final URL.
// GetJSON returns nil or parsed JSON to use in a short code.
func (ns *Namespace) GetJSON(urlParts ...string) interface{} {
	var v interface{}
	url := strings.Join(urlParts, "")

	for i := 0; i <= resRetries; i++ {
		c, err := ns.getResource(url)
		if err != nil {
			jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err