Make Page an interface

The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue  #5074, "pages from other data sources".

But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.

Most notable changes:

* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
  This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
  This improves the taxonomy template `.Title` situation and make common template constructs much simpler.

See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778

3 files changed, 418 insertions, 0 deletions

diff --git a/lazy/init.go b/lazy/init.go
new file mode 100644
index 000000000..5c1bee609
--- /dev/null
+++ b/lazy/init.go
@@ -0,0 +1,199 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lazy
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// New creates a new empty Init.
+func New() *Init {
+	return &Init{}
+}
+
+// Init holds a graph of lazily initialized dependencies.
+type Init struct {
+	mu sync.Mutex
+
+	prev     *Init
+	children []*Init
+
+	init onceMore
+	out  interface{}
+	err  error
+	f    func() (interface{}, error)
+}
+
+// Add adds a func as a new child dependency.
+func (ini *Init) Add(initFn func() (interface{}, error)) *Init {
+	if ini == nil {
+		ini = New()
+	}
+	return ini.add(false, initFn)
+}
+
+// AddWithTimeout is same as Add, but with a timeout that aborts initialization.
+func (ini *Init) AddWithTimeout(timeout time.Duration, f func(ctx context.Context) (interface{}, error)) *Init {
+	return ini.Add(func() (interface{}, error) {
+		return ini.withTimeout(timeout, f)
+	})
+}
+
+// Branch creates a new dependency branch based on an existing and adds
+// the given dependency as a child.
+func (ini *Init) Branch(initFn func() (interface{}, error)) *Init {
+	if ini == nil {
+		ini = New()
+	}
+	return ini.add(true, initFn)
+}
+
+// BranchdWithTimeout is same as Branch, but with a timeout.
+func (ini *Init) BranchdWithTimeout(timeout time.Duration, f func(ctx context.Context) (interface{}, error)) *Init {
+	return ini.Branch(func() (interface{}, error) {
+		return ini.withTimeout(timeout, f)
+	})
+}
+
+// Do initializes the entire dependency graph.
+func (ini *Init) Do() (interface{}, error) {
+	if ini == nil {
+		panic("init is nil")
+	}
+
+	ini.init.Do(func() {
+		var (
+			dependencies []*Init
+			children     []*Init
+		)
+
+		prev := ini.prev
+		for prev != nil {
+			if prev.shouldInitialize() {
+				dependencies = append(dependencies, prev)
+			}
+			prev = prev.prev
+		}
+
+		for _, child := range ini.children {
+			if child.shouldInitialize() {
+				children = append(children, child)
+			}
+		}
+
+		for _, dep := range dependencies {
+			_, err := dep.Do()
+			if err != nil {
+				ini.err = err
+				return
+			}
+		}
+
+		if ini.f != nil {
+			ini.out, ini.err = ini.f()
+		}
+
+		for _, dep := range children {
+			_, err := dep.Do()
+			if err != nil {
+				ini.err = err
+				return
+			}
+		}
+
+	})
+
+	var counter time.Duration
+	for !ini.init.Done() {
+		counter += 10
+		if counter > 600000000 {
+			panic("BUG: timed out in lazy init")
+		}
+		time.Sleep(counter * time.Microsecond)
+	}
+
+	return ini.out, ini.err
+}
+
+func (ini *Init) shouldInitialize() bool {
+	return !(ini == nil || ini.init.Done() || ini.init.InProgress())
+}
+
+// Reset resets the current and all its dependencies.
+func (ini *Init) Reset() {
+	mu := ini.init.ResetWithLock()
+	defer mu.Unlock()
+	for _, d := range ini.children {
+		d.Reset()
+	}
+}
+
+func (ini *Init) add(branch bool, initFn func() (interface{}, error)) *Init {
+	ini.mu.Lock()
+	defer ini.mu.Unlock()
+
+	if !branch {
+		ini.checkDone()
+	}
+
+	init := &Init{
+		f:    initFn,
+		prev: ini,
+	}
+
+	if !branch {
+		ini.children = append(ini.children, init)
+	}
+
+	return init
+}
+
+func (ini *Init) checkDone() {
+	if ini.init.Done() {
+		panic("init cannot be added to after it has run")
+	}
+}
+
+func (ini *Init) withTimeout(timeout time.Duration, f func(ctx context.Context) (interface{}, error)) (interface{}, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	c := make(chan verr, 1)
+
+	go func() {
+		v, err := f(ctx)
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			c <- verr{v: v, err: err}
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		return nil, errors.New("timed out initializing value. This is most likely a circular loop in a shortcode")
+	case ve := <-c:
+		return ve.v, ve.err
+	}
+
+}
+
+type verr struct {
+	v   interface{}
+	err error
+}
diff --git a/lazy/init_test.go b/lazy/init_test.go
new file mode 100644
index 000000000..bcb57acb3
--- /dev/null
+++ b/lazy/init_test.go
@@ -0,0 +1,150 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lazy
+
+import (
+	"context"
+	"errors"
+	"math/rand"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestInit(t *testing.T) {
+	assert := require.New(t)
+
+	var result string
+
+	bigOrSmall := func() int {
+		if rand.Intn(10) < 3 {
+			return 10000 + rand.Intn(100000)
+		}
+		return 1 + rand.Intn(50)
+	}
+
+	f1 := func(name string) func() (interface{}, error) {
+		return func() (interface{}, error) {
+			result += name + "|"
+			size := bigOrSmall()
+			_ = strings.Repeat("Hugo Rocks! ", size)
+			return name, nil
+		}
+	}
+
+	f2 := func() func() (interface{}, error) {
+		return func() (interface{}, error) {
+			size := bigOrSmall()
+			_ = strings.Repeat("Hugo Rocks! ", size)
+			return size, nil
+		}
+	}
+
+	root := New()
+
+	root.Add(f1("root(1)"))
+	root.Add(f1("root(2)"))
+
+	branch1 := root.Branch(f1("branch_1"))
+	branch1.Add(f1("branch_1_1"))
+	branch1_2 := branch1.Add(f1("branch_1_2"))
+	branch1_2_1 := branch1_2.Add(f1("branch_1_2_1"))
+
+	var wg sync.WaitGroup
+
+	// Add some concurrency and randomness to verify thread safety and
+	// init order.
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			var err error
+			if rand.Intn(10) < 5 {
+				_, err = root.Do()
+				assert.NoError(err)
+			}
+
+			// Add a new branch on the fly.
+			if rand.Intn(10) > 5 {
+				branch := branch1_2.Branch(f2())
+				init := branch.Add(f2())
+				_, err = init.Do()
+				assert.NoError(err)
+			} else {
+				_, err = branch1_2_1.Do()
+				assert.NoError(err)
+			}
+			_, err = branch1_2.Do()
+			assert.NoError(err)
+
+		}(i)
+
+		wg.Wait()
+
+		assert.Equal("root(1)|root(2)|branch_1|branch_1_1|branch_1_2|branch_1_2_1|", result)
+
+	}
+
+}
+
+func TestInitAddWithTimeout(t *testing.T) {
+	assert := require.New(t)
+
+	init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (interface{}, error) {
+		return nil, nil
+	})
+
+	_, err := init.Do()
+
+	assert.NoError(err)
+}
+
+func TestInitAddWithTimeoutTimeout(t *testing.T) {
+	assert := require.New(t)
+
+	init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (interface{}, error) {
+		time.Sleep(500 * time.Millisecond)
+		select {
+		case <-ctx.Done():
+			return nil, nil
+		default:
+		}
+		t.Fatal("slept")
+		return nil, nil
+	})
+
+	_, err := init.Do()
+
+	assert.Error(err)
+
+	assert.Contains(err.Error(), "timed out")
+
+	time.Sleep(1 * time.Second)
+
+}
+
+func TestInitAddWithTimeoutError(t *testing.T) {
+	assert := require.New(t)
+
+	init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (interface{}, error) {
+		return nil, errors.New("failed")
+	})
+
+	_, err := init.Do()
+
+	assert.Error(err)
+}
diff --git a/lazy/once.go b/lazy/once.go
new file mode 100644
index 000000000..c434bfa0b
--- /dev/null
+++ b/lazy/once.go
@@ -0,0 +1,69 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lazy
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// onceMore is similar to sync.Once.
+//
+// Additional features are:
+// * it can be reset, so the action can be repeated if needed
+// * it has methods to check if it's done or in progress
+//
+type onceMore struct {
+	mu   sync.Mutex
+	lock uint32
+	done uint32
+}
+
+func (t *onceMore) Do(f func()) {
+	if atomic.LoadUint32(&t.done) == 1 {
+		return
+	}
+
+	// f may call this Do and we would get a deadlock.
+	locked := atomic.CompareAndSwapUint32(&t.lock, 0, 1)
+	if !locked {
+		return
+	}
+	defer atomic.StoreUint32(&t.lock, 0)
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	// Double check
+	if t.done == 1 {
+		return
+	}
+	defer atomic.StoreUint32(&t.done, 1)
+	f()
+
+}
+
+func (t *onceMore) InProgress() bool {
+	return atomic.LoadUint32(&t.lock) == 1
+}
+
+func (t *onceMore) Done() bool {
+	return atomic.LoadUint32(&t.done) == 1
+}
+
+func (t *onceMore) ResetWithLock() *sync.Mutex {
+	t.mu.Lock()
+	defer atomic.StoreUint32(&t.done, 0)
+	return &t.mu
+}