Make Page an interface

The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue  #5074, "pages from other data sources".

But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.

Most notable changes:

* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
  This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
  This improves the taxonomy template `.Title` situation and make common template constructs much simpler.

See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778

1 files changed, 165 insertions, 114 deletions

diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go
index 78325344b..f62ea0905 100644
--- a/hugolib/pagecollections.go
+++ b/hugolib/pagecollections.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,43 +18,65 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/cache"
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/resources/page"
 )
 
+// Used in the page cache to mark more than one hit for a given key.
+var ambiguityFlag = &pageState{}
+
 // PageCollections contains the page collections for a site.
 type PageCollections struct {
-	// Includes only pages of all types, and only pages in the current language.
-	Pages Pages
 
-	// Includes all pages in all languages, including the current one.
-	// Includes pages of all types.
-	AllPages Pages
+	// Includes absolute all pages (of all types), including drafts etc.
+	rawAllPages pageStatePages
 
-	// A convenience cache for the traditional index types, taxonomies, home page etc.
-	// This is for the current language only.
-	indexPages Pages
+	// rawAllPages plus additional pages created during the build process.
+	workAllPages pageStatePages
 
-	// A convenience cache for the regular pages.
-	// This is for the current language only.
-	RegularPages Pages
+	// Includes headless bundles, i.e. bundles that produce no output for its content page.
+	headlessPages pageStatePages
 
-	// A convenience cache for the all the regular pages.
-	AllRegularPages Pages
+	// Lazy initialized page collections
+	pages           *lazyPagesFactory
+	regularPages    *lazyPagesFactory
+	allPages        *lazyPagesFactory
+	allRegularPages *lazyPagesFactory
 
-	// Includes absolute all pages (of all types), including drafts etc.
-	rawAllPages Pages
+	// The index for .Site.GetPage etc.
+	pageIndex *cache.Lazy
+}
 
-	// Includes headless bundles, i.e. bundles that produce no output for its content page.
-	headlessPages Pages
+// Pages returns all pages.
+// This is for the current language only.
+func (c *PageCollections) Pages() page.Pages {
+	return c.pages.get()
+}
 
-	pageIndex *cache.Lazy
+// RegularPages returns all the regular pages.
+// This is for the current language only.
+func (c *PageCollections) RegularPages() page.Pages {
+	return c.regularPages.get()
+}
+
+// AllPages returns all pages for all languages.
+func (c *PageCollections) AllPages() page.Pages {
+	return c.allPages.get()
+}
+
+// AllPages returns all regular pages for all languages.
+func (c *PageCollections) AllRegularPages() page.Pages {
+	return c.allRegularPages.get()
 }
 
 // Get initializes the index if not already done so, then
 // looks up the given page ref, returns nil if no value found.
-func (c *PageCollections) getFromCache(ref string) (*Page, error) {
+func (c *PageCollections) getFromCache(ref string) (page.Page, error) {
 	v, found, err := c.pageIndex.Get(ref)
 	if err != nil {
 		return nil, err
@@ -63,7 +85,7 @@ func (c *PageCollections) getFromCache(ref string) (*Page, error) {
 		return nil, nil
 	}
 
-	p := v.(*Page)
+	p := v.(page.Page)
 
 	if p != ambiguityFlag {
 		return p, nil
@@ -71,17 +93,49 @@ func (c *PageCollections) getFromCache(ref string) (*Page, error) {
 	return nil, fmt.Errorf("page reference %q is ambiguous", ref)
 }
 
-var ambiguityFlag = &Page{Kind: kindUnknown, title: "ambiguity flag"}
+type lazyPagesFactory struct {
+	pages page.Pages
 
-func (c *PageCollections) refreshPageCaches() {
-	c.indexPages = c.findPagesByKindNotIn(KindPage, c.Pages)
-	c.RegularPages = c.findPagesByKindIn(KindPage, c.Pages)
-	c.AllRegularPages = c.findPagesByKindIn(KindPage, c.AllPages)
+	init    sync.Once
+	factory page.PagesFactory
+}
 
-	indexLoader := func() (map[string]interface{}, error) {
+func (l *lazyPagesFactory) get() page.Pages {
+	l.init.Do(func() {
+		l.pages = l.factory()
+	})
+	return l.pages
+}
+
+func newLazyPagesFactory(factory page.PagesFactory) *lazyPagesFactory {
+	return &lazyPagesFactory{factory: factory}
+}
+
+func newPageCollections() *PageCollections {
+	return newPageCollectionsFromPages(nil)
+}
+
+func newPageCollectionsFromPages(pages pageStatePages) *PageCollections {
+
+	c := &PageCollections{rawAllPages: pages}
+
+	c.pages = newLazyPagesFactory(func() page.Pages {
+		pages := make(page.Pages, len(c.workAllPages))
+		for i, p := range c.workAllPages {
+			pages[i] = p
+		}
+		return pages
+	})
+
+	c.regularPages = newLazyPagesFactory(func() page.Pages {
+		return c.findPagesByKindInWorkPages(page.KindPage, c.workAllPages)
+	})
+
+	c.pageIndex = cache.NewLazy(func() (map[string]interface{}, error) {
 		index := make(map[string]interface{})
 
-		add := func(ref string, p *Page) {
+		add := func(ref string, p page.Page) {
+			ref = strings.ToLower(ref)
 			existing := index[ref]
 			if existing == nil {
 				index[ref] = p
@@ -90,71 +144,63 @@ func (c *PageCollections) refreshPageCaches() {
 			}
 		}
 
-		for _, pageCollection := range []Pages{c.RegularPages, c.headlessPages} {
+		for _, pageCollection := range []pageStatePages{c.workAllPages, c.headlessPages} {
 			for _, p := range pageCollection {
-				sourceRef := p.absoluteSourceRef()
+				if p.IsPage() {
+					sourceRef := p.sourceRef()
 
-				if sourceRef != "" {
-					// index the canonical ref
-					// e.g. /section/article.md
-					add(sourceRef, p)
-				}
+					if sourceRef != "" {
+						// index the canonical ref
+						// e.g. /section/article.md
+						add(sourceRef, p)
+					}
+
+					// Ref/Relref supports this potentially ambiguous lookup.
+					add(p.File().LogicalName(), p)
 
-				// Ref/Relref supports this potentially ambiguous lookup.
-				add(p.LogicalName(), p)
+					translationBaseName := p.File().TranslationBaseName()
 
-				translationBaseName := p.TranslationBaseName()
+					dir, _ := path.Split(sourceRef)
+					dir = strings.TrimSuffix(dir, "/")
 
-				dir, _ := path.Split(sourceRef)
-				dir = strings.TrimSuffix(dir, "/")
+					if translationBaseName == "index" {
+						add(dir, p)
+						add(path.Base(dir), p)
+					} else {
+						add(translationBaseName, p)
+					}
 
-				if translationBaseName == "index" {
-					add(dir, p)
-					add(path.Base(dir), p)
+					// We need a way to get to the current language version.
+					pathWithNoExtensions := path.Join(dir, translationBaseName)
+					add(pathWithNoExtensions, p)
 				} else {
-					add(translationBaseName, p)
+					// index the canonical, unambiguous ref for any backing file
+					// e.g. /section/_index.md
+					sourceRef := p.sourceRef()
+					if sourceRef != "" {
+						add(sourceRef, p)
+					}
+
+					ref := p.SectionsPath()
+
+					// index the canonical, unambiguous virtual ref
+					// e.g. /section
+					// (this may already have been indexed above)
+					add("/"+ref, p)
 				}
-
-				// We need a way to get to the current language version.
-				pathWithNoExtensions := path.Join(dir, translationBaseName)
-				add(pathWithNoExtensions, p)
-			}
-		}
-
-		for _, p := range c.indexPages {
-			// index the canonical, unambiguous ref for any backing file
-			// e.g. /section/_index.md
-			sourceRef := p.absoluteSourceRef()
-			if sourceRef != "" {
-				add(sourceRef, p)
 			}
-
-			ref := path.Join(p.sections...)
-
-			// index the canonical, unambiguous virtual ref
-			// e.g. /section
-			// (this may already have been indexed above)
-			add("/"+ref, p)
 		}
 
 		return index, nil
-	}
+	})
 
-	c.pageIndex = cache.NewLazy(indexLoader)
-}
-
-func newPageCollections() *PageCollections {
-	return &PageCollections{}
-}
-
-func newPageCollectionsFromPages(pages Pages) *PageCollections {
-	return &PageCollections{rawAllPages: pages}
+	return c
 }
 
 // This is an adapter func for the old API with Kind as first argument.
 // This is invoked when you do .Site.GetPage. We drop the Kind and fails
 // if there are more than 2 arguments, which would be ambigous.
-func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) {
+func (c *PageCollections) getPageOldVersion(ref ...string) (page.Page, error) {
 	var refs []string
 	for _, r := range ref {
 		// A common construct in the wild is
@@ -173,10 +219,10 @@ func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) {
 		return nil, fmt.Errorf(`too many arguments to .Site.GetPage: %v. Use lookups on the form {{ .Site.GetPage "/posts/mypage-md" }}`, ref)
 	}
 
-	if len(refs) == 0 || refs[0] == KindHome {
+	if len(refs) == 0 || refs[0] == page.KindHome {
 		key = "/"
 	} else if len(refs) == 1 {
-		if len(ref) == 2 && refs[0] == KindSection {
+		if len(ref) == 2 && refs[0] == page.KindSection {
 			// This is an old style reference to the "Home Page section".
 			// Typically fetched via {{ .Site.GetPage "section" .Section }}
 			// See https://github.com/gohugoio/hugo/issues/4989
@@ -197,17 +243,18 @@ func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) {
 }
 
 // 	Only used in tests.
-func (c *PageCollections) getPage(typ string, sections ...string) *Page {
+func (c *PageCollections) getPage(typ string, sections ...string) page.Page {
 	refs := append([]string{typ}, path.Join(sections...))
 	p, _ := c.getPageOldVersion(refs...)
 	return p
 }
 
-// Ref is either unix-style paths (i.e. callers responsible for
-// calling filepath.ToSlash as necessary) or shorthand refs.
-func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) {
+// Case insensitive page lookup.
+func (c *PageCollections) getPageNew(context page.Page, ref string) (page.Page, error) {
 	var anError error
 
+	ref = strings.ToLower(ref)
+
 	// Absolute (content root relative) reference.
 	if strings.HasPrefix(ref, "/") {
 		p, err := c.getFromCache(ref)
@@ -220,7 +267,7 @@ func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) {
 
 	} else if context != nil {
 		// Try the page-relative path.
-		ppath := path.Join("/", strings.Join(context.sections, "/"), ref)
+		ppath := path.Join("/", strings.ToLower(context.SectionsPath()), ref)
 		p, err := c.getFromCache(ppath)
 		if err == nil && p != nil {
 			return p, nil
@@ -236,7 +283,8 @@ func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) {
 		if err == nil && p != nil {
 			if context != nil {
 				// TODO(bep) remove this case and the message below when the storm has passed
-				helpers.DistinctFeedbackLog.Printf(`WARNING: make non-relative ref/relref page reference(s) in page %q absolute, e.g. {{< ref "/blog/my-post.md" >}}`, context.absoluteSourceRef())
+				err := wrapErr(errors.New(`make non-relative ref/relref page reference(s) in page %q absolute, e.g. {{< ref "/blog/my-post.md" >}}`), context)
+				helpers.DistinctWarnLog.Println(err)
 			}
 			return p, nil
 		}
@@ -253,49 +301,56 @@ func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) {
 	}
 
 	if p == nil && anError != nil {
-		if context != nil {
-			return nil, fmt.Errorf("failed to resolve path from page %q: %s", context.absoluteSourceRef(), anError)
-		}
-		return nil, fmt.Errorf("failed to resolve page: %s", anError)
+		return nil, wrapErr(errors.Wrap(anError, "failed to resolve ref"), context)
 	}
 
 	return p, nil
 }
 
-func (*PageCollections) findPagesByKindIn(kind string, inPages Pages) Pages {
-	var pages Pages
+func (*PageCollections) findPagesByKindIn(kind string, inPages page.Pages) page.Pages {
+	var pages page.Pages
 	for _, p := range inPages {
-		if p.Kind == kind {
+		if p.Kind() == kind {
 			pages = append(pages, p)
 		}
 	}
 	return pages
 }
 
-func (*PageCollections) findFirstPageByKindIn(kind string, inPages Pages) *Page {
-	for _, p := range inPages {
-		if p.Kind == kind {
-			return p
+func (c *PageCollections) findPagesByKind(kind string) page.Pages {
+	return c.findPagesByKindIn(kind, c.Pages())
+}
+
+func (c *PageCollections) findWorkPagesByKind(kind string) pageStatePages {
+	var pages pageStatePages
+	for _, p := range c.workAllPages {
+		if p.Kind() == kind {
+			pages = append(pages, p)
 		}
 	}
-	return nil
+	return pages
 }
 
-func (*PageCollections) findPagesByKindNotIn(kind string, inPages Pages) Pages {
-	var pages Pages
+func (*PageCollections) findPagesByKindInWorkPages(kind string, inPages pageStatePages) page.Pages {
+	var pages page.Pages
 	for _, p := range inPages {
-		if p.Kind != kind {
+		if p.Kind() == kind {
 			pages = append(pages, p)
 		}
 	}
 	return pages
 }
 
-func (c *PageCollections) findPagesByKind(kind string) Pages {
-	return c.findPagesByKindIn(kind, c.Pages)
+func (c *PageCollections) findFirstWorkPageByKindIn(kind string) *pageState {
+	for _, p := range c.workAllPages {
+		if p.Kind() == kind {
+			return p
+		}
+	}
+	return nil
 }
 
-func (c *PageCollections) addPage(page *Page) {
+func (c *PageCollections) addPage(page *pageState) {
 	c.rawAllPages = append(c.rawAllPages, page)
 }
 
@@ -307,35 +362,31 @@ func (c *PageCollections) removePageFilename(filename string) {
 
 }
 
-func (c *PageCollections) removePage(page *Page) {
+func (c *PageCollections) removePage(page *pageState) {
 	if i := c.rawAllPages.findPagePos(page); i >= 0 {
 		c.clearResourceCacheForPage(c.rawAllPages[i])
 		c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...)
 	}
-
 }
 
-func (c *PageCollections) findPagesByShortcode(shortcode string) Pages {
-	var pages Pages
-
+func (c *PageCollections) findPagesByShortcode(shortcode string) page.Pages {
+	var pages page.Pages
 	for _, p := range c.rawAllPages {
-		if p.shortcodeState != nil {
-			if _, ok := p.shortcodeState.nameSet[shortcode]; ok {
-				pages = append(pages, p)
-			}
+		if p.HasShortcode(shortcode) {
+			pages = append(pages, p)
 		}
 	}
 	return pages
 }
 
-func (c *PageCollections) replacePage(page *Page) {
+func (c *PageCollections) replacePage(page *pageState) {
 	// will find existing page that matches filepath and remove it
 	c.removePage(page)
 	c.addPage(page)
 }
 
-func (c *PageCollections) clearResourceCacheForPage(page *Page) {
-	if len(page.Resources) > 0 {
-		page.s.ResourceSpec.DeleteCacheByPrefix(page.relTargetPathBase)
+func (c *PageCollections) clearResourceCacheForPage(page *pageState) {
+	if len(page.resources) > 0 {
+		page.s.ResourceSpec.DeleteCacheByPrefix(page.targetPaths().SubResourceBaseTarget)
 	}
 }