Make Page an interface

The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue  #5074, "pages from other data sources".

But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.

Most notable changes:

* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
  This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
  This improves the taxonomy template `.Title` situation and make common template constructs much simpler.

See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778

1 files changed, 63 insertions, 60 deletions

diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
index 64d1ff96a..7de2280c7 100644
--- a/hugolib/testhelpers_test.go
+++ b/hugolib/testhelpers_test.go
@@ -14,11 +14,11 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/gohugoio/hugo/langs"
-	"github.com/sanity-io/litter"
-
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/deps"
+	"github.com/gohugoio/hugo/resources/page"
+	"github.com/sanity-io/litter"
 	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -387,6 +387,7 @@ func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder {
 		}
 	}
 	if err != nil && !shouldFail {
+		herrors.PrintStackTrace(err)
 		s.Fatalf("Build failed: %s", err)
 	} else if err == nil && shouldFail {
 		s.Fatalf("Expected error")
@@ -418,10 +419,10 @@ date: "2018-02-28"
 			"content/sect/doc1.nn.md", contentTemplate,
 		}
 
-		listTemplateCommon = "{{ $p := .Paginator }}{{ $p.PageNumber }}|{{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}"
+		listTemplateCommon = "{{ $p := .Paginator }}{{ $p.PageNumber }}|{{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}|Kind: {{ .Kind }}|Content: {{ .Content }}"
 
 		defaultTemplates = []string{
-			"_default/single.html", "Single: {{ .Title }}|{{ i18n \"hello\" }}|{{.Lang}}|{{ .Content }}",
+			"_default/single.html", "Single: {{ .Title }}|{{ i18n \"hello\" }}|{{.Language.Lang}}|RelPermalink: {{ .RelPermalink }}|Permalink: {{ .Permalink }}|{{ .Content }}|Resources: {{ range .Resources }}{{ .MediaType }}: {{ .RelPermalink}} -- {{ end }}|Summary: {{ .Summary }}|Truncated: {{ .Truncated }}",
 			"_default/list.html", "List Page " + listTemplateCommon,
 			"index.html", "{{ $p := .Paginator }}Default Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{  .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink  }}",
 			"index.fr.html", "{{ $p := .Paginator }}French Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{  .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink  }}",
@@ -432,6 +433,9 @@ date: "2018-02-28"
 			// A shortcode in multiple languages
 			"shortcodes/lingo.html", "LingoDefault",
 			"shortcodes/lingo.fr.html", "LingoFrench",
+			// Special templates
+			"404.html", "404|{{ .Lang }}|{{ .Title }}",
+			"robots.txt", "robots|{{ .Lang }}|{{ .Title }}",
 		}
 
 		defaultI18n = []string{
@@ -469,18 +473,25 @@ func (s *sitesBuilder) Fatalf(format string, args ...interface{}) {
 }
 
 func Fatalf(t testing.TB, format string, args ...interface{}) {
-	trace := trace()
+	trace := stackTrace()
 	format = format + "\n%s"
 	args = append(args, trace)
 	t.Fatalf(format, args...)
 }
 
-func trace() string {
+func stackTrace() string {
 	return strings.Join(assert.CallerInfo(), "\n\r\t\t\t")
 }
 
+func (s *sitesBuilder) AssertFileContentFn(filename string, f func(s string) bool) {
+	content := s.FileContent(filename)
+	if !f(content) {
+		s.Fatalf("Assert failed for %q", filename)
+	}
+}
+
 func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
-	content := readDestination(s.T, s.Fs, filename)
+	content := s.FileContent(filename)
 	for _, match := range matches {
 		if !strings.Contains(content, match) {
 			s.Fatalf("No match for %q in content for %s\n%s\n%q", match, filename, content, content)
@@ -488,6 +499,10 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
 	}
 }
 
+func (s *sitesBuilder) FileContent(filename string) string {
+	return readDestination(s.T, s.Fs, filename)
+}
+
 func (s *sitesBuilder) AssertObject(expected string, object interface{}) {
 	got := s.dumper.Sdump(object)
 	expected = strings.TrimSpace(expected)
@@ -502,7 +517,7 @@ func (s *sitesBuilder) AssertObject(expected string, object interface{}) {
 func (s *sitesBuilder) AssertFileContentRe(filename string, matches ...string) {
 	content := readDestination(s.T, s.Fs, filename)
 	for _, match := range matches {
-		r := regexp.MustCompile(match)
+		r := regexp.MustCompile("(?s)" + match)
 		if !r.MatchString(content) {
 			s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
 		}
@@ -555,32 +570,6 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string {
 	return value
 }
 
-func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *helpers.PathSpec {
-	l := langs.NewDefaultLanguage(v)
-	ps, _ := helpers.NewPathSpec(fs, l)
-	return ps
-}
-
-func newTestDefaultPathSpec(t *testing.T) *helpers.PathSpec {
-	v := viper.New()
-	// Easier to reason about in tests.
-	v.Set("disablePathToLower", true)
-	v.Set("contentDir", "content")
-	v.Set("dataDir", "data")
-	v.Set("i18nDir", "i18n")
-	v.Set("layoutDir", "layouts")
-	v.Set("archetypeDir", "archetypes")
-	v.Set("assetDir", "assets")
-	v.Set("resourceDir", "resources")
-	v.Set("publishDir", "public")
-	fs := hugofs.NewDefault(v)
-	ps, err := helpers.NewPathSpec(fs, v)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return ps
-}
-
 func newTestCfg() (*viper.Viper, *hugofs.Fs) {
 
 	v := viper.New()
@@ -597,27 +586,6 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) {
 
 }
 
-// newTestSite creates a new site in the  English language with in-memory Fs.
-// The site will have a template system loaded and ready to use.
-// Note: This is only used in single site tests.
-func newTestSite(t testing.TB, configKeyValues ...interface{}) *Site {
-
-	cfg, fs := newTestCfg()
-
-	for i := 0; i < len(configKeyValues); i += 2 {
-		cfg.Set(configKeyValues[i].(string), configKeyValues[i+1])
-	}
-
-	d := deps.DepsCfg{Fs: fs, Cfg: cfg}
-
-	s, err := NewSiteForCfg(d)
-
-	if err != nil {
-		Fatalf(t, "Failed to create Site: %s", err)
-	}
-	return s
-}
-
 func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layoutPathContentPairs ...string) (testHelper, *HugoSites) {
 	if len(layoutPathContentPairs)%2 != 0 {
 		Fatalf(t, "Layouts must be provided in pairs")
@@ -696,11 +664,28 @@ func writeSourcesToSource(t *testing.T, base string, fs *hugofs.Fs, sources ...[
 	}
 }
 
-func dumpPages(pages ...*Page) {
+func getPage(in page.Page, ref string) page.Page {
+	p, err := in.GetPage(ref)
+	if err != nil {
+		panic(err)
+	}
+	return p
+}
+
+func dumpPages(pages ...page.Page) {
+	fmt.Println("---------")
 	for i, p := range pages {
-		fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s Len Sections(): %d\n",
+		fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n",
 			i+1,
-			p.Kind, p.title, p.RelPermalink(), p.Path(), p.sections, len(p.Sections()))
+			p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath())
+	}
+}
+
+func dumpSPages(pages ...*pageState) {
+	for i, p := range pages {
+		fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n",
+			i+1,
+			p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath())
 	}
 }
 
@@ -722,8 +707,8 @@ func printStringIndexes(s string) {
 		fmt.Println()
 
 	}
-
 }
+
 func isCI() bool {
 	return os.Getenv("CI") != ""
 }
@@ -731,3 +716,21 @@ func isCI() bool {
 func isGo111() bool {
 	return strings.Contains(runtime.Version(), "1.11")
 }
+
+// See https://github.com/golang/go/issues/19280
+// Not in use.
+var parallelEnabled = true
+
+func parallel(t *testing.T) {
+	if parallelEnabled {
+		t.Parallel()
+	}
+}
+
+// Useful to debug nilpointers/panics in templates.
+// Put "defer recoverStack()" in top of the failing function.
+func recoverStack() {
+	if r := recover(); r != nil {
+		fmt.Println(printStackTrace(1000))
+	}
+}