Make Page an interface

The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue  #5074, "pages from other data sources".

But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.

Most notable changes:

* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
  This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
  This improves the taxonomy template `.Title` situation and make common template constructs much simpler.

See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778

1 files changed, 90 insertions, 85 deletions

diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index ec5070fa8..214f72c5f 100644
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -1,4 +1,4 @@
-// Copyright 2016-present The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,7 +15,12 @@ package hugolib
 
 import (
 	"bytes"
+	"context"
 	"fmt"
+	"runtime/trace"
+	"sort"
+
+	"github.com/gohugoio/hugo/output"
 
 	"errors"
 
@@ -26,6 +31,9 @@ import (
 // Build builds all sites. If filesystem events are provided,
 // this is considered to be a potential partial rebuild.
 func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
+	ctx, task := trace.NewTask(context.Background(), "Build")
+	defer task.End()
+
 	errCollector := h.StartErrorCollector()
 	errs := make(chan error)
 
@@ -71,22 +79,36 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 					return err
 				}
 			} else {
-				if err := h.init(conf); err != nil {
+				if err := h.initSites(conf); err != nil {
 					return err
 				}
 			}
 
-			if err := h.process(conf, events...); err != nil {
+			var err error
+
+			f := func() {
+				err = h.process(conf, events...)
+			}
+			trace.WithRegion(ctx, "process", f)
+			if err != nil {
 				return err
 			}
 
-			if err := h.assemble(conf); err != nil {
+			f = func() {
+				err = h.assemble(conf)
+			}
+			trace.WithRegion(ctx, "assemble", f)
+			if err != nil {
 				return err
 			}
+
 			return nil
 		}
 
-		prepareErr = prepare()
+		f := func() {
+			prepareErr = prepare()
+		}
+		trace.WithRegion(ctx, "prepare", f)
 		if prepareErr != nil {
 			h.SendError(prepareErr)
 		}
@@ -94,7 +116,12 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 	}
 
 	if prepareErr == nil {
-		if err := h.render(conf); err != nil {
+		var err error
+		f := func() {
+			err = h.render(conf)
+		}
+		trace.WithRegion(ctx, "render", f)
+		if err != nil {
 			h.SendError(err)
 		}
 	}
@@ -120,6 +147,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 		return err
 	}
 
+	if err := h.fatalErrorHandler.getErr(); err != nil {
+		return err
+	}
+
 	errorCount := h.Log.ErrorCounter.Count()
 	if errorCount > 0 {
 		return fmt.Errorf("logged %d error(s)", errorCount)
@@ -132,17 +163,8 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 // Build lifecycle methods below.
 // The order listed matches the order of execution.
 
-func (h *HugoSites) init(config *BuildCfg) error {
-
-	for _, s := range h.Sites {
-		if s.PageCollections == nil {
-			s.PageCollections = newPageCollections()
-		}
-	}
-
-	if config.ResetState {
-		h.reset()
-	}
+func (h *HugoSites) initSites(config *BuildCfg) error {
+	h.reset(config)
 
 	if config.NewConfig != nil {
 		if err := h.createSitesFromConfig(config.NewConfig); err != nil {
@@ -155,28 +177,22 @@ func (h *HugoSites) init(config *BuildCfg) error {
 
 func (h *HugoSites) initRebuild(config *BuildCfg) error {
 	if config.NewConfig != nil {
-		return errors.New("Rebuild does not support 'NewConfig'.")
+		return errors.New("rebuild does not support 'NewConfig'")
 	}
 
 	if config.ResetState {
-		return errors.New("Rebuild does not support 'ResetState'.")
+		return errors.New("rebuild does not support 'ResetState'")
 	}
 
 	if !h.running {
-		return errors.New("Rebuild called when not in watch mode")
-	}
-
-	if config.whatChanged.source {
-		// This is for the non-renderable content pages (rarely used, I guess).
-		// We could maybe detect if this is really needed, but it should be
-		// pretty fast.
-		h.TemplateHandler().RebuildClone()
+		return errors.New("rebuild called when not in watch mode")
 	}
 
 	for _, s := range h.Sites {
 		s.resetBuildState()
 	}
 
+	h.reset(config)
 	h.resetLogs()
 	helpers.InitLoggers()
 
@@ -203,14 +219,6 @@ func (h *HugoSites) process(config *BuildCfg, events ...fsnotify.Event) error {
 }
 
 func (h *HugoSites) assemble(config *BuildCfg) error {
-	if config.whatChanged.source {
-		for _, s := range h.Sites {
-			s.createTaxonomiesEntries()
-		}
-	}
-
-	// TODO(bep) we could probably wait and do this in one go later
-	h.setupTranslations()
 
 	if len(h.Sites) > 1 {
 		// The first is initialized during process; initialize the rest
@@ -221,47 +229,26 @@ func (h *HugoSites) assemble(config *BuildCfg) error {
 		}
 	}
 
+	if err := h.createPageCollections(); err != nil {
+		return err
+	}
+
 	if config.whatChanged.source {
 		for _, s := range h.Sites {
-			if err := s.buildSiteMeta(); err != nil {
+			if err := s.assembleTaxonomies(); err != nil {
 				return err
 			}
 		}
 	}
 
+	// Create pagexs for the section pages etc. without content file.
 	if err := h.createMissingPages(); err != nil {
 		return err
 	}
 
 	for _, s := range h.Sites {
-		for _, pages := range []Pages{s.Pages, s.headlessPages} {
-			for _, p := range pages {
-				// May have been set in front matter
-				if len(p.outputFormats) == 0 {
-					p.outputFormats = s.outputFormats[p.Kind]
-				}
-
-				if p.headless {
-					// headless = 1 output format only
-					p.outputFormats = p.outputFormats[:1]
-				}
-				for _, r := range p.Resources.ByType(pageResourceType) {
-					r.(*Page).outputFormats = p.outputFormats
-				}
-
-				if err := p.initPaths(); err != nil {
-					return err
-				}
-
-			}
-		}
-		s.assembleMenus()
-		s.refreshPageCaches()
 		s.setupSitePages()
-	}
-
-	if err := h.assignMissingTranslations(); err != nil {
-		return err
+		sort.Stable(s.workAllPages)
 	}
 
 	return nil
@@ -269,42 +256,60 @@ func (h *HugoSites) assemble(config *BuildCfg) error {
 }
 
 func (h *HugoSites) render(config *BuildCfg) error {
+	siteRenderContext := &siteRenderContext{cfg: config, multihost: h.multihost}
+
 	if !config.PartialReRender {
+		h.renderFormats = output.Formats{}
 		for _, s := range h.Sites {
 			s.initRenderFormats()
+			h.renderFormats = append(h.renderFormats, s.renderFormats...)
 		}
 	}
 
+	i := 0
 	for _, s := range h.Sites {
-		for i, rf := range s.renderFormats {
-			for _, s2 := range h.Sites {
-				// We render site by site, but since the content is lazily rendered
-				// and a site can "borrow" content from other sites, every site
-				// needs this set.
-				s2.rc = &siteRenderingContext{Format: rf}
-
-				isRenderingSite := s == s2
-
-				if !config.PartialReRender {
-					if err := s2.preparePagesForRender(isRenderingSite && i == 0); err != nil {
-						return err
+		for siteOutIdx, renderFormat := range s.renderFormats {
+			siteRenderContext.outIdx = siteOutIdx
+			siteRenderContext.sitesOutIdx = i
+			i++
+
+			select {
+			case <-h.Done():
+				return nil
+			default:
+				// For the non-renderable pages, we use the content iself as
+				// template and we may have to re-parse and execute it for
+				// each output format.
+				h.TemplateHandler().RebuildClone()
+
+				for _, s2 := range h.Sites {
+					// We render site by site, but since the content is lazily rendered
+					// and a site can "borrow" content from other sites, every site
+					// needs this set.
+					s2.rc = &siteRenderingContext{Format: renderFormat}
+
+					if !config.PartialReRender {
+						if err := s2.preparePagesForRender(siteRenderContext.sitesOutIdx); err != nil {
+							return err
+						}
 					}
 				}
 
-			}
-
-			if !config.SkipRender {
-				if config.PartialReRender {
-					if err := s.renderPages(config); err != nil {
-						return err
-					}
-				} else {
-					if err := s.render(config, i); err != nil {
-						return err
+				if !config.SkipRender {
+					if config.PartialReRender {
+						if err := s.renderPages(siteRenderContext); err != nil {
+							return err
+						}
+					} else {
+						if err := s.render(siteRenderContext); err != nil {
+							return err
+						}
 					}
 				}
 			}
+
 		}
+
 	}
 
 	if !config.SkipRender {