Make Page an interface

The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue  #5074, "pages from other data sources".

But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.

Most notable changes:

* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
  This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
  This improves the taxonomy template `.Title` situation and make common template constructs much simpler.

See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778

3 files changed, 39 insertions, 10 deletions

diff --git a/parser/pageparser/itemtype_string.go b/parser/pageparser/itemtype_string.go
new file mode 100644
index 000000000..632afaecc
--- /dev/null
+++ b/parser/pageparser/itemtype_string.go
@@ -0,0 +1,16 @@
+// Code generated by "stringer -type ItemType"; DO NOT EDIT.
+
+package pageparser
+
+import "strconv"
+
+const _ItemType_name = "tErrortEOFTypeHTMLStartTypeLeadSummaryDividerTypeFrontMatterYAMLTypeFrontMatterTOMLTypeFrontMatterJSONTypeFrontMatterORGTypeEmojiTypeIgnoretLeftDelimScNoMarkuptRightDelimScNoMarkuptLeftDelimScWithMarkuptRightDelimScWithMarkuptScClosetScNametScNameInlinetScParamtScParamValtTexttKeywordMarker"
+
+var _ItemType_index = [...]uint16{0, 6, 10, 23, 45, 64, 83, 102, 120, 129, 139, 159, 180, 202, 225, 233, 240, 253, 261, 272, 277, 291}
+
+func (i ItemType) String() string {
+	if i < 0 || i >= ItemType(len(_ItemType_index)-1) {
+		return "ItemType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _ItemType_name[_ItemType_index[i]:_ItemType_index[i+1]]
+}
diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go
index 14b341ee9..db563d44c 100644
--- a/parser/pageparser/pageparser.go
+++ b/parser/pageparser/pageparser.go
@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -36,16 +36,28 @@ type Result interface {
 var _ Result = (*pageLexer)(nil)
 
 // Parse parses the page in the given reader according to the given Config.
+// TODO(bep) now that we have improved the "lazy order" init, it *may* be
+// some potential saving in doing a buffered approach where the first pass does
+// the frontmatter only.
 func Parse(r io.Reader, cfg Config) (Result, error) {
+	return parseSection(r, cfg, lexIntroSection)
+}
+
+// ParseMain parses starting with the main section. Used in tests.
+func ParseMain(r io.Reader, cfg Config) (Result, error) {
+	return parseSection(r, cfg, lexMainSection)
+}
+
+func parseSection(r io.Reader, cfg Config, start stateFunc) (Result, error) {
 	b, err := ioutil.ReadAll(r)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read page content")
 	}
-	return parseBytes(b, cfg)
+	return parseBytes(b, cfg, start)
 }
 
-func parseBytes(b []byte, cfg Config) (Result, error) {
-	lexer := newPageLexer(b, lexIntroSection, cfg)
+func parseBytes(b []byte, cfg Config, start stateFunc) (Result, error) {
+	lexer := newPageLexer(b, start, cfg)
 	lexer.run()
 	return lexer, nil
 }
@@ -60,7 +72,7 @@ type Iterator struct {
 // consumes and returns the next item
 func (t *Iterator) Next() Item {
 	t.lastPos++
-	return t.current()
+	return t.Current()
 }
 
 // Input returns the input source.
@@ -70,7 +82,8 @@ func (t *Iterator) Input() []byte {
 
 var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")}
 
-func (t *Iterator) current() Item {
+// Current will repeatably return the current item.
+func (t *Iterator) Current() Item {
 	if t.lastPos >= len(t.l.items) {
 		return errIndexOutOfBounds
 	}
@@ -122,5 +135,5 @@ func (t *Iterator) Consume(cnt int) {
 
 // LineNumber returns the current line number. Used for logging.
 func (t *Iterator) LineNumber() int {
-	return bytes.Count(t.l.input[:t.current().Pos], lf) + 1
+	return bytes.Count(t.l.input[:t.Current().Pos], lf) + 1
 }
diff --git a/parser/pageparser/pageparser_test.go b/parser/pageparser/pageparser_test.go
index fad7082d2..f54376c33 100644
--- a/parser/pageparser/pageparser_test.go
+++ b/parser/pageparser/pageparser_test.go
@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -38,7 +38,7 @@ This is some summary. This is some summary. This is some summary. This is some s
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		if _, err := parseBytes(input, cfg); err != nil {
+		if _, err := parseBytes(input, cfg, lexIntroSection); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -64,7 +64,7 @@ This is some summary. This is some summary. This is some summary. This is some s
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		if _, err := parseBytes(input, cfg); err != nil {
+		if _, err := parseBytes(input, cfg, lexIntroSection); err != nil {
 			b.Fatal(err)
 		}
 	}