hugolib: Redo the summary delimiter logic

Now that we have a proper page parse tree, this can be greatly simplified.

See #5324

6 files changed, 45 insertions, 25 deletions

diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go
index 7527d7a08..280361a84 100644
--- a/parser/metadecoders/decoder.go
+++ b/parser/metadecoders/decoder.go
@@ -20,7 +20,7 @@ import (
 	"github.com/chaseadamsio/goorgeous"
 	"github.com/gohugoio/hugo/parser/pageparser"
 	"github.com/pkg/errors"
-	yaml "gopkg.in/yaml.v1"
+	yaml "gopkg.in/yaml.v2"
 )
 
 type Format string
diff --git a/parser/metadecoders/yaml.go b/parser/metadecoders/yaml.go
index 3a520ac07..21b23a9fd 100644
--- a/parser/metadecoders/yaml.go
+++ b/parser/metadecoders/yaml.go
@@ -19,7 +19,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cast"
-	yaml "gopkg.in/yaml.v1"
+	yaml "gopkg.in/yaml.v2"
 )
 
 // HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go
index d97fed734..afc3b5fab 100644
--- a/parser/pageparser/item.go
+++ b/parser/pageparser/item.go
@@ -13,10 +13,13 @@
 
 package pageparser
 
-import "fmt"
+import (
+	"bytes"
+	"fmt"
+)
 
 type Item struct {
-	Typ ItemType
+	Type ItemType
 	pos pos
 	Val []byte
 }
@@ -28,65 +31,69 @@ func (i Item) ValStr() string {
 }
 
 func (i Item) IsText() bool {
-	return i.Typ == tText
+	return i.Type == tText
+}
+
+func (i Item) IsNonWhitespace() bool {
+	return len(bytes.TrimSpace(i.Val)) > 0
 }
 
 func (i Item) IsShortcodeName() bool {
-	return i.Typ == tScName
+	return i.Type == tScName
 }
 
 func (i Item) IsLeftShortcodeDelim() bool {
-	return i.Typ == tLeftDelimScWithMarkup || i.Typ == tLeftDelimScNoMarkup
+	return i.Type == tLeftDelimScWithMarkup || i.Type == tLeftDelimScNoMarkup
 }
 
 func (i Item) IsRightShortcodeDelim() bool {
-	return i.Typ == tRightDelimScWithMarkup || i.Typ == tRightDelimScNoMarkup
+	return i.Type == tRightDelimScWithMarkup || i.Type == tRightDelimScNoMarkup
 }
 
 func (i Item) IsShortcodeClose() bool {
-	return i.Typ == tScClose
+	return i.Type == tScClose
 }
 
 func (i Item) IsShortcodeParam() bool {
-	return i.Typ == tScParam
+	return i.Type == tScParam
 }
 
 func (i Item) IsShortcodeParamVal() bool {
-	return i.Typ == tScParamVal
+	return i.Type == tScParamVal
 }
 
 func (i Item) IsShortcodeMarkupDelimiter() bool {
-	return i.Typ == tLeftDelimScWithMarkup || i.Typ == tRightDelimScWithMarkup
+	return i.Type == tLeftDelimScWithMarkup || i.Type == tRightDelimScWithMarkup
 }
 
 func (i Item) IsFrontMatter() bool {
-	return i.Typ >= TypeFrontMatterYAML && i.Typ <= TypeFrontMatterORG
+	return i.Type >= TypeFrontMatterYAML && i.Type <= TypeFrontMatterORG
 }
 
 func (i Item) IsDone() bool {
-	return i.Typ == tError || i.Typ == tEOF
+	return i.Type == tError || i.Type == tEOF
 }
 
 func (i Item) IsEOF() bool {
-	return i.Typ == tEOF
+	return i.Type == tEOF
 }
 
 func (i Item) IsError() bool {
-	return i.Typ == tError
+	return i.Type == tError
 }
 
 func (i Item) String() string {
 	switch {
-	case i.Typ == tEOF:
+	case i.Type == tEOF:
 		return "EOF"
-	case i.Typ == tError:
+	case i.Type == tError:
 		return string(i.Val)
-	case i.Typ > tKeywordMarker:
+	case i.Type > tKeywordMarker:
 		return fmt.Sprintf("<%s>", i.Val)
 	case len(i.Val) > 50:
-		return fmt.Sprintf("%v:%.20q...", i.Typ, i.Val)
+		return fmt.Sprintf("%v:%.20q...", i.Type, i.Val)
 	}
-	return fmt.Sprintf("%v:[%s]", i.Typ, i.Val)
+	return fmt.Sprintf("%v:[%s]", i.Type, i.Val)
 }
 
 type ItemType int
diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go
index 7768b0b2f..a6a26016b 100644
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -235,6 +235,7 @@ func lexMainSection(l *pageLexer) stateFunc {
 				}
 				l.summaryDividerChecked = true
 				l.pos += pos(len(summaryDivider))
+				//l.consumeCRLF()
 				l.emit(TypeLeadSummaryDivider)
 			} else if l.hasPrefix(summaryDividerOrg) {
 				if l.pos > l.start {
@@ -242,6 +243,7 @@ func lexMainSection(l *pageLexer) stateFunc {
 				}
 				l.summaryDividerChecked = true
 				l.pos += pos(len(summaryDividerOrg))
+				//l.consumeCRLF()
 				l.emit(TypeSummaryDividerOrg)
 			}
 		}
diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go
index b4cdef75c..bc6f55dd8 100644
--- a/parser/pageparser/pageparser.go
+++ b/parser/pageparser/pageparser.go
@@ -86,7 +86,7 @@ func (t *Iterator) Backup() {
 // check for non-error and non-EOF types coming next
 func (t *Iterator) IsValueNext() bool {
 	i := t.Peek()
-	return i.Typ != tError && i.Typ != tEOF
+	return i.Type != tError && i.Type != tEOF
 }
 
 // look at, but do not consume, the next item
@@ -95,12 +95,23 @@ func (t *Iterator) Peek() Item {
 	return t.l.items[t.lastPos+1]
 }
 
+// PeekWalk will feed the next items in the iterator to walkFn
+// until it returns false.
+func (t *Iterator) PeekWalk(walkFn func(item Item) bool) {
+	for i := t.lastPos + 1; i < pos(len(t.l.items)); i++ {
+		item := t.l.items[i]
+		if !walkFn(item) {
+			break
+		}
+	}
+}
+
 // Consume is a convencience method to consume the next n tokens,
 // but back off Errors and EOF.
 func (t *Iterator) Consume(cnt int) {
 	for i := 0; i < cnt; i++ {
 		token := t.Next()
-		if token.Typ == tError || token.Typ == tEOF {
+		if token.Type == tError || token.Type == tEOF {
 			t.Backup()
 			break
 		}
diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go
index bfd19c250..850254ac7 100644
--- a/parser/pageparser/pageparser_intro_test.go
+++ b/parser/pageparser/pageparser_intro_test.go
@@ -91,7 +91,7 @@ func collect(input []byte, skipFrontMatter bool, stateStart stateFunc) (items []
 	for {
 		item := t.Next()
 		items = append(items, item)
-		if item.Typ == tEOF || item.Typ == tError {
+		if item.Type == tEOF || item.Type == tError {
 			break
 		}
 	}
@@ -104,7 +104,7 @@ func equal(i1, i2 []Item) bool {
 		return false
 	}
 	for k := range i1 {
-		if i1[k].Typ != i2[k].Typ {
+		if i1[k].Type != i2[k].Type {
 			return false
 		}
 		if !reflect.DeepEqual(i1[k].Val, i2[k].Val) {