summaryrefslogtreecommitdiffstats
path: root/parser/pageparser
diff options
context:
space:
mode:
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2018-10-20 17:38:49 +0200
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2018-10-22 20:46:14 +0200
commiteb038cfa0a8ada29dfcba1204ec5c432da9ed7e0 (patch)
tree9ed33e3517e49ef16f349b70b328feff47ee60d3 /parser/pageparser
parent129c27ee6e9fed98dbfebeaa272fd52757b475b2 (diff)
Convert the rest to new page parser code paths
And remove some now unused code. See #5324
Diffstat (limited to 'parser/pageparser')
-rw-r--r--parser/pageparser/item.go4
-rw-r--r--parser/pageparser/pagelexer.go35
-rw-r--r--parser/pageparser/pageparser.go8
3 files changed, 22 insertions, 25 deletions
diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go
index c6f6c3f38..049db584a 100644
--- a/parser/pageparser/item.go
+++ b/parser/pageparser/item.go
@@ -20,8 +20,8 @@ import (
type Item struct {
Type ItemType
- Pos Pos
- Val []byte
+ Pos int
+ Val []byte
}
type Items []Item
diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go
index d3fc11bf2..b68850b10 100644
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -24,9 +24,6 @@ import (
"unicode/utf8"
)
-// position (in bytes)
-type Pos int
-
const eof = -1
// returns the next state in scanner.
@@ -47,9 +44,9 @@ type pageLexer struct {
input []byte
stateStart stateFunc
state stateFunc
- pos Pos // input position
- start Pos // item start position
- width Pos // width of last element
+ pos int // input position
+ start int // item start position
+ width int // width of last element
// Set when we have parsed any summary divider
summaryDividerChecked bool
@@ -73,7 +70,7 @@ func (l *pageLexer) Input() []byte {
// note: the input position here is normally 0 (start), but
// can be set if position of first shortcode is known
// TODO(bep) 2errors byte
-func newPageLexer(input []byte, inputPosition Pos, stateStart stateFunc) *pageLexer {
+func newPageLexer(input []byte, inputPosition int, stateStart stateFunc) *pageLexer {
lexer := &pageLexer{
input: input,
pos: inputPosition,
@@ -131,7 +128,7 @@ func (l *pageLexer) next() rune {
}
runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:])
- l.width = Pos(runeWidth)
+ l.width = runeWidth
l.pos += l.width
return runeValue
}
@@ -210,7 +207,7 @@ func lexMainSection(l *pageLexer) stateFunc {
l3 = l.index(leftDelimSc)
skip := minPositiveIndex(l1, l2, l3)
if skip > 0 {
- l.pos += Pos(skip)
+ l.pos += skip
}
for {
@@ -234,7 +231,7 @@ func lexMainSection(l *pageLexer) stateFunc {
l.emit(tText)
}
l.summaryDividerChecked = true
- l.pos += Pos(len(summaryDivider))
+ l.pos += len(summaryDivider)
//l.consumeCRLF()
l.emit(TypeLeadSummaryDivider)
} else if l.hasPrefix(summaryDividerOrg) {
@@ -242,7 +239,7 @@ func lexMainSection(l *pageLexer) stateFunc {
l.emit(tText)
}
l.summaryDividerChecked = true
- l.pos += Pos(len(summaryDividerOrg))
+ l.pos += len(summaryDividerOrg)
//l.consumeCRLF()
l.emit(TypeSummaryDividerOrg)
}
@@ -291,12 +288,12 @@ LOOP:
if right == -1 {
return l.errorf("starting HTML comment with no end")
}
- l.pos += Pos(right) + Pos(len(htmlCOmmentEnd))
+ l.pos += right + len(htmlCOmmentEnd)
l.emit(TypeHTMLComment)
} else {
// Not need to look further. Hugo treats this as plain HTML,
// no front matter, no shortcodes, no nothing.
- l.pos = Pos(len(l.input))
+ l.pos = len(l.input)
l.emit(TypeHTMLDocument)
}
}
@@ -434,7 +431,7 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string,
}
func lexShortcodeLeftDelim(l *pageLexer) stateFunc {
- l.pos += Pos(len(l.currentLeftShortcodeDelim()))
+ l.pos += len(l.currentLeftShortcodeDelim())
if l.hasPrefix(leftComment) {
return lexShortcodeComment
}
@@ -451,20 +448,20 @@ func lexShortcodeComment(l *pageLexer) stateFunc {
}
// we emit all as text, except the comment markers
l.emit(tText)
- l.pos += Pos(len(leftComment))
+ l.pos += len(leftComment)
l.ignore()
- l.pos += Pos(posRightComment - len(leftComment))
+ l.pos += posRightComment - len(leftComment)
l.emit(tText)
- l.pos += Pos(len(rightComment))
+ l.pos += len(rightComment)
l.ignore()
- l.pos += Pos(len(l.currentRightShortcodeDelim()))
+ l.pos += len(l.currentRightShortcodeDelim())
l.emit(tText)
return lexMainSection
}
func lexShortcodeRightDelim(l *pageLexer) stateFunc {
l.closingState = 0
- l.pos += Pos(len(l.currentRightShortcodeDelim()))
+ l.pos += len(l.currentRightShortcodeDelim())
l.emit(l.currentRightShortcodeDelimItem())
return lexMainSection
}
diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go
index 0d32c0e89..2cd141d37 100644
--- a/parser/pageparser/pageparser.go
+++ b/parser/pageparser/pageparser.go
@@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) {
}
func parseMainSection(input []byte, from int) Result {
- lexer := newPageLexer(input, Pos(from), lexMainSection) // TODO(bep) 2errors
+ lexer := newPageLexer(input, from, lexMainSection) // TODO(bep) 2errors
lexer.run()
return lexer
}
@@ -57,7 +57,7 @@ func parseMainSection(input []byte, from int) Result {
// if needed.
type Iterator struct {
l *pageLexer
- lastPos Pos // position of the last item returned by nextItem
+ lastPos int // position of the last item returned by nextItem
}
// consumes and returns the next item
@@ -69,7 +69,7 @@ func (t *Iterator) Next() Item {
var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")}
func (t *Iterator) current() Item {
- if t.lastPos >= Pos(len(t.l.items)) {
+ if t.lastPos >= len(t.l.items) {
return errIndexOutOfBounds
}
return t.l.items[t.lastPos]
@@ -98,7 +98,7 @@ func (t *Iterator) Peek() Item {
// PeekWalk will feed the next items in the iterator to walkFn
// until it returns false.
func (t *Iterator) PeekWalk(walkFn func(item Item) bool) {
- for i := t.lastPos + 1; i < Pos(len(t.l.items)); i++ {
+ for i := t.lastPos + 1; i < len(t.l.items); i++ {
item := t.l.items[i]
if !walkFn(item) {
break