diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-10-20 17:38:49 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-10-22 20:46:14 +0200 |
commit | eb038cfa0a8ada29dfcba1204ec5c432da9ed7e0 (patch) | |
tree | 9ed33e3517e49ef16f349b70b328feff47ee60d3 /parser/pageparser | |
parent | 129c27ee6e9fed98dbfebeaa272fd52757b475b2 (diff) |
Convert the rest to new page parser code paths
And remove some now unused code.
See #5324
Diffstat (limited to 'parser/pageparser')
-rw-r--r-- | parser/pageparser/item.go | 4 | ||||
-rw-r--r-- | parser/pageparser/pagelexer.go | 35 | ||||
-rw-r--r-- | parser/pageparser/pageparser.go | 8 |
3 files changed, 22 insertions, 25 deletions
diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go index c6f6c3f38..049db584a 100644 --- a/parser/pageparser/item.go +++ b/parser/pageparser/item.go @@ -20,8 +20,8 @@ import ( type Item struct { Type ItemType - Pos Pos - Val []byte + Pos int + Val []byte } type Items []Item diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index d3fc11bf2..b68850b10 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -24,9 +24,6 @@ import ( "unicode/utf8" ) -// position (in bytes) -type Pos int - const eof = -1 // returns the next state in scanner. @@ -47,9 +44,9 @@ type pageLexer struct { input []byte stateStart stateFunc state stateFunc - pos Pos // input position - start Pos // item start position - width Pos // width of last element + pos int // input position + start int // item start position + width int // width of last element // Set when we have parsed any summary divider summaryDividerChecked bool @@ -73,7 +70,7 @@ func (l *pageLexer) Input() []byte { // note: the input position here is normally 0 (start), but // can be set if position of first shortcode is known // TODO(bep) 2errors byte -func newPageLexer(input []byte, inputPosition Pos, stateStart stateFunc) *pageLexer { +func newPageLexer(input []byte, inputPosition int, stateStart stateFunc) *pageLexer { lexer := &pageLexer{ input: input, pos: inputPosition, @@ -131,7 +128,7 @@ func (l *pageLexer) next() rune { } runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:]) - l.width = Pos(runeWidth) + l.width = runeWidth l.pos += l.width return runeValue } @@ -210,7 +207,7 @@ func lexMainSection(l *pageLexer) stateFunc { l3 = l.index(leftDelimSc) skip := minPositiveIndex(l1, l2, l3) if skip > 0 { - l.pos += Pos(skip) + l.pos += skip } for { @@ -234,7 +231,7 @@ func lexMainSection(l *pageLexer) stateFunc { l.emit(tText) } l.summaryDividerChecked = true - l.pos += Pos(len(summaryDivider)) + l.pos += len(summaryDivider) //l.consumeCRLF() l.emit(TypeLeadSummaryDivider) } else if l.hasPrefix(summaryDividerOrg) { @@ -242,7 +239,7 @@ func lexMainSection(l *pageLexer) stateFunc { l.emit(tText) } l.summaryDividerChecked = true - l.pos += Pos(len(summaryDividerOrg)) + l.pos += len(summaryDividerOrg) //l.consumeCRLF() l.emit(TypeSummaryDividerOrg) } @@ -291,12 +288,12 @@ LOOP: if right == -1 { return l.errorf("starting HTML comment with no end") } - l.pos += Pos(right) + Pos(len(htmlCOmmentEnd)) + l.pos += right + len(htmlCOmmentEnd) l.emit(TypeHTMLComment) } else { // Not need to look further. Hugo treats this as plain HTML, // no front matter, no shortcodes, no nothing. - l.pos = Pos(len(l.input)) + l.pos = len(l.input) l.emit(TypeHTMLDocument) } } @@ -434,7 +431,7 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string, } func lexShortcodeLeftDelim(l *pageLexer) stateFunc { - l.pos += Pos(len(l.currentLeftShortcodeDelim())) + l.pos += len(l.currentLeftShortcodeDelim()) if l.hasPrefix(leftComment) { return lexShortcodeComment } @@ -451,20 +448,20 @@ func lexShortcodeComment(l *pageLexer) stateFunc { } // we emit all as text, except the comment markers l.emit(tText) - l.pos += Pos(len(leftComment)) + l.pos += len(leftComment) l.ignore() - l.pos += Pos(posRightComment - len(leftComment)) + l.pos += posRightComment - len(leftComment) l.emit(tText) - l.pos += Pos(len(rightComment)) + l.pos += len(rightComment) l.ignore() - l.pos += Pos(len(l.currentRightShortcodeDelim())) + l.pos += len(l.currentRightShortcodeDelim()) l.emit(tText) return lexMainSection } func lexShortcodeRightDelim(l *pageLexer) stateFunc { l.closingState = 0 - l.pos += Pos(len(l.currentRightShortcodeDelim())) + l.pos += len(l.currentRightShortcodeDelim()) l.emit(l.currentRightShortcodeDelimItem()) return lexMainSection } diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go index 0d32c0e89..2cd141d37 100644 --- a/parser/pageparser/pageparser.go +++ b/parser/pageparser/pageparser.go @@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) { } func parseMainSection(input []byte, from int) Result { - lexer := newPageLexer(input, Pos(from), lexMainSection) // TODO(bep) 2errors + lexer := newPageLexer(input, from, lexMainSection) // TODO(bep) 2errors lexer.run() return lexer } @@ -57,7 +57,7 @@ func parseMainSection(input []byte, from int) Result { // if needed. type Iterator struct { l *pageLexer - lastPos Pos // position of the last item returned by nextItem + lastPos int // position of the last item returned by nextItem } // consumes and returns the next item @@ -69,7 +69,7 @@ func (t *Iterator) Next() Item { var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")} func (t *Iterator) current() Item { - if t.lastPos >= Pos(len(t.l.items)) { + if t.lastPos >= len(t.l.items) { return errIndexOutOfBounds } return t.l.items[t.lastPos] @@ -98,7 +98,7 @@ func (t *Iterator) Peek() Item { // PeekWalk will feed the next items in the iterator to walkFn // until it returns false. func (t *Iterator) PeekWalk(walkFn func(item Item) bool) { - for i := t.lastPos + 1; i < Pos(len(t.l.items)); i++ { + for i := t.lastPos + 1; i < len(t.l.items); i++ { item := t.l.items[i] if !walkFn(item) { break |