Move the shortcode parser to the new pageparser package

See #5324

1 files changed, 0 insertions, 586 deletions

diff --git a/hugolib/shortcodeparser.go b/hugolib/shortcodeparser.go
deleted file mode 100644
index 32aa8b47a..000000000
--- a/hugolib/shortcodeparser.go
+++ /dev/null
@@ -1,586 +0,0 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugolib
-
-import (
-	"fmt"
-	"strings"
-	"unicode"
-	"unicode/utf8"
-)
-
-// The lexical scanning below is highly inspired by the great talk given by
-// Rob Pike called "Lexical Scanning in Go" (it's on YouTube, Google it!).
-// See slides here: http://cuddle.googlecode.com/hg/talk/lex.html
-
-// parsing
-
-type pageTokens struct {
-	lexer     *pagelexer
-	token     [3]item // 3-item look-ahead is what we currently need
-	peekCount int
-}
-
-func (t *pageTokens) next() item {
-	if t.peekCount > 0 {
-		t.peekCount--
-	} else {
-		t.token[0] = t.lexer.nextItem()
-	}
-	return t.token[t.peekCount]
-}
-
-// backs up one token.
-func (t *pageTokens) backup() {
-	t.peekCount++
-}
-
-// backs up two tokens.
-func (t *pageTokens) backup2(t1 item) {
-	t.token[1] = t1
-	t.peekCount = 2
-}
-
-// backs up three tokens.
-func (t *pageTokens) backup3(t2, t1 item) {
-	t.token[1] = t1
-	t.token[2] = t2
-	t.peekCount = 3
-}
-
-// check for non-error and non-EOF types coming next
-func (t *pageTokens) isValueNext() bool {
-	i := t.peek()
-	return i.typ != tError && i.typ != tEOF
-}
-
-// look at, but do not consume, the next item
-// repeated, sequential calls will return the same item
-func (t *pageTokens) peek() item {
-	if t.peekCount > 0 {
-		return t.token[t.peekCount-1]
-	}
-	t.peekCount = 1
-	t.token[0] = t.lexer.nextItem()
-	return t.token[0]
-}
-
-// convencience method to consume the next n tokens, but back off Errors and EOF
-func (t *pageTokens) consume(cnt int) {
-	for i := 0; i < cnt; i++ {
-		token := t.next()
-		if token.typ == tError || token.typ == tEOF {
-			t.backup()
-			break
-		}
-	}
-}
-
-// lexical scanning
-
-// position (in bytes)
-type pos int
-
-type item struct {
-	typ itemType
-	pos pos
-	val string
-}
-
-func (i item) String() string {
-	switch {
-	case i.typ == tEOF:
-		return "EOF"
-	case i.typ == tError:
-		return i.val
-	case i.typ > tKeywordMarker:
-		return fmt.Sprintf("<%s>", i.val)
-	case len(i.val) > 20:
-		return fmt.Sprintf("%.20q...", i.val)
-	}
-	return fmt.Sprintf("[%s]", i.val)
-}
-
-type itemType int
-
-const (
-	tError itemType = iota
-	tEOF
-
-	// shortcode items
-	tLeftDelimScNoMarkup
-	tRightDelimScNoMarkup
-	tLeftDelimScWithMarkup
-	tRightDelimScWithMarkup
-	tScClose
-	tScName
-	tScParam
-	tScParamVal
-
-	//itemIdentifier
-	tText // plain text, used for everything outside the shortcodes
-
-	// preserved for later - keywords come after this
-	tKeywordMarker
-)
-
-const eof = -1
-
-// returns the next state in scanner.
-type stateFunc func(*pagelexer) stateFunc
-
-type pagelexer struct {
-	name    string
-	input   string
-	state   stateFunc
-	pos     pos // input position
-	start   pos // item start position
-	width   pos // width of last element
-	lastPos pos // position of the last item returned by nextItem
-
-	// shortcode state
-	currLeftDelimItem  itemType
-	currRightDelimItem itemType
-	currShortcodeName  string          // is only set when a shortcode is in opened state
-	closingState       int             // > 0 = on its way to be closed
-	elementStepNum     int             // step number in element
-	paramElements      int             // number of elements (name + value = 2) found first
-	openShortcodes     map[string]bool // set of shortcodes in open state
-
-	// items delivered to client
-	items []item
-}
-
-// note: the input position here is normally 0 (start), but
-// can be set if position of first shortcode is known
-func newShortcodeLexer(name, input string, inputPosition pos) *pagelexer {
-	lexer := &pagelexer{
-		name:               name,
-		input:              input,
-		currLeftDelimItem:  tLeftDelimScNoMarkup,
-		currRightDelimItem: tRightDelimScNoMarkup,
-		pos:                inputPosition,
-		openShortcodes:     make(map[string]bool),
-		items:              make([]item, 0, 5),
-	}
-	lexer.runShortcodeLexer()
-	return lexer
-}
-
-// main loop
-// this looks kind of funky, but it works
-func (l *pagelexer) runShortcodeLexer() {
-	for l.state = lexTextOutsideShortcodes; l.state != nil; {
-		l.state = l.state(l)
-	}
-}
-
-// state functions
-
-const (
-	leftDelimScNoMarkup    = "{{<"
-	rightDelimScNoMarkup   = ">}}"
-	leftDelimScWithMarkup  = "{{%"
-	rightDelimScWithMarkup = "%}}"
-	leftComment            = "/*" // comments in this context us used to to mark shortcodes as "not really a shortcode"
-	rightComment           = "*/"
-)
-
-func (l *pagelexer) next() rune {
-	if int(l.pos) >= len(l.input) {
-		l.width = 0
-		return eof
-	}
-
-	// looks expensive, but should produce the same iteration sequence as the string range loop
-	// see: http://blog.golang.org/strings
-	runeValue, runeWidth := utf8.DecodeRuneInString(l.input[l.pos:])
-	l.width = pos(runeWidth)
-	l.pos += l.width
-	return runeValue
-}
-
-// peek, but no consume
-func (l *pagelexer) peek() rune {
-	r := l.next()
-	l.backup()
-	return r
-}
-
-// steps back one
-func (l *pagelexer) backup() {
-	l.pos -= l.width
-}
-
-// sends an item back to the client.
-func (l *pagelexer) emit(t itemType) {
-	l.items = append(l.items, item{t, l.start, l.input[l.start:l.pos]})
-	l.start = l.pos
-}
-
-// special case, do not send '\\' back to client
-func (l *pagelexer) ignoreEscapesAndEmit(t itemType) {
-	val := strings.Map(func(r rune) rune {
-		if r == '\\' {
-			return -1
-		}
-		return r
-	}, l.input[l.start:l.pos])
-	l.items = append(l.items, item{t, l.start, val})
-	l.start = l.pos
-}
-
-// gets the current value (for debugging and error handling)
-func (l *pagelexer) current() string {
-	return l.input[l.start:l.pos]
-}
-
-// ignore current element
-func (l *pagelexer) ignore() {
-	l.start = l.pos
-}
-
-// nice to have in error logs
-func (l *pagelexer) lineNum() int {
-	return strings.Count(l.input[:l.lastPos], "\n") + 1
-}
-
-// nil terminates the parser
-func (l *pagelexer) errorf(format string, args ...interface{}) stateFunc {
-	l.items = append(l.items, item{tError, l.start, fmt.Sprintf(format, args...)})
-	return nil
-}
-
-// consumes and returns the next item
-func (l *pagelexer) nextItem() item {
-	item := l.items[0]
-	l.items = l.items[1:]
-	l.lastPos = item.pos
-	return item
-}
-
-// scans until an opening shortcode opening bracket.
-// if no shortcodes, it will keep on scanning until EOF
-func lexTextOutsideShortcodes(l *pagelexer) stateFunc {
-	for {
-		if strings.HasPrefix(l.input[l.pos:], leftDelimScWithMarkup) || strings.HasPrefix(l.input[l.pos:], leftDelimScNoMarkup) {
-			if l.pos > l.start {
-				l.emit(tText)
-			}
-			if strings.HasPrefix(l.input[l.pos:], leftDelimScWithMarkup) {
-				l.currLeftDelimItem = tLeftDelimScWithMarkup
-				l.currRightDelimItem = tRightDelimScWithMarkup
-			} else {
-				l.currLeftDelimItem = tLeftDelimScNoMarkup
-				l.currRightDelimItem = tRightDelimScNoMarkup
-			}
-			return lexShortcodeLeftDelim
-
-		}
-		if l.next() == eof {
-			break
-		}
-	}
-	// Done!
-	if l.pos > l.start {
-		l.emit(tText)
-	}
-	l.emit(tEOF)
-	return nil
-}
-
-func lexShortcodeLeftDelim(l *pagelexer) stateFunc {
-	l.pos += pos(len(l.currentLeftShortcodeDelim()))
-	if strings.HasPrefix(l.input[l.pos:], leftComment) {
-		return lexShortcodeComment
-	}
-	l.emit(l.currentLeftShortcodeDelimItem())
-	l.elementStepNum = 0
-	l.paramElements = 0
-	return lexInsideShortcode
-}
-
-func lexShortcodeComment(l *pagelexer) stateFunc {
-	posRightComment := strings.Index(l.input[l.pos:], rightComment+l.currentRightShortcodeDelim())
-	if posRightComment <= 1 {
-		return l.errorf("comment must be closed")
-	}
-	// we emit all as text, except the comment markers
-	l.emit(tText)
-	l.pos += pos(len(leftComment))
-	l.ignore()
-	l.pos += pos(posRightComment - len(leftComment))
-	l.emit(tText)
-	l.pos += pos(len(rightComment))
-	l.ignore()
-	l.pos += pos(len(l.currentRightShortcodeDelim()))
-	l.emit(tText)
-	return lexTextOutsideShortcodes
-}
-
-func lexShortcodeRightDelim(l *pagelexer) stateFunc {
-	l.closingState = 0
-	l.pos += pos(len(l.currentRightShortcodeDelim()))
-	l.emit(l.currentRightShortcodeDelimItem())
-	return lexTextOutsideShortcodes
-}
-
-// either:
-// 1. param
-// 2. "param" or "param\"
-// 3. param="123" or param="123\"
-// 4. param="Some \"escaped\" text"
-func lexShortcodeParam(l *pagelexer, escapedQuoteStart bool) stateFunc {
-
-	first := true
-	nextEq := false
-
-	var r rune
-
-	for {
-		r = l.next()
-		if first {
-			if r == '"' {
-				// a positional param with quotes
-				if l.paramElements == 2 {
-					return l.errorf("got quoted positional parameter. Cannot mix named and positional parameters")
-				}
-				l.paramElements = 1
-				l.backup()
-				return lexShortcodeQuotedParamVal(l, !escapedQuoteStart, tScParam)
-			}
-			first = false
-		} else if r == '=' {
-			// a named param
-			l.backup()
-			nextEq = true
-			break
-		}
-
-		if !isAlphaNumericOrHyphen(r) {
-			l.backup()
-			break
-		}
-	}
-
-	if l.paramElements == 0 {
-		l.paramElements++
-
-		if nextEq {
-			l.paramElements++
-		}
-	} else {
-		if nextEq && l.paramElements == 1 {
-			return l.errorf("got named parameter '%s'. Cannot mix named and positional parameters", l.current())
-		} else if !nextEq && l.paramElements == 2 {
-			return l.errorf("got positional parameter '%s'. Cannot mix named and positional parameters", l.current())
-		}
-	}
-
-	l.emit(tScParam)
-	return lexInsideShortcode
-
-}
-
-func lexShortcodeQuotedParamVal(l *pagelexer, escapedQuotedValuesAllowed bool, typ itemType) stateFunc {
-	openQuoteFound := false
-	escapedInnerQuoteFound := false
-	escapedQuoteState := 0
-
-Loop:
-	for {
-		switch r := l.next(); {
-		case r == '\\':
-			if l.peek() == '"' {
-				if openQuoteFound && !escapedQuotedValuesAllowed {
-					l.backup()
-					break Loop
-				} else if openQuoteFound {
-					// the coming quoute is inside
-					escapedInnerQuoteFound = true
-					escapedQuoteState = 1
-				}
-			}
-		case r == eof, r == '\n':
-			return l.errorf("unterminated quoted string in shortcode parameter-argument: '%s'", l.current())
-		case r == '"':
-			if escapedQuoteState == 0 {
-				if openQuoteFound {
-					l.backup()
-					break Loop
-
-				} else {
-					openQuoteFound = true
-					l.ignore()
-				}
-			} else {
-				escapedQuoteState = 0
-			}
-
-		}
-	}
-
-	if escapedInnerQuoteFound {
-		l.ignoreEscapesAndEmit(typ)
-	} else {
-		l.emit(typ)
-	}
-
-	r := l.next()
-
-	if r == '\\' {
-		if l.peek() == '"' {
-			// ignore the escaped closing quote
-			l.ignore()
-			l.next()
-			l.ignore()
-		}
-	} else if r == '"' {
-		// ignore closing quote
-		l.ignore()
-	} else {
-		// handled by next state
-		l.backup()
-	}
-
-	return lexInsideShortcode
-}
-
-// scans an alphanumeric inside shortcode
-func lexIdentifierInShortcode(l *pagelexer) stateFunc {
-	lookForEnd := false
-Loop:
-	for {
-		switch r := l.next(); {
-		case isAlphaNumericOrHyphen(r):
-		// Allow forward slash inside names to make it possible to create namespaces.
-		case r == '/':
-		default:
-			l.backup()
-			word := l.input[l.start:l.pos]
-			if l.closingState > 0 && !l.openShortcodes[word] {
-				return l.errorf("closing tag for shortcode '%s' does not match start tag", word)
-			} else if l.closingState > 0 {
-				l.openShortcodes[word] = false
-				lookForEnd = true
-			}
-
-			l.closingState = 0
-			l.currShortcodeName = word
-			l.openShortcodes[word] = true
-			l.elementStepNum++
-			l.emit(tScName)
-			break Loop
-		}
-	}
-
-	if lookForEnd {
-		return lexEndOfShortcode
-	}
-	return lexInsideShortcode
-}
-
-func lexEndOfShortcode(l *pagelexer) stateFunc {
-	if strings.HasPrefix(l.input[l.pos:], l.currentRightShortcodeDelim()) {
-		return lexShortcodeRightDelim
-	}
-	switch r := l.next(); {
-	case isSpace(r):
-		l.ignore()
-	default:
-		return l.errorf("unclosed shortcode")
-	}
-	return lexEndOfShortcode
-}
-
-// scans the elements inside shortcode tags
-func lexInsideShortcode(l *pagelexer) stateFunc {
-	if strings.HasPrefix(l.input[l.pos:], l.currentRightShortcodeDelim()) {
-		return lexShortcodeRightDelim
-	}
-	switch r := l.next(); {
-	case r == eof:
-		// eol is allowed inside shortcodes; this may go to end of document before it fails
-		return l.errorf("unclosed shortcode action")
-	case isSpace(r), isEndOfLine(r):
-		l.ignore()
-	case r == '=':
-		l.ignore()
-		return lexShortcodeQuotedParamVal(l, l.peek() != '\\', tScParamVal)
-	case r == '/':
-		if l.currShortcodeName == "" {
-			return l.errorf("got closing shortcode, but none is open")
-		}
-		l.closingState++
-		l.emit(tScClose)
-	case r == '\\':
-		l.ignore()
-		if l.peek() == '"' {
-			return lexShortcodeParam(l, true)
-		}
-	case l.elementStepNum > 0 && (isAlphaNumericOrHyphen(r) || r == '"'): // positional params can have quotes
-		l.backup()
-		return lexShortcodeParam(l, false)
-	case isAlphaNumeric(r):
-		l.backup()
-		return lexIdentifierInShortcode
-	default:
-		return l.errorf("unrecognized character in shortcode action: %#U. Note: Parameters with non-alphanumeric args must be quoted", r)
-	}
-	return lexInsideShortcode
-}
-
-// state helpers
-
-func (l *pagelexer) currentLeftShortcodeDelimItem() itemType {
-	return l.currLeftDelimItem
-}
-
-func (l *pagelexer) currentRightShortcodeDelimItem() itemType {
-	return l.currRightDelimItem
-}
-
-func (l *pagelexer) currentLeftShortcodeDelim() string {
-	if l.currLeftDelimItem == tLeftDelimScWithMarkup {
-		return leftDelimScWithMarkup
-	}
-	return leftDelimScNoMarkup
-
-}
-
-func (l *pagelexer) currentRightShortcodeDelim() string {
-	if l.currRightDelimItem == tRightDelimScWithMarkup {
-		return rightDelimScWithMarkup
-	}
-	return rightDelimScNoMarkup
-}
-
-// helper functions
-
-func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
-}
-
-func isAlphaNumericOrHyphen(r rune) bool {
-	// let unquoted YouTube ids as positional params slip through (they contain hyphens)
-	return isAlphaNumeric(r) || r == '-'
-}
-
-func isEndOfLine(r rune) bool {
-	return r == '\r' || r == '\n'
-}
-
-func isAlphaNumeric(r rune) bool {
-	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
-}