From c35b2b3bf0f4acfa4927a3044751aead8bb65389 Mon Sep 17 00:00:00 2001 From: Richard Burke Date: Sat, 2 Mar 2019 12:25:50 +0000 Subject: Add configReader to allow multiple unreads of runes --- cmd/grv/config_scan.go | 103 ++++++++++++++++++++++++++++++++++++-------- cmd/grv/config_scan_test.go | 47 ++++++++++++++++++++ 2 files changed, 133 insertions(+), 17 deletions(-) diff --git a/cmd/grv/config_scan.go b/cmd/grv/config_scan.go index 7128278..7d19788 100644 --- a/cmd/grv/config_scan.go +++ b/cmd/grv/config_scan.go @@ -27,6 +27,10 @@ const ( CtkCount ) +const ( + configScannerLookAhead = 2 +) + var configTokenNames = map[ConfigTokenType]string{ CtkInvalid: "Invalid", CtkWord: "Word", @@ -38,6 +42,49 @@ var configTokenNames = map[ConfigTokenType]string{ CtkEOF: "EOF", } +type configReader struct { + reader *bufio.Reader + buffer []rune + bufferIndex int + unreadCount int +} + +func newConfigReader(reader io.Reader, bufferSize int) *configReader { + return &configReader{ + reader: bufio.NewReader(reader), + buffer: make([]rune, bufferSize, bufferSize), + } +} + +func (configReader *configReader) readRune() (char rune, err error) { + if configReader.unreadCount > 0 { + char = configReader.buffer[configReader.bufferIndex] + configReader.bufferIndex = (configReader.bufferIndex + 1) % len(configReader.buffer) + configReader.unreadCount-- + } else if char, _, err = configReader.reader.ReadRune(); err == nil { + configReader.buffer[configReader.bufferIndex] = char + configReader.bufferIndex = (configReader.bufferIndex + 1) % len(configReader.buffer) + } + + return +} + +func (configReader *configReader) unreadRune() (err error) { + if configReader.unreadCount+1 > len(configReader.buffer) { + return fmt.Errorf("Only %v consecutive unreads can be performed", len(configReader.buffer)) + } + + configReader.unreadCount++ + + if configReader.bufferIndex == 0 { + configReader.bufferIndex = len(configReader.buffer) - 1 + } else { + configReader.bufferIndex-- + } + + return +} + // ConfigScannerPos represents a position in the config scanner input stream type ConfigScannerPos struct { line uint @@ -57,7 +104,7 @@ type ConfigToken struct { // ConfigScanner scans an input stream and generates a stream of config tokens type ConfigScanner struct { - reader *bufio.Reader + reader *configReader pos ConfigScannerPos lastCharLineEnd bool lastLineEndCol uint @@ -71,6 +118,7 @@ func (token *ConfigToken) Equal(other *ConfigToken) bool { return token.tokenType == other.tokenType && token.value == other.value && + token.rawValue == other.rawValue && token.startPos == other.startPos && token.endPos == other.endPos && ((token.err == nil && other.err == nil) || @@ -94,7 +142,7 @@ func ConfigTokenName(tokenType ConfigTokenType) string { // NewConfigScanner creates a new scanner which uses the provided reader func NewConfigScanner(reader io.Reader) *ConfigScanner { return &ConfigScanner{ - reader: bufio.NewReader(reader), + reader: newConfigReader(reader, configScannerLookAhead), pos: ConfigScannerPos{ line: 1, col: 0, @@ -103,7 +151,7 @@ func NewConfigScanner(reader io.Reader) *ConfigScanner { } func (scanner *ConfigScanner) read() (char rune, eof bool, err error) { - char, _, err = scanner.reader.ReadRune() + char, err = scanner.reader.readRune() if err == io.EOF { eof = true @@ -127,8 +175,18 @@ func (scanner *ConfigScanner) read() (char rune, eof bool, err error) { return } +func (scanner *ConfigScanner) unreadRunes(runeNum int) (err error) { + for i := 0; i < runeNum; i++ { + if err = scanner.unread(); err != nil { + return + } + } + + return +} + func (scanner *ConfigScanner) unread() (err error) { - if err = scanner.reader.UnreadRune(); err != nil { + if err = scanner.reader.unreadRune(); err != nil { return } @@ -182,23 +240,27 @@ func (scanner *ConfigScanner) Scan() (token *ConfigToken, err error) { token, err = scanner.scanShellCommand() case char == '-': - var nextBytes []byte - nextBytes, err = scanner.reader.Peek(1) + unreadCount := 2 + var nextChar rune + nextChar, eof, err = scanner.read() if err != nil { break - } else if len(nextBytes) == 1 && nextBytes[0] == '-' { + } else if eof { + unreadCount = 1 + } else if nextChar == '-' { token, err = scanner.scanWord() if token != nil && token.tokenType != CtkInvalid { token.tokenType = CtkOption - token.value = "-" + token.value + token.value = "--" + token.value + token.rawValue = "--" + token.rawValue } break } - if err = scanner.unread(); err != nil { + if err = scanner.unreadRunes(unreadCount); err != nil { break } @@ -242,21 +304,28 @@ OuterLoop: case eof: break OuterLoop case char == '\\': - if _, err = rawBuffer.WriteRune(char); err != nil { - return - } - - var nextBytes []byte - nextBytes, err = scanner.reader.Peek(1) + unreadCount := 2 + var nextChar rune + nextChar, eof, err = scanner.read() if err != nil { return - } else if len(nextBytes) == 1 && nextBytes[0] == '\n' { + } else if eof { + unreadCount = 1 + } else if !eof && nextChar == '\n' { escape = true + if err = scanner.unread(); err != nil { + return + } + + if _, err = rawBuffer.WriteRune(char); err != nil { + return + } + continue } - if err = scanner.unread(); err != nil { + if err = scanner.unreadRunes(unreadCount); err != nil { return } diff --git a/cmd/grv/config_scan_test.go b/cmd/grv/config_scan_test.go index 88a5a9e..94d5ff8 100644 --- a/cmd/grv/config_scan_test.go +++ b/cmd/grv/config_scan_test.go @@ -170,6 +170,22 @@ func TestScanSingleConfigToken(t *testing.T) { err: errors.New("Unterminated string"), }, }, + { + input: " \\\n ", + expectedToken: ConfigToken{ + tokenType: CtkWhiteSpace, + value: " ", + rawValue: " \\\n ", + startPos: ConfigScannerPos{ + line: 1, + col: 1, + }, + endPos: ConfigScannerPos{ + line: 2, + col: 1, + }, + }, + }, } for _, singleTokenTest := range singleTokenTests { @@ -1003,6 +1019,37 @@ func TestScanMultipleConfigTokens(t *testing.T) { }, }, }, + { + input: ` \test`, + expectedTokens: []ConfigToken{ + { + tokenType: CtkWhiteSpace, + value: " ", + rawValue: " ", + startPos: ConfigScannerPos{ + line: 1, + col: 1, + }, + endPos: ConfigScannerPos{ + line: 1, + col: 1, + }, + }, + { + tokenType: CtkWord, + value: `\test`, + rawValue: `\test`, + startPos: ConfigScannerPos{ + line: 1, + col: 2, + }, + endPos: ConfigScannerPos{ + line: 1, + col: 6, + }, + }, + }, + }, } for _, multiTokenTest := range multiTokenTests { -- cgit v1.2.3