Fix --with-nth performance; use simpler regular expression

Related #317

1 files changed, 10 insertions, 1 deletions

diff --git a/src/tokenizer.go b/src/tokenizer.go
index 72deb2b1..a8d04003 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -145,7 +145,16 @@ func Tokenize(runes []rune, delimiter Delimiter) []Token {
 			tokens[i] = tokens[i] + *delimiter.str
 		}
 	} else if delimiter.regex != nil {
-		tokens = delimiter.regex.FindAllString(string(runes), -1)
+		str := string(runes)
+		for len(str) > 0 {
+			loc := delimiter.regex.FindStringIndex(str)
+			if loc == nil {
+				loc = []int{0, len(str)}
+			}
+			last := util.Max(loc[1], 1)
+			tokens = append(tokens, str[:last])
+			str = str[last:]
+		}
 	}
 	asRunes := make([][]rune, len(tokens))
 	for i, token := range tokens {