From d0f2c00f9f0a2ded5a94703c30ea127b8f8cb847 Mon Sep 17 00:00:00 2001
From: Junegunn Choi <junegunn.c@gmail.com>
Date: Mon, 10 Aug 2015 23:47:03 +0900
Subject: Fix --with-nth performance; use simpler regular expression

Related #317
---
 src/tokenizer.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'src/tokenizer.go')

diff --git a/src/tokenizer.go b/src/tokenizer.go
index 72deb2b1..a8d04003 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -145,7 +145,16 @@ func Tokenize(runes []rune, delimiter Delimiter) []Token {
 			tokens[i] = tokens[i] + *delimiter.str
 		}
 	} else if delimiter.regex != nil {
-		tokens = delimiter.regex.FindAllString(string(runes), -1)
+		str := string(runes)
+		for len(str) > 0 {
+			loc := delimiter.regex.FindStringIndex(str)
+			if loc == nil {
+				loc = []int{0, len(str)}
+			}
+			last := util.Max(loc[1], 1)
+			tokens = append(tokens, str[:last])
+			str = str[last:]
+		}
 	}
 	asRunes := make([][]rune, len(tokens))
 	for i, token := range tokens {
-- 
cgit v1.2.3