Fix delimiter regex to properly support caret (^)

Fix #2861

1 files changed, 8 insertions, 8 deletions

diff --git a/src/tokenizer.go b/src/tokenizer.go
index 26f42d25..9f9e2c19 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -156,14 +156,14 @@ func Tokenize(text string, delimiter Delimiter) []Token {
 	// FIXME performance
 	var tokens []string
 	if delimiter.regex != nil {
-		for len(text) > 0 {
-			loc := delimiter.regex.FindStringIndex(text)
-			if len(loc) < 2 {
-				loc = []int{0, len(text)}
-			}
-			last := util.Max(loc[1], 1)
-			tokens = append(tokens, text[:last])
-			text = text[last:]
+		locs := delimiter.regex.FindAllStringIndex(text, -1)
+		begin := 0
+		for _, loc := range locs {
+			tokens = append(tokens, text[begin:loc[1]])
+			begin = loc[1]
+		}
+		if begin < len(text) {
+			tokens = append(tokens, text[begin:])
 		}
 	}
 	return withPrefixLengths(tokens, 0)