diff options
author | Junegunn Choi <junegunn.c@gmail.com> | 2015-08-10 23:47:03 +0900 |
---|---|---|
committer | Junegunn Choi <junegunn.c@gmail.com> | 2015-08-11 00:15:41 +0900 |
commit | d0f2c00f9f0a2ded5a94703c30ea127b8f8cb847 (patch) | |
tree | b30e5f42cec30491af9f2ab6270e2ba012f8682a /src/tokenizer.go | |
parent | 766427de0c04c64085c5ed907e3fdcc6124fa2dd (diff) |
Fix --with-nth performance; use simpler regular expression
Related #317
Diffstat (limited to 'src/tokenizer.go')
-rw-r--r-- | src/tokenizer.go | 11 |
1 files changed, 10 insertions, 1 deletions
diff --git a/src/tokenizer.go b/src/tokenizer.go index 72deb2b1..a8d04003 100644 --- a/src/tokenizer.go +++ b/src/tokenizer.go @@ -145,7 +145,16 @@ func Tokenize(runes []rune, delimiter Delimiter) []Token { tokens[i] = tokens[i] + *delimiter.str } } else if delimiter.regex != nil { - tokens = delimiter.regex.FindAllString(string(runes), -1) + str := string(runes) + for len(str) > 0 { + loc := delimiter.regex.FindStringIndex(str) + if loc == nil { + loc = []int{0, len(str)} + } + last := util.Max(loc[1], 1) + tokens = append(tokens, str[:last]) + str = str[last:] + } } asRunes := make([][]rune, len(tokens)) for i, token := range tokens { |