diff options
author | Junegunn Choi <junegunn.c@gmail.com> | 2016-08-14 04:23:37 +0900 |
---|---|---|
committer | Junegunn Choi <junegunn.c@gmail.com> | 2016-08-14 04:30:55 +0900 |
commit | d9c8a9a880dd9fc28c9fb466bd1c81cd6a8a159c (patch) | |
tree | 98a2018200734947aa6fa40552f3ccc7463404a2 /src/tokenizer.go | |
parent | ddc7bb9064042a0d5da9546eaf6ff888dca63f0c (diff) |
[perf] Remove memory copy when using string delimiter
Diffstat (limited to 'src/tokenizer.go')
-rw-r--r-- | src/tokenizer.go | 12 |
1 files changed, 6 insertions, 6 deletions
diff --git a/src/tokenizer.go b/src/tokenizer.go index eec19898..e48f48bf 100644 --- a/src/tokenizer.go +++ b/src/tokenizer.go @@ -140,13 +140,13 @@ func Tokenize(text util.Chars, delimiter Delimiter) []Token { return withPrefixLengths(tokens, prefixLength) } - var tokens []string if delimiter.str != nil { - tokens = strings.Split(text.ToString(), *delimiter.str) - for i := 0; i < len(tokens)-1; i++ { - tokens[i] = tokens[i] + *delimiter.str - } - } else if delimiter.regex != nil { + return withPrefixLengths(text.Split(*delimiter.str), 0) + } + + // FIXME performance + var tokens []string + if delimiter.regex != nil { str := text.ToString() for len(str) > 0 { loc := delimiter.regex.FindStringIndex(str) |