diff options
| author | Junegunn Choi <junegunn.c@gmail.com> | 2017-08-26 03:24:42 +0900 |
|---|---|---|
| committer | Junegunn Choi <junegunn.c@gmail.com> | 2017-08-26 03:24:42 +0900 |
| commit | af809c966150f4623f9316e8b63e146491134e50 (patch) | |
| tree | 6154e9deb5e0278694a48874724abbc609163ba8 /src/tokenizer.go | |
| parent | 329de8f41664ab0e10ad934abe3623a07988e83f (diff) | |
Minor refactorings
Diffstat (limited to 'src/tokenizer.go')
| -rw-r--r-- | src/tokenizer.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/src/tokenizer.go b/src/tokenizer.go index 5b7a8b6f..6c1d8cab 100644 --- a/src/tokenizer.go +++ b/src/tokenizer.go @@ -147,7 +147,7 @@ func Tokenize(text string, delimiter Delimiter) []Token { if delimiter.regex != nil { for len(text) > 0 { loc := delimiter.regex.FindStringIndex(text) - if loc == nil { + if len(loc) < 2 { loc = []int{0, len(text)} } last := util.Max(loc[1], 1) |