diff options
| author | Junegunn Choi <junegunn.c@gmail.com> | 2015-01-09 02:35:20 +0900 |
|---|---|---|
| committer | Junegunn Choi <junegunn.c@gmail.com> | 2015-01-09 02:35:20 +0900 |
| commit | d303c5b3ebc6d56af6d3a03c6b4cdb361a2b022c (patch) | |
| tree | e8963b7f3fb3d16fae7befe45095c2d86d6ca9ed /src/tokenizer.go | |
| parent | f401c42f9c22de9df7a40ee31727ff0eab5dd30e (diff) | |
Minor refactoring
Diffstat (limited to 'src/tokenizer.go')
| -rw-r--r-- | src/tokenizer.go | 3 |
1 files changed, 1 insertions, 2 deletions
diff --git a/src/tokenizer.go b/src/tokenizer.go index bc1ca3a4..d62f395d 100644 --- a/src/tokenizer.go +++ b/src/tokenizer.go @@ -118,14 +118,13 @@ func awkTokenizer(input *string) ([]string, int) { } func Tokenize(str *string, delimiter *regexp.Regexp) []Token { - prefixLength := 0 if delimiter == nil { // AWK-style (\S+\s*) tokens, prefixLength := awkTokenizer(str) return withPrefixLengths(tokens, prefixLength) } else { tokens := delimiter.FindAllString(*str, -1) - return withPrefixLengths(tokens, prefixLength) + return withPrefixLengths(tokens, 0) } } |