diff options
Diffstat (limited to 'src/tokenizer.go')
| -rw-r--r-- | src/tokenizer.go | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/tokenizer.go b/src/tokenizer.go index 9f9e2c19..defe8cce 100644 --- a/src/tokenizer.go +++ b/src/tokenizer.go @@ -91,7 +91,7 @@ func withPrefixLengths(tokens []string, begin int) []Token { prefixLength := begin for idx := range tokens { - chars := util.ToChars([]byte(tokens[idx])) + chars := util.ToChars(sbytes(tokens[idx])) ret[idx] = Token{&chars, int32(prefixLength)} prefixLength += chars.Length() } @@ -187,7 +187,7 @@ func Transform(tokens []Token, withNth []Range) []Token { if r.begin == r.end { idx := r.begin if idx == rangeEllipsis { - chars := util.ToChars([]byte(joinTokens(tokens))) + chars := util.ToChars(sbytes(joinTokens(tokens))) parts = append(parts, &chars) } else { if idx < 0 { |