diff options
author | Junegunn Choi <junegunn.c@gmail.com> | 2017-06-02 13:25:35 +0900 |
---|---|---|
committer | Junegunn Choi <junegunn.c@gmail.com> | 2017-06-02 13:25:35 +0900 |
commit | 2e3dc75425d23b4d9e8e05a901395914cf8d3120 (patch) | |
tree | baccbb0305c7337b95207945c55d5e33f726c55b /src/tokenizer.go | |
parent | 5d6eb5bfd64c6d5d773c71159d819b651dd5f7f1 (diff) |
Fix inconsistent tiebreak scores when --nth is used
Make sure to consistently calculate tiebreak scores based on the
original line.
This change may not be preferable if you filter aligned tabular input on
a subset of columns using --nth. However, if we calculate length
tiebreak only on the matched components instead of the entire line, the
result can be very confusing when multiple --nth components are
specified, so let's keep it simple and consistent.
Close #926
Diffstat (limited to 'src/tokenizer.go')
-rw-r--r-- | src/tokenizer.go | 5 |
1 files changed, 2 insertions, 3 deletions
diff --git a/src/tokenizer.go b/src/tokenizer.go index ed873622..0e216ac7 100644 --- a/src/tokenizer.go +++ b/src/tokenizer.go @@ -20,7 +20,6 @@ type Range struct { type Token struct { text *util.Chars prefixLength int32 - trimLength int32 } // Delimiter for tokenizing the input @@ -81,7 +80,7 @@ func withPrefixLengths(tokens []util.Chars, begin int) []Token { prefixLength := begin for idx, token := range tokens { // NOTE: &tokens[idx] instead of &tokens - ret[idx] = Token{&tokens[idx], int32(prefixLength), int32(token.TrimLength())} + ret[idx] = Token{&tokens[idx], int32(prefixLength)} prefixLength += token.Length() } return ret @@ -242,7 +241,7 @@ func Transform(tokens []Token, withNth []Range) []Token { } else { prefixLength = 0 } - transTokens[idx] = Token{&merged, prefixLength, int32(merged.TrimLength())} + transTokens[idx] = Token{&merged, prefixLength} } return transTokens } |