Use trimmed length when --nth is used with --tiebreak=length

This change improves sort ordering for aligned tabular input.
Given the following input:

    apple   juice   100
    apple   pie     200

fzf --nth=2 will now prefer the one with pie. Before this change fzf
compared "juice   " and "pie     ", both of which have the same length.

1 files changed, 3 insertions, 2 deletions

diff --git a/src/tokenizer.go b/src/tokenizer.go
index a8d04003..4b89b38e 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -20,6 +20,7 @@ type Range struct {
 type Token struct {
 	text         []rune
 	prefixLength int
+	trimLength   int
 }
 
 // Delimiter for tokenizing the input
@@ -81,7 +82,7 @@ func withPrefixLengths(tokens [][]rune, begin int) []Token {
 	for idx, token := range tokens {
 		// Need to define a new local variable instead of the reused token to take
 		// the pointer to it
-		ret[idx] = Token{text: token, prefixLength: prefixLength}
+		ret[idx] = Token{token, prefixLength, util.TrimLen(token)}
 		prefixLength += len(token)
 	}
 	return ret
@@ -233,7 +234,7 @@ func Transform(tokens []Token, withNth []Range) []Token {
 		} else {
 			prefixLength = 0
 		}
-		transTokens[idx] = Token{part, prefixLength}
+		transTokens[idx] = Token{part, prefixLength, util.TrimLen(part)}
 	}
 	return transTokens
 }