From d0f2c00f9f0a2ded5a94703c30ea127b8f8cb847 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Mon, 10 Aug 2015 23:47:03 +0900 Subject: Fix --with-nth performance; use simpler regular expression Related #317 --- src/tokenizer_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'src/tokenizer_test.go') diff --git a/src/tokenizer_test.go b/src/tokenizer_test.go index 61017b8c..0f95aa13 100644 --- a/src/tokenizer_test.go +++ b/src/tokenizer_test.go @@ -53,6 +53,15 @@ func TestTokenize(t *testing.T) { if string(tokens[0].text) != " abc:" || tokens[0].prefixLength != 0 { t.Errorf("%s", tokens) } + + // With delimiter regex + tokens = Tokenize([]rune(input), delimiterRegexp("\\s+")) + if string(tokens[0].text) != " " || tokens[0].prefixLength != 0 || + string(tokens[1].text) != "abc: " || tokens[1].prefixLength != 2 || + string(tokens[2].text) != "def: " || tokens[2].prefixLength != 8 || + string(tokens[3].text) != "ghi " || tokens[3].prefixLength != 14 { + t.Errorf("%s", tokens) + } } func TestTransform(t *testing.T) { -- cgit v1.2.3