diff options
Diffstat (limited to 'src/tokenizer_test.go')
| -rw-r--r-- | src/tokenizer_test.go | 9 |
1 files changed, 9 insertions, 0 deletions
diff --git a/src/tokenizer_test.go b/src/tokenizer_test.go index 61017b8c..0f95aa13 100644 --- a/src/tokenizer_test.go +++ b/src/tokenizer_test.go @@ -53,6 +53,15 @@ func TestTokenize(t *testing.T) { if string(tokens[0].text) != " abc:" || tokens[0].prefixLength != 0 { t.Errorf("%s", tokens) } + + // With delimiter regex + tokens = Tokenize([]rune(input), delimiterRegexp("\\s+")) + if string(tokens[0].text) != " " || tokens[0].prefixLength != 0 || + string(tokens[1].text) != "abc: " || tokens[1].prefixLength != 2 || + string(tokens[2].text) != "def: " || tokens[2].prefixLength != 8 || + string(tokens[3].text) != "ghi " || tokens[3].prefixLength != 14 { + t.Errorf("%s", tokens) + } } func TestTransform(t *testing.T) { |