diff options
| author | Dan Davison <dandavison7@gmail.com> | 2020-06-11 17:02:01 -0400 |
|---|---|---|
| committer | Dan Davison <dandavison7@gmail.com> | 2020-06-11 17:03:07 -0400 |
| commit | aee83cf87a8033aaed040dc1cf02589a5244b05c (patch) | |
| tree | e268830542b96e19b9939e983bb64b59e431c53e /src/edits.rs | |
| parent | f469492bb63763bf54f99559ad40138d3f8e24fd (diff) | |
Add test of minimal tokenization edge cases
Diffstat (limited to 'src/edits.rs')
| -rw-r--r-- | src/edits.rs | 11 |
1 files changed, 11 insertions, 0 deletions
diff --git a/src/edits.rs b/src/edits.rs index 7b6e4674..79ec0252 100644 --- a/src/edits.rs +++ b/src/edits.rs @@ -244,6 +244,17 @@ mod tests { use EditOperation::*; #[test] + fn test_tokenize_0() { + assert_tokenize("", &[]); + assert_tokenize(";", &["", ";"]); + assert_tokenize(";;", &["", ";", ";"]); + assert_tokenize(";;a", &["", ";", ";", "a"]); + assert_tokenize(";;ab", &["", ";", ";", "ab"]); + assert_tokenize(";;ab;", &["", ";", ";", "ab", ";"]); + assert_tokenize(";;ab;;", &["", ";", ";", "ab", ";", ";"]); + } + + #[test] fn test_tokenize_1() { assert_eq!(tokenize("aaa bbb"), vec!["aaa", " ", "bbb"]) } |