diff options
| author | Dan Davison <dandavison7@gmail.com> | 2020-06-11 17:58:20 -0400 |
|---|---|---|
| committer | Dan Davison <dandavison7@gmail.com> | 2020-06-11 19:27:07 -0400 |
| commit | 12e251d2d0a3117929c5f097071f5361a2be4d1a (patch) | |
| tree | 040ee092137a47c8cbc7428a27c54765c9d7d684 /src/edits.rs | |
| parent | 30c5fc9d361287765163c283520b152c26acbaaa (diff) | |
Add test of tokenization of non-ASCII input
Diffstat (limited to 'src/edits.rs')
| -rw-r--r-- | src/edits.rs | 50 |
1 files changed, 50 insertions, 0 deletions
diff --git a/src/edits.rs b/src/edits.rs index 474071ef..a7142d6c 100644 --- a/src/edits.rs +++ b/src/edits.rs @@ -381,6 +381,56 @@ mod tests { ) } + #[test] + fn test_tokenize_6() { + assert_tokenize( + " (minus_file, plus_file) => format!(\"renamed: {} ⟶ {}\", minus_file, plus_file),", + &["", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "(", + "minus_file", + ",", + " ", + "plus_file", + ")", + " ", + "=", + ">", + " ", + "format", + "!", + "(", + "\"", + "renamed", + ":", + " ", + "{", + "}", + " ", + "⟶", + " ", + " ", + "{", + "}", + "\"", + ",", + " ", + "minus_file", + ",", + " ", + "plus_file", + ")", + ","]) + } + fn assert_tokenize(text: &str, expected_tokens: &[&str]) { let actual_tokens = tokenize(text); assert_eq!(text, expected_tokens.iter().join("")); |