From 12e251d2d0a3117929c5f097071f5361a2be4d1a Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 11 Jun 2020 17:58:20 -0400 Subject: Add test of tokenization of non-ASCII input --- src/edits.rs | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) (limited to 'src/edits.rs') diff --git a/src/edits.rs b/src/edits.rs index 474071ef..a7142d6c 100644 --- a/src/edits.rs +++ b/src/edits.rs @@ -381,6 +381,56 @@ mod tests { ) } + #[test] + fn test_tokenize_6() { + assert_tokenize( + " (minus_file, plus_file) => format!(\"renamed: {} ⟶ {}\", minus_file, plus_file),", + &["", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "(", + "minus_file", + ",", + " ", + "plus_file", + ")", + " ", + "=", + ">", + " ", + "format", + "!", + "(", + "\"", + "renamed", + ":", + " ", + "{", + "}", + " ", + "⟶", + " ", + " ", + "{", + "}", + "\"", + ",", + " ", + "minus_file", + ",", + " ", + "plus_file", + ")", + ","]) + } + fn assert_tokenize(text: &str, expected_tokens: &[&str]) { let actual_tokens = tokenize(text); assert_eq!(text, expected_tokens.iter().join("")); -- cgit v1.2.3