diff options
| author | Dan Davison <dandavison7@gmail.com> | 2019-08-11 11:02:04 -0700 |
|---|---|---|
| committer | Dan Davison <dandavison7@gmail.com> | 2019-08-11 18:39:11 -0700 |
| commit | a7c91b6a17a47852d8a9d77b3eda125f00e04432 (patch) | |
| tree | 0c1cd1047034b33ac8cf2f081dcdca70f661f30e /src/edits.rs | |
| parent | 13a210b7004ce431a487ade076f594c4a5f157fe (diff) | |
Align separating text as multiple single-character tokens.
Diffstat (limited to 'src/edits.rs')
| -rw-r--r-- | src/edits.rs | 54 |
1 files changed, 45 insertions, 9 deletions
diff --git a/src/edits.rs b/src/edits.rs index 0c2e4894..072e4eca 100644 --- a/src/edits.rs +++ b/src/edits.rs @@ -79,7 +79,10 @@ fn tokenize(line: &str) -> Vec<&str> { let mut offset = 0; for m in separators.find_iter(line) { tokens.push(&line[offset..m.start()]); - tokens.push(m.as_str()); + // Align separating text as multiple single-character tokens. + for i in m.start()..m.end() { + tokens.push(&line[i..i + 1]); + } offset = m.end(); } if offset < line.len() { @@ -222,9 +225,11 @@ mod tests { "coalesce_edits", "<", "'a", - ", ", + ",", + " ", "EditOperation", - ">(" + ">", + "(" ] ); } @@ -239,11 +244,14 @@ mod tests { "coalesce_edits", "<", "'a", - ", ", + ",", + " ", "'b", - ", ", + ",", + " ", "EditOperation", - ">(" + ">", + "(" ] ); } @@ -258,11 +266,16 @@ mod tests { "push", "(", "vec!", - "[(", + "[", + "(", "noop_insertion", - ", ", + ",", + " ", "plus_line", - ")]);" + ")", + "]", + ")", + ";" ] ); } @@ -425,6 +438,29 @@ mod tests { ) } + #[test] + fn test_infer_edits_8() { + assert_edits( + vec!["for _ in range(0, options[\"count\"]):"], + vec!["for _ in range(0, int(options[\"count\"])):"], + ( + vec![vec![ + (MinusNoop, "for _ in range(0, "), + (MinusNoop, "options[\"count\"]"), + (MinusNoop, "):"), + ]], + vec![vec![ + (PlusNoop, "for _ in range(0, "), + (Insertion, "int("), + (PlusNoop, "options[\"count\"]"), + (Insertion, ")"), + (PlusNoop, "):"), + ]], + ), + 0.3, + ) + } + fn assert_edits( minus_lines: Vec<&str>, plus_lines: Vec<&str>, |