From a7c91b6a17a47852d8a9d77b3eda125f00e04432 Mon Sep 17 00:00:00 2001
From: Dan Davison <dandavison7@gmail.com>
Date: Sun, 11 Aug 2019 11:02:04 -0700
Subject: Align separating text as multiple single-character tokens.

---
 src/edits.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 45 insertions(+), 9 deletions(-)

(limited to 'src/edits.rs')

diff --git a/src/edits.rs b/src/edits.rs
index 0c2e4894..072e4eca 100644
--- a/src/edits.rs
+++ b/src/edits.rs
@@ -79,7 +79,10 @@ fn tokenize(line: &str) -> Vec<&str> {
     let mut offset = 0;
     for m in separators.find_iter(line) {
         tokens.push(&line[offset..m.start()]);
-        tokens.push(m.as_str());
+        // Align separating text as multiple single-character tokens.
+        for i in m.start()..m.end() {
+            tokens.push(&line[i..i + 1]);
+        }
         offset = m.end();
     }
     if offset < line.len() {
@@ -222,9 +225,11 @@ mod tests {
                 "coalesce_edits",
                 "<",
                 "'a",
-                ", ",
+                ",",
+                " ",
                 "EditOperation",
-                ">("
+                ">",
+                "("
             ]
         );
     }
@@ -239,11 +244,14 @@ mod tests {
                 "coalesce_edits",
                 "<",
                 "'a",
-                ", ",
+                ",",
+                " ",
                 "'b",
-                ", ",
+                ",",
+                " ",
                 "EditOperation",
-                ">("
+                ">",
+                "("
             ]
         );
     }
@@ -258,11 +266,16 @@ mod tests {
                 "push",
                 "(",
                 "vec!",
-                "[(",
+                "[",
+                "(",
                 "noop_insertion",
-                ", ",
+                ",",
+                " ",
                 "plus_line",
-                ")]);"
+                ")",
+                "]",
+                ")",
+                ";"
             ]
         );
     }
@@ -425,6 +438,29 @@ mod tests {
         )
     }
 
+    #[test]
+    fn test_infer_edits_8() {
+        assert_edits(
+            vec!["for _ in range(0, options[\"count\"]):"],
+            vec!["for _ in range(0, int(options[\"count\"])):"],
+            (
+                vec![vec![
+                    (MinusNoop, "for _ in range(0, "),
+                    (MinusNoop, "options[\"count\"]"),
+                    (MinusNoop, "):"),
+                ]],
+                vec![vec![
+                    (PlusNoop, "for _ in range(0, "),
+                    (Insertion, "int("),
+                    (PlusNoop, "options[\"count\"]"),
+                    (Insertion, ")"),
+                    (PlusNoop, "):"),
+                ]],
+            ),
+            0.3,
+        )
+    }
+
     fn assert_edits(
         minus_lines: Vec<&str>,
         plus_lines: Vec<&str>,
-- 
cgit v1.2.3