Preserve leading whitespace when constructing comment nodes0.8

Display relies on accurate content values. If we transform comment
contents, we would colour the wrong part of the source code when
displaying. See before/after.java sample files.

3 files changed, 49 insertions, 71 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c4a6fc3bc..2cd4eb82d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,9 @@ Removed Scheme support, as there's no tree-sitter parser available.
 
 Fixed crashes on files with non-ASCII characters on long lines.
 
+Fixed an issue where multiline comments were not highlighted
+correctly.
+
 Improved display to better use the whole width when whole files are
 added or removed.
 
diff --git a/src/regex_parser.rs b/src/regex_parser.rs
index f4289159c..25204859f 100644
--- a/src/regex_parser.rs
+++ b/src/regex_parser.rs
@@ -680,11 +680,18 @@ pub mod tests {
             &[
                 Syntax::new_comment(
                     &arena,
-                    vec![SingleLineSpan {
-                        line: 0.into(),
-                        start_col: 0,
-                        end_col: 6,
-                    }],
+                    vec![
+                        SingleLineSpan {
+                            line: 0.into(),
+                            start_col: 0,
+                            end_col: 6,
+                        },
+                        SingleLineSpan {
+                            line: 1.into(),
+                            start_col: 0,
+                            end_col: 0,
+                        },
+                    ],
                     "// foo\n",
                 ),
                 Syntax::new_atom(
@@ -726,34 +733,6 @@ pub mod tests {
     }
 
     #[test]
-    fn test_parse_indented_multiline_comment() {
-        let arena = Arena::new();
-
-        assert_syntaxes(
-            &parse(&arena, "  /* foo\n  bar */", &lang()),
-            // Deliberately construct an Atom directly, because
-            // Syntax::new_comment has the multiline logic.
-            &[&Atom {
-                info: crate::syntax::SyntaxInfo::new(0),
-                content: "/* foo\nbar */".into(),
-                is_comment: true,
-                position: vec![
-                    SingleLineSpan {
-                        line: 0.into(),
-                        start_col: 2,
-                        end_col: 8,
-                    },
-                    SingleLineSpan {
-                        line: 1.into(),
-                        start_col: 2,
-                        end_col: 8,
-                    },
-                ],
-            }],
-        );
-    }
-
-    #[test]
     fn test_parse_list() {
         let arena = Arena::new();
 
diff --git a/src/syntax.rs b/src/syntax.rs
index a324db31e..b9522dc31 100644
--- a/src/syntax.rs
+++ b/src/syntax.rs
@@ -4,7 +4,7 @@
 
 use itertools::{EitherOrBoth, Itertools};
 use std::cell::Cell;
-use std::cmp::{max, min};
+use std::cmp::min;
 use std::collections::hash_map::DefaultHasher;
 use std::collections::HashMap;
 use std::hash::{Hash, Hasher};
@@ -149,21 +149,6 @@ impl<'a> fmt::Debug for Syntax<'a> {
     }
 }
 
-fn trim_left(max_trim: usize, content: &str, pos: SingleLineSpan) -> (String, SingleLineSpan) {
-    let chars: Vec<_> = content.chars().collect();
-
-    match chars.iter().position(|c| *c != ' ' && *c != '\t') {
-        Some(first_non_whitespace) => {
-            let skip_num = max(max_trim, first_non_whitespace);
-
-            let mut new_pos = pos;
-            new_pos.start_col += skip_num;
-            (chars.iter().skip(skip_num).collect(), new_pos)
-        }
-        None => (content.to_string(), pos),
-    }
-}
-
 impl<'a> Syntax<'a> {
     pub fn new_list(
         arena: &'a Arena<Syntax<'a>>,
@@ -217,27 +202,7 @@ impl<'a> Syntax<'a> {
         position: Vec<SingleLineSpan>,
         content: &str,
     ) -> &'a Syntax<'a> {
-        // Ignore leading whitespace in multiline comments, so changes
-        // in comment indentation are ignored.
-        let first_line_indent = match position.first() {
-            Some(line_pos) => line_pos.start_col,
-            None => 0,
-        };
-
-        let mut new_lines: Vec<String> = vec![];
-        let mut new_position = vec![];
-        for (i, (line, span)) in content.lines().zip(position).enumerate() {
-            if i == 0 {
-                new_lines.push(line.to_string());
-                new_position.push(span);
-            } else {
-                let (new_line, new_span) = trim_left(first_line_indent, line, span);
-                new_lines.push(new_line);
-                new_position.push(new_span);
-            }
-        }
-
-        Self::new_atom_(arena, new_position, &new_lines.join("\n"), true)
+        Self::new_atom_(arena, position, content, true)
     }
 
     #[allow(clippy::mut_from_ref)] // Clippy doesn't understand arenas.
@@ -357,7 +322,22 @@ impl<'a> Syntax<'a> {
                     is_comment: rhs_is_comment,
                     ..
                 },
-            ) => lhs_content == rhs_content && lhs_is_comment == rhs_is_comment,
+            ) => {
+                if lhs_is_comment != rhs_is_comment {
+                    return false;
+                }
+
+                if *lhs_is_comment {
+                    let is_multiline = lhs_content.lines().count() > 1;
+
+                    if is_multiline {
+                        let lhs_lines = lhs_content.lines().map(|l| l.trim_start()).collect_vec();
+                        let rhs_lines = rhs_content.lines().map(|l| l.trim_start()).collect_vec();
+                        return lhs_lines == rhs_lines;
+                    }
+                }
+                lhs_content == rhs_content
+            }
             (
                 List {
                     open_content: lhs_open_content,
@@ -1134,6 +1114,22 @@ mod tests {
     }
 
     #[test]
+    fn test_multiline_comment_ignores_leading_whitespace() {
+        let pos = vec![SingleLineSpan {
+            line: 0.into(),
+            start_col: 2,
+            end_col: 3,
+        }];
+
+        let arena = Arena::new();
+
+        let x = Syntax::new_comment(&arena, pos.clone(), "foo\nbar");
+        let y = Syntax::new_comment(&arena, pos, "foo\n    bar");
+
+        assert_eq!(x, y);
+    }
+
+    #[test]
     fn test_atom_equality_ignores_change() {
         assert_eq!(
             Atom {