Bug fix: fix tokenization of non-ASCII text

1 files changed, 5 insertions, 4 deletions

diff --git a/src/edits.rs b/src/edits.rs
index a7142d6c..3707d195 100644
--- a/src/edits.rs
+++ b/src/edits.rs
@@ -1,6 +1,7 @@
 use regex::Regex;
 
 use lazy_static::lazy_static;
+use unicode_segmentation::UnicodeSegmentation;
 use unicode_width::UnicodeWidthStr;
 
 use crate::align;
@@ -90,8 +91,8 @@ fn tokenize(line: &str) -> Vec<&str> {
             tokens.push("");
         }
         // Align separating text as multiple single-character tokens.
-        for i in offset..m.start() {
-            tokens.push(&line[i..i + 1]);
+        for t in line[offset..m.start()].graphemes(true) {
+            tokens.push(t);
         }
         tokens.push(&line[m.start()..m.end()]);
         offset = m.end();
@@ -100,8 +101,8 @@ fn tokenize(line: &str) -> Vec<&str> {
         if offset == 0 {
             tokens.push("");
         }
-        for i in offset..line.len() {
-            tokens.push(&line[i..i + 1]);
+        for t in line[offset..line.len()].graphemes(true) {
+            tokens.push(t);
         }
     }
     tokens