Call pygments.Lexer.get_tokens_unprocessed (This doesn't replace \r.)

This should fix: https://github.com/ipython/ipython/issues/9737

1 files changed, 11 insertions, 3 deletions

diff --git a/prompt_toolkit/layout/lexers.py b/prompt_toolkit/layout/lexers.py
index 552518f6..95af2a4d 100644
--- a/prompt_toolkit/layout/lexers.py
+++ b/prompt_toolkit/layout/lexers.py
@@ -230,9 +230,17 @@ class PygmentsLexer(Lexer):
             Create a generator that yields the lexed lines.
             Each iteration it yields a (line_number, [(token, text), ...]) tuple.
             """
-            text = '\n'.join(document.lines[start_lineno:])[column:]
-            return enumerate(split_lines(self.pygments_lexer.get_tokens(text)),
-                                  start_lineno)
+            def get_tokens():
+                text = '\n'.join(document.lines[start_lineno:])[column:]
+
+                # We call `get_tokens_unprocessed`, because `get_tokens` will
+                # still replace \r\n and \r by \n.  (We don't want that,
+                # Pygments should return exactly the same amount of text, as we
+                # have given as input.)
+                for _, t, v in self.pygments_lexer.get_tokens_unprocessed(text):
+                    yield t, v
+
+            return enumerate(split_lines(get_tokens()), start_lineno)
 
         def get_generator(i):
             """