termcolor: support ANSI in Windows terminals

This commit uses the new virtual terminal processing feature in Windows 10
to enable the use of ANSI escape codes to color ripgrep's output. This
technique is preferred over the console APIs because it seems like where
the future is heading, but also because it avoids needing to use an
intermediate buffer to deal with the Windows console in a multithreaded
environment.

2 files changed, 30 insertions, 13 deletions

diff --git a/termcolor/src/lib.rs b/termcolor/src/lib.rs
index 72b276f9..13b9fe24 100644
--- a/termcolor/src/lib.rs
+++ b/termcolor/src/lib.rs
@@ -239,7 +239,7 @@ impl io::Write for IoStandardStream {
     }
 }
 
-/// Same rigmarole for the locked variants of the standard streams.
+// Same rigmarole for the locked variants of the standard streams.
 
 enum IoStandardStreamLock<'a> {
     StdoutLock(io::StdoutLock<'a>),
@@ -328,14 +328,17 @@ impl StandardStream {
     /// the `WriteColor` trait.
     #[cfg(windows)]
     fn create(sty: StandardStreamType, choice: ColorChoice) -> StandardStream {
-        let con = match sty {
+        let mut con = match sty {
             StandardStreamType::Stdout => wincolor::Console::stdout(),
             StandardStreamType::Stderr => wincolor::Console::stderr(),
         };
         let is_win_console = con.is_ok();
+        let is_console_virtual = con.as_mut().map(|con| {
+            con.set_virtual_terminal_processing(true).is_ok()
+        }).unwrap_or(false);
         let wtr =
             if choice.should_attempt_color() {
-                if choice.should_ansi() {
+                if choice.should_ansi() || is_console_virtual {
                     WriterInner::Ansi(Ansi(IoStandardStream::new(sty)))
                 } else if let Ok(console) = con {
                     WriterInner::Windows {
@@ -612,10 +615,18 @@ impl BufferWriter {
     /// the buffers themselves.
     #[cfg(windows)]
     fn create(sty: StandardStreamType, choice: ColorChoice) -> BufferWriter {
-        let con = match sty {
+        let mut con = match sty {
             StandardStreamType::Stdout => wincolor::Console::stdout(),
             StandardStreamType::Stderr => wincolor::Console::stderr(),
-        }.ok().map(Mutex::new);
+        }.ok();
+        let is_console_virtual = con.as_mut().map(|con| {
+            con.set_virtual_terminal_processing(true).is_ok()
+        }).unwrap_or(false);
+        // If we can enable ANSI on Windows, then we don't need the console
+        // anymore.
+        if is_console_virtual {
+            con = None;
+        }
         let stream = LossyStandardStream::new(IoStandardStream::new(sty))
             .is_console(con.is_some());
         BufferWriter {
@@ -623,7 +634,7 @@ impl BufferWriter {
             printed: AtomicBool::new(false),
             separator: None,
             color_choice: choice,
-            console: con,
+            console: con.map(Mutex::new),
         }
     }
 
diff --git a/wincolor/src/win.rs b/wincolor/src/win.rs
index 43ccdcdd..9542704c 100644
--- a/wincolor/src/win.rs
+++ b/wincolor/src/win.rs
@@ -130,17 +130,23 @@ impl Console {
         &mut self,
         yes: bool,
     ) -> io::Result<()> {
-        let mut lpmode = 0;
+        let vt = wincon::ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+
+        let mut old_mode = 0;
         let handle = unsafe { processenv::GetStdHandle(self.handle_id) };
-        if unsafe { consoleapi::GetConsoleMode(handle, &mut lpmode) } == 0 {
+        if unsafe { consoleapi::GetConsoleMode(handle, &mut old_mode) } == 0 {
             return Err(io::Error::last_os_error());
         }
-        if yes {
-            lpmode |= wincon::ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-        } else {
-            lpmode &= !wincon::ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+        let new_mode =
+            if yes {
+                old_mode | vt
+            } else {
+                old_mode & !vt
+            };
+        if old_mode == new_mode {
+            return Ok(());
         }
-        if unsafe { consoleapi::SetConsoleMode(handle, lpmode) } == 0 {
+        if unsafe { consoleapi::SetConsoleMode(handle, new_mode) } == 0 {
             return Err(io::Error::last_os_error());
         }
         Ok(())