patch 9.0.1834: Some problems with xxd coloringv9.0.1834

Problem:  Some problems with xxd coloring
Solution: Fix the following problems:

* Support colored output on Windows.
  SetConsoleMode() is required to enable ANSI color sequences.
* Support "NO_COLOR" environment variable.
  If "NO_COLOR" is defined and not empty, colored output should be
  disabled.
  See https://no-color.org/
* "-R" should only accept "always", "never" or "auto" as the parameter.
* Adjust help and documentation. "-R" cannot omit the parameter. Remove
  surrounding brackets.

Related #12131
closes: #12997
closes: #12991
closes: #12986

Signed-off-by: Christian Brabandt <cb@256bit.org>
Co-authored-by: K.Takata <kentkt@csc.jp>

3 files changed, 54 insertions, 18 deletions

diff --git a/runtime/doc/xxd.1 b/runtime/doc/xxd.1
index 9ae132b324..e52ef888d3 100644
--- a/runtime/doc/xxd.1
+++ b/runtime/doc/xxd.1
@@ -135,9 +135,9 @@ to read plain hexadecimal dumps without line number information and without a
 particular column layout. Additional whitespace and line breaks are allowed
 anywhere.
 .TP
-.IR \-R " "[WHEN]
+.IR \-R " " when
 In output the hex-value and the value are both colored with the same color depending on the hex-value. Mostly helping to differentiate printable and non-printable characters.
-.I WHEN
+.I \fIwhen\fP
 is
 .BR never ", " always ", or " auto .
 .TP
diff --git a/src/version.c b/src/version.c
index 1e528504a5..658bf608ef 100644
--- a/src/version.c
+++ b/src/version.c
@@ -700,6 +700,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    1834,
+/**/
     1833,
 /**/
     1832,
diff --git a/src/xxd/xxd.c b/src/xxd/xxd.c
index 590cd44aad..f8601eb09d 100644
--- a/src/xxd/xxd.c
+++ b/src/xxd/xxd.c
@@ -87,10 +87,10 @@
 #endif
 #if defined(WIN32) || defined(CYGWIN)
 # include <io.h>	/* for setmode() */
-#else
-# ifdef UNIX
-#  include <unistd.h>
-# endif
+# include <windows.h>
+#endif
+#ifdef UNIX
+# include <unistd.h>
 #endif
 #include <stdlib.h>
 #include <string.h>	/* for strncmp() */
@@ -135,7 +135,7 @@ extern void perror __P((char *));
 # endif
 #endif
 
-char version[] = "xxd 2023-08-31 by Juergen Weigert et al.";
+char version[] = "xxd 2023-09-01 by Juergen Weigert et al.";
 #ifdef WIN32
 char osver[] = " (Win32)";
 #else
@@ -256,7 +256,7 @@ exit_with_usage(void)
 	  "", "");
 #endif
   fprintf(stderr, "    -u          use upper case hex letters.\n");
-  fprintf(stderr, "    -R [WHEN]   colorize the output; WHEN can be 'always', 'auto' or 'never'. Default: 'auto'.\n"),
+  fprintf(stderr, "    -R when     colorize the output; <when> can be 'always', 'auto' or 'never'. Default: 'auto'.\n"),
   fprintf(stderr, "    -v          show version: \"%s%s\".\n", version, osver);
   exit(1);
 }
@@ -502,7 +502,7 @@ static unsigned char etoa64[] =
     0070,0071,0372,0373,0374,0375,0376,0377
 };
 
-   static void
+  static void
 begin_coloring_char (char *l, int *c, int e, int ebcdic)
 {
   if (ebcdic)
@@ -547,6 +547,27 @@ begin_coloring_char (char *l, int *c, int e, int ebcdic)
   l[(*c)++] = 'm';
 }
 
+  static int
+enable_color(void)
+{
+#ifdef WIN32
+  DWORD   mode;
+  HANDLE  out;
+
+  if (!isatty(1))
+    return 0;
+
+  out = GetStdHandle(STD_OUTPUT_HANDLE);
+  GetConsoleMode(out, &mode);
+  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+  return (int)SetConsoleMode(out, mode);
+#elif defined(UNIX)
+  return isatty(STDOUT_FILENO);
+#else
+  return 0;
+#endif
+}
+
   int
 main(int argc, char *argv[])
 {
@@ -564,10 +585,11 @@ main(int argc, char *argv[])
   char *varname = NULL;
   int addrlen = 9;
   int color = 0;
+  char *no_color;
 
-#ifdef UNIX
-  color = isatty(STDOUT_FILENO);
-#endif
+  no_color = getenv("NO_COLOR");
+  if (no_color == NULL || no_color[0] == '\0')
+    color = enable_color();
 
 #ifdef AMIGA
   /* This program doesn't work when started from the Workbench */
@@ -720,14 +742,26 @@ main(int argc, char *argv[])
         }
       else if (!STRNCMP(pp, "-R", 2))
         {
-	  if (!argv[2])
+	  char *pw = pp + 2;
+	  if (!pw[0])
+	    {
+	      pw = argv[2];
+	      argv++;
+	      argc--;
+	    }
+	  if (!pw)
 	    exit_with_usage();
-	  if (!STRNCMP(argv[2], "always", 2))
-	    color = 1;
-	  else if (!STRNCMP(argv[2], "never", 1))
+	  if (!STRNCMP(pw, "always", 6))
+	    {
+	      (void)enable_color();
+	      color = 1;
+	    }
+	  else if (!STRNCMP(pw, "never", 5))
 	    color = 0;
-	  argv++;
-	  argc--;
+	  else if (!STRNCMP(pw, "auto", 4))
+	    ;	/* Do nothing. */
+	  else
+	    exit_with_usage();
         }
       else if (!strcmp(pp, "--"))	/* end of options */
 	{