Make it possible to have RFC2254 escapes with ASN1_STRING_print_ex()

Also adds 'esc_2254' to the possible command line name options

RT#1466

Reviewed-by: Rich Salz <rsalz@openssl.org>
(cherry picked from commit bc776510982b3768761d32c1160e79cb45a561c3)

3 files changed, 28 insertions, 9 deletions

diff --git a/crypto/asn1/a_strex.c b/crypto/asn1/a_strex.c
index 35fd44cd22..5fa7a31500 100644
--- a/crypto/asn1/a_strex.c
+++ b/crypto/asn1/a_strex.c
@@ -75,6 +75,7 @@
 #define CHARTYPE_BS_ESC         (ASN1_STRFLGS_ESC_2253 | CHARTYPE_FIRST_ESC_2253 | CHARTYPE_LAST_ESC_2253)
 
 #define ESC_FLAGS (ASN1_STRFLGS_ESC_2253 | \
+                  ASN1_STRFLGS_ESC_2254 | \
                   ASN1_STRFLGS_ESC_QUOTE | \
                   ASN1_STRFLGS_ESC_CTRL | \
                   ASN1_STRFLGS_ESC_MSB)
@@ -124,7 +125,8 @@ typedef int char_io (void *arg, const void *buf, int len);
 static int do_esc_char(unsigned long c, unsigned char flags, char *do_quotes,
                        char_io *io_ch, void *arg)
 {
-    unsigned char chflgs, chtmp;
+    unsigned short chflgs;
+    unsigned char chtmp;
     char tmphex[HEX_SIZE(long) + 3];
 
     if (c > 0xffffffffL)
@@ -161,7 +163,9 @@ static int do_esc_char(unsigned long c, unsigned char flags, char *do_quotes,
             return -1;
         return 2;
     }
-    if (chflgs & (ASN1_STRFLGS_ESC_CTRL | ASN1_STRFLGS_ESC_MSB)) {
+    if (chflgs & (ASN1_STRFLGS_ESC_CTRL
+                  | ASN1_STRFLGS_ESC_MSB
+                  | ASN1_STRFLGS_ESC_2254)) {
         BIO_snprintf(tmphex, 11, "\\%02X", chtmp);
         if (!io_ch(arg, tmphex, 3))
             return -1;
@@ -191,11 +195,12 @@ static int do_esc_char(unsigned long c, unsigned char flags, char *do_quotes,
  */
 
 static int do_buf(unsigned char *buf, int buflen,
-                  int type, unsigned char flags, char *quotes, char_io *io_ch,
+                  int type, unsigned short flags, char *quotes, char_io *io_ch,
                   void *arg)
 {
     int i, outlen, len;
-    unsigned char orflags, *p, *q;
+    unsigned short orflags;
+    unsigned char *p, *q;
     unsigned long c;
     p = buf;
     q = buf + buflen;
@@ -245,7 +250,7 @@ static int do_buf(unsigned char *buf, int buflen,
                  * character will never be escaped on first and last.
                  */
                 len =
-                    do_esc_char(utfbuf[i], (unsigned char)(flags | orflags),
+                    do_esc_char(utfbuf[i], (unsigned short)(flags | orflags),
                                 quotes, io_ch, arg);
                 if (len < 0)
                     return -1;
@@ -253,7 +258,7 @@ static int do_buf(unsigned char *buf, int buflen,
             }
         } else {
             len =
-                do_esc_char(c, (unsigned char)(flags | orflags), quotes,
+                do_esc_char(c, (unsigned short)(flags | orflags), quotes,
                             io_ch, arg);
             if (len < 0)
                 return -1;
@@ -355,10 +360,10 @@ static int do_print_ex(char_io *io_ch, void *arg, unsigned long lflags,
     int outlen, len;
     int type;
     char quotes;
-    unsigned char flags;
+    unsigned short flags;
     quotes = 0;
     /* Keep a copy of escape flags */
-    flags = (unsigned char)(lflags & ESC_FLAGS);
+    flags = (unsigned short)(lflags & ESC_FLAGS);
 
     type = str->type;
 
diff --git a/crypto/asn1/asn1.h b/crypto/asn1/asn1.h
index 68e791fcdb..09335a92b1 100644
--- a/crypto/asn1/asn1.h
+++ b/crypto/asn1/asn1.h
@@ -505,6 +505,11 @@ typedef const ASN1_ITEM *ASN1_ITEM_EXP (void);
 # define ASN1_STRFLGS_DUMP_DER           0x200
 
 /*
+ * This flag specifies that RC2254 escaping shall be performed.
+ */
+#define ASN1_STRFLGS_ESC_2254           0x400
+
+/*
  * All the string flags consistent with RFC2253, escaping control characters
  * isn't essential in RFC2253 but it is advisable anyway.
  */
diff --git a/crypto/asn1/charmap.pl b/crypto/asn1/charmap.pl
index 25ebf2c205..6c6eb34ea2 100644
--- a/crypto/asn1/charmap.pl
+++ b/crypto/asn1/charmap.pl
@@ -18,6 +18,7 @@ my $NOESC_QUOTE	= 8;	# Not escaped if quoted
 my $PSTRING_CHAR = 0x10;	# Valid PrintableString character
 my $RFC2253_FIRST_ESC = 0x20; # Escaped with \ if first character
 my $RFC2253_LAST_ESC = 0x40;  # Escaped with \ if last character
+my $RFC2254_ESC = 0x400;	# Character escaped \XX
 
 for($i = 0; $i < 128; $i++) {
 	# Set the RFC2253 escape characters (control)
@@ -49,6 +50,14 @@ $arr[ord("<")] |= $NOESC_QUOTE | $RFC2253_ESC;
 $arr[ord(">")] |= $NOESC_QUOTE | $RFC2253_ESC;
 $arr[ord(";")] |= $NOESC_QUOTE | $RFC2253_ESC;
 
+# Remaining RFC2254 characters
+
+$arr[0] |= $RFC2254_ESC;
+$arr[ord("(")] |= $RFC2254_ESC;
+$arr[ord(")")] |= $RFC2254_ESC;
+$arr[ord("*")] |= $RFC2254_ESC;
+$arr[ord("\\")] |= $RFC2254_ESC;
+
 # Remaining PrintableString characters
 
 $arr[ord(" ")] |= $PSTRING_CHAR;
@@ -71,7 +80,7 @@ print <<EOF;
  * Mask of various character properties
  */
 
-static unsigned char char_type[] = {
+static unsigned short char_type[] = {
 EOF
 
 for($i = 0; $i < 128; $i++) {