Fix false positives of IS_*() macros for 8-bit ASCII characters

Fixes #5778, #5840

The various IS_*() macros did not work correctly for 8-bit ASCII
characters with the high bit set, because the CVT(a) preprocessor
macro and'ed the given ASCII value with 0x7F, effectively folding
the high value range 128-255 over the low value range 0-127.
As a consequence, some of the IS_*() erroneously returned TRUE.

This commit fixes the issue by adding range checks instead of
cutting off high order bits using a mask. In order avoid multiple
evaluation of macro arguments, most of the implementation was moved
from macros into a static function is_keytype().

Thanks to Румен Петров for reporting and analyzing the UTF-8 parsing
issue #5840.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5903)

3 files changed, 45 insertions, 34 deletions

diff --git a/crypto/conf/conf_def.c b/crypto/conf/conf_def.c
index 05ba1c9358..aa9f2faa9a 100644
--- a/crypto/conf/conf_def.c
+++ b/crypto/conf/conf_def.c
@@ -33,6 +33,7 @@
  */
 #define MAX_CONF_VALUE_LENGTH       65536
 
+static int is_keytype(const CONF *conf, char c, unsigned short type);
 static char *eat_ws(CONF *conf, char *p);
 static void trim_ws(CONF *conf, char *start);
 static char *eat_alpha_numeric(CONF *conf, char *p);
@@ -732,6 +733,30 @@ static BIO *get_next_file(const char *path, OPENSSL_DIR_CTX **dirctx)
 }
 #endif
 
+static int is_keytype(const CONF *conf, char c, unsigned short type)
+{
+    const unsigned short * keytypes = (const unsigned short *) conf->meth_data;
+    unsigned char key = (unsigned char)c;
+
+#ifdef CHARSET_EBCDIC
+# if CHAR_BIT > 8
+    if (key > 255) {
+        /* key is out of range for os_toascii table */
+        return 0;
+    }
+# endif
+    /* convert key from ebcdic to ascii */
+    key = os_toascii[key];
+#endif
+
+    if (key > 127) {
+        /* key is not a seven bit ascii character */
+        return 0;
+    }
+
+    return (keytypes[key] & type) ? 1 : 0;
+}
+
 static char *eat_ws(CONF *conf, char *p)
 {
     while (IS_WS(conf, *p) && (!IS_EOF(conf, *p)))
diff --git a/crypto/conf/conf_def.h b/crypto/conf/conf_def.h
index aa14d4ad06..73e88baa8b 100644
--- a/crypto/conf/conf_def.h
+++ b/crypto/conf/conf_def.h
@@ -25,24 +25,17 @@
 #define CONF_ALNUM        (CONF_ALPHA|CONF_NUMBER|CONF_UNDER)
 #define CONF_ALNUM_PUNCT  (CONF_ALPHA|CONF_NUMBER|CONF_UNDER|CONF_PUNCT)
 
-#define KEYTYPES(c)       ((const unsigned short *)((c)->meth_data))
 
-#ifndef CHARSET_EBCDIC
-# define CVT(a) ((a) & 0x7F)
-#else
-# define CVT(a) os_toascci[(a) & 0x7F]
-#endif
-
-#define IS_COMMENT(c,a)     (KEYTYPES(c)[CVT(a)] & CONF_COMMENT)
-#define IS_FCOMMENT(c,a)    (KEYTYPES(c)[CVT(a)] & CONF_FCOMMENT)
-#define IS_EOF(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_EOF)
-#define IS_ESC(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_ESC)
-#define IS_NUMBER(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_NUMBER)
-#define IS_WS(c,a)          (KEYTYPES(c)[CVT(a)] & CONF_WS)
-#define IS_ALNUM(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_ALNUM)
-#define IS_ALNUM_PUNCT(c,a) (KEYTYPES(c)[CVT(a)] & CONF_ALNUM_PUNCT)
-#define IS_QUOTE(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_QUOTE)
-#define IS_DQUOTE(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_DQUOTE)
+#define IS_COMMENT(conf,c)     is_keytype(conf, c, CONF_COMMENT)
+#define IS_FCOMMENT(conf,c)    is_keytype(conf, c, CONF_FCOMMENT)
+#define IS_EOF(conf,c)         is_keytype(conf, c, CONF_EOF)
+#define IS_ESC(conf,c)         is_keytype(conf, c, CONF_ESC)
+#define IS_NUMBER(conf,c)      is_keytype(conf, c, CONF_NUMBER)
+#define IS_WS(conf,c)          is_keytype(conf, c, CONF_WS)
+#define IS_ALNUM(conf,c)       is_keytype(conf, c, CONF_ALNUM)
+#define IS_ALNUM_PUNCT(conf,c) is_keytype(conf, c, CONF_ALNUM_PUNCT)
+#define IS_QUOTE(conf,c)       is_keytype(conf, c, CONF_QUOTE)
+#define IS_DQUOTE(conf,c)      is_keytype(conf, c, CONF_DQUOTE)
 
 static const unsigned short CONF_type_default[128] = {
     0x0008, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
diff --git a/crypto/conf/keysets.pl b/crypto/conf/keysets.pl
index e2af26c89f..cfa230ec3a 100644
--- a/crypto/conf/keysets.pl
+++ b/crypto/conf/keysets.pl
@@ -85,24 +85,17 @@ print <<"EOF";
 #define CONF_ALNUM        (CONF_ALPHA|CONF_NUMBER|CONF_UNDER)
 #define CONF_ALNUM_PUNCT  (CONF_ALPHA|CONF_NUMBER|CONF_UNDER|CONF_PUNCT)
 
-#define KEYTYPES(c)       ((const unsigned short *)((c)->meth_data))
 
-#ifndef CHARSET_EBCDIC
-# define CVT(a) ((a) & 0x7F)
-#else
-# define CVT(a) os_toascci[(a) & 0x7F]
-#endif
-
-#define IS_COMMENT(c,a)     (KEYTYPES(c)[CVT(a)] & CONF_COMMENT)
-#define IS_FCOMMENT(c,a)    (KEYTYPES(c)[CVT(a)] & CONF_FCOMMENT)
-#define IS_EOF(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_EOF)
-#define IS_ESC(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_ESC)
-#define IS_NUMBER(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_NUMBER)
-#define IS_WS(c,a)          (KEYTYPES(c)[CVT(a)] & CONF_WS)
-#define IS_ALNUM(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_ALNUM)
-#define IS_ALNUM_PUNCT(c,a) (KEYTYPES(c)[CVT(a)] & CONF_ALNUM_PUNCT)
-#define IS_QUOTE(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_QUOTE)
-#define IS_DQUOTE(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_DQUOTE)
+#define IS_COMMENT(conf,c)     is_keytype(conf, c, CONF_COMMENT)
+#define IS_FCOMMENT(conf,c)    is_keytype(conf, c, CONF_FCOMMENT)
+#define IS_EOF(conf,c)         is_keytype(conf, c, CONF_EOF)
+#define IS_ESC(conf,c)         is_keytype(conf, c, CONF_ESC)
+#define IS_NUMBER(conf,c)      is_keytype(conf, c, CONF_NUMBER)
+#define IS_WS(conf,c)          is_keytype(conf, c, CONF_WS)
+#define IS_ALNUM(conf,c)       is_keytype(conf, c, CONF_ALNUM)
+#define IS_ALNUM_PUNCT(conf,c) is_keytype(conf, c, CONF_ALNUM_PUNCT)
+#define IS_QUOTE(conf,c)       is_keytype(conf, c, CONF_QUOTE)
+#define IS_DQUOTE(conf,c)      is_keytype(conf, c, CONF_DQUOTE)
 
 EOF