Fix false positives of IS_*() macros for 8-bit ASCII characters

Fixes #5778, #5840

The various IS_*() macros did not work correctly for 8-bit ASCII
characters with the high bit set, because the CVT(a) preprocessor
macro and'ed the given ASCII value with 0x7F, effectively folding
the high value range 128-255 over the low value range 0-127.
As a consequence, some of the IS_*() erroneously returned TRUE.

This commit fixes the issue by adding range checks instead of
cutting off high order bits using a mask. In order avoid multiple
evaluation of macro arguments, most of the implementation was moved
from macros into a static function is_keytype().

Thanks to Румен Петров for reporting and analyzing the UTF-8 parsing
issue #5840.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5903)

1 files changed, 25 insertions, 0 deletions

diff --git a/crypto/conf/conf_def.c b/crypto/conf/conf_def.c
index 05ba1c9358..aa9f2faa9a 100644
--- a/crypto/conf/conf_def.c
+++ b/crypto/conf/conf_def.c
@@ -33,6 +33,7 @@
  */
 #define MAX_CONF_VALUE_LENGTH       65536
 
+static int is_keytype(const CONF *conf, char c, unsigned short type);
 static char *eat_ws(CONF *conf, char *p);
 static void trim_ws(CONF *conf, char *start);
 static char *eat_alpha_numeric(CONF *conf, char *p);
@@ -732,6 +733,30 @@ static BIO *get_next_file(const char *path, OPENSSL_DIR_CTX **dirctx)
 }
 #endif
 
+static int is_keytype(const CONF *conf, char c, unsigned short type)
+{
+    const unsigned short * keytypes = (const unsigned short *) conf->meth_data;
+    unsigned char key = (unsigned char)c;
+
+#ifdef CHARSET_EBCDIC
+# if CHAR_BIT > 8
+    if (key > 255) {
+        /* key is out of range for os_toascii table */
+        return 0;
+    }
+# endif
+    /* convert key from ebcdic to ascii */
+    key = os_toascii[key];
+#endif
+
+    if (key > 127) {
+        /* key is not a seven bit ascii character */
+        return 0;
+    }
+
+    return (keytypes[key] & type) ? 1 : 0;
+}
+
 static char *eat_ws(CONF *conf, char *p)
 {
     while (IS_WS(conf, *p) && (!IS_EOF(conf, *p)))