Remove asn1 module dependency from RSASSA-PKCS1-v1_5 implementation.

Replace use of the asn1 module (X509_SIG, X509_ALGOR, ASN1_TYPE,
ASN1_OCTET_STRING, i2d_X509_SIG(), etc.) as well as OID lookups using
OBJ_nid2obj() with pre-generated DigestInfo encodings for MD2, MD5, MDC-2,
SHA-1, SHA-2 and SHA-3; the encoding is selected based on the NID. This is
similar to the approach used by the old FOM.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9138)

1 files changed, 186 insertions, 50 deletions

diff --git a/crypto/rsa/rsa_sign.c b/crypto/rsa/rsa_sign.c
index 0ed9acf431..6479d12684 100644
--- a/crypto/rsa/rsa_sign.c
+++ b/crypto/rsa/rsa_sign.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -14,68 +14,199 @@
 #include <openssl/objects.h>
 #include <openssl/x509.h>
 #include "crypto/x509.h"
+#ifndef OPENSSL_NO_MD2
+# include <openssl/md2.h> /* uses MD2_DIGEST_LENGTH */
+#endif
+#ifndef OPENSSL_NO_MD5
+# include <openssl/md5.h> /* uses MD5_DIGEST_LENGTH */
+#endif
+#ifndef OPENSSL_NO_MDC2
+# include <openssl/mdc2.h> /* uses MDC2_DIGEST_LENGTH */
+#endif
+#include <openssl/sha.h> /* uses SHA???_DIGEST_LENGTH */
 #include "rsa_local.h"
 
+/*
+ * The general purpose ASN1 code is not available inside the FIPS provider.
+ * To remove the dependency RSASSA-PKCS1-v1_5 DigestInfo encodings can be
+ * treated as a special case by pregenerating the required ASN1 encoding.
+ * This encoding will also be shared by the default provider.
+ *
+ * The EMSA-PKCS1-v1_5 encoding method includes an ASN.1 value of type
+ * DigestInfo, where the type DigestInfo has the syntax
+ *
+ *     DigestInfo ::= SEQUENCE {
+ *         digestAlgorithm DigestAlgorithm,
+ *         digest OCTET STRING
+ *     }
+ *
+ *     DigestAlgorithm ::= AlgorithmIdentifier {
+ *         {PKCS1-v1-5DigestAlgorithms}
+ *     }
+ *
+ * The AlgorithmIdentifier is a sequence containing the digest OID and
+ * parameters (a value of type NULL).
+ *
+ * The ENCODE_DIGESTINFO_SHA() and ENCODE_DIGESTINFO_MD() macros define an
+ * initialized array containing the DER encoded DigestInfo for the specified
+ * SHA or MD digest. The content of the OCTET STRING is not included.
+ * |name| is the digest name.
+ * |n| is last byte in the encoded OID for the digest.
+ * |sz| is the digest length in bytes. It must not be greater than 110.
+ */
+
+#define ASN1_SEQUENCE 0x30
+#define ASN1_OCTET_STRING 0x04
+#define ASN1_NULL 0x05
+#define ASN1_OID 0x06
+
+/* SHA OIDs are of the form: (2 16 840 1 101 3 4 2 |n|) */
+#define ENCODE_DIGESTINFO_SHA(name, n, sz)                                     \
+static const unsigned char digestinfo_##name##_der[] = {                       \
+    ASN1_SEQUENCE, 0x11 + sz,                                                  \
+      ASN1_SEQUENCE, 0x0d,                                                     \
+        ASN1_OID, 0x09, 2 * 40 + 16, 0x86, 0x48, 1, 101, 3, 4, 2, n,           \
+        ASN1_NULL, 0x00,                                                       \
+      ASN1_OCTET_STRING, sz                                                    \
+};
+
+/* MD2 and MD5 OIDs are of the form: (1 2 840 113549 2 |n|) */
+#define ENCODE_DIGESTINFO_MD(name, n, sz)                                      \
+static const unsigned char digestinfo_##name##_der[] = {                       \
+    ASN1_SEQUENCE, 0x10 + sz,                                                  \
+      ASN1_SEQUENCE, 0x0c,                                                     \
+        ASN1_OID, 0x08, 1 * 40 + 2, 0x86, 0x48, 0x86, 0xf7, 0x0d, 2, n,        \
+        ASN1_NULL, 0x00,                                                       \
+      ASN1_OCTET_STRING, sz                                                    \
+};
+
+#ifndef FIPS_MODE
+# ifndef OPENSSL_NO_MD2
+ENCODE_DIGESTINFO_MD(md2, 0x02, MD2_DIGEST_LENGTH)
+# endif
+# ifndef OPENSSL_NO_MD5
+ENCODE_DIGESTINFO_MD(md5, 0x05, MD5_DIGEST_LENGTH)
+# endif
+# ifndef OPENSSL_NO_MDC2
+/* MDC-2 (2 5 8 3 101) */
+static const unsigned char digestinfo_mdc2_der[] = {
+    ASN1_SEQUENCE, 0x0c + MDC2_DIGEST_LENGTH,
+      ASN1_SEQUENCE, 0x08,
+        ASN1_OID, 0x04, 2 * 40 + 5, 8, 3, 101,
+        ASN1_NULL, 0x00,
+      ASN1_OCTET_STRING, MDC2_DIGEST_LENGTH
+};
+# endif
+/* SHA-1 (1 3 14 3 2 26) */
+static const unsigned char digestinfo_sha1_der[] = {
+    ASN1_SEQUENCE, 0x0d + SHA_DIGEST_LENGTH,
+      ASN1_SEQUENCE, 0x09,
+        ASN1_OID, 0x05, 1 * 40 + 3, 14, 3, 2, 26,
+        ASN1_NULL, 0x00,
+      ASN1_OCTET_STRING, SHA_DIGEST_LENGTH
+};
+
+#endif /* FIPS_MODE */
+
+ENCODE_DIGESTINFO_SHA(sha256, 0x01, SHA256_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha384, 0x02, SHA384_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha512, 0x03, SHA512_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha224, 0x04, SHA224_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha512_224, 0x05, SHA224_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha512_256, 0x06, SHA256_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha3_224, 0x07, SHA224_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha3_256, 0x08, SHA256_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha3_384, 0x09, SHA384_DIGEST_LENGTH)
+ENCODE_DIGESTINFO_SHA(sha3_512, 0x0a, SHA512_DIGEST_LENGTH)
+
+#define MD_CASE(name)                                                          \
+    case NID_##name:                                                           \
+        *len = sizeof(digestinfo_##name##_der);                                \
+        return digestinfo_##name##_der;
+
+static const unsigned char *digestinfo_encoding(int nid, size_t *len)
+{
+    switch (nid) {
+#ifndef FIPS_MODE
+# ifndef OPENSSL_NO_MDC2
+    MD_CASE(mdc2)
+# endif
+# ifndef OPENSSL_NO_MD2
+    MD_CASE(md2)
+# endif
+# ifndef OPENSSL_NO_MD5
+    MD_CASE(md5)
+# endif
+    MD_CASE(sha1)
+#endif /* FIPS_MODE */
+    MD_CASE(sha224)
+    MD_CASE(sha256)
+    MD_CASE(sha384)
+    MD_CASE(sha512)
+    MD_CASE(sha512_224)
+    MD_CASE(sha512_256)
+    MD_CASE(sha3_224)
+    MD_CASE(sha3_256)
+    MD_CASE(sha3_384)
+    MD_CASE(sha3_512)
+    default:
+        return NULL;
+    }
+}
+
 /* Size of an SSL signature: MD5+SHA1 */
 #define SSL_SIG_LENGTH  36
 
 /*
- * encode_pkcs1 encodes a DigestInfo prefix of hash |type| and digest |m|, as
+ * Encodes a DigestInfo prefix of hash |type| and digest |m|, as
  * described in EMSA-PKCS1-v1_5-ENCODE, RFC 3447 section 9.2 step 2. This
  * encodes the DigestInfo (T and tLen) but does not add the padding.
  *
  * On success, it returns one and sets |*out| to a newly allocated buffer
  * containing the result and |*out_len| to its length. The caller must free
- * |*out| with |OPENSSL_free|. Otherwise, it returns zero.
+ * |*out| with OPENSSL_free(). Otherwise, it returns zero.
  */
-static int encode_pkcs1(unsigned char **out, int *out_len, int type,
-                        const unsigned char *m, unsigned int m_len)
+static int encode_pkcs1(unsigned char **out, size_t *out_len, int type,
+                        const unsigned char *m, size_t m_len)
 {
-    X509_SIG sig;
-    X509_ALGOR algor;
-    ASN1_TYPE parameter;
-    ASN1_OCTET_STRING digest;
-    uint8_t *der = NULL;
-    int len;
-
-    sig.algor = &algor;
-    sig.algor->algorithm = OBJ_nid2obj(type);
-    if (sig.algor->algorithm == NULL) {
+    size_t di_prefix_len, dig_info_len;
+    const unsigned char *di_prefix;
+    unsigned char *dig_info;
+
+    if (type == NID_undef) {
         RSAerr(RSA_F_ENCODE_PKCS1, RSA_R_UNKNOWN_ALGORITHM_TYPE);
         return 0;
     }
-    if (OBJ_length(sig.algor->algorithm) == 0) {
+    di_prefix = digestinfo_encoding(type, &di_prefix_len);
+    if (di_prefix == NULL) {
         RSAerr(RSA_F_ENCODE_PKCS1,
                RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
         return 0;
     }
-    parameter.type = V_ASN1_NULL;
-    parameter.value.ptr = NULL;
-    sig.algor->parameter = &parameter;
-
-    sig.digest = &digest;
-    sig.digest->data = (unsigned char *)m;
-    sig.digest->length = m_len;
-
-    len = i2d_X509_SIG(&sig, &der);
-    if (len < 0)
+    dig_info_len = di_prefix_len + m_len;
+    dig_info = OPENSSL_malloc(dig_info_len);
+    if (dig_info == NULL) {
+        RSAerr(RSA_F_ENCODE_PKCS1, ERR_R_MALLOC_FAILURE);
         return 0;
+    }
+    memcpy(dig_info, di_prefix, di_prefix_len);
+    memcpy(dig_info + di_prefix_len, m, m_len);
 
-    *out = der;
-    *out_len = len;
+    *out = dig_info;
+    *out_len = dig_info_len;
     return 1;
 }
 
 int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
              unsigned char *sigret, unsigned int *siglen, RSA *rsa)
 {
-    int encrypt_len, encoded_len = 0, ret = 0;
+    int encrypt_len, ret = 0;
+    size_t encoded_len = 0;
     unsigned char *tmps = NULL;
     const unsigned char *encoded = NULL;
 
-    if (rsa->meth->rsa_sign) {
+    if (rsa->meth->rsa_sign != NULL)
         return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
-    }
 
     /* Compute the encoded digest. */
     if (type == NID_md5_sha1) {
@@ -96,11 +227,11 @@ int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
         encoded = tmps;
     }
 
-    if (encoded_len > RSA_size(rsa) - RSA_PKCS1_PADDING_SIZE) {
+    if (encoded_len + RSA_PKCS1_PADDING_SIZE > (size_t)RSA_size(rsa)) {
         RSAerr(RSA_F_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
         goto err;
     }
-    encrypt_len = RSA_private_encrypt(encoded_len, encoded, sigret, rsa,
+    encrypt_len = RSA_private_encrypt((int)encoded_len, encoded, sigret, rsa,
                                       RSA_PKCS1_PADDING);
     if (encrypt_len <= 0)
         goto err;
@@ -109,23 +240,25 @@ int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
     ret = 1;
 
 err:
-    OPENSSL_clear_free(tmps, (size_t)encoded_len);
+    OPENSSL_clear_free(tmps, encoded_len);
     return ret;
 }
 
 /*
- * int_rsa_verify verifies an RSA signature in |sigbuf| using |rsa|. It may be
- * called in two modes. If |rm| is NULL, it verifies the signature for digest
- * |m|. Otherwise, it recovers the digest from the signature, writing the digest
- * to |rm| and the length to |*prm_len|. |type| is the NID of the digest
- * algorithm to use. It returns one on successful verification and zero
- * otherwise.
+ * Verify an RSA signature in |sigbuf| using |rsa|.
+ * |type| is the NID of the digest algorithm to use.
+ * If |rm| is NULL, it verifies the signature for digest |m|, otherwise
+ * it recovers the digest from the signature, writing the digest to |rm| and
+ * the length to |*prm_len|.
+ *
+ * It returns one on successful verification or zero otherwise.
  */
 int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
                    unsigned char *rm, size_t *prm_len,
                    const unsigned char *sigbuf, size_t siglen, RSA *rsa)
 {
-    int decrypt_len, ret = 0, encoded_len = 0;
+    int len, ret = 0;
+    size_t decrypt_len, encoded_len = 0;
     unsigned char *decrypt_buf = NULL, *encoded = NULL;
 
     if (siglen != (size_t)RSA_size(rsa)) {
@@ -140,10 +273,11 @@ int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
         goto err;
     }
 
-    decrypt_len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
-                                     RSA_PKCS1_PADDING);
-    if (decrypt_len <= 0)
+    len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
+                             RSA_PKCS1_PADDING);
+    if (len <= 0)
         goto err;
+    decrypt_len = len;
 
     if (type == NID_md5_sha1) {
         /*
@@ -203,8 +337,11 @@ int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
                 goto err;
             }
 
-            m_len = EVP_MD_size(md);
-            if (m_len > (size_t)decrypt_len) {
+            len = EVP_MD_size(md);
+            if (len <= 0)
+                goto err;
+            m_len = (unsigned int)len;
+            if (m_len > decrypt_len) {
                 RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_DIGEST_LENGTH);
                 goto err;
             }
@@ -216,7 +353,7 @@ int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
             goto err;
 
         if (encoded_len != decrypt_len
-            || memcmp(encoded, decrypt_buf, encoded_len) != 0) {
+                || memcmp(encoded, decrypt_buf, encoded_len) != 0) {
             RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
             goto err;
         }
@@ -231,7 +368,7 @@ int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
     ret = 1;
 
 err:
-    OPENSSL_clear_free(encoded, (size_t)encoded_len);
+    OPENSSL_clear_free(encoded, encoded_len);
     OPENSSL_clear_free(decrypt_buf, siglen);
     return ret;
 }
@@ -240,9 +377,8 @@ int RSA_verify(int type, const unsigned char *m, unsigned int m_len,
                const unsigned char *sigbuf, unsigned int siglen, RSA *rsa)
 {
 
-    if (rsa->meth->rsa_verify) {
+    if (rsa->meth->rsa_verify != NULL)
         return rsa->meth->rsa_verify(type, m, m_len, sigbuf, siglen, rsa);
-    }
 
     return int_rsa_verify(type, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
 }