PEM: Unlock MSBLOB and PVK functions from 'no-dsa' and 'no-rc4'

All these functions are usable with RSA keys, there's no reason why
they should be unaccessible when DSA or RC4 are disabled.

When DSA is disabled, it's not possible to use these functions for
DSA EVP_PKEYs.  That's fine, and supported.

When RC4 is disabled, it's not possible to use these functions to
write encrypted PVK output.  That doesn't even depend on the
definition of OPENSSL_NO_RC4, but if the RC4 algorithm is accessible
via EVP, something that isn't known when building libcrypto.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13648)

1 files changed, 118 insertions, 72 deletions

diff --git a/crypto/pem/pvkfmt.c b/crypto/pem/pvkfmt.c
index d10cf7a349..b0aa76b3f5 100644
--- a/crypto/pem/pvkfmt.c
+++ b/crypto/pem/pvkfmt.c
@@ -13,7 +13,7 @@
  */
 
 /*
- * DSA low level APIs are deprecated for public use, but still ok for
+ * RSA and DSA low level APIs are deprecated for public use, but still ok for
  * internal use.
  */
 #include "internal/deprecated.h"
@@ -23,9 +23,8 @@
 #include "crypto/pem.h"
 #include <openssl/rand.h>
 #include <openssl/bn.h>
-#if !defined(OPENSSL_NO_RSA) && !defined(OPENSSL_NO_DSA)
-# include <openssl/dsa.h>
-# include <openssl/rsa.h>
+#include <openssl/dsa.h>
+#include <openssl/rsa.h>
 
 /*
  * Utility function: read a DWORD (4 byte unsigned integer) in little endian
@@ -36,6 +35,7 @@ static unsigned int read_ledword(const unsigned char **in)
 {
     const unsigned char *p = *in;
     unsigned int ret;
+
     ret = (unsigned int)*p++;
     ret |= (unsigned int)*p++ << 8;
     ret |= (unsigned int)*p++ << 16;
@@ -87,14 +87,17 @@ static int read_lebn(const unsigned char **in, unsigned int nbyte, BIGNUM **r)
 
 static EVP_PKEY *b2i_rsa(const unsigned char **in,
                          unsigned int bitlen, int ispub);
+#ifndef OPENSSL_NO_DSA
 static EVP_PKEY *b2i_dss(const unsigned char **in,
                          unsigned int bitlen, int ispub);
+#endif
 
 int ossl_do_blob_header(const unsigned char **in, unsigned int length,
                         unsigned int *pmagic, unsigned int *pbitlen,
                         int *pisdss, int *pispub)
 {
     const unsigned char *p = *in;
+
     if (length < 16)
         return 0;
     /* bType */
@@ -110,8 +113,9 @@ int ossl_do_blob_header(const unsigned char **in, unsigned int length,
             return 0;
         }
         *pispub = 0;
-    } else
+    } else {
         return 0;
+    }
     p++;
     /* Version */
     if (*p++ != 0x2) {
@@ -155,9 +159,9 @@ int ossl_do_blob_header(const unsigned char **in, unsigned int length,
 
 static unsigned int blob_length(unsigned bitlen, int isdss, int ispub)
 {
-    unsigned int nbyte, hnbyte;
-    nbyte = (bitlen + 7) >> 3;
-    hnbyte = (bitlen + 15) >> 4;
+    unsigned int nbyte = (bitlen + 7) >> 3;
+    unsigned int hnbyte = (bitlen + 15) >> 4;
+
     if (isdss) {
 
         /*
@@ -191,6 +195,7 @@ EVP_PKEY *ossl_b2i(const unsigned char **in, unsigned int length, int *ispub)
     const unsigned char *p = *in;
     unsigned int bitlen, magic;
     int isdss;
+
     if (ossl_do_blob_header(&p, length, &magic, &bitlen, &isdss, ispub) <= 0) {
         ERR_raise(ERR_LIB_PEM, PEM_R_KEYBLOB_HEADER_PARSE_ERROR);
         return NULL;
@@ -200,10 +205,15 @@ EVP_PKEY *ossl_b2i(const unsigned char **in, unsigned int length, int *ispub)
         ERR_raise(ERR_LIB_PEM, PEM_R_KEYBLOB_TOO_SHORT);
         return NULL;
     }
-    if (isdss)
-        return b2i_dss(&p, bitlen, *ispub);
-    else
+    if (!isdss)
         return b2i_rsa(&p, bitlen, *ispub);
+#ifndef OPENSSL_NO_DSA
+    else
+        return b2i_dss(&p, bitlen, *ispub);
+#endif
+
+    ERR_raise(ERR_LIB_PEM, PEM_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
+    return NULL;
 }
 
 EVP_PKEY *ossl_b2i_bio(BIO *in, int *ispub)
@@ -213,6 +223,7 @@ EVP_PKEY *ossl_b2i_bio(BIO *in, int *ispub)
     unsigned int bitlen, magic, length;
     int isdss;
     EVP_PKEY *ret = NULL;
+
     if (BIO_read(in, hdr_buf, 16) != 16) {
         ERR_raise(ERR_LIB_PEM, PEM_R_KEYBLOB_TOO_SHORT);
         return NULL;
@@ -237,16 +248,22 @@ EVP_PKEY *ossl_b2i_bio(BIO *in, int *ispub)
         goto err;
     }
 
-    if (isdss)
-        ret = b2i_dss(&p, bitlen, *ispub);
-    else
+    if (!isdss)
         ret = b2i_rsa(&p, bitlen, *ispub);
+#ifndef OPENSSL_NO_DSA
+    else
+        ret = b2i_dss(&p, bitlen, *ispub);
+#endif
+
+    if (ret == NULL)
+        ERR_raise(ERR_LIB_PEM, PEM_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
 
  err:
     OPENSSL_free(buf);
     return ret;
 }
 
+#ifndef OPENSSL_NO_DSA
 static EVP_PKEY *b2i_dss(const unsigned char **in,
                          unsigned int bitlen, int ispub)
 {
@@ -254,11 +271,9 @@ static EVP_PKEY *b2i_dss(const unsigned char **in,
     EVP_PKEY *ret = NULL;
     DSA *dsa = NULL;
     BN_CTX *ctx = NULL;
-    unsigned int nbyte;
     BIGNUM *pbn = NULL, *qbn = NULL, *gbn = NULL, *priv_key = NULL;
     BIGNUM *pub_key = NULL;
-
-    nbyte = (bitlen + 7) >> 3;
+    unsigned int nbyte = (bitlen + 7) >> 3;
 
     dsa = DSA_new();
     ret = EVP_PKEY_new();
@@ -321,6 +336,7 @@ static EVP_PKEY *b2i_dss(const unsigned char **in,
     BN_CTX_free(ctx);
     return NULL;
 }
+#endif
 
 static EVP_PKEY *b2i_rsa(const unsigned char **in,
                          unsigned int bitlen, int ispub)
@@ -330,9 +346,9 @@ static EVP_PKEY *b2i_rsa(const unsigned char **in,
     BIGNUM *e = NULL, *n = NULL, *d = NULL;
     BIGNUM *p = NULL, *q = NULL, *dmp1 = NULL, *dmq1 = NULL, *iqmp = NULL;
     RSA *rsa = NULL;
-    unsigned int nbyte, hnbyte;
-    nbyte = (bitlen + 7) >> 3;
-    hnbyte = (bitlen + 15) >> 4;
+    unsigned int nbyte = (bitlen + 7) >> 3;
+    unsigned int hnbyte = (bitlen + 15) >> 4;
+
     rsa = RSA_new();
     ret = EVP_PKEY_new();
     if (rsa == NULL || ret == NULL)
@@ -419,6 +435,7 @@ EVP_PKEY *b2i_PublicKey_bio(BIO *in)
 static void write_ledword(unsigned char **out, unsigned int dw)
 {
     unsigned char *p = *out;
+
     *p++ = dw & 0xff;
     *p++ = (dw >> 8) & 0xff;
     *p++ = (dw >> 16) & 0xff;
@@ -433,25 +450,29 @@ static void write_lebn(unsigned char **out, const BIGNUM *bn, int len)
 }
 
 static int check_bitlen_rsa(RSA *rsa, int ispub, unsigned int *magic);
-static int check_bitlen_dsa(DSA *dsa, int ispub, unsigned int *magic);
-
 static void write_rsa(unsigned char **out, RSA *rsa, int ispub);
+
+#ifndef OPENSSL_NO_DSA
+static int check_bitlen_dsa(DSA *dsa, int ispub, unsigned int *magic);
 static void write_dsa(unsigned char **out, DSA *dsa, int ispub);
+#endif
 
 static int do_i2b(unsigned char **out, const EVP_PKEY *pk, int ispub)
 {
     unsigned char *p;
-    unsigned int bitlen, magic = 0, keyalg;
+    unsigned int bitlen = 0, magic = 0, keyalg = 0;
     int outlen, noinc = 0;
     int pktype = EVP_PKEY_id(pk);
-    if (pktype == EVP_PKEY_DSA) {
-        bitlen = check_bitlen_dsa(EVP_PKEY_get0_DSA(pk), ispub, &magic);
-        keyalg = MS_KEYALG_DSS_SIGN;
-    } else if (pktype == EVP_PKEY_RSA) {
+
+    if (pktype == EVP_PKEY_RSA) {
         bitlen = check_bitlen_rsa(EVP_PKEY_get0_RSA(pk), ispub, &magic);
         keyalg = MS_KEYALG_RSA_KEYX;
-    } else
-        return -1;
+#ifndef OPENSSL_NO_DSA
+    } else if (pktype == EVP_PKEY_DSA) {
+        bitlen = check_bitlen_dsa(EVP_PKEY_get0_DSA(pk), ispub, &magic);
+        keyalg = MS_KEYALG_DSS_SIGN;
+#endif
+    }
     if (bitlen == 0)
         return -1;
     outlen = 16 + blob_length(bitlen,
@@ -478,10 +499,12 @@ static int do_i2b(unsigned char **out, const EVP_PKEY *pk, int ispub)
     write_ledword(&p, keyalg);
     write_ledword(&p, magic);
     write_ledword(&p, bitlen);
-    if (keyalg == MS_KEYALG_DSS_SIGN)
-        write_dsa(&p, EVP_PKEY_get0_DSA(pk), ispub);
-    else
+    if (keyalg == MS_KEYALG_RSA_KEYX)
         write_rsa(&p, EVP_PKEY_get0_RSA(pk), ispub);
+#ifndef OPENSSL_NO_DSA
+    else
+        write_dsa(&p, EVP_PKEY_get0_DSA(pk), ispub);
+#endif
     if (!noinc)
         *out += outlen;
     return outlen;
@@ -491,6 +514,7 @@ static int do_i2b_bio(BIO *out, const EVP_PKEY *pk, int ispub)
 {
     unsigned char *tmp = NULL;
     int outlen, wrlen;
+
     outlen = do_i2b(&tmp, pk, ispub);
     if (outlen < 0)
         return -1;
@@ -501,34 +525,6 @@ static int do_i2b_bio(BIO *out, const EVP_PKEY *pk, int ispub)
     return -1;
 }
 
-static int check_bitlen_dsa(DSA *dsa, int ispub, unsigned int *pmagic)
-{
-    int bitlen;
-    const BIGNUM *p = NULL, *q = NULL, *g = NULL;
-    const BIGNUM *pub_key = NULL, *priv_key = NULL;
-
-    DSA_get0_pqg(dsa, &p, &q, &g);
-    DSA_get0_key(dsa, &pub_key, &priv_key);
-    bitlen = BN_num_bits(p);
-    if ((bitlen & 7) || (BN_num_bits(q) != 160)
-        || (BN_num_bits(g) > bitlen))
-        goto badkey;
-    if (ispub) {
-        if (BN_num_bits(pub_key) > bitlen)
-            goto badkey;
-        *pmagic = MS_DSS1MAGIC;
-    } else {
-        if (BN_num_bits(priv_key) > 160)
-            goto badkey;
-        *pmagic = MS_DSS2MAGIC;
-    }
-
-    return bitlen;
- badkey:
-    ERR_raise(ERR_LIB_PEM, PEM_R_UNSUPPORTED_KEY_COMPONENTS);
-    return 0;
-}
-
 static int check_bitlen_rsa(RSA *rsa, int ispub, unsigned int *pmagic)
 {
     int nbyte, hnbyte, bitlen;
@@ -591,6 +587,35 @@ static void write_rsa(unsigned char **out, RSA *rsa, int ispub)
     write_lebn(out, d, nbyte);
 }
 
+#ifndef OPENSSL_NO_DSA
+static int check_bitlen_dsa(DSA *dsa, int ispub, unsigned int *pmagic)
+{
+    int bitlen;
+    const BIGNUM *p = NULL, *q = NULL, *g = NULL;
+    const BIGNUM *pub_key = NULL, *priv_key = NULL;
+
+    DSA_get0_pqg(dsa, &p, &q, &g);
+    DSA_get0_key(dsa, &pub_key, &priv_key);
+    bitlen = BN_num_bits(p);
+    if ((bitlen & 7) || (BN_num_bits(q) != 160)
+        || (BN_num_bits(g) > bitlen))
+        goto badkey;
+    if (ispub) {
+        if (BN_num_bits(pub_key) > bitlen)
+            goto badkey;
+        *pmagic = MS_DSS1MAGIC;
+    } else {
+        if (BN_num_bits(priv_key) > 160)
+            goto badkey;
+        *pmagic = MS_DSS2MAGIC;
+    }
+
+    return bitlen;
+ badkey:
+    ERR_raise(ERR_LIB_PEM, PEM_R_UNSUPPORTED_KEY_COMPONENTS);
+    return 0;
+}
+
 static void write_dsa(unsigned char **out, DSA *dsa, int ispub)
 {
     int nbyte;
@@ -612,6 +637,7 @@ static void write_dsa(unsigned char **out, DSA *dsa, int ispub)
     *out += 24;
     return;
 }
+#endif
 
 int i2b_PrivateKey_bio(BIO *out, const EVP_PKEY *pk)
 {
@@ -623,8 +649,6 @@ int i2b_PublicKey_bio(BIO *out, const EVP_PKEY *pk)
     return do_i2b_bio(out, pk, 1);
 }
 
-# ifndef OPENSSL_NO_RC4
-
 int ossl_do_PVK_header(const unsigned char **in, unsigned int length,
                        int skip_magic,
                        unsigned int *psaltlen, unsigned int *pkeylen)
@@ -669,12 +693,14 @@ int ossl_do_PVK_header(const unsigned char **in, unsigned int length,
     return 1;
 }
 
+#ifndef OPENSSL_NO_RC4
 static int derive_pvk_key(unsigned char *key,
                           const unsigned char *salt, unsigned int saltlen,
                           const unsigned char *pass, int passlen)
 {
     EVP_MD_CTX *mctx = EVP_MD_CTX_new();
     int rv = 1;
+
     if (mctx == NULL
         || !EVP_DigestInit_ex(mctx, EVP_sha1(), NULL)
         || !EVP_DigestUpdate(mctx, salt, saltlen)
@@ -685,6 +711,7 @@ static int derive_pvk_key(unsigned char *key,
     EVP_MD_CTX_free(mctx);
     return rv;
 }
+#endif
 
 static EVP_PKEY *do_PVK_body(const unsigned char **in,
                              unsigned int saltlen, unsigned int keylen,
@@ -692,14 +719,17 @@ static EVP_PKEY *do_PVK_body(const unsigned char **in,
 {
     EVP_PKEY *ret = NULL;
     const unsigned char *p = *in;
-    unsigned int magic;
-    unsigned char *enctmp = NULL, *q;
+    unsigned char *enctmp = NULL;
     unsigned char keybuf[20];
 
     EVP_CIPHER_CTX *cctx = EVP_CIPHER_CTX_new();
     if (saltlen) {
+#ifndef OPENSSL_NO_RC4
+        unsigned int magic;
         char psbuf[PEM_BUFSIZE];
         int enctmplen, inlen;
+        unsigned char *q;
+
         if (cb)
             inlen = cb(psbuf, PEM_BUFSIZE, 0, u);
         else
@@ -749,6 +779,10 @@ static EVP_PKEY *do_PVK_body(const unsigned char **in,
             }
         }
         p = enctmp;
+#else
+        ERR_raise(ERR_LIB_PEM, PEM_R_UNSUPPORTED_CIPHER);
+        goto err;
+#endif
     }
 
     ret = b2i_PrivateKey(&p, keylen);
@@ -768,6 +802,7 @@ EVP_PKEY *b2i_PVK_bio(BIO *in, pem_password_cb *cb, void *u)
     int buflen;
     EVP_PKEY *ret = NULL;
     unsigned int saltlen, keylen;
+
     if (BIO_read(in, pvk_hdr, 24) != 24) {
         ERR_raise(ERR_LIB_PEM, PEM_R_PVK_DATA_TOO_SHORT);
         return NULL;
@@ -798,8 +833,12 @@ static int i2b_PVK(unsigned char **out, const EVP_PKEY *pk, int enclevel,
                    pem_password_cb *cb, void *u)
 {
     int outlen = 24, pklen;
-    unsigned char *p = NULL, *start = NULL, *salt = NULL;
+    unsigned char *p = NULL, *start = NULL;
     EVP_CIPHER_CTX *cctx = NULL;
+#ifndef OPENSSL_NO_RC4
+    unsigned char *salt = NULL;
+#endif
+
     if (enclevel)
         outlen += PVK_SALTLEN;
     pklen = do_i2b(NULL, pk, 0);
@@ -824,24 +863,30 @@ static int i2b_PVK(unsigned char **out, const EVP_PKEY *pk, int enclevel,
 
     write_ledword(&p, MS_PVKMAGIC);
     write_ledword(&p, 0);
-    if (EVP_PKEY_id(pk) == EVP_PKEY_DSA)
-        write_ledword(&p, MS_KEYTYPE_SIGN);
-    else
+    if (EVP_PKEY_id(pk) == EVP_PKEY_RSA)
         write_ledword(&p, MS_KEYTYPE_KEYX);
+#ifndef OPENSSL_NO_DSA
+    else
+        write_ledword(&p, MS_KEYTYPE_SIGN);
+#endif
     write_ledword(&p, enclevel ? 1 : 0);
     write_ledword(&p, enclevel ? PVK_SALTLEN : 0);
     write_ledword(&p, pklen);
     if (enclevel) {
+#ifndef OPENSSL_NO_RC4
         if (RAND_bytes(p, PVK_SALTLEN) <= 0)
             goto error;
         salt = p;
         p += PVK_SALTLEN;
+#endif
     }
     do_i2b(&p, pk, 0);
     if (enclevel != 0) {
+#ifndef OPENSSL_NO_RC4
         char psbuf[PEM_BUFSIZE];
         unsigned char keybuf[20];
         int enctmplen, inlen;
+
         if (cb)
             inlen = cb(psbuf, PEM_BUFSIZE, 1, u);
         else
@@ -863,6 +908,10 @@ static int i2b_PVK(unsigned char **out, const EVP_PKEY *pk, int enclevel,
             goto error;
         if (!EVP_EncryptFinal_ex(cctx, p + enctmplen, &enctmplen))
             goto error;
+#else
+        ERR_raise(ERR_LIB_PEM, PEM_R_UNSUPPORTED_CIPHER);
+        goto error;
+#endif
     }
 
     EVP_CIPHER_CTX_free(cctx);
@@ -884,6 +933,7 @@ int i2b_PVK_bio(BIO *out, const EVP_PKEY *pk, int enclevel,
 {
     unsigned char *tmp = NULL;
     int outlen, wrlen;
+
     outlen = i2b_PVK(&tmp, pk, enclevel, cb, u);
     if (outlen < 0)
         return -1;
@@ -895,7 +945,3 @@ int i2b_PVK_bio(BIO *out, const EVP_PKEY *pk, int enclevel,
     ERR_raise(ERR_LIB_PEM, PEM_R_BIO_WRITE_FAILURE);
     return -1;
 }
-
-# endif
-
-#endif