Deprecate RSA harder

This deprecates all functions that deal with the types RSA and RSA_METHOD

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13096)

5 files changed, 127 insertions, 108 deletions

diff --git a/apps/genrsa.c b/apps/genrsa.c
index f471814e08..32f088238d 100644
--- a/apps/genrsa.c
+++ b/apps/genrsa.c
@@ -79,9 +79,7 @@ int genrsa_main(int argc, char **argv)
     BN_GENCB *cb = BN_GENCB_new();
     ENGINE *eng = NULL;
     BIGNUM *bn = BN_new();
-    RSA *rsa;
     BIO *out = NULL;
-    const BIGNUM *e;
     EVP_PKEY *pkey = NULL;
     EVP_PKEY_CTX *ctx = NULL;
     const EVP_CIPHER *enc = NULL;
@@ -205,9 +203,11 @@ opthelp:
     }
 
     if (verbose) {
-        if ((rsa = EVP_PKEY_get0_RSA(pkey)) != NULL) {
-            RSA_get0_key(rsa, NULL, &e, NULL);
-        } else {
+        BIGNUM *e = NULL;
+
+        /* Every RSA key has an 'e' */
+        EVP_PKEY_get_bn_param(pkey, "e", &e);
+        if (e == NULL) {
             BIO_printf(bio_err, "Error cannot access RSA e\n");
             goto end;
         }
@@ -218,6 +218,7 @@ opthelp:
         }
         OPENSSL_free(hexe);
         OPENSSL_free(dece);
+        BN_free(e);
     }
     if (traditional) {
         if (!PEM_write_bio_PrivateKey_traditional(out, pkey, enc, NULL, 0,
diff --git a/apps/req.c b/apps/req.c
index 9fa3429baf..41a78593b0 100644
--- a/apps/req.c
+++ b/apps/req.c
@@ -939,10 +939,13 @@ int req_main(int argc, char **argv)
         }
         fprintf(stdout, "Modulus=");
 #ifndef OPENSSL_NO_RSA
-        if (EVP_PKEY_base_id(tpubkey) == EVP_PKEY_RSA) {
-            const BIGNUM *n;
-            RSA_get0_key(EVP_PKEY_get0_RSA(tpubkey), &n, NULL, NULL);
+        if (EVP_PKEY_is_a(tpubkey, "RSA")) {
+            BIGNUM *n;
+
+            /* Every RSA key has an 'n' */
+            EVP_PKEY_get_bn_param(pkey, "n", &n);
             BN_print(out, n);
+            BN_free(n);
         } else
 #endif
             fprintf(stdout, "Wrong Algorithm type");
diff --git a/apps/rsa.c b/apps/rsa.c
index 558b126560..da1342b4c0 100644
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -22,6 +22,13 @@
 #include <openssl/x509.h>
 #include <openssl/pem.h>
 #include <openssl/bn.h>
+#include <openssl/encoder.h>
+
+/*
+ * TODO: This include is to get OSSL_KEYMGMT_SELECT_*, which feels a bit
+ * much just for those macros...  they might serve better as EVP macros.
+ */
+#include <openssl/core_dispatch.h>
 
 typedef enum OPTION_choice {
     OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
@@ -62,12 +69,10 @@ const OPTIONS rsa_options[] = {
     {"traditional", OPT_TRADITIONAL, '-',
      "Use traditional format for private keys"},
 
-#if !defined(OPENSSL_NO_DSA) && !defined(OPENSSL_NO_RC4)
     OPT_SECTION("PVK"),
     {"pvk-strong", OPT_PVK_STRONG, '-', "Enable 'Strong' PVK encoding level (default)"},
     {"pvk-weak", OPT_PVK_WEAK, '-', "Enable 'Weak' PVK encoding level"},
     {"pvk-none", OPT_PVK_NONE, '-', "Don't enforce PVK encoding"},
-#endif
 
     OPT_PROV_OPTIONS,
     {NULL}
@@ -77,20 +82,21 @@ int rsa_main(int argc, char **argv)
 {
     ENGINE *e = NULL;
     BIO *out = NULL;
-    RSA *rsa = NULL;
     EVP_PKEY *pkey = NULL;
     EVP_PKEY_CTX *pctx;
     const EVP_CIPHER *enc = NULL;
     char *infile = NULL, *outfile = NULL, *prog;
     char *passin = NULL, *passout = NULL, *passinarg = NULL, *passoutarg = NULL;
-    int i, private = 0;
+    int private = 0;
     int informat = FORMAT_PEM, outformat = FORMAT_PEM, text = 0, check = 0;
     int noout = 0, modulus = 0, pubin = 0, pubout = 0, ret = 1;
-#if !defined(OPENSSL_NO_DSA) && !defined(OPENSSL_NO_RC4)
     int pvk_encr = 2;
-#endif
     OPTION_CHOICE o;
     int traditional = 0;
+    const char *output_type = NULL;
+    const char *output_structure = NULL;
+    int selection = 0;
+    OSSL_ENCODER_CTX *ectx = NULL;
 
     prog = opt_init(argc, argv, rsa_options);
     while ((o = opt_next()) != OPT_EOF) {
@@ -142,9 +148,7 @@ int rsa_main(int argc, char **argv)
         case OPT_PVK_STRONG:    /* pvk_encr:= 2 */
         case OPT_PVK_WEAK:      /* pvk_encr:= 1 */
         case OPT_PVK_NONE:      /* pvk_encr:= 0 */
-#if !defined(OPENSSL_NO_DSA) && !defined(OPENSSL_NO_RC4)
             pvk_encr = (o - OPT_PVK_NONE);
-#endif
             break;
         case OPT_NOOUT:
             noout = 1;
@@ -203,13 +207,14 @@ int rsa_main(int argc, char **argv)
         pkey = load_key(infile, informat, 1, passin, e, "private key");
     }
 
-    if (pkey != NULL)
-        rsa = EVP_PKEY_get1_RSA(pkey);
-
-    if (rsa == NULL) {
+    if (pkey == NULL) {
         ERR_print_errors(bio_err);
         goto end;
     }
+    if (!EVP_PKEY_is_a(pkey, "RSA")) {
+        BIO_printf(bio_err, "Not an RSA key\n");
+        goto end;
+    }
 
     out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
@@ -226,11 +231,14 @@ int rsa_main(int argc, char **argv)
     }
 
     if (modulus) {
-        const BIGNUM *n;
-        RSA_get0_key(rsa, &n, NULL, NULL);
+        BIGNUM *n = NULL;
+
+        /* Every RSA key has an 'n' */
+        EVP_PKEY_get_bn_param(pkey, "n", &n);
         BIO_printf(out, "Modulus=");
         BN_print(out, n);
         BIO_printf(out, "\n");
+        BN_free(n);
     }
 
     if (check) {
@@ -268,77 +276,81 @@ int rsa_main(int argc, char **argv)
         goto end;
     }
     BIO_printf(bio_err, "writing RSA key\n");
+
+    /* Choose output type for the format */
     if (outformat == FORMAT_ASN1) {
-        if (pubout || pubin) {
-            if (pubout == 2)
-                i = i2d_RSAPublicKey_bio(out, rsa);
-            else
-                i = i2d_RSA_PUBKEY_bio(out, rsa);
-        } else {
-            assert(private);
-            i = i2d_RSAPrivateKey_bio(out, rsa);
-        }
+        output_type = "DER";
     } else if (outformat == FORMAT_PEM) {
+        output_type = "PEM";
+    } else if (outformat == FORMAT_MSBLOB) {
+        output_type = "MSBLOB";
+    } else if (outformat == FORMAT_PVK) {
+        if (pubin) {
+            BIO_printf(bio_err, "PVK form impossible with public key input\n");
+            goto end;
+        }
+        output_type = "PVK";
+    } else {
+        BIO_printf(bio_err, "bad output format specified for outfile\n");
+        goto end;
+    }
+
+    /* Select what you want in the output */
+    if (pubout || pubin) {
+        selection = OSSL_KEYMGMT_SELECT_PUBLIC_KEY;
+    } else {
+        assert(private);
+        selection = (OSSL_KEYMGMT_SELECT_KEYPAIR
+                     | OSSL_KEYMGMT_SELECT_ALL_PARAMETERS);
+    }
+
+    /* For DER based output, select the desired output structure */
+    if (outformat == FORMAT_ASN1 || outformat == FORMAT_PEM) {
         if (pubout || pubin) {
             if (pubout == 2)
-                i = PEM_write_bio_RSAPublicKey(out, rsa);
+                output_structure = "SubjectPublicKeyInfo";
             else
-                i = PEM_write_bio_RSA_PUBKEY(out, rsa);
+                output_structure = "pkcs1"; /* "type-specific" would work too */
         } else {
             assert(private);
-            if (traditional) {
-                i = PEM_write_bio_PrivateKey_traditional(out, pkey, enc, NULL, 0,
-                                                         NULL, passout);
-            } else {
-                i = PEM_write_bio_PrivateKey(out, pkey,
-                                             enc, NULL, 0, NULL, passout);
-            }
+            if (traditional)
+                output_structure = "pkcs1"; /* "type-specific" would work too */
+            else
+                output_structure = "pkcs8";
         }
-#ifndef OPENSSL_NO_DSA
-    } else if (outformat == FORMAT_MSBLOB || outformat == FORMAT_PVK) {
-        EVP_PKEY *pk;
-        pk = EVP_PKEY_new();
-        if (pk == NULL)
-            goto end;
+    }
 
-        EVP_PKEY_set1_RSA(pk, rsa);
-        if (outformat == FORMAT_PVK) {
-            if (pubin) {
-                BIO_printf(bio_err, "PVK form impossible with public key input\n");
-                EVP_PKEY_free(pk);
-                goto end;
-            }
-            assert(private);
-# ifdef OPENSSL_NO_RC4
-            BIO_printf(bio_err, "PVK format not supported\n");
-            EVP_PKEY_free(pk);
+    /* Now, perform the encoding */
+    ectx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, selection,
+                                            output_type, output_structure,
+                                            NULL, NULL);
+    if (OSSL_ENCODER_CTX_get_num_encoders(ectx) == 0) {
+        BIO_printf(bio_err, "%s format not supported\n", output_type);
+        goto end;
+    }
+
+    /* PVK is a bit special... */
+    if (outformat == FORMAT_PVK) {
+        OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END };
+
+        params[0] = OSSL_PARAM_construct_int("encrypt-level", &pvk_encr);
+        if (!OSSL_ENCODER_CTX_set_params(ectx, params)) {
+            BIO_printf(bio_err, "invalid PVK encryption level\n");
             goto end;
-# else
-            i = i2b_PVK_bio(out, pk, pvk_encr, 0, passout);
-# endif
-        } else if (pubin || pubout) {
-            i = i2b_PublicKey_bio(out, pk);
-        } else {
-            assert(private);
-            i = i2b_PrivateKey_bio(out, pk);
         }
-        EVP_PKEY_free(pk);
-#endif
-    } else {
-        BIO_printf(bio_err, "bad output format specified for outfile\n");
-        goto end;
     }
-    if (i <= 0) {
+
+    if (!OSSL_ENCODER_to_bio(ectx, out)) {
         BIO_printf(bio_err, "unable to write key\n");
         ERR_print_errors(bio_err);
-    } else {
-        ret = 0;
+        goto end;
     }
+    ret = 0;
  end:
+    OSSL_ENCODER_CTX_free(ectx);
     release_engine(e);
     BIO_free_all(out);
     EVP_PKEY_free(pkey);
-    RSA_free(rsa);
     OPENSSL_free(passin);
     OPENSSL_free(passout);
     return ret;
diff --git a/apps/rsautl.c b/apps/rsautl.c
index 9b5456cb89..8fefaee8f5 100644
--- a/apps/rsautl.c
+++ b/apps/rsautl.c
@@ -7,9 +7,6 @@
  * https://www.openssl.org/source/license.html
  */
 
-/* We need to use the deprecated RSA low level calls */
-#define OPENSSL_SUPPRESS_DEPRECATED
-
 #include <openssl/opensslconf.h>
 
 #include "apps.h"
@@ -78,14 +75,15 @@ int rsautl_main(int argc, char **argv)
     BIO *in = NULL, *out = NULL;
     ENGINE *e = NULL;
     EVP_PKEY *pkey = NULL;
-    RSA *rsa = NULL;
+    EVP_PKEY_CTX *ctx = NULL;
     X509 *x;
     char *infile = NULL, *outfile = NULL, *keyfile = NULL;
     char *passinarg = NULL, *passin = NULL, *prog;
     char rsa_mode = RSA_VERIFY, key_type = KEY_PRIVKEY;
     unsigned char *rsa_in = NULL, *rsa_out = NULL, pad = RSA_PKCS1_PADDING;
-    int rsa_inlen, keyformat = FORMAT_PEM, keysize, ret = 1;
-    int rsa_outlen = 0, hexdump = 0, asn1parse = 0, need_priv = 0, rev = 0;
+    size_t rsa_inlen, rsa_outlen = 0;
+    int keyformat = FORMAT_PEM, keysize, ret = 1, rv;
+    int hexdump = 0, asn1parse = 0, need_priv = 0, rev = 0;
     OPTION_CHOICE o;
 
     prog = opt_init(argc, argv, rsautl_options);
@@ -208,15 +206,6 @@ int rsautl_main(int argc, char **argv)
     if (pkey == NULL)
         return 1;
 
-    rsa = EVP_PKEY_get1_RSA(pkey);
-    EVP_PKEY_free(pkey);
-
-    if (rsa == NULL) {
-        BIO_printf(bio_err, "Error getting RSA key\n");
-        ERR_print_errors(bio_err);
-        goto end;
-    }
-
     in = bio_open_default(infile, 'r', FORMAT_BINARY);
     if (in == NULL)
         goto end;
@@ -224,48 +213,58 @@ int rsautl_main(int argc, char **argv)
     if (out == NULL)
         goto end;
 
-    keysize = RSA_size(rsa);
+    keysize = EVP_PKEY_size(pkey);
 
     rsa_in = app_malloc(keysize * 2, "hold rsa key");
     rsa_out = app_malloc(keysize, "output rsa key");
+    rsa_outlen = keysize;
 
     /* Read the input data */
-    rsa_inlen = BIO_read(in, rsa_in, keysize * 2);
-    if (rsa_inlen < 0) {
+    rv = BIO_read(in, rsa_in, keysize * 2);
+    if (rv < 0) {
         BIO_printf(bio_err, "Error reading input Data\n");
         goto end;
     }
+    rsa_inlen = rv;
     if (rev) {
-        int i;
+        size_t i;
         unsigned char ctmp;
+
         for (i = 0; i < rsa_inlen / 2; i++) {
             ctmp = rsa_in[i];
             rsa_in[i] = rsa_in[rsa_inlen - 1 - i];
             rsa_in[rsa_inlen - 1 - i] = ctmp;
         }
     }
-    switch (rsa_mode) {
 
+    if ((ctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL)) == NULL)
+        goto end;
+
+    switch (rsa_mode) {
     case RSA_VERIFY:
-        rsa_outlen = RSA_public_decrypt(rsa_inlen, rsa_in, rsa_out, rsa, pad);
+        rv = EVP_PKEY_verify_recover_init(ctx)
+            && EVP_PKEY_CTX_set_rsa_padding(ctx, pad)
+            && EVP_PKEY_verify_recover(ctx, rsa_out, &rsa_outlen,
+                                       rsa_in, rsa_inlen);
         break;
-
     case RSA_SIGN:
-        rsa_outlen =
-            RSA_private_encrypt(rsa_inlen, rsa_in, rsa_out, rsa, pad);
+        rv = EVP_PKEY_sign_init(ctx)
+            && EVP_PKEY_CTX_set_rsa_padding(ctx, pad)
+            && EVP_PKEY_sign(ctx, rsa_out, &rsa_outlen, rsa_in, rsa_inlen);
         break;
-
     case RSA_ENCRYPT:
-        rsa_outlen = RSA_public_encrypt(rsa_inlen, rsa_in, rsa_out, rsa, pad);
+        rv = EVP_PKEY_encrypt_init(ctx)
+            && EVP_PKEY_CTX_set_rsa_padding(ctx, pad)
+            && EVP_PKEY_encrypt(ctx, rsa_out, &rsa_outlen, rsa_in, rsa_inlen);
         break;
-
     case RSA_DECRYPT:
-        rsa_outlen =
-            RSA_private_decrypt(rsa_inlen, rsa_in, rsa_out, rsa, pad);
+        rv = EVP_PKEY_decrypt_init(ctx)
+            && EVP_PKEY_CTX_set_rsa_padding(ctx, pad)
+            && EVP_PKEY_decrypt(ctx, rsa_out, &rsa_outlen, rsa_in, rsa_inlen);
         break;
     }
 
-    if (rsa_outlen < 0) {
+    if (!rv) {
         BIO_printf(bio_err, "RSA operation error\n");
         ERR_print_errors(bio_err);
         goto end;
@@ -281,7 +280,8 @@ int rsautl_main(int argc, char **argv)
         BIO_write(out, rsa_out, rsa_outlen);
     }
  end:
-    RSA_free(rsa);
+    EVP_PKEY_CTX_free(ctx);
+    EVP_PKEY_free(pkey);
     release_engine(e);
     BIO_free(in);
     BIO_free_all(out);
diff --git a/apps/x509.c b/apps/x509.c
index 0d0d93edc0..ad627f4558 100644
--- a/apps/x509.c
+++ b/apps/x509.c
@@ -757,10 +757,13 @@ int x509_main(int argc, char **argv)
                 }
                 BIO_printf(out, "Modulus=");
 #ifndef OPENSSL_NO_RSA
-                if (EVP_PKEY_id(pkey) == EVP_PKEY_RSA) {
-                    const BIGNUM *n;
-                    RSA_get0_key(EVP_PKEY_get0_RSA(pkey), &n, NULL, NULL);
+                if (EVP_PKEY_is_a(pkey, "RSA")) {
+                    BIGNUM *n;
+
+                    /* Every RSA key has an 'n' */
+                    EVP_PKEY_get_bn_param(pkey, "n", &n);
                     BN_print(out, n);
+                    BN_free(n);
                 } else
 #endif
 #ifndef OPENSSL_NO_DSA