rsa: Try legacy encoding functions for pubkey

If there are no suitable encoders it might mean the key is in
an engine and thus it is a legacy key. Try legacy encoding
routines to encode the public key. We do not attempt encoding
a private key as it would be in most cases impossible anyway.

Fixes #16256

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16289)

1 files changed, 38 insertions, 1 deletions

diff --git a/apps/rsa.c b/apps/rsa.c
index 3e9d320ea3..05a091ce4b 100644
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -7,6 +7,9 @@
  * https://www.openssl.org/source/license.html
  */
 
+/* Necessary for legacy RSA public key export */
+#define OPENSSL_SUPPRESS_DEPRECATED
+
 #include <openssl/opensslconf.h>
 
 #include <stdio.h>
@@ -86,6 +89,36 @@ const OPTIONS rsa_options[] = {
     {NULL}
 };
 
+static int try_legacy_encoding(EVP_PKEY *pkey, int outformat, int pubout,
+                               BIO *out)
+{
+    int ret = 0;
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    const RSA *rsa = EVP_PKEY_get0_RSA(pkey);
+
+    if (rsa == NULL)
+        return 0;
+
+    if (outformat == FORMAT_ASN1) {
+        if (pubout == 2)
+            ret = i2d_RSAPublicKey_bio(out, rsa) > 0;
+        else
+            ret = i2d_RSA_PUBKEY_bio(out, rsa) > 0;
+    } else if (outformat == FORMAT_PEM) {
+        if (pubout == 2)
+            ret = PEM_write_bio_RSAPublicKey(out, rsa) > 0;
+        else
+            ret = PEM_write_bio_RSA_PUBKEY(out, rsa) > 0;
+# ifndef OPENSSL_NO_DSA
+    } else if (outformat == FORMAT_MSBLOB || outformat == FORMAT_PVK) {
+        ret = i2b_PublicKey_bio(out, pkey) > 0;
+# endif
+    }
+#endif
+
+    return ret;
+}
+
 int rsa_main(int argc, char **argv)
 {
     ENGINE *e = NULL;
@@ -331,7 +364,11 @@ int rsa_main(int argc, char **argv)
                                          output_type, output_structure,
                                          NULL);
     if (OSSL_ENCODER_CTX_get_num_encoders(ectx) == 0) {
-        BIO_printf(bio_err, "%s format not supported\n", output_type);
+        if ((!pubout && !pubin)
+            || !try_legacy_encoding(pkey, outformat, pubout, out))
+            BIO_printf(bio_err, "%s format not supported\n", output_type);
+        else
+            ret = 0;
         goto end;
     }