Deprecate RSA harder

This deprecates all functions that deal with the types RSA and RSA_METHOD

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13096)

2 files changed, 250 insertions, 23 deletions

diff --git a/doc/man3/d2i_RSAPrivateKey.pod b/doc/man3/d2i_RSAPrivateKey.pod
new file mode 100644
index 0000000000..e7cf3989ab
--- /dev/null
+++ b/doc/man3/d2i_RSAPrivateKey.pod
@@ -0,0 +1,242 @@
+=pod
+
+=begin comment
+
+Any deprecated keypair function from d2i_X509.pod are collected in this file.
+
+=end comment
+
+=head1 NAME
+
+d2i_RSAPrivateKey,
+d2i_RSAPrivateKey_bio,
+d2i_RSAPrivateKey_fp,
+d2i_RSAPublicKey,
+d2i_RSAPublicKey_bio,
+d2i_RSAPublicKey_fp,
+d2i_RSA_PUBKEY,
+d2i_RSA_PUBKEY_bio,
+d2i_RSA_PUBKEY_fp,
+i2d_RSAPrivateKey,
+i2d_RSAPrivateKey_bio,
+i2d_RSAPrivateKey_fp,
+i2d_RSAPublicKey,
+i2d_RSAPublicKey_bio,
+i2d_RSAPublicKey_fp,
+i2d_RSA_PUBKEY,
+i2d_RSA_PUBKEY_bio,
+i2d_RSA_PUBKEY_fp
+- DEPRECATED
+
+=head1 SYNOPSIS
+
+=for openssl generic
+
+Deprecated since OpenSSL 3.0, can be hidden entirely by defining
+B<OPENSSL_API_COMPAT> with a suitable version value, see
+L<openssl_user_macros(7)>:
+
+ TYPE *d2i_TYPEPrivateKey(TYPE **a, const unsigned char **ppin, long length);
+ TYPE *d2i_TYPEPrivateKey_bio(BIO *bp, TYPE **a);
+ TYPE *d2i_TYPEPrivateKey_fp(FILE *fp, TYPE **a);
+ TYPE *d2i_TYPEPublicKey(TYPE **a, const unsigned char **ppin, long length);
+ TYPE *d2i_TYPEPublicKey_bio(BIO *bp, TYPE **a);
+ TYPE *d2i_TYPEPublicKey_fp(FILE *fp, TYPE **a);
+ TYPE *d2i_TYPEparams(TYPE **a, const unsigned char **ppin, long length);
+ TYPE *d2i_TYPEparams_bio(BIO *bp, TYPE **a);
+ TYPE *d2i_TYPEparams_fp(FILE *fp, TYPE **a);
+ TYPE *d2i_TYPE_PUBKEY(TYPE **a, const unsigned char **ppin, long length);
+ TYPE *d2i_TYPE_PUBKEY_bio(BIO *bp, TYPE **a);
+ TYPE *d2i_TYPE_PUBKEY_fp(FILE *fp, TYPE **a);
+
+ int i2d_TYPEPrivateKey(const TYPE *a, unsigned char **ppout);
+ int i2d_TYPEPrivateKey(TYPE *a, unsigned char **ppout);
+ int i2d_TYPEPrivateKey_fp(FILE *fp, const TYPE *a);
+ int i2d_TYPEPrivateKey_fp(FILE *fp, TYPE *a);
+ int i2d_TYPEPrivateKey_bio(BIO *bp, const TYPE *a);
+ int i2d_TYPEPrivateKey_bio(BIO *bp, TYPE *a);
+ int i2d_TYPEPublicKey(const TYPE *a, unsigned char **ppout);
+ int i2d_TYPEPublicKey(TYPE *a, unsigned char **ppout);
+ int i2d_TYPEPublicKey_fp(FILE *fp, const TYPE *a);
+ int i2d_TYPEPublicKey_fp(FILE *fp, TYPE *a);
+ int i2d_TYPEPublicKey_bio(BIO *bp, const TYPE *a);
+ int i2d_TYPEPublicKey_bio(BIO *bp, TYPE *a);
+ int i2d_TYPEparams(const TYPE *a, unsigned char **ppout);
+ int i2d_TYPEparams(TYPE *a, unsigned char **ppout);
+ int i2d_TYPEparams_fp(FILE *fp, const TYPE *a);
+ int i2d_TYPEparams_fp(FILE *fp, TYPE *a);
+ int i2d_TYPEparams_bio(BIO *bp, const TYPE *a);
+ int i2d_TYPEparams_bio(BIO *bp, TYPE *a);
+ int i2d_TYPE_PUBKEY(const TYPE *a, unsigned char **ppout);
+ int i2d_TYPE_PUBKEY(TYPE *a, unsigned char **ppout);
+ int i2d_TYPE_PUBKEY_fp(FILE *fp, const TYPE *a);
+ int i2d_TYPE_PUBKEY_fp(FILE *fp, TYPE *a);
+ int i2d_TYPE_PUBKEY_bio(BIO *bp, const TYPE *a);
+ int i2d_TYPE_PUBKEY_bio(BIO *bp, TYPE *a);
+
+=head1 DESCRIPTION
+
+All functions described here are deprecated.  Please use L<OSSL_DECODER(3)>
+instead of the B<d2i> functions and L<OSSL_ENCODER(3)> instead of the B<i2d>
+functions.  See L</Migration> below.
+
+In the description here, B<I<TYPE>> is used a placeholder for any of the
+OpenSSL datatypes, such as B<RSA>.
+The function parameters I<ppin> and I<ppout> are generally either both named
+I<pp> in the headers, or I<in> and I<out>.
+
+All the functions here behave the way that's described in L<d2i_X509(3)>.
+
+Please note that not all functions in the synopsis are available for all key
+types.  For example, there are no d2i_RSAparams() or i2d_RSAparams(),
+because the PKCS#1 B<RSA> structure doesn't include any key parameters.
+
+B<d2i_I<TYPE>PrivateKey>() and derivates thereof decode DER encoded
+B<I<TYPE>> private key data organized in a type specific structure.
+
+B<d2i_I<TYPE>PublicKey>() and derivates thereof decode DER encoded
+B<I<TYPE>> public key data organized in a type specific structure.
+
+B<d2i_I<TYPE>params>() and derivates thereof decode DER encoded B<I<TYPE>>
+key parameters organized in a type specific structure.
+
+B<d2i_I<TYPE>_PUBKEY>() and derivates thereof decode DER encoded B<I<TYPE>>
+public key data organized in a B<SubjectPublicKeyInfo> structure.
+
+B<i2d_I<TYPE>PrivateKey>() and derivates thereof encode the private key
+B<I<TYPE>> data into a type specific DER encoded structure.
+
+B<i2d_I<TYPE>PublicKey>() and derivates thereof encode the public key
+B<I<TYPE>> data into a type specific DER encoded structure.
+
+B<i2d_I<TYPE>params>() and derivates thereof encode the B<I<TYPE>> key
+parameters data into a type specific DER encoded structure.
+
+B<i2d_I<TYPE>_PUBKEY>() and derivates thereof encode the public key
+B<I<TYPE>> data into a DER encoded B<SubjectPublicKeyInfo> structure.
+
+For example, d2i_RSAPrivateKey() and d2i_RSAPublicKey() expects the
+structure defined by PKCS#1.
+Similarly, i2d_RSAPrivateKey() and  i2d_RSAPublicKey() produce DER encoded
+string organized according to PKCS#1.
+
+=head2 Migration
+
+Migration from the diverse B<I<TYPE>>s requires using corresponding new
+OpenSSL types.  For all B<I<TYPE>>s described here, the corresponding new
+type is B<EVP_PKEY>.  The rest of this section assumes that this has been
+done, exactly how to do that is described elsewhere.
+
+There are two migration paths:
+
+=over 4
+
+=item *
+
+Replace
+b<d2i_I<TYPE>PrivateKey()> with L<d2i_PrivateKey(3)>, 
+b<d2i_I<TYPE>PublicKey()> with L<d2i_PublicKey(3)>, 
+b<d2i_I<TYPE>params()> with L<d2i_KeyParams(3)>, 
+b<d2i_I<TYPE>_PUBKEY()> with L<d2i_PUBKEY(3)>, 
+b<i2d_I<TYPE>PrivateKey()> with L<i2d_PrivateKey(3)>, 
+b<i2d_I<TYPE>PublicKey()> with L<i2d_PublicKey(3)>, 
+b<i2d_I<TYPE>params()> with L<i2d_KeyParams(3)>, 
+b<i2d_I<TYPE>_PUBKEY()> with L<i2d_PUBKEY(3)>.
+A caveat is that L<i2d_PrivateKey(3)> may output a DER encoded PKCS#8
+outermost structure instead of the type specific structure, and that
+L<d2i_PrivateKey(3)> recognises and unpacks a PKCS#8 structures.
+
+=item *
+
+Use L<OSSL_DECODER(3)> and L<OSSL_ENCODER(3)>.  How to migrate is described
+below.  All those descriptions assume that the key to be encoded is in the
+variable I<pkey>.
+
+=back
+
+=head3 Migrating B<i2d> functions to B<OSSL_ENCODER>
+
+The exact L<OSSL_ENCODER(3)> output is driven by arguments rather than by
+function names.  The sample code to get DER encoded output in a type
+specific structure is uniform, the only things that vary are the selection
+of what part of the B<EVP_PKEY> should be output, and the structure.  The
+B<i2d> functions names can therefore be translated into two variables,
+I<selection> and I<structure> as follows:
+
+=over 4
+
+=item B<i2d_I<TYPE>PrivateKey>() translates into:
+
+ int selection = EVP_PKEY_PRIVATE_KEY;
+ const char *structure = "type-specific";
+
+=item B<i2d_I<TYPE>PublicKey>() translates into:
+
+ int selection = EVP_PKEY_PUBLIC_KEY;
+ const char *structure = "type-specific";
+
+=item B<i2d_I<TYPE>params>() translates into:
+
+ int selection = EVP_PKEY_PARAMETERS;
+ const char *structure = "type-specific";
+
+=item B<i2d_I<TYPE>_PUBKEY>() translates into:
+
+ int selection = EVP_PKEY_PUBLIC_KEY;
+ const char *structure = "SubjectPublicKeyInfo";
+
+=back
+
+The following sample code does the rest of the work:
+
+ unsigned char *p = buffer;     /* |buffer| is supplied by the caller */
+ size_t len = buffer_size;      /* assumed be the size of |buffer| */
+ OSSL_ENCODER_CTX *ctx =
+     OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, selection, "DER", structure,
+                                      NULL, NULL);
+ if (ctx == NULL) {
+     /* fatal error handling */
+ }
+ if (OSSL_ENCODER_CTX_get_num_encoders(ctx) == 0) {
+     OSSL_ENCODER_CTX_free(ctx);
+     /* non-fatal error handling */
+ }
+ if (!OSSL_ENCODER_to_data(ctx, &p, &len)) {
+     OSSL_ENCODER_CTX_free(ctx);
+     /* error handling */
+ }
+ OSSL_ENCODER_CTX_free(ctx);
+
+=for comment TODO: a similar section on OSSL_DECODER is to be added
+
+=head1 RETURN VALUES
+
+B<d2i_I<TYPE>>(), B<d2i_I<TYPE>_bio>() and B<d2i_I<TYPE>_fp>() return a valid
+B<I<TYPE>> structure or NULL if an error occurs.  If the "reuse" capability has
+been used with a valid structure being passed in via I<a>, then the object is
+freed in the event of error and I<*a> is set to NULL.
+
+B<i2d_I<TYPE>>() returns the number of bytes successfully encoded or a negative
+value if an error occurs.
+
+B<i2d_I<TYPE>_bio>() and B<i2d_I<TYPE>_fp>() return 1 for success and 0 if an
+error occurs.
+
+=head1 SEE ALSO
+
+L<OSSL_ENCODER(3)>, L<OSSL_DECODER(3)>,
+L<d2i_PrivateKey(3)>, L<d2i_PublicKey(3)>, L<d2i_KeyParams(3)>,
+L<d2i_PUBKEY(3)>,
+L<i2d_PrivateKey(3)>, L<i2d_PublicKey(3)>, L<i2d_KeyParams(3)>,
+L<i2d_PUBKEY(3)>
+
+=head1 COPYRIGHT
+
+Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut
diff --git a/doc/man3/d2i_X509.pod b/doc/man3/d2i_X509.pod
index a46977bc93..0b3414ba8f 100644
--- a/doc/man3/d2i_X509.pod
+++ b/doc/man3/d2i_X509.pod
@@ -1,5 +1,12 @@
 =pod
 
+=begin comment
+
+Any keypair function here that gets deprecated should be moved to
+d2i_RSAPrivateKey.pod.
+
+=end comment
+
 =head1 NAME
 
 d2i_ACCESS_DESCRIPTION,
@@ -141,17 +148,8 @@ d2i_POLICYQUALINFO,
 d2i_PROFESSION_INFO,
 d2i_PROXY_CERT_INFO_EXTENSION,
 d2i_PROXY_POLICY,
-d2i_RSAPrivateKey,
-d2i_RSAPrivateKey_bio,
-d2i_RSAPrivateKey_fp,
-d2i_RSAPublicKey,
-d2i_RSAPublicKey_bio,
-d2i_RSAPublicKey_fp,
 d2i_RSA_OAEP_PARAMS,
 d2i_RSA_PSS_PARAMS,
-d2i_RSA_PUBKEY,
-d2i_RSA_PUBKEY_bio,
-d2i_RSA_PUBKEY_fp,
 d2i_SCRYPT_PARAMS,
 d2i_SCT_LIST,
 d2i_SXNET,
@@ -337,17 +335,8 @@ i2d_POLICYQUALINFO,
 i2d_PROFESSION_INFO,
 i2d_PROXY_CERT_INFO_EXTENSION,
 i2d_PROXY_POLICY,
-i2d_RSAPrivateKey,
-i2d_RSAPrivateKey_bio,
-i2d_RSAPrivateKey_fp,
-i2d_RSAPublicKey,
-i2d_RSAPublicKey_bio,
-i2d_RSAPublicKey_fp,
 i2d_RSA_OAEP_PARAMS,
 i2d_RSA_PSS_PARAMS,
-i2d_RSA_PUBKEY,
-i2d_RSA_PUBKEY_bio,
-i2d_RSA_PUBKEY_fp,
 i2d_SCRYPT_PARAMS,
 i2d_SCT_LIST,
 i2d_SXNET,
@@ -411,7 +400,7 @@ i2d_X509_VAL,
 =head1 DESCRIPTION
 
 In the description here, B<I<TYPE>> is used a placeholder
-for any of the OpenSSL datatypes, such as I<X509_CRL>.
+for any of the OpenSSL datatypes, such as B<X509_CRL>.
 The function parameters I<ppin> and I<ppout> are generally
 either both named I<pp> in the headers, or I<in> and I<out>.
 
@@ -512,10 +501,6 @@ L<PEM_write_PrivateKey(3)>, or similar instead.
 
 Represents an ECDSA signature.
 
-=item B<RSAPublicKey>
-
-Represents a PKCS#1 RSA public key structure.
-
 =item B<X509_ALGOR>
 
 Represents an B<AlgorithmIdentifier> structure as used in IETF RFC 6960 and