EVP: Adapt EVP_PKEY Seal and Open for provider keys

This affects the following function, which can now deal with provider
side keys:

- EVP_SealInit()
- EVP_OpenInit()

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/10808)

6 files changed, 50 insertions, 29 deletions

diff --git a/CHANGES b/CHANGES
index 2b281f8da9..d64163d4e0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -9,6 +9,12 @@
 
  Changes between 1.1.1 and 3.0.0 [xx XXX xxxx]
 
+  *) Deprecated EVP_PKEY_decrypt_old(), please use EVP_PKEY_decrypt_init()
+     and EVP_PKEY_decrypt() instead.
+     Deprecated EVP_PKEY_encrypt_old(), please use EVP_PKEY_encrypt_init()
+     and EVP_PKEY_encrypt() instead.
+     [Richard Levitte]
+
   *) Enhanced the documentation of EVP_PKEY_size(), EVP_PKEY_bits()
      and EVP_PKEY_security_bits().  Especially EVP_PKEY_size() needed
      a new formulation to include all the things it can be used for,
diff --git a/crypto/evp/build.info b/crypto/evp/build.info
index 7f566b80ce..d3ebac9f4e 100644
--- a/crypto/evp/build.info
+++ b/crypto/evp/build.info
@@ -8,7 +8,7 @@ SOURCE[../../libcrypto]=$COMMON\
         e_des.c e_bf.c e_idea.c e_des3.c e_camellia.c\
         e_rc4.c e_aes.c names.c e_seed.c e_aria.c e_sm4.c \
         e_xcbc_d.c e_rc2.c e_cast.c e_rc5.c m_null.c \
-        p_open.c p_seal.c p_sign.c p_verify.c p_enc.c p_dec.c \
+        p_open.c p_seal.c p_sign.c p_verify.c \
         bio_md.c bio_b64.c bio_enc.c evp_err.c e_null.c \
         c_allc.c c_alld.c bio_ok.c \
         evp_pkey.c evp_pbe.c p5_crpt.c p5_crpt2.c pbe_scrypt.c \
@@ -18,6 +18,10 @@ SOURCE[../../libcrypto]=$COMMON\
         pkey_mac.c \
         legacy_sha.c
 
+IF[{- !$disabled{deprecated} || $config{api} < 30000 -}]
+  SOURCE[../../libcrypto]=p_enc.c p_dec.c
+ENDIF
+
 IF[{- !$disabled{md2} -}]
   SOURCE[../../libcrypto]=legacy_md2.c
 ENDIF
diff --git a/crypto/evp/p_open.c b/crypto/evp/p_open.c
index 8cc72ebbf2..bcc01a7817 100644
--- a/crypto/evp/p_open.c
+++ b/crypto/evp/p_open.c
@@ -23,41 +23,44 @@ int EVP_OpenInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
                  EVP_PKEY *priv)
 {
     unsigned char *key = NULL;
-    int i, size = 0, ret = 0;
+    size_t keylen = 0;
+    int ret = 0;
+    EVP_PKEY_CTX *pctx = NULL;
 
     if (type) {
         EVP_CIPHER_CTX_reset(ctx);
         if (!EVP_DecryptInit_ex(ctx, type, NULL, NULL, NULL))
-            return 0;
+            goto err;
     }
 
     if (priv == NULL)
         return 1;
 
-    if (EVP_PKEY_id(priv) != EVP_PKEY_RSA) {
-        EVPerr(EVP_F_EVP_OPENINIT, EVP_R_PUBLIC_KEY_NOT_RSA);
+    if ((pctx = EVP_PKEY_CTX_new(priv, NULL)) == NULL) {
+        ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
         goto err;
     }
 
-    size = EVP_PKEY_size(priv);
-    key = OPENSSL_malloc(size);
-    if (key == NULL) {
-        /* ERROR */
-        EVPerr(EVP_F_EVP_OPENINIT, ERR_R_MALLOC_FAILURE);
+    if (EVP_PKEY_decrypt_init(pctx) <= 0
+        || EVP_PKEY_decrypt(pctx, NULL, &keylen, ek, ekl) <= 0)
         goto err;
-    }
 
-    i = EVP_PKEY_decrypt_old(key, ek, ekl, priv);
-    if ((i <= 0) || !EVP_CIPHER_CTX_set_key_length(ctx, i)) {
-        /* ERROR */
+    if ((key = OPENSSL_malloc(keylen)) == NULL) {
+        ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
         goto err;
     }
-    if (!EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv))
+
+    if (EVP_PKEY_decrypt(pctx, key, &keylen, ek, ekl) <= 0)
+        goto err;
+
+    if (!EVP_CIPHER_CTX_set_key_length(ctx, keylen)
+        || !EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv))
         goto err;
 
     ret = 1;
  err:
-    OPENSSL_clear_free(key, size);
+    EVP_PKEY_CTX_free(pctx);
+    OPENSSL_clear_free(key, keylen);
     return ret;
 }
 
diff --git a/crypto/evp/p_seal.c b/crypto/evp/p_seal.c
index 26e0e7c38d..3f855d08dc 100644
--- a/crypto/evp/p_seal.c
+++ b/crypto/evp/p_seal.c
@@ -30,6 +30,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
     }
     if ((npubk <= 0) || !pubk)
         return 1;
+
     if (EVP_CIPHER_CTX_rand_key(ctx, key) <= 0)
         return 0;
 
@@ -41,13 +42,19 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
         goto err;
 
     for (i = 0; i < npubk; i++) {
-        ekl[i] =
-            EVP_PKEY_encrypt_old(ek[i], key, EVP_CIPHER_CTX_key_length(ctx),
-                                 pubk[i]);
-        if (ekl[i] <= 0) {
-            rv = -1;
+        size_t keylen = EVP_CIPHER_CTX_key_length(ctx);
+        EVP_PKEY_CTX *pctx = NULL;
+
+        if ((pctx = EVP_PKEY_CTX_new(pubk[i], NULL)) == NULL) {
+            ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
             goto err;
         }
+
+        if (EVP_PKEY_encrypt_init(pctx) <= 0
+            || EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
+            goto err;
+        ekl[i] = (int)keylen;
+        EVP_PKEY_CTX_free(pctx);
     }
     rv = npubk;
 err:
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 6c042d3765..7fc16807b9 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -1089,12 +1089,13 @@ void EVP_MAC_names_do_all(const EVP_MAC *mac,
                           void *data);
 
 /* PKEY stuff */
-int EVP_PKEY_decrypt_old(unsigned char *dec_key,
-                         const unsigned char *enc_key, int enc_key_len,
-                         EVP_PKEY *private_key);
-int EVP_PKEY_encrypt_old(unsigned char *enc_key,
-                         const unsigned char *key, int key_len,
-                         EVP_PKEY *pub_key);
+DEPRECATEDIN_3_0(int EVP_PKEY_decrypt_old(unsigned char *dec_key,
+                                          const unsigned char *enc_key,
+                                          int enc_key_len,
+                                          EVP_PKEY *private_key))
+DEPRECATEDIN_3_0(int EVP_PKEY_encrypt_old(unsigned char *enc_key,
+                                          const unsigned char *key,
+                                          int key_len, EVP_PKEY *pub_key))
 int EVP_PKEY_type(int type);
 int EVP_PKEY_id(const EVP_PKEY *pkey);
 int EVP_PKEY_base_id(const EVP_PKEY *pkey);
diff --git a/util/libcrypto.num b/util/libcrypto.num
index 8c3fdc0e7f..64b2ed277c 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -1044,7 +1044,7 @@ X509_VERIFY_PARAM_set_flags             1070	3_0_0	EXIST::FUNCTION:
 X509_EXTENSION_set_data                 1071	3_0_0	EXIST::FUNCTION:
 ENGINE_get_EC                           1072	3_0_0	EXIST::FUNCTION:ENGINE
 ASN1_STRING_copy                        1073	3_0_0	EXIST::FUNCTION:
-EVP_PKEY_encrypt_old                    1074	3_0_0	EXIST::FUNCTION:
+EVP_PKEY_encrypt_old                    1074	3_0_0	EXIST::FUNCTION:DEPRECATEDIN_3_0
 OPENSSL_LH_free                         1075	3_0_0	EXIST::FUNCTION:
 DES_is_weak_key                         1076	3_0_0	EXIST::FUNCTION:DEPRECATEDIN_3_0,DES
 EVP_PKEY_verify                         1077	3_0_0	EXIST::FUNCTION:
@@ -3606,7 +3606,7 @@ X509_VERIFY_PARAM_inherit               3685	3_0_0	EXIST::FUNCTION:
 EC_GROUP_get_curve_name                 3686	3_0_0	EXIST::FUNCTION:EC
 RSA_print                               3687	3_0_0	EXIST::FUNCTION:RSA
 i2d_ASN1_BMPSTRING                      3688	3_0_0	EXIST::FUNCTION:
-EVP_PKEY_decrypt_old                    3689	3_0_0	EXIST::FUNCTION:
+EVP_PKEY_decrypt_old                    3689	3_0_0	EXIST::FUNCTION:DEPRECATEDIN_3_0
 ASN1_UTCTIME_cmp_time_t                 3690	3_0_0	EXIST::FUNCTION:
 X509_VERIFY_PARAM_set1_ip               3691	3_0_0	EXIST::FUNCTION:
 OTHERNAME_free                          3692	3_0_0	EXIST::FUNCTION: