EVP: Adapt EVP_PKEY Seal and Open for provider keys

This affects the following function, which can now deal with provider
side keys:

- EVP_SealInit()
- EVP_OpenInit()

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/10808)

3 files changed, 35 insertions, 21 deletions

diff --git a/crypto/evp/build.info b/crypto/evp/build.info
index 7f566b80ce..d3ebac9f4e 100644
--- a/crypto/evp/build.info
+++ b/crypto/evp/build.info
@@ -8,7 +8,7 @@ SOURCE[../../libcrypto]=$COMMON\
         e_des.c e_bf.c e_idea.c e_des3.c e_camellia.c\
         e_rc4.c e_aes.c names.c e_seed.c e_aria.c e_sm4.c \
         e_xcbc_d.c e_rc2.c e_cast.c e_rc5.c m_null.c \
-        p_open.c p_seal.c p_sign.c p_verify.c p_enc.c p_dec.c \
+        p_open.c p_seal.c p_sign.c p_verify.c \
         bio_md.c bio_b64.c bio_enc.c evp_err.c e_null.c \
         c_allc.c c_alld.c bio_ok.c \
         evp_pkey.c evp_pbe.c p5_crpt.c p5_crpt2.c pbe_scrypt.c \
@@ -18,6 +18,10 @@ SOURCE[../../libcrypto]=$COMMON\
         pkey_mac.c \
         legacy_sha.c
 
+IF[{- !$disabled{deprecated} || $config{api} < 30000 -}]
+  SOURCE[../../libcrypto]=p_enc.c p_dec.c
+ENDIF
+
 IF[{- !$disabled{md2} -}]
   SOURCE[../../libcrypto]=legacy_md2.c
 ENDIF
diff --git a/crypto/evp/p_open.c b/crypto/evp/p_open.c
index 8cc72ebbf2..bcc01a7817 100644
--- a/crypto/evp/p_open.c
+++ b/crypto/evp/p_open.c
@@ -23,41 +23,44 @@ int EVP_OpenInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
                  EVP_PKEY *priv)
 {
     unsigned char *key = NULL;
-    int i, size = 0, ret = 0;
+    size_t keylen = 0;
+    int ret = 0;
+    EVP_PKEY_CTX *pctx = NULL;
 
     if (type) {
         EVP_CIPHER_CTX_reset(ctx);
         if (!EVP_DecryptInit_ex(ctx, type, NULL, NULL, NULL))
-            return 0;
+            goto err;
     }
 
     if (priv == NULL)
         return 1;
 
-    if (EVP_PKEY_id(priv) != EVP_PKEY_RSA) {
-        EVPerr(EVP_F_EVP_OPENINIT, EVP_R_PUBLIC_KEY_NOT_RSA);
+    if ((pctx = EVP_PKEY_CTX_new(priv, NULL)) == NULL) {
+        ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
         goto err;
     }
 
-    size = EVP_PKEY_size(priv);
-    key = OPENSSL_malloc(size);
-    if (key == NULL) {
-        /* ERROR */
-        EVPerr(EVP_F_EVP_OPENINIT, ERR_R_MALLOC_FAILURE);
+    if (EVP_PKEY_decrypt_init(pctx) <= 0
+        || EVP_PKEY_decrypt(pctx, NULL, &keylen, ek, ekl) <= 0)
         goto err;
-    }
 
-    i = EVP_PKEY_decrypt_old(key, ek, ekl, priv);
-    if ((i <= 0) || !EVP_CIPHER_CTX_set_key_length(ctx, i)) {
-        /* ERROR */
+    if ((key = OPENSSL_malloc(keylen)) == NULL) {
+        ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
         goto err;
     }
-    if (!EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv))
+
+    if (EVP_PKEY_decrypt(pctx, key, &keylen, ek, ekl) <= 0)
+        goto err;
+
+    if (!EVP_CIPHER_CTX_set_key_length(ctx, keylen)
+        || !EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv))
         goto err;
 
     ret = 1;
  err:
-    OPENSSL_clear_free(key, size);
+    EVP_PKEY_CTX_free(pctx);
+    OPENSSL_clear_free(key, keylen);
     return ret;
 }
 
diff --git a/crypto/evp/p_seal.c b/crypto/evp/p_seal.c
index 26e0e7c38d..3f855d08dc 100644
--- a/crypto/evp/p_seal.c
+++ b/crypto/evp/p_seal.c
@@ -30,6 +30,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
     }
     if ((npubk <= 0) || !pubk)
         return 1;
+
     if (EVP_CIPHER_CTX_rand_key(ctx, key) <= 0)
         return 0;
 
@@ -41,13 +42,19 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
         goto err;
 
     for (i = 0; i < npubk; i++) {
-        ekl[i] =
-            EVP_PKEY_encrypt_old(ek[i], key, EVP_CIPHER_CTX_key_length(ctx),
-                                 pubk[i]);
-        if (ekl[i] <= 0) {
-            rv = -1;
+        size_t keylen = EVP_CIPHER_CTX_key_length(ctx);
+        EVP_PKEY_CTX *pctx = NULL;
+
+        if ((pctx = EVP_PKEY_CTX_new(pubk[i], NULL)) == NULL) {
+            ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
             goto err;
         }
+
+        if (EVP_PKEY_encrypt_init(pctx) <= 0
+            || EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
+            goto err;
+        ekl[i] = (int)keylen;
+        EVP_PKEY_CTX_free(pctx);
     }
     rv = npubk;
 err: