Adapt EVP_CIPHER_{param_to_asn1,asn1_to_param} for use with provider.

So far, these two funtions have depended on legacy EVP_CIPHER
implementations to be able to do their work.  This change adapts them
to work with provided implementations as well, in one of two possible
ways:

1.  If the implementation's set_asn1_parameters or get_asn1_parameters
    function pointers are non-NULL, this is a legacy implementation,
    and that function is called.
2.  Otherwise, if the cipher doesn't have EVP_CIPH_FLAG_CUSTOM_ASN1
    set, the default AlgorithmIdentifier parameter code in libcrypto
    is executed.
3.  Otherwise, if the cipher is a provided implementation, the ASN1
    type structure is converted to a DER blob which is then passed to
    the implementation as a parameter (param_to_asn1) or the DER blob
    is retrieved from the implementation as a parameter and converted
    locally to a ASN1_TYPE (asn1_to_param).

With this, the old flag EVP_CIPH_FLAG_DEFAULT_ASN1 has become
irrelevant and is simply ignored.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10008)

1 files changed, 91 insertions, 37 deletions

diff --git a/crypto/evp/evp_lib.c b/crypto/evp/evp_lib.c
index 6a3ad8553b..5c935075dc 100644
--- a/crypto/evp/evp_lib.c
+++ b/crypto/evp/evp_lib.c
@@ -21,29 +21,31 @@
 #if !defined(FIPS_MODE)
 int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 {
-    int ret;
+    int ret = -1;                /* Assume the worst */
     const EVP_CIPHER *cipher = c->cipher;
 
-    if (cipher->prov != NULL) {
-        /*
-         * The cipher has come from a provider and won't have the default flags.
-         * Find the implicit form so we can check the flags.
-         * TODO(3.0): This won't work for 3rd party ciphers we know nothing about
-         * We'll need to think of something else for those.
-         */
-        cipher = EVP_get_cipherbynid(cipher->nid);
-        if (cipher == NULL) {
-            EVPerr(EVP_F_EVP_CIPHER_PARAM_TO_ASN1, ASN1_R_UNSUPPORTED_CIPHER);
-            return -1;
-        }
-    }
-
-    if (cipher->set_asn1_parameters != NULL)
+    /*
+     * For legacy implementations, we detect custom AlgorithmIdentifier
+     * parameter handling by checking if the function pointer
+     * cipher->set_asn1_parameters is set.  We know that this pointer
+     * is NULL for provided implementations.
+     *
+     * Otherwise, for any implementation, we check the flag
+     * EVP_CIPH_FLAG_CUSTOM_ASN1.  If it isn't set, we apply
+     * default AI parameter extraction.
+     *
+     * Otherwise, for provided implementations, we convert |type| to
+     * a DER encoded blob and pass to the implementation in OSSL_PARAM
+     * form.
+     *
+     * If none of the above applies, this operation is unsupported.
+     */
+    if (cipher->set_asn1_parameters != NULL) {
         ret = cipher->set_asn1_parameters(c, type);
-    else if (cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) {
+    } else if ((EVP_CIPHER_flags(cipher) & EVP_CIPH_FLAG_CUSTOM_ASN1) == 0) {
         switch (EVP_CIPHER_mode(cipher)) {
         case EVP_CIPH_WRAP_MODE:
-            if (EVP_CIPHER_nid(cipher) == NID_id_smime_alg_CMS3DESwrap)
+            if (EVP_CIPHER_is_a(cipher, SN_id_smime_alg_CMS3DESwrap))
                 ASN1_TYPE_set(type, V_ASN1_NULL, NULL);
             ret = 1;
             break;
@@ -58,8 +60,40 @@ int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
         default:
             ret = EVP_CIPHER_set_asn1_iv(c, type);
         }
-    } else
-        ret = -1;
+    } else if (cipher->prov != NULL) {
+        OSSL_PARAM params[3], *p = params;
+        unsigned char *der = NULL, *derp;
+
+        /*
+         * We make two passes, the first to get the appropriate buffer size,
+         * and the second to get the actual value.
+         */
+        *p++ = OSSL_PARAM_construct_octet_string(OSSL_CIPHER_PARAM_ALG_ID,
+                                                 NULL, 0);
+        *p = OSSL_PARAM_construct_end();
+
+        if (!EVP_CIPHER_CTX_get_params(c, params))
+            goto err;
+
+        /* ... but, we should get a return size too! */
+        if (params[0].return_size != 0
+            && (der = OPENSSL_malloc(params[0].return_size)) != NULL) {
+            params[0].data = der;
+            params[0].data_size = params[0].return_size;
+            params[0].return_size = 0;
+            derp = der;
+            if (EVP_CIPHER_CTX_get_params(c, params)
+                && d2i_ASN1_TYPE(&type, (const unsigned char **)&derp,
+                                 params[0].return_size) != NULL) {
+                ret = 1;
+            }
+            OPENSSL_free(der);
+        }
+    } else {
+        ret = -2;
+    }
+
+ err:
     if (ret == -2)
         EVPerr(EVP_F_EVP_CIPHER_PARAM_TO_ASN1, ASN1_R_UNSUPPORTED_CIPHER);
     else if (ret <= 0)
@@ -71,24 +105,29 @@ int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 
 int EVP_CIPHER_asn1_to_param(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 {
-    int ret;
+    int ret = -1;                /* Assume the worst */
     const EVP_CIPHER *cipher = c->cipher;
 
-    if (cipher->prov != NULL) {
-        /*
-         * The cipher has come from a provider and won't have the default flags.
-         * Find the implicit form so we can check the flags.
-         */
-        cipher = EVP_get_cipherbynid(cipher->nid);
-        if (cipher == NULL)
-            return -1;
-    }
-
-    if (cipher->get_asn1_parameters != NULL)
+    /*
+     * For legacy implementations, we detect custom AlgorithmIdentifier
+     * parameter handling by checking if there the function pointer
+     * cipher->get_asn1_parameters is set.  We know that this pointer
+     * is NULL for provided implementations.
+     *
+     * Otherwise, for any implementation, we check the flag
+     * EVP_CIPH_FLAG_CUSTOM_ASN1.  If it isn't set, we apply
+     * default AI parameter creation.
+     *
+     * Otherwise, for provided implementations, we get the AI parameter
+     * in DER encoded form from the implementation by requesting the
+     * appropriate OSSL_PARAM and converting the result to a ASN1_TYPE.
+     *
+     * If none of the above applies, this operation is unsupported.
+     */
+    if (cipher->get_asn1_parameters != NULL) {
         ret = cipher->get_asn1_parameters(c, type);
-    else if (cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) {
+    } else if ((EVP_CIPHER_flags(cipher) & EVP_CIPH_FLAG_CUSTOM_ASN1) == 0) {
         switch (EVP_CIPHER_mode(cipher)) {
-
         case EVP_CIPH_WRAP_MODE:
             ret = 1;
             break;
@@ -102,10 +141,25 @@ int EVP_CIPHER_asn1_to_param(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 
         default:
             ret = EVP_CIPHER_get_asn1_iv(c, type);
-            break;
         }
-    } else
-        ret = -1;
+    } else if (cipher->prov != NULL) {
+        OSSL_PARAM params[3], *p = params;
+        unsigned char *der = NULL;
+        int derl = -1;
+
+        if ((derl = i2d_ASN1_TYPE(type, &der)) >= 0) {
+            *p++ =
+                OSSL_PARAM_construct_octet_string(OSSL_CIPHER_PARAM_ALG_ID,
+                                                  der, (size_t)derl);
+            *p = OSSL_PARAM_construct_end();
+            if (EVP_CIPHER_CTX_set_params(c, params))
+                ret = 1;
+            OPENSSL_free(der);
+        }
+    } else {
+        ret = -2;
+    }
+
     if (ret == -2)
         EVPerr(EVP_F_EVP_CIPHER_ASN1_TO_PARAM, EVP_R_UNSUPPORTED_CIPHER);
     else if (ret <= 0)