Make EVP_PKEY_[get1|set1]_tls_encodedpoint work with provided keys

EVP_PKEY_[get1|set1]_tls_encodedpoint() only worked if an ameth was present
which isn't the case for provided keys. Support has been added to dh,
ec and ecx keys.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11898)

3 files changed, 26 insertions, 19 deletions

diff --git a/crypto/dh/dh_ameth.c b/crypto/dh/dh_ameth.c
index d93d519444..d5e5f72517 100644
--- a/crypto/dh/dh_ameth.c
+++ b/crypto/dh/dh_ameth.c
@@ -438,7 +438,7 @@ static int dh_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
     case ASN1_PKEY_CTRL_SET1_TLS_ENCPT:
         return dh_buf2key(EVP_PKEY_get0_DH(pkey), arg2, arg1);
     case ASN1_PKEY_CTRL_GET1_TLS_ENCPT:
-        return dh_key2buf(EVP_PKEY_get0_DH(pkey), arg2);
+        return dh_key2buf(EVP_PKEY_get0_DH(pkey), arg2, 0, 1);
     default:
         return -2;
     }
diff --git a/crypto/dh/dh_key.c b/crypto/dh/dh_key.c
index 1893b487ca..5d2acca25c 100644
--- a/crypto/dh/dh_key.c
+++ b/crypto/dh/dh_key.c
@@ -351,10 +351,10 @@ err:
     return 0;
 }
 
-size_t dh_key2buf(const DH *dh, unsigned char **pbuf_out)
+size_t dh_key2buf(const DH *dh, unsigned char **pbuf_out, size_t size, int alloc)
 {
     const BIGNUM *pubkey;
-    unsigned char *pbuf;
+    unsigned char *pbuf = NULL;
     const BIGNUM *p;
     int p_size;
 
@@ -366,19 +366,29 @@ size_t dh_key2buf(const DH *dh, unsigned char **pbuf_out)
         DHerr(DH_F_DH_KEY2BUF, DH_R_INVALID_PUBKEY);
         return 0;
     }
-    if ((pbuf = OPENSSL_malloc(p_size)) == NULL) {
-        DHerr(DH_F_DH_KEY2BUF, ERR_R_MALLOC_FAILURE);
-        return 0;
-    }
-    /*
-     * As per Section 4.2.8.1 of RFC 8446 left pad public
-     * key with zeros to the size of p
-     */
-    if (BN_bn2binpad(pubkey, pbuf, p_size) < 0) {
-        OPENSSL_free(pbuf);
-        DHerr(DH_F_DH_KEY2BUF, DH_R_BN_ERROR);
-        return 0;
+    if (pbuf_out != NULL && (alloc || *pbuf_out != NULL)) {
+        if (!alloc) {
+            if (size >= (size_t)p_size)
+                pbuf = *pbuf_out;
+        } else {
+            pbuf = OPENSSL_malloc(p_size);
+        }
+
+        if (pbuf == NULL) {
+            DHerr(DH_F_DH_KEY2BUF, ERR_R_MALLOC_FAILURE);
+            return 0;
+        }
+        /*
+         * As per Section 4.2.8.1 of RFC 8446 left pad public
+         * key with zeros to the size of p
+         */
+        if (BN_bn2binpad(pubkey, pbuf, p_size) < 0) {
+            if (alloc)
+                OPENSSL_free(pbuf);
+            DHerr(DH_F_DH_KEY2BUF, DH_R_BN_ERROR);
+            return 0;
+        }
+        *pbuf_out = pbuf;
     }
-    *pbuf_out = pbuf;
     return p_size;
 }
diff --git a/crypto/dh/dh_local.h b/crypto/dh/dh_local.h
index a54d25f487..51c3f974e1 100644
--- a/crypto/dh/dh_local.h
+++ b/crypto/dh/dh_local.h
@@ -58,6 +58,3 @@ struct dh_method {
     int (*generate_params) (DH *dh, int prime_len, int generator,
                             BN_GENCB *cb);
 };
-
-int dh_buf2key(DH *key, const unsigned char *buf, size_t len);
-size_t dh_key2buf(const DH *dh, unsigned char **pbuf);