ENCODER / DECODER: Add functions to encode/decode to/from a buffer

This adds OSSL_ENCODER_to_data() and OSSL_DECODER_from_data().  These
functions allow fairly simple rewrites of type-specific i2d and d2i
calls.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/13094)

2 files changed, 69 insertions, 0 deletions

diff --git a/crypto/encode_decode/decoder_lib.c b/crypto/encode_decode/decoder_lib.c
index e44ca8cbd4..192d33089b 100644
--- a/crypto/encode_decode/decoder_lib.c
+++ b/crypto/encode_decode/decoder_lib.c
@@ -77,6 +77,27 @@ int OSSL_DECODER_from_fp(OSSL_DECODER_CTX *ctx, FILE *fp)
 }
 #endif
 
+int OSSL_DECODER_from_data(OSSL_DECODER_CTX *ctx, const unsigned char **pdata,
+                           size_t *pdata_len)
+{
+    BIO *membio;
+    int ret = 0;
+
+    if (pdata == NULL || *pdata == NULL || pdata_len == NULL) {
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
+    membio = BIO_new_mem_buf(*pdata, (int)*pdata_len);
+    if (OSSL_DECODER_from_bio(ctx, membio)) {
+        *pdata_len = (size_t)BIO_get_mem_data(membio, pdata);
+        ret = 1;
+    }
+    BIO_free(membio);
+
+    return ret;
+}
+
 int OSSL_DECODER_CTX_set_input_type(OSSL_DECODER_CTX *ctx,
                                     const char *input_type)
 {
diff --git a/crypto/encode_decode/encoder_lib.c b/crypto/encode_decode/encoder_lib.c
index 179c6d3dc3..6d3aa279d7 100644
--- a/crypto/encode_decode/encoder_lib.c
+++ b/crypto/encode_decode/encoder_lib.c
@@ -49,6 +49,54 @@ int OSSL_ENCODER_to_fp(OSSL_ENCODER_CTX *ctx, FILE *fp)
 }
 #endif
 
+int OSSL_ENCODER_to_data(OSSL_ENCODER_CTX *ctx, unsigned char **pdata,
+                         size_t *pdata_len)
+{
+    BIO *out = BIO_new(BIO_s_mem());
+    BUF_MEM *buf = NULL;
+    int ret = 0;
+
+    if (pdata_len == NULL) {
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
+    if (OSSL_ENCODER_to_bio(ctx, out)
+        && BIO_get_mem_ptr(out, &buf) > 0) {
+        ret = 1; /* Hope for the best. A too small buffer will clear this */
+
+        if (pdata != NULL && *pdata != NULL) {
+            if (*pdata_len < buf->length)
+                /*
+                 * It's tempting to do |*pdata_len = (size_t)buf->length|
+                 * However, it's believed to be confusing more than helpful,
+                 * so we don't.
+                 */
+                ret = 0;
+            else
+                *pdata_len -= buf->length;
+        } else {
+            /* The buffer with the right size is already allocated for us */
+            *pdata_len = (size_t)buf->length;
+        }
+
+        if (ret) {
+            if (pdata != NULL) {
+                if (*pdata != NULL) {
+                    memcpy(*pdata, buf->data, buf->length);
+                    *pdata += buf->length;
+                } else {
+                    /* In this case, we steal the data from BIO_s_mem() */
+                    *pdata = (unsigned char *)buf->data;
+                    buf->data = NULL;
+                }
+            }
+        }
+    }
+    BIO_free(out);
+    return ret;
+}
+
 int OSSL_ENCODER_CTX_set_output_type(OSSL_ENCODER_CTX *ctx,
                                      const char *output_type)
 {