TEST: Add new serializer and deserializer test

This test revolves around a central function that will first serialize
an EVP_PKEY, then deserialize the result into a new EVP_PKEY and
compare the two.

The following tests are currently implemented:

1.  EVP_PKEY (RSA) -> DER, then DER -> EVP_PKEY (RSA).
2.  EVP_PKEY (RSA) -> PEM, then PEM -> EVP_PKEY (RSA).
    This one exercises deserializer chains, as we know that there is a
    PEM -> DER and a DER -> EVP_PKEY (RSA) deserializer, but no direct
    PEM -> EVP_PKEY (RSA) deserializer.

Additionally, a small fix in test_fail_string_common(), as strcmp()
could run past a buffer if one of the strings isn't terminated with
a null byte within the given length.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12410)

4 files changed, 221 insertions, 1 deletions

diff --git a/test/build.info b/test/build.info
index a49d9c6d6e..d15ee75814 100644
--- a/test/build.info
+++ b/test/build.info
@@ -777,6 +777,11 @@ IF[{- !$disabled{tests} -}]
   INCLUDE[hexstr_test]=.. ../include ../apps/include
   DEPEND[hexstr_test]=../libcrypto.a libtestutil.a
 
+  PROGRAMS{noinst}=serdes_test
+  SOURCE[serdes_test]=serdes_test.c
+  INCLUDE[serdes_test]=.. ../include ../apps/include
+  DEPEND[serdes_test]=../libcrypto.a libtestutil.a
+
   PROGRAMS{noinst}=namemap_internal_test
   SOURCE[namemap_internal_test]=namemap_internal_test.c
   INCLUDE[namemap_internal_test]=.. ../include ../apps/include
diff --git a/test/recipes/04-test_serializer_deserializer.t b/test/recipes/04-test_serializer_deserializer.t
new file mode 100644
index 0000000000..8da6ffb09f
--- /dev/null
+++ b/test/recipes/04-test_serializer_deserializer.t
@@ -0,0 +1,15 @@
+#! /usr/bin/env perl
+# Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test::Simple;
+use OpenSSL::Test;
+
+simple_test("test_serializer_deserializer", "serdes_test");
diff --git a/test/serdes_test.c b/test/serdes_test.c
new file mode 100644
index 0000000000..5870ceae81
--- /dev/null
+++ b/test/serdes_test.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <openssl/evp.h>
+#include <openssl/pem.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
+#include <openssl/serializer.h>
+#include <openssl/deserializer.h>
+
+#include "testutil.h"
+
+static EVP_PKEY *key_RSA = NULL;
+
+static EVP_PKEY *make_RSA(const char *rsa_type)
+{
+    EVP_PKEY *pkey = NULL;
+    EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_from_name(NULL, rsa_type, NULL);
+
+    /*
+     * No real need to check the errors other than for the cascade
+     * effect.  |pkey| will imply remain NULL if something goes wrong.
+     */
+    (void)(ctx != NULL
+           && EVP_PKEY_keygen_init(ctx) > 0
+           && EVP_PKEY_keygen(ctx, &pkey) > 0);
+    EVP_PKEY_CTX_free(ctx);
+
+    return pkey;
+}
+
+/* Main test driver */
+
+typedef int (serializer)(void **serialized, long *serialized_len,
+                         void *object, const char *ser_propq);
+typedef int (deserializer)(void **object,
+                           void *serialized, long serialized_len);
+typedef int (checker)(int type, const void *data, size_t data_len);
+typedef void (dumper)(const char *label, const void *data, size_t data_len);
+
+static int test_serialize_deserialize(EVP_PKEY *pkey,
+                                      serializer *serialize_cb,
+                                      deserializer *deserialize_cb,
+                                      checker *check_cb, dumper *dump_cb,
+                                      const char *ser_propq)
+{
+    void *serialized = NULL;
+    long serialized_len = 0;
+    EVP_PKEY *pkey2 = NULL;
+    void *serialized2 = NULL;
+    long serialized2_len = 0;
+    int ok = 0;
+
+    if (!serialize_cb(&serialized, &serialized_len, pkey, ser_propq)
+        || !check_cb(EVP_PKEY_base_id(pkey), serialized, serialized_len)
+        || !deserialize_cb((void **)&pkey2, serialized, serialized_len)
+        || !TEST_int_eq(EVP_PKEY_eq(pkey, pkey2), 1))
+        goto end;
+
+    /*
+     * Double check the serialization.
+     */
+    if (!serialize_cb(&serialized2, &serialized2_len, pkey2, ser_propq)
+        || !TEST_mem_eq(serialized, serialized_len,
+                        serialized2, serialized2_len))
+        goto end;
+
+    ok = 1;
+ end:
+    if (!ok)
+        dump_cb("serialized result", serialized, serialized_len);
+
+    OPENSSL_free(serialized);
+    OPENSSL_free(serialized2);
+    EVP_PKEY_free(pkey2);
+    return ok;
+}
+
+/* Serializing and desserializing methods */
+
+static int serialize_EVP_PKEY(void **serialized, long *serialized_len,
+                              void *object, const char *ser_propq)
+{
+    EVP_PKEY *pkey = object;
+    OSSL_SERIALIZER_CTX *sctx = NULL;
+    BIO *mem_ser = NULL;
+    BUF_MEM *mem_buf = NULL;
+    int ok = 0;
+
+    if (!TEST_ptr(sctx = OSSL_SERIALIZER_CTX_new_by_EVP_PKEY(pkey, ser_propq))
+        || !TEST_ptr(mem_ser = BIO_new(BIO_s_mem()))
+        || !TEST_true(OSSL_SERIALIZER_to_bio(sctx, mem_ser))
+        || !TEST_true(BIO_get_mem_ptr(mem_ser, &mem_buf) > 0)
+        || !TEST_ptr(*serialized = mem_buf->data)
+        || !TEST_long_gt(*serialized_len = mem_buf->length, 0))
+        goto end;
+
+    /* Detach the serialized output */
+    mem_buf->data = NULL;
+    mem_buf->length = 0;
+    ok = 1;
+ end:
+    BIO_free(mem_ser);
+    OSSL_SERIALIZER_CTX_free(sctx);
+    return ok;
+}
+
+static int deserialize_EVP_PKEY(void **object,
+                                void *serialized, long serialized_len)
+{
+    EVP_PKEY *pkey = NULL;
+    OSSL_DESERIALIZER_CTX *dctx = NULL;
+    BIO *mem_deser = NULL;
+    int ok = 0;
+
+    if (!TEST_ptr(dctx = OSSL_DESERIALIZER_CTX_new_by_EVP_PKEY(&pkey, NULL,
+                                                               NULL, NULL))
+        || !TEST_ptr(mem_deser = BIO_new_mem_buf(serialized, serialized_len))
+        || !TEST_true(OSSL_DESERIALIZER_from_bio(dctx, mem_deser)))
+        goto end;
+    ok = 1;
+    *object = pkey;
+ end:
+    BIO_free(mem_deser);
+    OSSL_DESERIALIZER_CTX_free(dctx);
+    return ok;
+}
+
+/* Test cases and their dumpers / checkers */
+
+static void dump_der(const char *label, const void *data, size_t data_len)
+{
+    test_output_memory(label, data, data_len);
+}
+
+static void dump_pem(const char *label, const void *data, size_t data_len)
+{
+    test_output_string(label, data, data_len - 1);
+}
+
+static int check_PKCS8_DER(int type, const void *data, size_t data_len)
+{
+    const unsigned char *datap = data;
+    PKCS8_PRIV_KEY_INFO *p8inf =
+        d2i_PKCS8_PRIV_KEY_INFO(NULL, &datap, data_len);
+    int ok = 0;
+
+    if (TEST_ptr(p8inf)) {
+        EVP_PKEY *pkey = EVP_PKCS82PKEY(p8inf);
+
+        ok = (TEST_ptr(pkey) && TEST_true(EVP_PKEY_is_a(pkey, "RSA")));
+        EVP_PKEY_free(pkey);
+    }
+    PKCS8_PRIV_KEY_INFO_free(p8inf);
+    return ok;
+}
+
+static int test_RSA_via_DER(void)
+{
+    return test_serialize_deserialize(key_RSA,
+                                      serialize_EVP_PKEY,
+                                      deserialize_EVP_PKEY,
+                                      check_PKCS8_DER, dump_der,
+                                      OSSL_SERIALIZER_PrivateKey_TO_DER_PQ);
+}
+
+static int check_PKCS8_PEM(int type, const void *data, size_t data_len)
+{
+    static const char pem_header[] = "-----BEGIN " PEM_STRING_PKCS8INF "-----";
+
+    return TEST_strn_eq(data, pem_header, sizeof(pem_header) - 1);
+}
+
+static int test_RSA_via_PEM(void)
+{
+    return test_serialize_deserialize(key_RSA,
+                                      serialize_EVP_PKEY,
+                                      deserialize_EVP_PKEY,
+                                      check_PKCS8_PEM, dump_pem,
+                                      OSSL_SERIALIZER_PrivateKey_TO_PEM_PQ);
+}
+
+int setup_tests(void)
+{
+    TEST_info("Generating key...");
+    if (!TEST_ptr(key_RSA = make_RSA("RSA")))
+        return 0;
+    TEST_info("Generating key... done");
+
+    ADD_TEST(test_RSA_via_DER);
+    ADD_TEST(test_RSA_via_PEM);
+
+    return 1;
+}
diff --git a/test/testutil/format_output.c b/test/testutil/format_output.c
index 069a6a03a5..e2ee98cfd8 100644
--- a/test/testutil/format_output.c
+++ b/test/testutil/format_output.c
@@ -65,7 +65,7 @@ static void test_fail_string_common(const char *prefix, const char *file,
         goto fin;
     }
 
-    if (l1 != l2 || strcmp(m1, m2) != 0)
+    if (l1 != l2 || strncmp(m1, m2, l1) != 0)
         test_diff_header(left, right);
 
     while (l1 > 0 || l2 > 0) {