Backport key redirection test from master branch

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4520)

1 files changed, 204 insertions, 0 deletions

diff --git a/test/enginetest.c b/test/enginetest.c
index 21cd20af05..0a8c1855e2 100644
--- a/test/enginetest.c
+++ b/test/enginetest.c
@@ -22,6 +22,8 @@ int main(int argc, char *argv[])
 # include <openssl/crypto.h>
 # include <openssl/engine.h>
 # include <openssl/err.h>
+# include <openssl/rsa.h>
+# include <openssl/bn.h>
 
 static void display_engine_list(void)
 {
@@ -44,6 +46,206 @@ static void display_engine_list(void)
     ENGINE_free(h);
 }
 
+/* Test EVP_PKEY method */
+static EVP_PKEY_METHOD *test_rsa = NULL;
+
+static int called_encrypt = 0;
+
+/* Test function to check operation has been redirected */
+static int test_encrypt(EVP_PKEY_CTX *ctx, unsigned char *sig,
+                        size_t *siglen, const unsigned char *tbs, size_t tbslen)
+{
+    called_encrypt = 1;
+    return 1;
+}
+
+static int test_pkey_meths(ENGINE *e, EVP_PKEY_METHOD **pmeth,
+                           const int **pnids, int nid)
+{
+    static const int rnid = EVP_PKEY_RSA;
+    if (pmeth == NULL) {
+        *pnids = &rnid;
+        return 1;
+    }
+
+    if (nid == EVP_PKEY_RSA) {
+        *pmeth = test_rsa;
+        return 1;
+    }
+
+    *pmeth = NULL;
+    return 0;
+}
+
+/* Return a test EVP_PKEY value */
+
+static EVP_PKEY *get_test_pkey(void)
+{
+    static unsigned char n[] =
+        "\x00\xAA\x36\xAB\xCE\x88\xAC\xFD\xFF\x55\x52\x3C\x7F\xC4\x52\x3F"
+        "\x90\xEF\xA0\x0D\xF3\x77\x4A\x25\x9F\x2E\x62\xB4\xC5\xD9\x9C\xB5"
+        "\xAD\xB3\x00\xA0\x28\x5E\x53\x01\x93\x0E\x0C\x70\xFB\x68\x76\x93"
+        "\x9C\xE6\x16\xCE\x62\x4A\x11\xE0\x08\x6D\x34\x1E\xBC\xAC\xA0\xA1"
+        "\xF5";
+    static unsigned char e[] = "\x11";
+
+    RSA *rsa = RSA_new();
+    EVP_PKEY *pk = EVP_PKEY_new();
+
+    if (rsa == NULL || pk == NULL || !EVP_PKEY_assign_RSA(pk, rsa)) {
+        RSA_free(rsa);
+        EVP_PKEY_free(pk);
+        return NULL;
+    }
+
+    if (!RSA_set0_key(rsa, BN_bin2bn(n, sizeof(n)-1, NULL),
+                      BN_bin2bn(e, sizeof(e)-1, NULL), NULL)) {
+        EVP_PKEY_free(pk);
+        return NULL;
+    }
+
+    return pk;
+}
+
+static int test_redirect(void)
+{
+    const unsigned char pt[] = "Hello World\n";
+    unsigned char *tmp = NULL;
+    size_t len;
+    EVP_PKEY_CTX *ctx = NULL;
+    ENGINE *e = NULL;
+    EVP_PKEY *pkey = NULL;
+
+    int to_return = 0;
+
+    printf("\nRedirection test\n");
+
+    if ((pkey = get_test_pkey()) == NULL) {
+        printf("Get test key failed\n");
+        goto err;
+    }
+
+    len = EVP_PKEY_size(pkey);
+    if ((tmp = OPENSSL_malloc(len)) == NULL) {
+        printf("Buffer alloc failed\n");
+        goto err;
+    }
+
+    if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL) {
+        printf("Key context allocation failure\n");
+        goto err;
+    }
+    printf("EVP_PKEY_encrypt test: no redirection\n");
+    /* Encrypt some data: should succeed but not be redirected */
+    if (EVP_PKEY_encrypt_init(ctx) <= 0
+            || EVP_PKEY_encrypt(ctx, tmp, &len, pt, sizeof(pt)) <= 0
+            || called_encrypt) {
+        printf("Test encryption failure\n");
+        goto err;
+    }
+    EVP_PKEY_CTX_free(ctx);
+    ctx = NULL;
+
+    /* Create a test ENGINE */
+    if ((e = ENGINE_new()) == NULL
+            || !ENGINE_set_id(e, "Test redirect engine")
+            || !ENGINE_set_name(e, "Test redirect engine")) {
+        printf("Redirection engine setup failure\n");
+        goto err;
+    }
+
+    /*
+     * Try to create a context for this engine and test key.
+     * Try setting test key engine. Both should fail because the
+     * engine has no public key methods.
+     */
+    if (EVP_PKEY_CTX_new(pkey, e) != NULL
+            || EVP_PKEY_set1_engine(pkey, e) > 0) {
+        printf("Unexpected redirection success\n");
+        goto err;
+    }
+
+    /* Setup an empty test EVP_PKEY_METHOD and set callback to return it */
+    if ((test_rsa = EVP_PKEY_meth_new(EVP_PKEY_RSA, 0)) == NULL) {
+        printf("Test RSA algorithm setup failure\n");
+        goto err;
+    }
+    ENGINE_set_pkey_meths(e, test_pkey_meths);
+
+    /* Getting a context for test ENGINE should now succeed */
+    if ((ctx = EVP_PKEY_CTX_new(pkey, e)) == NULL) {
+        printf("Redirected context allocation failed\n");
+        goto err;
+    }
+    /* Encrypt should fail because operation is not supported */
+    if (EVP_PKEY_encrypt_init(ctx) > 0) {
+        printf("Encryption redirect unexpected success\n");
+        goto err;
+    }
+    EVP_PKEY_CTX_free(ctx);
+    ctx = NULL;
+
+    /* Add test encrypt operation to method */
+    EVP_PKEY_meth_set_encrypt(test_rsa, 0, test_encrypt);
+
+    printf("EVP_PKEY_encrypt test: redirection via EVP_PKEY_CTX_new()\n");
+    if ((ctx = EVP_PKEY_CTX_new(pkey, e)) == NULL) {
+        printf("Redirected context allocation failed\n");
+        goto err;
+    }
+    /* Encrypt some data: should succeed and be redirected */
+    if (EVP_PKEY_encrypt_init(ctx) <= 0
+            || EVP_PKEY_encrypt(ctx, tmp, &len, pt, sizeof(pt)) <= 0
+            || !called_encrypt) {
+        printf("Redirected key context encryption failed\n");
+        goto err;
+    }
+
+    EVP_PKEY_CTX_free(ctx);
+    ctx = NULL;
+    called_encrypt = 0;
+
+    printf("EVP_PKEY_encrypt test: check default operation not redirected\n");
+
+    /* Create context with default engine: should not be redirected */
+    if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL
+            || EVP_PKEY_encrypt_init(ctx) <= 0
+            || EVP_PKEY_encrypt(ctx, tmp, &len, pt, sizeof(pt)) <= 0
+            || called_encrypt) {
+        printf("Unredirected key context encryption failed\n");
+        goto err;
+    }
+
+    EVP_PKEY_CTX_free(ctx);
+    ctx = NULL;
+
+    /* Set engine explicitly for test key */
+    if (!EVP_PKEY_set1_engine(pkey, e)) {
+        printf("Key engine set failed\n");
+        goto err;
+    }
+
+    printf("EVP_PKEY_encrypt test: redirection via EVP_PKEY_set1_engine()\n");
+
+    /* Create context with default engine: should be redirected now */
+    if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL
+            || EVP_PKEY_encrypt_init(ctx) <= 0
+            || EVP_PKEY_encrypt(ctx, tmp, &len, pt, sizeof(pt)) <= 0
+            || !called_encrypt) {
+        printf("Key redirection failure\n");
+        goto err;
+    }
+
+    to_return = 1;
+
+ err:
+    EVP_PKEY_CTX_free(ctx);
+    EVP_PKEY_free(pkey);
+    ENGINE_free(e);
+    OPENSSL_free(tmp);
+    return to_return;
+}
+
 int main(int argc, char *argv[])
 {
     ENGINE *block[512];
@@ -183,6 +385,8 @@ int main(int argc, char *argv[])
         OPENSSL_free((void *)ENGINE_get_id(block[loop]));
         OPENSSL_free((void *)ENGINE_get_name(block[loop]));
     }
+    if (!test_redirect())
+        goto end;
     printf("\nTests completed happily\n");
     to_return = 0;
  end: