Convert RSA tests to new framework

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3179)

1 files changed, 107 insertions, 99 deletions

diff --git a/test/rsa_test.c b/test/rsa_test.c
index a38be333ac..af912eb198 100644
--- a/test/rsa_test.c
+++ b/test/rsa_test.c
@@ -18,11 +18,14 @@
 #include <openssl/err.h>
 #include <openssl/rand.h>
 #include <openssl/bn.h>
+
+#include "test_main.h"
+#include "testutil.h"
+
 #ifdef OPENSSL_NO_RSA
-int main(int argc, char *argv[])
+void register_tests(void)
 {
-    printf("No RSA support\n");
-    return (0);
+    /* No tests */
 }
 #else
 # include <openssl/rsa.h>
@@ -220,10 +223,9 @@ static int pad_unknown(void)
 static const char rnd_seed[] =
     "string to make the random number generator think it has entropy";
 
-int main(int argc, char *argv[])
+static int test_rsa_pkcs1(int idx)
 {
-    int err = 0;
-    int v;
+    int ret = 0;
     RSA *key;
     unsigned char ptext[256];
     unsigned char ctext[256];
@@ -232,109 +234,115 @@ int main(int argc, char *argv[])
     int plen;
     int clen = 0;
     int num;
-    int n;
 
-    CRYPTO_set_mem_debug(1);
-    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
+    plen = sizeof(ptext_ex) - 1;
+
+    key = RSA_new();
+    switch (idx) {
+    case 0:
+        clen = key1(key, ctext_ex);
+        break;
+    case 1:
+        clen = key2(key, ctext_ex);
+        break;
+    case 2:
+        clen = key3(key, ctext_ex);
+        break;
+    }
+
+    num = RSA_public_encrypt(plen, ptext_ex, ctext, key,
+                             RSA_PKCS1_PADDING);
+    if (!TEST_int_eq(num, clen))
+        goto err;
+
+    num = RSA_private_decrypt(num, ctext, ptext, key, RSA_PKCS1_PADDING);
+    if (!TEST_mem_eq(ptext, num, ptext_ex, plen))
+        goto err;
+
+    ret = 1;
+err:
+    RSA_free(key);
+    return ret;
+}
+
+static int test_rsa_oaep(int idx)
+{
+    int ret = 0;
+    RSA *key;
+    unsigned char ptext[256];
+    unsigned char ctext[256];
+    static unsigned char ptext_ex[] = "\x54\x85\x9b\x34\x2c\x49\xea\x2a";
+    unsigned char ctext_ex[256];
+    int plen;
+    int clen = 0;
+    int num;
+    int n;
 
     RAND_seed(rnd_seed, sizeof rnd_seed); /* or OAEP may fail */
 
     plen = sizeof(ptext_ex) - 1;
 
-    for (v = 0; v < 3; v++) {
-        key = RSA_new();
-        switch (v) {
-        case 0:
-            clen = key1(key, ctext_ex);
-            break;
-        case 1:
-            clen = key2(key, ctext_ex);
-            break;
-        case 2:
-            clen = key3(key, ctext_ex);
-            break;
-        }
-
-        num = RSA_public_encrypt(plen, ptext_ex, ctext, key,
-                                 RSA_PKCS1_PADDING);
-        if (num != clen) {
-            printf("PKCS#1 v1.5 encryption failed!\n");
-            err = 1;
-            goto oaep;
-        }
-
-        num = RSA_private_decrypt(num, ctext, ptext, key, RSA_PKCS1_PADDING);
-        if (num != plen || memcmp(ptext, ptext_ex, num) != 0) {
-            printf("PKCS#1 v1.5 decryption failed!\n");
-            err = 1;
-        } else
-            printf("PKCS #1 v1.5 encryption/decryption ok\n");
-
- oaep:
-        ERR_clear_error();
-        num = RSA_public_encrypt(plen, ptext_ex, ctext, key,
-                                 RSA_PKCS1_OAEP_PADDING);
-        if (num == -1 && pad_unknown()) {
-            printf("No OAEP support\n");
-            goto next;
-        }
-        if (num != clen) {
-            printf("OAEP encryption failed!\n");
-            err = 1;
-            goto next;
-        }
-
-        num = RSA_private_decrypt(num, ctext, ptext, key,
-                                  RSA_PKCS1_OAEP_PADDING);
-        if (num != plen || memcmp(ptext, ptext_ex, num) != 0) {
-            printf("OAEP decryption (encrypted data) failed!\n");
-            err = 1;
-        } else if (memcmp(ctext, ctext_ex, num) == 0)
-            printf("OAEP test vector %d passed!\n", v);
-
-        /* Different ciphertexts. Try decrypting ctext_ex */
-        num = RSA_private_decrypt(clen, ctext_ex, ptext, key,
-                                  RSA_PKCS1_OAEP_PADDING);
+    key = RSA_new();
+    switch (idx) {
+    case 0:
+        clen = key1(key, ctext_ex);
+        break;
+    case 1:
+        clen = key2(key, ctext_ex);
+        break;
+    case 2:
+        clen = key3(key, ctext_ex);
+        break;
+    }
 
-        if (num != plen || memcmp(ptext, ptext_ex, num) != 0) {
-            printf("OAEP decryption (test vector data) failed!\n");
-            err = 1;
-        } else
-            printf("OAEP encryption/decryption ok\n");
-
-        /* Try decrypting corrupted ciphertexts. */
-        for (n = 0; n < clen; ++n) {
-            ctext[n] ^= 1;
-            num = RSA_private_decrypt(clen, ctext, ptext, key,
-                                          RSA_PKCS1_OAEP_PADDING);
-            if (num > 0) {
-                printf("Corrupt data decrypted!\n");
-                err = 1;
-                break;
-            }
-            ctext[n] ^= 1;
-        }
-
-        /* Test truncated ciphertexts, as well as negative length. */
-        for (n = -1; n < clen; ++n) {
-            num = RSA_private_decrypt(n, ctext, ptext, key,
+    num = RSA_public_encrypt(plen, ptext_ex, ctext, key,
+                             RSA_PKCS1_OAEP_PADDING);
+    if (num == -1 && pad_unknown()) {
+        TEST_info("Skipping: No OAEP support");
+        ret = 1;
+        goto err;
+    }
+    if (!TEST_int_eq(num, clen))
+        goto err;
+
+    num = RSA_private_decrypt(num, ctext, ptext, key,
+                              RSA_PKCS1_OAEP_PADDING);
+    if (!TEST_mem_eq(ptext, num, ptext_ex, plen))
+        goto err;
+
+    /* Different ciphertexts. Try decrypting ctext_ex */
+    num = RSA_private_decrypt(clen, ctext_ex, ptext, key,
+                              RSA_PKCS1_OAEP_PADDING);
+    if (!TEST_mem_eq(ptext, num, ptext_ex, plen))
+        goto err;
+
+    /* Try decrypting corrupted ciphertexts. */
+    for (n = 0; n < clen; ++n) {
+        ctext[n] ^= 1;
+        num = RSA_private_decrypt(clen, ctext, ptext, key,
                                       RSA_PKCS1_OAEP_PADDING);
-            if (num > 0) {
-                printf("Truncated data decrypted!\n");
-                err = 1;
-                break;
-            }
-        }
-
- next:
-        RSA_free(key);
+        if (!TEST_int_le(num, 0))
+            goto err;
+        ctext[n] ^= 1;
     }
 
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-    if (CRYPTO_mem_leaks_fp(stderr) <= 0)
-        err = 1;
-#endif
+    /* Test truncated ciphertexts, as well as negative length. */
+    for (n = -1; n < clen; ++n) {
+        num = RSA_private_decrypt(n, ctext, ptext, key,
+                                  RSA_PKCS1_OAEP_PADDING);
+        if (!TEST_int_le(num, 0))
+            goto err;
+    }
+
+    ret = 1;
+err:
+    RSA_free(key);
+    return ret;
+}
 
-    return err;
+void register_tests(void)
+{
+    ADD_ALL_TESTS(test_rsa_pkcs1, 3);
+    ADD_ALL_TESTS(test_rsa_oaep, 3);
 }
 #endif