Update RSA keygen to use sp800-56b by default

Fixes #11742
Fixes #11764

The newer RSA sp800-56b algorithm is being used for the normal case of a non multiprime key of at least length 2048.
Insecure key lengths and mutltiprime RSA will use the old method.

Bad public exponents are no longer allowed (i.e values less than 65537 or even). Values such as 2 that would cause a infinite loop now result in an error. The value of 3 has been marked as deprecated but is still allowed for legacy purposes.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/11765)

1 files changed, 19 insertions, 4 deletions

diff --git a/test/recipes/15-test_genrsa.t b/test/recipes/15-test_genrsa.t
index bfe000a26d..17b0cbc1a0 100644
--- a/test/recipes/15-test_genrsa.t
+++ b/test/recipes/15-test_genrsa.t
@@ -16,7 +16,7 @@ use OpenSSL::Test::Utils;
 
 setup("test_genrsa");
 
-plan tests => 9;
+plan tests => 12;
 
 # We want to know that an absurdly small number of bits isn't support
 if (disabled("deprecated-3.0")) {
@@ -43,7 +43,7 @@ while ($good > $bad + 1) {
     my $bits = 2 ** $checked;
     if (disabled("deprecated-3.0")) {
         $fin = run(app([ 'openssl', 'genpkey', '-out', 'genrsatest.pem',
-                         '-algorithm', 'RSA', '-pkeyopt', 'rsa_keygen_pubexp:3',
+                         '-algorithm', 'RSA', '-pkeyopt', 'rsa_keygen_pubexp:65537',
                          '-pkeyopt', "rsa_keygen_bits:$bits",
                        ], stderr => undef));
     } else {
@@ -64,7 +64,7 @@ $good = 2 ** $good;
 note "Found lowest allowed amount of bits to be $good";
 
 ok(run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
-             '-pkeyopt', 'rsa_keygen_pubexp:3',
+             '-pkeyopt', 'rsa_keygen_pubexp:65537',
              '-pkeyopt', "rsa_keygen_bits:$good",
              '-out', 'genrsatest.pem' ])),
    "genpkey -3 $good");
@@ -75,9 +75,24 @@ ok(run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
              '-pkeyopt', "rsa_keygen_bits:$good",
              '-out', 'genrsatest.pem' ])),
    "genpkey -f4 $good");
-ok(run(app([ 'openssl', 'pkey', '-check', '-in', 'genrsatest.pem', '-noout' ])),
+
+ok(run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
+             '-pkeyopt', 'rsa_keygen_bits:2048',
+             '-out', 'genrsatest2048.pem' ])),
+   "genpkey 2048 bits");
+ok(run(app([ 'openssl', 'pkey', '-check', '-in', 'genrsatest2048.pem', '-noout' ])),
    "pkey -check");
 
+ok(!run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
+             '-pkeyopt', 'hexe:02',
+             '-out', 'genrsatest.pem' ])),
+   "genpkey with a bad public exponent should fail");
+ok(!run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
+             '-pkeyopt', 'e:65538',
+             '-out', 'genrsatest.pem' ])),
+   "genpkey with a even public exponent should fail");
+
+
  SKIP: {
     skip "Skipping rsa command line test", 4 if disabled("deprecated-3.0");