rsa: Do not allow less than 512 bit RSA keys

As per documentation, the RSA keys should not be smaller than 64bit (the
documentation mentions something about a quirk in the prime generation
algorithm). I am adding check into the code which used to be 16 for some
reason.
My primary motivation is to get rid of the last sentence in the
documentation which suggest that typical keys have 1024 bits (instead
updating it to the now default 2048).
I *assume* that keys less than the 2048 bits (say 512) are used for
education purposes.
The 512 bits as the minimum have been suggested by Bernd Edlinger.

Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>

Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4547)

5 files changed, 7 insertions, 17 deletions

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index eda23b5481..4b9296e46c 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -72,11 +72,7 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, int primes, BIGNUM *e_value,
     BN_CTX *ctx = NULL;
     BN_ULONG bitst = 0;
 
-    /*
-     * When generating ridiculously small keys, we can get stuck
-     * continually regenerating the same prime values.
-     */
-    if (bits < 16) {
+    if (bits < RSA_MIN_MODULUS_BITS) {
         ok = 0;             /* we set our own err */
         RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
         goto err;
diff --git a/crypto/rsa/rsa_locl.h b/crypto/rsa/rsa_locl.h
index 52d839d73a..9bd53bec5a 100644
--- a/crypto/rsa/rsa_locl.h
+++ b/crypto/rsa/rsa_locl.h
@@ -12,6 +12,7 @@
 
 #define RSA_MAX_PRIME_NUM 16
 #define RSA_MIN_PRIME_SIZE 64
+#define RSA_MIN_MODULUS_BITS	512
 
 typedef struct rsa_prime_info_st {
     BIGNUM *r;
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 8a114cff89..e11ed1f034 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -459,7 +459,7 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         return 1;
 
     case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
-        if (p1 < 512) {
+        if (p1 < RSA_MIN_MODULUS_BITS) {
             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_KEY_SIZE_TOO_SMALL);
             return -2;
         }
diff --git a/doc/man1/genrsa.pod b/doc/man1/genrsa.pod
index 3e42c98f5d..25562dc866 100644
--- a/doc/man1/genrsa.pod
+++ b/doc/man1/genrsa.pod
@@ -94,7 +94,7 @@ RSA key, which is defined in RFC 8017.
 =item B<numbits>
 
 The size of the private key to generate in bits. This must be the last option
-specified. The default is 2048.
+specified. The default is 2048 and values less than 512 are not allowed.
 
 =back
 
@@ -112,13 +112,6 @@ Because key generation is a random process the time taken to generate a key
 may vary somewhat. But in general, more primes lead to less generation time
 of a key.
 
-=head1 BUGS
-
-A quirk of the prime generation algorithm is that it cannot generate small
-primes. Therefore the number of bits should not be less that 64. For typical
-private keys this will not matter because for security reasons they will
-be much larger (typically 1024 bits).
-
 =head1 SEE ALSO
 
 L<gendsa(1)>
diff --git a/test/recipes/15-test_genrsa.t b/test/recipes/15-test_genrsa.t
index cc74e303f1..72a58bcd69 100644
--- a/test/recipes/15-test_genrsa.t
+++ b/test/recipes/15-test_genrsa.t
@@ -18,9 +18,9 @@ setup("test_genrsa");
 
 plan tests => 5;
 
-is(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '8'])), 0, "genrsa -3 8");
-ok(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '16'])), "genrsa -3 16");
+is(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '256'])), 0, "genrsa -3 256");
+ok(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '512'])), "genrsa -3 512");
 ok(run(app([ 'openssl', 'rsa', '-check', '-in', 'genrsatest.pem', '-noout'])), "rsa -check");
-ok(run(app([ 'openssl', 'genrsa', '-f4', '-out', 'genrsatest.pem', '16'])), "genrsa -f4 16");
+ok(run(app([ 'openssl', 'genrsa', '-f4', '-out', 'genrsatest.pem', '512'])), "genrsa -f4 512");
 ok(run(app([ 'openssl', 'rsa', '-check', '-in', 'genrsatest.pem', '-noout'])), "rsa -check");
 unlink 'genrsatest.pem';