DRBG: delay initialization of DRBG method until instantiation

Previously, the initialization was done immediately in RAND_DRBG_set(),
which is also called in RAND_DRBG_uninstantiate().

This made it difficult for the FIPS DRBG self test to verify that the
internal state had been zeroized, because it had the side effect that
the drbg->data structure was reinitialized immediately.

To solve the problem, RAND_DRBG_set() has been split in two parts

    static int rand_drbg_set(RAND_DRBG *drbg, int type, unsigned int flags);
    static int rand_drbg_init_method(RAND_DRBG *drbg);

and only the first part is called from RAND_DRBG_uninstantiate().

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11111)

2 files changed, 46 insertions, 31 deletions

diff --git a/crypto/rand/drbg_lib.c b/crypto/rand/drbg_lib.c
index 029cc6e77c..0334dee99d 100644
--- a/crypto/rand/drbg_lib.c
+++ b/crypto/rand/drbg_lib.c
@@ -115,6 +115,9 @@ static RAND_DRBG *rand_drbg_new(OPENSSL_CTX *ctx,
                                 unsigned int flags,
                                 RAND_DRBG *parent);
 
+static int rand_drbg_set(RAND_DRBG *drbg, int type, unsigned int flags);
+static int rand_drbg_init_method(RAND_DRBG *drbg);
+
 static int is_ctr(int type)
 {
     switch (type) {
@@ -341,8 +344,11 @@ void *RAND_DRBG_get_callback_data(RAND_DRBG *drbg)
  */
 int RAND_DRBG_set(RAND_DRBG *drbg, int type, unsigned int flags)
 {
-    int ret = 1;
+    return rand_drbg_set(drbg, type, flags) && rand_drbg_init_method(drbg);
+}
 
+static int rand_drbg_set(RAND_DRBG *drbg, int type, unsigned int flags)
+{
     if (type == 0 && flags == 0) {
         type = rand_drbg_type[RAND_DRBG_TYPE_MASTER];
         flags = rand_drbg_flags[RAND_DRBG_TYPE_MASTER];
@@ -350,7 +356,8 @@ int RAND_DRBG_set(RAND_DRBG *drbg, int type, unsigned int flags)
 
     /* If set is called multiple times - clear the old one */
     if (drbg->type != 0 && (type != drbg->type || flags != drbg->flags)) {
-        drbg->meth->uninstantiate(drbg);
+        if (drbg->meth != NULL)
+            drbg->meth->uninstantiate(drbg);
         rand_pool_free(drbg->adin_pool);
         drbg->adin_pool = NULL;
     }
@@ -358,29 +365,43 @@ int RAND_DRBG_set(RAND_DRBG *drbg, int type, unsigned int flags)
     drbg->state = DRBG_UNINITIALISED;
     drbg->flags = flags;
     drbg->type = type;
+    drbg->meth = NULL;
 
-    if (type == 0) {
-        /* Uninitialized; that's okay. */
-        drbg->meth = NULL;
+    if (type == 0 || is_ctr(type) || is_digest(type))
         return 1;
-    } else if (is_ctr(type)) {
+
+    drbg->type = 0;
+    drbg->flags = 0;
+    RANDerr(RAND_F_RAND_DRBG_SET, RAND_R_UNSUPPORTED_DRBG_TYPE);
+
+    return 0;
+}
+
+static int rand_drbg_init_method(RAND_DRBG *drbg)
+{
+    int ret;
+
+    if (drbg->meth != NULL)
+        return 1;
+
+    if (is_ctr(drbg->type)) {
         ret = drbg_ctr_init(drbg);
-    } else if (is_digest(type)) {
-        if (flags & RAND_DRBG_FLAG_HMAC)
+    } else if (is_digest(drbg->type)) {
+        if (drbg->flags & RAND_DRBG_FLAG_HMAC)
             ret = drbg_hmac_init(drbg);
         else
             ret = drbg_hash_init(drbg);
     } else {
+        /* other cases should already be excluded */
+        RANDerr(RAND_F_RAND_DRBG_INIT_METHOD, ERR_R_INTERNAL_ERROR);
         drbg->type = 0;
         drbg->flags = 0;
-        drbg->meth = NULL;
-        RANDerr(RAND_F_RAND_DRBG_SET, RAND_R_UNSUPPORTED_DRBG_TYPE);
         return 0;
     }
 
     if (ret == 0) {
         drbg->state = DRBG_ERROR;
-        RANDerr(RAND_F_RAND_DRBG_SET, RAND_R_ERROR_INITIALISING_DRBG);
+        RANDerr(RAND_F_RAND_DRBG_INIT_METHOD, RAND_R_ERROR_INITIALISING_DRBG);
     }
     return ret;
 }
@@ -554,19 +575,21 @@ int RAND_DRBG_instantiate(RAND_DRBG *drbg,
 {
     unsigned char *nonce = NULL, *entropy = NULL;
     size_t noncelen = 0, entropylen = 0;
-    size_t min_entropy = drbg->strength;
-    size_t min_entropylen = drbg->min_entropylen;
-    size_t max_entropylen = drbg->max_entropylen;
+    size_t min_entropy, min_entropylen, max_entropylen;
 
-    if (perslen > drbg->max_perslen) {
+    if (drbg->meth == NULL && !rand_drbg_init_method(drbg)) {
         RANDerr(RAND_F_RAND_DRBG_INSTANTIATE,
-                RAND_R_PERSONALISATION_STRING_TOO_LONG);
+                RAND_R_NO_DRBG_IMPLEMENTATION_SELECTED);
         goto end;
     }
 
-    if (drbg->meth == NULL) {
+    min_entropy = drbg->strength;
+    min_entropylen = drbg->min_entropylen;
+    max_entropylen = drbg->max_entropylen;
+
+    if (perslen > drbg->max_perslen) {
         RANDerr(RAND_F_RAND_DRBG_INSTANTIATE,
-                RAND_R_NO_DRBG_IMPLEMENTATION_SELECTED);
+                RAND_R_PERSONALISATION_STRING_TOO_LONG);
         goto end;
     }
 
@@ -648,19 +671,11 @@ int RAND_DRBG_instantiate(RAND_DRBG *drbg,
 int RAND_DRBG_uninstantiate(RAND_DRBG *drbg)
 {
     int index = -1, type, flags;
-    if (drbg->meth == NULL) {
-        drbg->state = DRBG_ERROR;
-        RANDerr(RAND_F_RAND_DRBG_UNINSTANTIATE,
-                RAND_R_NO_DRBG_IMPLEMENTATION_SELECTED);
-        return 0;
+    if (drbg->meth != NULL) {
+        drbg->meth->uninstantiate(drbg);
+        drbg->meth = NULL;
     }
 
-    /* Clear the entire drbg->ctr struct, then reset some important
-     * members of the drbg->ctr struct (e.g. keysize, df_ks) to their
-     * initial values.
-     */
-    drbg->meth->uninstantiate(drbg);
-
     /* The reset uses the default values for type and flags */
     if (drbg->flags & RAND_DRBG_FLAG_MASTER)
         index = RAND_DRBG_TYPE_MASTER;
@@ -676,7 +691,7 @@ int RAND_DRBG_uninstantiate(RAND_DRBG *drbg)
         flags = drbg->flags;
         type = drbg->type;
     }
-    return RAND_DRBG_set(drbg, type, flags);
+    return rand_drbg_set(drbg, type, flags);
 }
 
 /*
diff --git a/crypto/rand/rand_err.c b/crypto/rand/rand_err.c
index 53d329380a..73b488fb90 100644
--- a/crypto/rand/rand_err.c
+++ b/crypto/rand/rand_err.c
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy