hashbn: check for NULL result when allocating bin and return an error if it fails all (in)direct callers of hashbn: propagate potential error in hashbn

Signed-off-by: Kurt Roeckx <kurt@openssl.org>
Reviewed-by: Dr. Stephen Henson <steve@openssl.org>

1 files changed, 47 insertions, 21 deletions

diff --git a/crypto/jpake/jpake.c b/crypto/jpake/jpake.c
index 424929471f..736d3c0916 100644
--- a/crypto/jpake/jpake.c
+++ b/crypto/jpake/jpake.c
@@ -149,19 +149,23 @@ static void hashstring(SHA_CTX *sha, const char *string)
     SHA1_Update(sha, string, l);
     }
 
-static void hashbn(SHA_CTX *sha, const BIGNUM *bn)
+static int hashbn(SHA_CTX *sha, const BIGNUM *bn)
     {
     size_t l = BN_num_bytes(bn);
     unsigned char *bin = OPENSSL_malloc(l);
 
+    if (bin == NULL)
+	return 0;
+
     hashlength(sha, l);
     BN_bn2bin(bn, bin);
     SHA1_Update(sha, bin, l);
     OPENSSL_free(bin);
+    return 1;
     }
 
 /* h=hash(g, g^r, g^x, name) */
-static void zkp_hash(BIGNUM *h, const BIGNUM *zkpg, const JPAKE_STEP_PART *p,
+static int zkp_hash(BIGNUM *h, const BIGNUM *zkpg, const JPAKE_STEP_PART *p,
 		     const char *proof_name)
     {
     unsigned char md[SHA_DIGEST_LENGTH];
@@ -172,22 +176,27 @@ static void zkp_hash(BIGNUM *h, const BIGNUM *zkpg, const JPAKE_STEP_PART *p,
     * is flawed in this respect. Length encoding seems simplest.
     */
     SHA1_Init(&sha);
-    hashbn(&sha, zkpg);
+    if (!hashbn(&sha, zkpg))
+	return 0;
     OPENSSL_assert(!BN_is_zero(p->zkpx.gr));
-    hashbn(&sha, p->zkpx.gr);
-    hashbn(&sha, p->gx);
+    if (!hashbn(&sha, p->zkpx.gr))
+	return 0;
+    if (!hashbn(&sha, p->gx))
+	return 0;
     hashstring(&sha, proof_name);
     SHA1_Final(md, &sha);
     BN_bin2bn(md, SHA_DIGEST_LENGTH, h);
+    return 1;
     }
 
 /*
  * Prove knowledge of x
  * Note that p->gx has already been calculated
  */
-static void generate_zkp(JPAKE_STEP_PART *p, const BIGNUM *x,
+static int generate_zkp(JPAKE_STEP_PART *p, const BIGNUM *x,
 			 const BIGNUM *zkpg, JPAKE_CTX *ctx)
     {
+    int res = 0;
     BIGNUM *r = BN_new();
     BIGNUM *h = BN_new();
     BIGNUM *t = BN_new();
@@ -201,16 +210,20 @@ static void generate_zkp(JPAKE_STEP_PART *p, const BIGNUM *x,
     BN_mod_exp(p->zkpx.gr, zkpg, r, ctx->p.p, ctx->ctx);
 
    /* h=hash... */
-    zkp_hash(h, zkpg, p, ctx->p.name);
+    if (!zkp_hash(h, zkpg, p, ctx->p.name))
+	goto end;
 
    /* b = r - x*h */
     BN_mod_mul(t, x, h, ctx->p.q, ctx->ctx);
     BN_mod_sub(p->zkpx.b, r, t, ctx->p.q, ctx->ctx);
 
+    res = 1;
+end:
    /* cleanup */
     BN_free(t);
     BN_free(h);
     BN_free(r);
+    return res;
     }
 
 static int verify_zkp(const JPAKE_STEP_PART *p, const BIGNUM *zkpg,
@@ -222,7 +235,8 @@ static int verify_zkp(const JPAKE_STEP_PART *p, const BIGNUM *zkpg,
     BIGNUM *t3 = BN_new();
     int ret = 0;
 
-    zkp_hash(h, zkpg, p, ctx->p.peer_name);
+    if (!zkp_hash(h, zkpg, p, ctx->p.peer_name))
+	goto end;
 
    /* t1 = g^b */
     BN_mod_exp(t1, zkpg, p->zkpx.b, ctx->p.p, ctx->ctx);
@@ -237,6 +251,7 @@ static int verify_zkp(const JPAKE_STEP_PART *p, const BIGNUM *zkpg,
     else
 	JPAKEerr(JPAKE_F_VERIFY_ZKP, JPAKE_R_ZKP_VERIFY_FAILED);
 
+end:
    /* cleanup */
     BN_free(t3);
     BN_free(t2);
@@ -246,11 +261,13 @@ static int verify_zkp(const JPAKE_STEP_PART *p, const BIGNUM *zkpg,
     return ret;
     }    
 
-static void generate_step_part(JPAKE_STEP_PART *p, const BIGNUM *x,
+static int generate_step_part(JPAKE_STEP_PART *p, const BIGNUM *x,
 			       const BIGNUM *g, JPAKE_CTX *ctx)
     {
     BN_mod_exp(p->gx, g, x, ctx->p.p, ctx->ctx);
-    generate_zkp(p, x, g, ctx);
+    if (!generate_zkp(p, x, g, ctx))
+	return 0;
+    return 1;
     }
 
 /* Generate each party's random numbers. xa is in [0, q), xb is in [1, q). */
@@ -278,8 +295,10 @@ static void genrand(JPAKE_CTX *ctx)
 int JPAKE_STEP1_generate(JPAKE_STEP1 *send, JPAKE_CTX *ctx)
     {
     genrand(ctx);
-    generate_step_part(&send->p1, ctx->xa, ctx->p.g, ctx);
-    generate_step_part(&send->p2, ctx->xb, ctx->p.g, ctx);
+    if (!generate_step_part(&send->p1, ctx->xa, ctx->p.g, ctx))
+	return 0;
+    if (!generate_step_part(&send->p2, ctx->xb, ctx->p.g, ctx))
+	return 0;
 
     return 1;
     }
@@ -346,6 +365,8 @@ int JPAKE_STEP1_process(JPAKE_CTX *ctx, const JPAKE_STEP1 *received)
 
 int JPAKE_STEP2_generate(JPAKE_STEP2 *send, JPAKE_CTX *ctx)
     {
+    int ret;
+
     BIGNUM *t1 = BN_new();
     BIGNUM *t2 = BN_new();
 
@@ -370,13 +391,13 @@ int JPAKE_STEP2_generate(JPAKE_STEP2 *send, JPAKE_CTX *ctx)
     * as the generator, which means X is g'^{xb * s}
     * X = t1^{t2} = t1^{xb * s} = g^{(xa + xc + xd) * xb * s}
     */
-    generate_step_part(send, t2, t1, ctx);
+    ret = generate_step_part(send, t2, t1, ctx);
 
    /* cleanup */
     BN_free(t1);
     BN_free(t2);
 
-    return 1;
+    return ret;
     }
 
 /* gx = g^{xc + xa + xb} * xd * s */
@@ -443,13 +464,15 @@ int JPAKE_STEP2_process(JPAKE_CTX *ctx, const JPAKE_STEP2 *received)
     return ret;
     }
 
-static void quickhashbn(unsigned char *md, const BIGNUM *bn)
+static int quickhashbn(unsigned char *md, const BIGNUM *bn)
     {
     SHA_CTX sha;
 
     SHA1_Init(&sha);
-    hashbn(&sha, bn);
+    if (!hashbn(&sha, bn))
+	return 0;
     SHA1_Final(md, &sha);
+    return 1;
     }
 
 void JPAKE_STEP3A_init(JPAKE_STEP3A *s3a)
@@ -457,7 +480,8 @@ void JPAKE_STEP3A_init(JPAKE_STEP3A *s3a)
 
 int JPAKE_STEP3A_generate(JPAKE_STEP3A *send, JPAKE_CTX *ctx)
     {
-    quickhashbn(send->hhk, ctx->key);
+    if (!quickhashbn(send->hhk, ctx->key))
+	return 0;
     SHA1(send->hhk, sizeof send->hhk, send->hhk);
 
     return 1;
@@ -467,7 +491,8 @@ int JPAKE_STEP3A_process(JPAKE_CTX *ctx, const JPAKE_STEP3A *received)
     {
     unsigned char hhk[SHA_DIGEST_LENGTH];
 
-    quickhashbn(hhk, ctx->key);
+    if (!quickhashbn(hhk, ctx->key))
+	return 0;
     SHA1(hhk, sizeof hhk, hhk);
     if(memcmp(hhk, received->hhk, sizeof hhk))
 	{
@@ -485,8 +510,8 @@ void JPAKE_STEP3B_init(JPAKE_STEP3B *s3b)
 
 int JPAKE_STEP3B_generate(JPAKE_STEP3B *send, JPAKE_CTX *ctx)
     {
-    quickhashbn(send->hk, ctx->key);
-
+    if (!quickhashbn(send->hk, ctx->key))
+	return 0;
     return 1;
     }
 
@@ -494,7 +519,8 @@ int JPAKE_STEP3B_process(JPAKE_CTX *ctx, const JPAKE_STEP3B *received)
     {
     unsigned char hk[SHA_DIGEST_LENGTH];
 
-    quickhashbn(hk, ctx->key);
+    if (!quickhashbn(hk, ctx->key))
+	return 0;
     if(memcmp(hk, received->hk, sizeof hk))
 	{
 	JPAKEerr(JPAKE_F_JPAKE_STEP3B_PROCESS, JPAKE_R_HASH_OF_KEY_MISMATCH);