- djm@cvs.openbsd.org 2014/06/24 01:13:21

     [Makefile.in auth-bsdauth.c auth-chall.c auth-options.c auth-rsa.c
     [auth2-none.c auth2-pubkey.c authfile.c authfile.h cipher-3des1.c
     [cipher-chachapoly.c cipher-chachapoly.h cipher.c cipher.h
     [digest-libc.c digest-openssl.c digest.h dns.c entropy.c hmac.h
     [hostfile.c key.c key.h krl.c monitor.c packet.c rsa.c rsa.h
     [ssh-add.c ssh-agent.c ssh-dss.c ssh-ecdsa.c ssh-ed25519.c
     [ssh-keygen.c ssh-pkcs11-client.c ssh-pkcs11-helper.c ssh-pkcs11.c
     [ssh-rsa.c sshbuf-misc.c sshbuf.h sshconnect.c sshconnect1.c
     [sshconnect2.c sshd.c sshkey.c sshkey.h
     [openbsd-compat/openssl-compat.c openbsd-compat/openssl-compat.h]
     New key API: refactor key-related functions to be more library-like,
     existing API is offered as a set of wrappers.

     with and ok markus@

     Thanks also to Ben Hawkes, David Tomaschik, Ivan Fratric, Matthew
     Dempsky and Ron Bowes for a detailed review a few months ago.

     NB. This commit also removes portable OpenSSH support for OpenSSL
     <0.9.8e.

1 files changed, 75 insertions, 38 deletions

diff --git a/rsa.c b/rsa.c
index d0b5bbf5..5ecacef9 100644
--- a/rsa.c
+++ b/rsa.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa.c,v 1.31 2014/02/02 03:44:31 djm Exp $ */
+/* $OpenBSD: rsa.c,v 1.32 2014/06/24 01:13:21 djm Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -67,85 +67,122 @@
 #include <stdarg.h>
 #include <string.h>
 
-#include "xmalloc.h"
 #include "rsa.h"
 #include "log.h"
+#include "ssherr.h"
 
-void
+int
 rsa_public_encrypt(BIGNUM *out, BIGNUM *in, RSA *key)
 {
-	u_char *inbuf, *outbuf;
-	int len, ilen, olen;
+	u_char *inbuf = NULL, *outbuf = NULL;
+	int len, ilen, olen, r = SSH_ERR_INTERNAL_ERROR;
 
 	if (BN_num_bits(key->e) < 2 || !BN_is_odd(key->e))
-		fatal("rsa_public_encrypt() exponent too small or not odd");
+		return SSH_ERR_INVALID_ARGUMENT;
 
 	olen = BN_num_bytes(key->n);
-	outbuf = xmalloc(olen);
+	if ((outbuf = malloc(olen)) == NULL) {
+		r = SSH_ERR_ALLOC_FAIL;
+		goto out;
+	}
 
 	ilen = BN_num_bytes(in);
-	inbuf = xmalloc(ilen);
+	if ((inbuf = malloc(ilen)) == NULL) {
+		r = SSH_ERR_ALLOC_FAIL;
+		goto out;
+	}
 	BN_bn2bin(in, inbuf);
 
 	if ((len = RSA_public_encrypt(ilen, inbuf, outbuf, key,
-	    RSA_PKCS1_PADDING)) <= 0)
-		fatal("rsa_public_encrypt() failed");
+	    RSA_PKCS1_PADDING)) <= 0) {
+		r = SSH_ERR_LIBCRYPTO_ERROR;
+		goto out;
+	}
 
-	if (BN_bin2bn(outbuf, len, out) == NULL)
-		fatal("rsa_public_encrypt: BN_bin2bn failed");
+	if (BN_bin2bn(outbuf, len, out) == NULL) {
+		r = SSH_ERR_LIBCRYPTO_ERROR;
+		goto out;
+	}
+	r = 0;
 
-	explicit_bzero(outbuf, olen);
-	explicit_bzero(inbuf, ilen);
-	free(outbuf);
-	free(inbuf);
+ out:
+	if (outbuf != NULL) {
+		explicit_bzero(outbuf, olen);
+		free(outbuf);
+	}
+	if (inbuf != NULL) {
+		explicit_bzero(inbuf, ilen);
+		free(inbuf);
+	}
+	return r;
 }
 
 int
 rsa_private_decrypt(BIGNUM *out, BIGNUM *in, RSA *key)
 {
-	u_char *inbuf, *outbuf;
-	int len, ilen, olen;
+	u_char *inbuf = NULL, *outbuf = NULL;
+	int len, ilen, olen, r = SSH_ERR_INTERNAL_ERROR;
 
 	olen = BN_num_bytes(key->n);
-	outbuf = xmalloc(olen);
+	if ((outbuf = malloc(olen)) == NULL) {
+		r = SSH_ERR_ALLOC_FAIL;
+		goto out;
+	}
 
 	ilen = BN_num_bytes(in);
-	inbuf = xmalloc(ilen);
+	if ((inbuf = malloc(ilen)) == NULL) {
+		r = SSH_ERR_ALLOC_FAIL;
+		goto out;
+	}
 	BN_bn2bin(in, inbuf);
 
 	if ((len = RSA_private_decrypt(ilen, inbuf, outbuf, key,
 	    RSA_PKCS1_PADDING)) <= 0) {
-		error("rsa_private_decrypt() failed");
-	} else {
-		if (BN_bin2bn(outbuf, len, out) == NULL)
-			fatal("rsa_private_decrypt: BN_bin2bn failed");
+		r = SSH_ERR_LIBCRYPTO_ERROR;
+		goto out;
+	} else if (BN_bin2bn(outbuf, len, out) == NULL) {
+		r = SSH_ERR_LIBCRYPTO_ERROR;
+		goto out;
+	}
+	r = 0;
+ out:
+	if (outbuf != NULL) {
+		explicit_bzero(outbuf, olen);
+		free(outbuf);
+	}
+	if (inbuf != NULL) {
+		explicit_bzero(inbuf, ilen);
+		free(inbuf);
 	}
-	explicit_bzero(outbuf, olen);
-	explicit_bzero(inbuf, ilen);
-	free(outbuf);
-	free(inbuf);
-	return len;
+	return r;
 }
 
 /* calculate p-1 and q-1 */
-void
+int
 rsa_generate_additional_parameters(RSA *rsa)
 {
-	BIGNUM *aux;
-	BN_CTX *ctx;
+	BIGNUM *aux = NULL;
+	BN_CTX *ctx = NULL;
+	int r;
 
-	if ((aux = BN_new()) == NULL)
-		fatal("rsa_generate_additional_parameters: BN_new failed");
 	if ((ctx = BN_CTX_new()) == NULL)
-		fatal("rsa_generate_additional_parameters: BN_CTX_new failed");
+		return SSH_ERR_ALLOC_FAIL;
+	if ((aux = BN_new()) == NULL) {
+		r = SSH_ERR_ALLOC_FAIL;
+		goto out;
+	}
 
 	if ((BN_sub(aux, rsa->q, BN_value_one()) == 0) ||
 	    (BN_mod(rsa->dmq1, rsa->d, aux, ctx) == 0) ||
 	    (BN_sub(aux, rsa->p, BN_value_one()) == 0) ||
-	    (BN_mod(rsa->dmp1, rsa->d, aux, ctx) == 0))
-		fatal("rsa_generate_additional_parameters: BN_sub/mod failed");
-
+	    (BN_mod(rsa->dmp1, rsa->d, aux, ctx) == 0)) {
+		r = SSH_ERR_LIBCRYPTO_ERROR;
+		goto out;
+	}
+	r = 0;
+ out:
 	BN_clear_free(aux);
 	BN_CTX_free(ctx);
+	return r;
 }