Add support for signed BIGNUMs in the OSSL_PARAM API

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17162)

1 files changed, 38 insertions, 19 deletions

diff --git a/crypto/params.c b/crypto/params.c
index cf86eea3c7..a1a04a6b4d 100644
--- a/crypto/params.c
+++ b/crypto/params.c
@@ -1018,24 +1018,32 @@ OSSL_PARAM OSSL_PARAM_construct_time_t(const char *key, time_t *buf)
 
 int OSSL_PARAM_get_BN(const OSSL_PARAM *p, BIGNUM **val)
 {
-    BIGNUM *b;
+    BIGNUM *b = NULL;
 
     if (val == NULL || p == NULL) {
         err_null_argument;
         return 0;
     }
-    if (p->data_type != OSSL_PARAM_UNSIGNED_INTEGER) {
+
+    switch (p->data_type) {
+    case OSSL_PARAM_UNSIGNED_INTEGER:
+        b = BN_native2bn(p->data, (int)p->data_size, *val);
+        break;
+    case OSSL_PARAM_INTEGER:
+        b = BN_signed_native2bn(p->data, (int)p->data_size, *val);
+        break;
+    default:
         err_bad_type;
-        return 0;
+        break;
     }
 
-    b = BN_native2bn(p->data, (int)p->data_size, *val);
-    if (b != NULL) {
-        *val = b;
-        return 1;
+    if (b == NULL) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+        return 0;
     }
-    ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
-    return 0;
+
+    *val = b;
+    return 1;
 }
 
 int OSSL_PARAM_set_BN(OSSL_PARAM *p, const BIGNUM *val)
@@ -1051,26 +1059,37 @@ int OSSL_PARAM_set_BN(OSSL_PARAM *p, const BIGNUM *val)
         err_null_argument;
         return 0;
     }
-    if (p->data_type != OSSL_PARAM_UNSIGNED_INTEGER) {
+    if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER && BN_is_negative(val)) {
         err_bad_type;
         return 0;
     }
 
-    /* For the moment, only positive values are permitted */
-    if (BN_is_negative(val)) {
-        err_unsigned_negative;
-        return 0;
-    }
-
     bytes = (size_t)BN_num_bytes(val);
+    /* We add 1 byte for signed numbers, to make space for a sign extension */
+    if (p->data_type == OSSL_PARAM_INTEGER)
+        bytes++;
+
     p->return_size = bytes;
     if (p->data == NULL)
         return 1;
     if (p->data_size >= bytes) {
         p->return_size = p->data_size;
-        if (BN_bn2nativepad(val, p->data, p->data_size) >= 0)
-            return 1;
-        ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_INTEGER_OVERFLOW);
+
+        switch (p->data_type) {
+        case OSSL_PARAM_UNSIGNED_INTEGER:
+            if (BN_bn2nativepad(val, p->data, p->data_size) >= 0)
+                return 1;
+            ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_INTEGER_OVERFLOW);
+            break;
+        case OSSL_PARAM_INTEGER:
+            if (BN_signed_bn2native(val, p->data, p->data_size) >= 0)
+                return 1;
+            ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_INTEGER_OVERFLOW);
+            break;
+        default:
+            err_bad_type;
+            break;
+        }
         return 0;
     }
     err_too_small;