Fix range_should_be_prefix() to actually return the correct result

range_should_be_prefix() was misidentifying whether an IP address range
should in fact be represented as a prefix. This was due to a bug introduced
in commit 42d7d7dd which made this incorrect change:

-    OPENSSL_assert(memcmp(min, max, length) <= 0);
+    if (memcmp(min, max, length) <= 0)
+        return -1;

This error leads to incorrect DER being encoded/accepted.

Reported by Theo Buehler (@botovq)

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18524)

(cherry picked from commit 30532e59f475e0066c030693e4d614311a9e0cae)

1 files changed, 12 insertions, 2 deletions

diff --git a/crypto/x509/v3_addr.c b/crypto/x509/v3_addr.c
index 83752cd4b9..4205e7d7af 100644
--- a/crypto/x509/v3_addr.c
+++ b/crypto/x509/v3_addr.c
@@ -13,6 +13,8 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <assert.h>
+#include <string.h>
 
 #include "internal/cryptlib.h"
 #include <openssl/conf.h>
@@ -343,8 +345,13 @@ static int range_should_be_prefix(const unsigned char *min,
     unsigned char mask;
     int i, j;
 
-    if (memcmp(min, max, length) <= 0)
-        return -1;
+    /*
+     * It is the responsibility of the caller to confirm min <= max. We don't
+     * use ossl_assert() here since we have no way of signalling an error from
+     * this function - so we just use a plain assert instead.
+     */
+    assert(memcmp(min, max, length) <= 0);
+
     for (i = 0; i < length && min[i] == max[i]; i++) ;
     for (j = length - 1; j >= 0 && min[j] == 0x00 && max[j] == 0xFF; j--) ;
     if (i < j)
@@ -427,6 +434,9 @@ static int make_addressRange(IPAddressOrRange **result,
     IPAddressOrRange *aor;
     int i, prefixlen;
 
+    if (memcmp(min, max, length) > 0)
+        return 0;
+
     if ((prefixlen = range_should_be_prefix(min, max, length)) >= 0)
         return make_addressPrefix(result, min, prefixlen);