punycode: update to use WPACKET instead of using custom range checking

Add test for `.' overflows, remove the output size argument from
ossl_a2ulabel() since it was never used and greatly complicated the code.
Convert ossl_a2ulabel() to use WPACKET for building the output string.
Update the documentation to match the new definition of ossl_a2ulabel().

x509: let punycode handle the '\0' string termination.  Saves a memset(3)
and some size fiddling.  Also update to deal with the modified parameters.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/19591)

(cherry picked from commit 905ba924398f474e647de70345b4ae4089fedba7)

1 files changed, 53 insertions, 16 deletions

diff --git a/test/punycode_test.c b/test/punycode_test.c
index 9d8171346c..5b8b0bd272 100644
--- a/test/punycode_test.c
+++ b/test/punycode_test.c
@@ -12,6 +12,7 @@
 
 #include "crypto/punycode.h"
 #include "internal/nelem.h"
+#include "internal/packet.h"
 #include "testutil.h"
 
 
@@ -166,29 +167,17 @@ static int test_punycode(int n)
 static int test_a2ulabel(void)
 {
     char out[50];
-    size_t outlen;
 
     /*
-     * Test that no buffer correctly returns the true length.
      * The punycode being passed in and parsed is malformed but we're not
      * verifying that behaviour here.
      */
-    if (!TEST_int_eq(ossl_a2ulabel("xn--a.b.c", NULL, &outlen), 0)
-            || !TEST_size_t_eq(outlen, 7)
-            || !TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, &outlen), 1))
-        return 0;
-    /* Test that a short input length returns the true length */
-    outlen = 1;
-    if (!TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, &outlen), 0)
-            || !TEST_size_t_eq(outlen, 7)
-            || !TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, &outlen), 1)
-            || !TEST_str_eq(out,"\xc2\x80.b.c"))
+    if (!TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, 1), 0)
+            || !TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, 7), 1))
         return 0;
     /* Test for an off by one on the buffer size works */
-    outlen = 6;
-    if (!TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, &outlen), 0)
-            || !TEST_size_t_eq(outlen, 7)
-            || !TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, &outlen), 1)
+    if (!TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, 6), 0)
+            || !TEST_int_eq(ossl_a2ulabel("xn--a.b.c", out, 7), 1)
             || !TEST_str_eq(out,"\xc2\x80.b.c"))
         return 0;
     return 1;
@@ -211,9 +200,57 @@ static int test_puny_overrun(void)
     return 1;
 }
 
+static int test_dotted_overflow(void)
+{
+    static const char string[] = "a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a";
+    const size_t num_reps = OSSL_NELEM(string) / 2;
+    WPACKET p;
+    BUF_MEM *in;
+    char *out = NULL;
+    size_t i;
+    int res = 0;
+
+    /* Create out input punycode string */
+    if (!TEST_ptr(in = BUF_MEM_new()))
+        return 0;
+    if (!TEST_true(WPACKET_init_len(&p, in, 0))) {
+        BUF_MEM_free(in);
+        return 0;
+    }
+    for (i = 0; i < num_reps; i++) {
+        if (i > 1 && !TEST_true(WPACKET_put_bytes_u8(&p, '.')))
+            goto err;
+        if (!TEST_true(WPACKET_memcpy(&p, "xn--a", sizeof("xn--a") - 1)))
+            goto err;
+    }
+    if (!TEST_true(WPACKET_put_bytes_u8(&p, '\0')))
+            goto err;
+    if (!TEST_ptr(out = OPENSSL_malloc(in->length)))
+        goto err;
+
+    /* Test the decode into an undersized buffer */
+    memset(out, 0x7f, in->length - 1);
+    if (!TEST_int_le(ossl_a2ulabel(in->data, out, num_reps), 0)
+            || !TEST_int_eq(out[num_reps], 0x7f))
+        goto err;
+
+    /* Test the decode works into a full size buffer */
+    if (!TEST_int_gt(ossl_a2ulabel(in->data, out, in->length), 0)
+            || !TEST_size_t_eq(strlen(out), num_reps * 3))
+        goto err;
+
+    res = 1;
+ err:
+    WPACKET_cleanup(&p);
+    BUF_MEM_free(in);
+    OPENSSL_free(out);
+    return res;
+}
+
 int setup_tests(void)
 {
     ADD_ALL_TESTS(test_punycode, OSSL_NELEM(puny_cases));
+    ADD_TEST(test_dotted_overflow);
     ADD_TEST(test_a2ulabel);
     ADD_TEST(test_puny_overrun);
     return 1;