Add OPENSSL_hexstr2buf_ex() and OPENSSL_buf2hexstr_ex()

They do the same thing as OPENSSL_hexstr2buf() and OPENSSL_buf2hexstr(),
except they take a result buffer from the caller.

We take the opportunity to break out the documentation of the hex to /
from buffer conversion routines from the OPENSSL_malloc() file to its
own file.  These routines aren't memory allocation routines per se.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9303)

8 files changed, 175 insertions, 58 deletions

diff --git a/crypto/cpt_err.c b/crypto/cpt_err.c
index fdf0e6ebce..012f181d2a 100644
--- a/crypto/cpt_err.c
+++ b/crypto/cpt_err.c
@@ -40,6 +40,8 @@ static const ERR_STRING_DATA CRYPTO_str_reasons[] = {
     {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_TOO_MANY_BYTES), "too many bytes"},
     {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_TOO_MANY_RECORDS),
     "too many records"},
+    {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_TOO_SMALL_BUFFER),
+    "too small buffer"},
     {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_ZERO_LENGTH_NUMBER),
     "zero length number"},
     {0, NULL}
diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index f6e5a7593e..61ad994a8d 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -391,8 +391,10 @@ CRYPTO_F_GET_AND_LOCK:113:get_and_lock
 CRYPTO_F_GET_PROVIDER_STORE:133:get_provider_store
 CRYPTO_F_OPENSSL_ATEXIT:114:OPENSSL_atexit
 CRYPTO_F_OPENSSL_BUF2HEXSTR:117:OPENSSL_buf2hexstr
+CRYPTO_F_OPENSSL_BUF2HEXSTR_EX:153:
 CRYPTO_F_OPENSSL_FOPEN:119:openssl_fopen
 CRYPTO_F_OPENSSL_HEXSTR2BUF:118:OPENSSL_hexstr2buf
+CRYPTO_F_OPENSSL_HEXSTR2BUF_EX:154:
 CRYPTO_F_OPENSSL_INIT_CRYPTO:116:OPENSSL_init_crypto
 CRYPTO_F_OPENSSL_LH_NEW:126:OPENSSL_LH_new
 CRYPTO_F_OPENSSL_SK_DEEP_COPY:127:OPENSSL_sk_deep_copy
@@ -2243,6 +2245,7 @@ CRYPTO_R_SECURE_MALLOC_FAILURE:111:secure malloc failure
 CRYPTO_R_STRING_TOO_LONG:112:string too long
 CRYPTO_R_TOO_MANY_BYTES:113:too many bytes
 CRYPTO_R_TOO_MANY_RECORDS:114:too many records
+CRYPTO_R_TOO_SMALL_BUFFER:116:too small buffer
 CRYPTO_R_ZERO_LENGTH_NUMBER:115:zero length number
 CT_R_BASE64_DECODE_ERROR:108:base64 decode error
 CT_R_INVALID_LOG_ID_LENGTH:100:invalid log id length
diff --git a/crypto/o_str.c b/crypto/o_str.c
index c24524f892..6780188cda 100644
--- a/crypto/o_str.c
+++ b/crypto/o_str.c
@@ -132,76 +132,125 @@ int OPENSSL_hexchar2int(unsigned char c)
 /*
  * Give a string of hex digits convert to a buffer
  */
-unsigned char *OPENSSL_hexstr2buf(const char *str, long *len)
+int OPENSSL_hexstr2buf_ex(unsigned char *buf, size_t buf_n, size_t *buflen,
+                          const char *str)
 {
-    unsigned char *hexbuf, *q;
+    unsigned char *q;
     unsigned char ch, cl;
     int chi, cli;
     const unsigned char *p;
-    size_t s;
+    size_t cnt;
 
-    s = strlen(str);
-    if ((hexbuf = OPENSSL_malloc(s >> 1)) == NULL) {
-        CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF, ERR_R_MALLOC_FAILURE);
-        return NULL;
-    }
-    for (p = (const unsigned char *)str, q = hexbuf; *p; ) {
+    for (p = (const unsigned char *)str, q = buf, cnt = 0; *p; ) {
         ch = *p++;
         if (ch == ':')
             continue;
         cl = *p++;
         if (!cl) {
-            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF,
+            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF_EX,
                       CRYPTO_R_ODD_NUMBER_OF_DIGITS);
-            OPENSSL_free(hexbuf);
-            return NULL;
+            return 0;
         }
         cli = OPENSSL_hexchar2int(cl);
         chi = OPENSSL_hexchar2int(ch);
         if (cli < 0 || chi < 0) {
-            OPENSSL_free(hexbuf);
-            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF, CRYPTO_R_ILLEGAL_HEX_DIGIT);
-            return NULL;
+            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF_EX,
+                      CRYPTO_R_ILLEGAL_HEX_DIGIT);
+            return 0;
+        }
+        cnt++;
+        if (q != NULL) {
+            if (cnt > buf_n) {
+                CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF_EX,
+                          CRYPTO_R_TOO_SMALL_BUFFER);
+                return 0;
+            }
+            *q++ = (unsigned char)((chi << 4) | cli);
         }
-        *q++ = (unsigned char)((chi << 4) | cli);
     }
 
-    if (len)
-        *len = q - hexbuf;
-    return hexbuf;
+    if (buflen != NULL)
+        *buflen = cnt;
+    return 1;
 }
 
-/*
- * Given a buffer of length 'len' return a OPENSSL_malloc'ed string with its
- * hex representation @@@ (Contents of buffer are always kept in ASCII, also
- * on EBCDIC machines)
- */
-char *OPENSSL_buf2hexstr(const unsigned char *buffer, long len)
+unsigned char *OPENSSL_hexstr2buf(const char *str, long *buflen)
+{
+    unsigned char *buf;
+    size_t buf_n, tmp_buflen;
+
+    buf_n = strlen(str) >> 1;
+    if ((buf = OPENSSL_malloc(buf_n)) == NULL) {
+        CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF, ERR_R_MALLOC_FAILURE);
+        return NULL;
+    }
+
+    if (buflen != NULL)
+        *buflen = 0;
+    tmp_buflen = 0;
+    if (OPENSSL_hexstr2buf_ex(buf, buf_n, &tmp_buflen, str)) {
+        if (buflen != NULL)
+            *buflen = (long)tmp_buflen;
+        return buf;
+    }
+    OPENSSL_free(buf);
+    return NULL;
+}
+
+int OPENSSL_buf2hexstr_ex(char *str, size_t str_n, size_t *strlen,
+                          const unsigned char *buf, size_t buflen)
 {
     static const char hexdig[] = "0123456789ABCDEF";
-    char *tmp, *q;
     const unsigned char *p;
-    int i;
+    char *q;
+    size_t i;
 
-    if (len == 0)
-        return OPENSSL_zalloc(1);
+    if (strlen != NULL)
+        *strlen = buflen * 3;
+    if (str == NULL)
+        return 1;
 
-    if ((tmp = OPENSSL_malloc(len * 3)) == NULL) {
-        CRYPTOerr(CRYPTO_F_OPENSSL_BUF2HEXSTR, ERR_R_MALLOC_FAILURE);
-        return NULL;
+    if (str_n < (unsigned long)buflen * 3) {
+        CRYPTOerr(CRYPTO_F_OPENSSL_BUF2HEXSTR_EX, CRYPTO_R_TOO_SMALL_BUFFER);
+        return 0;
     }
-    q = tmp;
-    for (i = 0, p = buffer; i < len; i++, p++) {
+
+    q = str;
+    for (i = 0, p = buf; i < buflen; i++, p++) {
         *q++ = hexdig[(*p >> 4) & 0xf];
         *q++ = hexdig[*p & 0xf];
         *q++ = ':';
     }
     q[-1] = 0;
 #ifdef CHARSET_EBCDIC
-    ebcdic2ascii(tmp, tmp, q - tmp - 1);
+    ebcdic2ascii(str, str, q - str - 1);
 #endif
+    return 1;
+}
+
+/*
+ * Given a buffer of length 'len' return a OPENSSL_malloc'ed string with its
+ * hex representation @@@ (Contents of buffer are always kept in ASCII, also
+ * on EBCDIC machines)
+ */
+char *OPENSSL_buf2hexstr(const unsigned char *buf, long buflen)
+{
+    char *tmp;
+    size_t tmp_n;
+
+    if (buflen == 0)
+        return OPENSSL_zalloc(1);
+
+    tmp_n = buflen * 3;
+    if ((tmp = OPENSSL_malloc(tmp_n)) == NULL) {
+        CRYPTOerr(CRYPTO_F_OPENSSL_BUF2HEXSTR, ERR_R_MALLOC_FAILURE);
+        return NULL;
+    }
 
-    return tmp;
+    if (OPENSSL_buf2hexstr_ex(tmp, tmp_n, NULL, buf, buflen))
+        return tmp;
+    OPENSSL_free(tmp);
+    return NULL;
 }
 
 int openssl_strerror_r(int errnum, char *buf, size_t buflen)
diff --git a/doc/man3/OPENSSL_hexchar2int.pod b/doc/man3/OPENSSL_hexchar2int.pod
new file mode 100644
index 0000000000..930b32b61f
--- /dev/null
+++ b/doc/man3/OPENSSL_hexchar2int.pod
@@ -0,0 +1,74 @@
+=pod
+
+=head1 NAME
+
+OPENSSL_hexchar2int,
+OPENSSL_hexstr2buf_ex, OPENSSL_hexstr2buf,
+OPENSSL_buf2hexstr_ex, OPENSSL_buf2hexstr
+- Hex encoding and decoding functions
+
+=head1 SYNOPSIS
+
+ #include <openssl/crypto.h>
+
+ int OPENSSL_hexchar2int(unsigned char c);
+ int OPENSSL_hexstr2buf_ex(unsigned char *buf, size_t buf_n, long *buflen,
+                           const char *str);
+ unsigned char *OPENSSL_hexstr2buf(const char *str, long *len);
+ int OPENSSL_buf2hexstr_ex(char *str, size_t str_n, size_t *strlen,
+                           const unsigned char *buf, long buflen);
+ char *OPENSSL_buf2hexstr(const unsigned char *buf, long buflen);
+
+=head1 DESCRIPTION
+
+OPENSSL_hexchar2int() converts a hexadecimal character to its numeric
+equivalent.
+
+OPENSSL_hexstr2buf_ex() decodes the hex string B<str> and places the
+resulting string of bytes in the given I<buf>.
+I<buf_n> gives the size of the buffer.
+If I<buflen> is not NULL, it is filled in with the result length.
+To find out how large the result will be, call this function with NULL
+for I<buf>.
+Colons between two-character hex "bytes" are accepted and ignored.
+An odd number of hex digits is an error.
+
+OPENSSL_hexstr2buf() does the same thing as OPENSSL_hexstr2buf_ex(),
+but allocates the space for the result, and returns the result.
+The memory is allocated by calling OPENSSL_malloc() and should be
+released by calling OPENSSL_free().
+
+OPENSSL_buf2hexstr_ex() encodes the contents of the given I<buf> with
+length I<buflen> and places the resulting hexadecimal character string
+in the given I<str>.
+I<str_n> gives the size of the of the string buffer.
+If I<strlen> is not NULL, it is filled in with the result length.
+To find out how large the result will be, call this function with NULL
+for I<str>.
+
+OPENSSL_buf2hexstr() does the same thing as OPENSSL_buf2hexstr_ex(),
+but allocates the space for the result, and returns the result.
+The memory is allocated by calling OPENSSL_malloc() and should be
+released by calling OPENSSL_free().
+
+=head1 RETURN VALUES
+
+OPENSSL_hexchar2int returns the value of a decoded hex character,
+or -1 on error.
+
+OPENSSL_buf2hexstr() and OPENSSL_hexstr2buf()
+return a pointer to allocated memory, or NULL on error.
+
+OPENSSL_buf2hexstr_ex() and OPENSSL_hexstr2buf_ex() return 1 on
+success, or 0 on error.
+
+=head1 COPYRIGHT
+
+Copyright 2016-2019 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut
diff --git a/doc/man3/OPENSSL_malloc.pod b/doc/man3/OPENSSL_malloc.pod
index 38edf49d4d..198251fcd7 100644
--- a/doc/man3/OPENSSL_malloc.pod
+++ b/doc/man3/OPENSSL_malloc.pod
@@ -8,7 +8,6 @@ OPENSSL_clear_realloc, OPENSSL_clear_free, OPENSSL_cleanse,
 CRYPTO_malloc, CRYPTO_zalloc, CRYPTO_realloc, CRYPTO_free,
 OPENSSL_strdup, OPENSSL_strndup,
 OPENSSL_memdup, OPENSSL_strlcpy, OPENSSL_strlcat,
-OPENSSL_hexstr2buf, OPENSSL_buf2hexstr, OPENSSL_hexchar2int,
 CRYPTO_strdup, CRYPTO_strndup,
 OPENSSL_mem_debug_push, OPENSSL_mem_debug_pop,
 CRYPTO_mem_debug_push, CRYPTO_mem_debug_pop,
@@ -40,10 +39,6 @@ OPENSSL_MALLOC_FD
  void OPENSSL_clear_free(void *str, size_t num)
  void OPENSSL_cleanse(void *ptr, size_t len);
 
- unsigned char *OPENSSL_hexstr2buf(const char *str, long *len);
- char *OPENSSL_buf2hexstr(const unsigned char *buffer, long len);
- int OPENSSL_hexchar2int(unsigned char c);
-
  void *CRYPTO_malloc(size_t num, const char *file, int line)
  void *CRYPTO_zalloc(size_t num, const char *file, int line)
  void *CRYPTO_realloc(void *p, size_t num, const char *file, int line)
@@ -119,20 +114,6 @@ OPENSSL_strlcpy(),
 OPENSSL_strlcat() and OPENSSL_strnlen() are equivalents of the common C
 library functions and are provided for portability.
 
-OPENSSL_hexstr2buf() parses B<str> as a hex string and returns a
-pointer to the parsed value. The memory is allocated by calling
-OPENSSL_malloc() and should be released by calling OPENSSL_free().
-If B<len> is not NULL, it is filled in with the output length.
-Colons between two-character hex "bytes" are ignored.
-An odd number of hex digits is an error.
-
-OPENSSL_buf2hexstr() takes the specified buffer and length, and returns
-a hex string for value, or NULL on error.
-B<Buffer> cannot be NULL; if B<len> is 0 an empty string is returned.
-
-OPENSSL_hexchar2int() converts a character to the hexadecimal equivalent,
-or returns -1 on error.
-
 If no allocations have been done, it is possible to "swap out" the default
 implementations for OPENSSL_malloc(), OPENSSL_realloc and OPENSSL_free()
 and replace them with alternate versions (hooks).
@@ -216,7 +197,6 @@ OPENSSL_malloc(), OPENSSL_zalloc(), OPENSSL_realloc(),
 OPENSSL_clear_realloc(),
 CRYPTO_malloc(), CRYPTO_zalloc(), CRYPTO_realloc(),
 CRYPTO_clear_realloc(),
-OPENSSL_buf2hexstr(), OPENSSL_hexstr2buf(),
 OPENSSL_strdup(), and OPENSSL_strndup()
 return a pointer to allocated memory or NULL on error.
 
diff --git a/include/openssl/crypto.h b/include/openssl/crypto.h
index 875ee556c1..accb22f3b1 100644
--- a/include/openssl/crypto.h
+++ b/include/openssl/crypto.h
@@ -148,8 +148,12 @@ int CRYPTO_mem_ctrl(int mode);
 size_t OPENSSL_strlcpy(char *dst, const char *src, size_t siz);
 size_t OPENSSL_strlcat(char *dst, const char *src, size_t siz);
 size_t OPENSSL_strnlen(const char *str, size_t maxlen);
-char *OPENSSL_buf2hexstr(const unsigned char *buffer, long len);
-unsigned char *OPENSSL_hexstr2buf(const char *str, long *len);
+int OPENSSL_buf2hexstr_ex(char *str, size_t str_n, size_t *strlen,
+                          const unsigned char *buf, size_t buflen);
+char *OPENSSL_buf2hexstr(const unsigned char *buf, long buflen);
+int OPENSSL_hexstr2buf_ex(unsigned char *buf, size_t buf_n, size_t *buflen,
+                          const char *str);
+unsigned char *OPENSSL_hexstr2buf(const char *str, long *buflen);
 int OPENSSL_hexchar2int(unsigned char c);
 
 # define OPENSSL_MALLOC_MAX_NELEMS(type)  (((1U<<(sizeof(int)*8-1))-1)/sizeof(type))
diff --git a/include/openssl/cryptoerr.h b/include/openssl/cryptoerr.h
index 9fdf52c0c1..15071e2741 100644
--- a/include/openssl/cryptoerr.h
+++ b/include/openssl/cryptoerr.h
@@ -40,8 +40,10 @@ int ERR_load_CRYPTO_strings(void);
 #  define CRYPTO_F_GET_PROVIDER_STORE                      0
 #  define CRYPTO_F_OPENSSL_ATEXIT                          0
 #  define CRYPTO_F_OPENSSL_BUF2HEXSTR                      0
+#  define CRYPTO_F_OPENSSL_BUF2HEXSTR_EX                   0
 #  define CRYPTO_F_OPENSSL_FOPEN                           0
 #  define CRYPTO_F_OPENSSL_HEXSTR2BUF                      0
+#  define CRYPTO_F_OPENSSL_HEXSTR2BUF_EX                   0
 #  define CRYPTO_F_OPENSSL_INIT_CRYPTO                     0
 #  define CRYPTO_F_OPENSSL_LH_NEW                          0
 #  define CRYPTO_F_OPENSSL_SK_DEEP_COPY                    0
@@ -89,6 +91,7 @@ int ERR_load_CRYPTO_strings(void);
 # define CRYPTO_R_STRING_TOO_LONG                         112
 # define CRYPTO_R_TOO_MANY_BYTES                          113
 # define CRYPTO_R_TOO_MANY_RECORDS                        114
+# define CRYPTO_R_TOO_SMALL_BUFFER                        116
 # define CRYPTO_R_ZERO_LENGTH_NUMBER                      115
 
 #endif
diff --git a/util/libcrypto.num b/util/libcrypto.num
index c99a69ab22..c6f1f2d7ad 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -4706,3 +4706,5 @@ EC_GROUP_new_ex                         4815	3_0_0	EXIST::FUNCTION:EC
 EC_GROUP_new_by_curve_name_ex           4816	3_0_0	EXIST::FUNCTION:EC
 EC_KEY_new_ex                           4817	3_0_0	EXIST::FUNCTION:EC
 EC_KEY_new_by_curve_name_ex             4818	3_0_0	EXIST::FUNCTION:EC
+OPENSSL_hexstr2buf_ex                   4819	3_0_0	EXIST::FUNCTION:
+OPENSSL_buf2hexstr_ex                   4820	3_0_0	EXIST::FUNCTION: