Add OPENSSL_hexstr2buf_ex() and OPENSSL_buf2hexstr_ex()

They do the same thing as OPENSSL_hexstr2buf() and OPENSSL_buf2hexstr(),
except they take a result buffer from the caller.

We take the opportunity to break out the documentation of the hex to /
from buffer conversion routines from the OPENSSL_malloc() file to its
own file.  These routines aren't memory allocation routines per se.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9303)

1 files changed, 85 insertions, 36 deletions

diff --git a/crypto/o_str.c b/crypto/o_str.c
index c24524f892..6780188cda 100644
--- a/crypto/o_str.c
+++ b/crypto/o_str.c
@@ -132,76 +132,125 @@ int OPENSSL_hexchar2int(unsigned char c)
 /*
  * Give a string of hex digits convert to a buffer
  */
-unsigned char *OPENSSL_hexstr2buf(const char *str, long *len)
+int OPENSSL_hexstr2buf_ex(unsigned char *buf, size_t buf_n, size_t *buflen,
+                          const char *str)
 {
-    unsigned char *hexbuf, *q;
+    unsigned char *q;
     unsigned char ch, cl;
     int chi, cli;
     const unsigned char *p;
-    size_t s;
+    size_t cnt;
 
-    s = strlen(str);
-    if ((hexbuf = OPENSSL_malloc(s >> 1)) == NULL) {
-        CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF, ERR_R_MALLOC_FAILURE);
-        return NULL;
-    }
-    for (p = (const unsigned char *)str, q = hexbuf; *p; ) {
+    for (p = (const unsigned char *)str, q = buf, cnt = 0; *p; ) {
         ch = *p++;
         if (ch == ':')
             continue;
         cl = *p++;
         if (!cl) {
-            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF,
+            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF_EX,
                       CRYPTO_R_ODD_NUMBER_OF_DIGITS);
-            OPENSSL_free(hexbuf);
-            return NULL;
+            return 0;
         }
         cli = OPENSSL_hexchar2int(cl);
         chi = OPENSSL_hexchar2int(ch);
         if (cli < 0 || chi < 0) {
-            OPENSSL_free(hexbuf);
-            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF, CRYPTO_R_ILLEGAL_HEX_DIGIT);
-            return NULL;
+            CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF_EX,
+                      CRYPTO_R_ILLEGAL_HEX_DIGIT);
+            return 0;
+        }
+        cnt++;
+        if (q != NULL) {
+            if (cnt > buf_n) {
+                CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF_EX,
+                          CRYPTO_R_TOO_SMALL_BUFFER);
+                return 0;
+            }
+            *q++ = (unsigned char)((chi << 4) | cli);
         }
-        *q++ = (unsigned char)((chi << 4) | cli);
     }
 
-    if (len)
-        *len = q - hexbuf;
-    return hexbuf;
+    if (buflen != NULL)
+        *buflen = cnt;
+    return 1;
 }
 
-/*
- * Given a buffer of length 'len' return a OPENSSL_malloc'ed string with its
- * hex representation @@@ (Contents of buffer are always kept in ASCII, also
- * on EBCDIC machines)
- */
-char *OPENSSL_buf2hexstr(const unsigned char *buffer, long len)
+unsigned char *OPENSSL_hexstr2buf(const char *str, long *buflen)
+{
+    unsigned char *buf;
+    size_t buf_n, tmp_buflen;
+
+    buf_n = strlen(str) >> 1;
+    if ((buf = OPENSSL_malloc(buf_n)) == NULL) {
+        CRYPTOerr(CRYPTO_F_OPENSSL_HEXSTR2BUF, ERR_R_MALLOC_FAILURE);
+        return NULL;
+    }
+
+    if (buflen != NULL)
+        *buflen = 0;
+    tmp_buflen = 0;
+    if (OPENSSL_hexstr2buf_ex(buf, buf_n, &tmp_buflen, str)) {
+        if (buflen != NULL)
+            *buflen = (long)tmp_buflen;
+        return buf;
+    }
+    OPENSSL_free(buf);
+    return NULL;
+}
+
+int OPENSSL_buf2hexstr_ex(char *str, size_t str_n, size_t *strlen,
+                          const unsigned char *buf, size_t buflen)
 {
     static const char hexdig[] = "0123456789ABCDEF";
-    char *tmp, *q;
     const unsigned char *p;
-    int i;
+    char *q;
+    size_t i;
 
-    if (len == 0)
-        return OPENSSL_zalloc(1);
+    if (strlen != NULL)
+        *strlen = buflen * 3;
+    if (str == NULL)
+        return 1;
 
-    if ((tmp = OPENSSL_malloc(len * 3)) == NULL) {
-        CRYPTOerr(CRYPTO_F_OPENSSL_BUF2HEXSTR, ERR_R_MALLOC_FAILURE);
-        return NULL;
+    if (str_n < (unsigned long)buflen * 3) {
+        CRYPTOerr(CRYPTO_F_OPENSSL_BUF2HEXSTR_EX, CRYPTO_R_TOO_SMALL_BUFFER);
+        return 0;
     }
-    q = tmp;
-    for (i = 0, p = buffer; i < len; i++, p++) {
+
+    q = str;
+    for (i = 0, p = buf; i < buflen; i++, p++) {
         *q++ = hexdig[(*p >> 4) & 0xf];
         *q++ = hexdig[*p & 0xf];
         *q++ = ':';
     }
     q[-1] = 0;
 #ifdef CHARSET_EBCDIC
-    ebcdic2ascii(tmp, tmp, q - tmp - 1);
+    ebcdic2ascii(str, str, q - str - 1);
 #endif
+    return 1;
+}
+
+/*
+ * Given a buffer of length 'len' return a OPENSSL_malloc'ed string with its
+ * hex representation @@@ (Contents of buffer are always kept in ASCII, also
+ * on EBCDIC machines)
+ */
+char *OPENSSL_buf2hexstr(const unsigned char *buf, long buflen)
+{
+    char *tmp;
+    size_t tmp_n;
+
+    if (buflen == 0)
+        return OPENSSL_zalloc(1);
+
+    tmp_n = buflen * 3;
+    if ((tmp = OPENSSL_malloc(tmp_n)) == NULL) {
+        CRYPTOerr(CRYPTO_F_OPENSSL_BUF2HEXSTR, ERR_R_MALLOC_FAILURE);
+        return NULL;
+    }
 
-    return tmp;
+    if (OPENSSL_buf2hexstr_ex(tmp, tmp_n, NULL, buf, buflen))
+        return tmp;
+    OPENSSL_free(tmp);
+    return NULL;
 }
 
 int openssl_strerror_r(int errnum, char *buf, size_t buflen)