Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

5 files changed, 45 insertions, 64 deletions

diff --git a/crypto/dso/dso_dl.c b/crypto/dso/dso_dl.c
index f4e6e5f457..ac94254807 100644
--- a/crypto/dso/dso_dl.c
+++ b/crypto/dso/dso_dl.c
@@ -165,20 +165,16 @@ static char *dl_merger(DSO *dso, const char *filespec1, const char *filespec2)
      */
     if (!filespec2 || filespec1[0] == '/') {
         merged = OPENSSL_strdup(filespec1);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
     }
     /*
      * If the first file specification is missing, the second one rules.
      */
     else if (!filespec1) {
         merged = OPENSSL_strdup(filespec2);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
     } else
         /*
          * This part isn't as trivial as it looks.  It assumes that the
@@ -198,10 +194,8 @@ static char *dl_merger(DSO *dso, const char *filespec1, const char *filespec2)
             len--;
         }
         merged = OPENSSL_malloc(len + 2);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
         strcpy(merged, filespec2);
         merged[spec2len] = '/';
         strcpy(&merged[spec2len + 1], filespec1);
diff --git a/crypto/dso/dso_dlfcn.c b/crypto/dso/dso_dlfcn.c
index c292b41c43..2befd67248 100644
--- a/crypto/dso/dso_dlfcn.c
+++ b/crypto/dso/dso_dlfcn.c
@@ -207,20 +207,16 @@ static char *dlfcn_merger(DSO *dso, const char *filespec1,
      */
     if (!filespec2 || (filespec1 != NULL && filespec1[0] == '/')) {
         merged = OPENSSL_strdup(filespec1);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
     }
     /*
      * If the first file specification is missing, the second one rules.
      */
     else if (!filespec1) {
         merged = OPENSSL_strdup(filespec2);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
     } else {
         /*
          * This part isn't as trivial as it looks.  It assumes that the
@@ -239,10 +235,8 @@ static char *dlfcn_merger(DSO *dso, const char *filespec1,
             len--;
         }
         merged = OPENSSL_malloc(len + 2);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
         strcpy(merged, filespec2);
         merged[spec2len] = '/';
         strcpy(&merged[spec2len + 1], filespec1);
diff --git a/crypto/dso/dso_lib.c b/crypto/dso/dso_lib.c
index e093b77a27..a73d91e839 100644
--- a/crypto/dso/dso_lib.c
+++ b/crypto/dso/dso_lib.c
@@ -15,14 +15,12 @@ static DSO *DSO_new_method(DSO_METHOD *meth)
     DSO *ret;
 
     ret = OPENSSL_zalloc(sizeof(*ret));
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
     ret->meth_data = sk_void_new_null();
     if (ret->meth_data == NULL) {
         /* sk_new doesn't generate any errors so we do */
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSO, ERR_R_CRYPTO_LIB);
         OPENSSL_free(ret);
         return NULL;
     }
@@ -30,7 +28,7 @@ static DSO *DSO_new_method(DSO_METHOD *meth)
     ret->references = 1;
     ret->lock = CRYPTO_THREAD_lock_new();
     if (ret->lock == NULL) {
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSO, ERR_R_CRYPTO_LIB);
         sk_void_free(ret->meth_data);
         OPENSSL_free(ret);
         return NULL;
@@ -114,7 +112,7 @@ DSO *DSO_load(DSO *dso, const char *filename, DSO_METHOD *meth, int flags)
     if (dso == NULL) {
         ret = DSO_new_method(meth);
         if (ret == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_DSO, ERR_R_DSO_LIB);
             goto err;
         }
         allocated = 1;
@@ -241,10 +239,8 @@ int DSO_set_filename(DSO *dso, const char *filename)
     }
     /* We'll duplicate filename */
     copied = OPENSSL_strdup(filename);
-    if (copied == NULL) {
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+    if (copied == NULL)
         return 0;
-    }
     OPENSSL_free(dso->filename);
     dso->filename = copied;
     return 1;
@@ -289,10 +285,8 @@ char *DSO_convert_filename(DSO *dso, const char *filename)
     }
     if (result == NULL) {
         result = OPENSSL_strdup(filename);
-        if (result == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (result == NULL)
             return NULL;
-        }
     }
     return result;
 }
diff --git a/crypto/dso/dso_vms.c b/crypto/dso/dso_vms.c
index aa2dfaa4d1..1afa222dbc 100644
--- a/crypto/dso/dso_vms.c
+++ b/crypto/dso/dso_vms.c
@@ -27,8 +27,21 @@
 #   pragma pointer_size save
 #   pragma pointer_size 32
 void *_malloc32(__size_t);
+static void *dso_malloc(__size_t num, const char *file, int line)
+{
+    void *ret = _malloc32(num);
+    if (ret == NULL && (file != NULL || line != 0)) {
+        ERR_new();
+        ERR_set_debug(file, line, NULL);
+        ERR_set_error(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE, NULL);
+    }
+    return ret;
+}
+#   define DSO_MALLOC(num) dso_malloc((num), OPENSSL_FILE, OPENSSL_LINE)
 #   pragma pointer_size restore
-#  endif                        /* __INITIAL_POINTER_SIZE == 64 */
+#  else                         /* __INITIAL_POINTER_SIZE == 64 */
+#   define DSO_MALLOC OPENSSL_malloc
+#  endif                        /* __INITIAL_POINTER_SIZE == 64 [else] */
 # endif                         /* __INITIAL_POINTER_SIZE && defined
                                  * _ANSI_C_SOURCE */
 
@@ -88,19 +101,19 @@ static int vms_load(DSO *dso)
     char *filename = DSO_convert_filename(dso, NULL);
 
 /* Ensure 32-bit pointer for "p", and appropriate malloc() function. */
-# if __INITIAL_POINTER_SIZE == 64
-#  define DSO_MALLOC _malloc32
-#  pragma pointer_size save
-#  pragma pointer_size 32
-# else                          /* __INITIAL_POINTER_SIZE == 64 */
-#  define DSO_MALLOC OPENSSL_malloc
-# endif                         /* __INITIAL_POINTER_SIZE == 64 [else] */
+# if __INITIAL_POINTER_SIZE && defined _ANSI_C_SOURCE
+#  if __INITIAL_POINTER_SIZE == 64
+#   pragma pointer_size save
+#   pragma pointer_size 32
+#  endif                        /* __INITIAL_POINTER_SIZE == 64 */
 
     DSO_VMS_INTERNAL *p = NULL;
 
-# if __INITIAL_POINTER_SIZE == 64
-#  pragma pointer_size restore
-# endif                         /* __INITIAL_POINTER_SIZE == 64 */
+#  if __INITIAL_POINTER_SIZE == 64
+#   pragma pointer_size restore
+#  endif                        /* __INITIAL_POINTER_SIZE == 64 */
+# endif                         /* __INITIAL_POINTER_SIZE && defined
+                                 * _ANSI_C_SOURCE */
 
     const char *sp1, *sp2;      /* Search result */
     const char *ext = NULL;     /* possible extension to add */
@@ -174,10 +187,8 @@ static int vms_load(DSO *dso)
     }
 
     p = DSO_MALLOC(sizeof(*p));
-    if (p == NULL) {
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+    if (p == NULL)
         goto err;
-    }
 
     strncpy(p->filename, sp1, sp2 - sp1);
     p->filename[sp2 - sp1] = '\0';
@@ -443,12 +454,10 @@ static char *vms_merger(DSO *dso, const char *filespec1,
 
     merged = OPENSSL_malloc(nam.NAMX_ESL + 1);
     if (merged == NULL)
-        goto malloc_err;
+        return NULL;
     strncpy(merged, nam.NAMX_ESA, nam.NAMX_ESL);
     merged[nam.NAMX_ESL] = '\0';
     return merged;
- malloc_err:
-    ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
 }
 
 static char *vms_name_converter(DSO *dso, const char *filename)
diff --git a/crypto/dso/dso_win32.c b/crypto/dso/dso_win32.c
index 20fa3dce7d..43210e3d98 100644
--- a/crypto/dso/dso_win32.c
+++ b/crypto/dso/dso_win32.c
@@ -110,10 +110,8 @@ static int win32_load(DSO *dso)
         goto err;
     }
     p = OPENSSL_malloc(sizeof(*p));
-    if (p == NULL) {
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+    if (p == NULL)
         goto err;
-    }
     *p = h;
     if (!sk_void_push(dso->meth_data, p)) {
         ERR_raise(ERR_LIB_DSO, DSO_R_STACK_ERROR);
@@ -214,10 +212,8 @@ static struct file_st *win32_splitter(DSO *dso, const char *filename,
     }
 
     result = OPENSSL_zalloc(sizeof(*result));
-    if (result == NULL) {
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+    if (result == NULL)
         return NULL;
-    }
 
     position = IN_DEVICE;
 
@@ -333,10 +329,8 @@ static char *win32_joiner(DSO *dso, const struct file_st *file_split)
     }
 
     result = OPENSSL_malloc(len + 1);
-    if (result == NULL) {
-        ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+    if (result == NULL)
         return NULL;
-    }
 
     if (file_split->node) {
         strcpy(&result[offset], "\\\\");
@@ -399,25 +393,21 @@ static char *win32_merger(DSO *dso, const char *filespec1,
     }
     if (!filespec2) {
         merged = OPENSSL_strdup(filespec1);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
     } else if (!filespec1) {
         merged = OPENSSL_strdup(filespec2);
-        if (merged == NULL) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+        if (merged == NULL)
             return NULL;
-        }
     } else {
         filespec1_split = win32_splitter(dso, filespec1, 0);
         if (!filespec1_split) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_DSO, ERR_R_DSO_LIB);
             return NULL;
         }
         filespec2_split = win32_splitter(dso, filespec2, 1);
         if (!filespec2_split) {
-            ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_DSO, ERR_R_DSO_LIB);
             OPENSSL_free(filespec1_split);
             return NULL;
         }