test/memleaktest.c: Modify for use with address/leak sanitizer

Detects if leak sanitizing is on, and directs the exit code accordingly.

Note that this program is designed to fail when leaking, as that's
expected, so to make it easy for wrapper scripts, we also make it look
like it fails when sanitizing isn't on.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9294)

2 files changed, 32 insertions, 25 deletions

diff --git a/test/memleaktest.c b/test/memleaktest.c
index 419d6464fa..1e07723a6f 100644
--- a/test/memleaktest.c
+++ b/test/memleaktest.c
@@ -13,44 +13,47 @@
 
 #include "testutil.h"
 
+/* __has_feature is a clang-ism, while __SANITIZE_ADDRESS__ is a gcc-ism */
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+#  define __SANITIZE_ADDRESS__ 1
+# endif
+#endif
+/* If __SANITIZE_ADDRESS__ isn't defined, define it to be false */
+#ifndef __SANITIZE_ADDRESS__
+# define __SANITIZE_ADDRESS__ 0
+#endif
+
 /*
  * We use a proper main function here instead of the custom main from the
- * test framework because the CRYPTO_mem_leaks_fp function cannot be called
- * a second time without trying to use a null pointer.  The test framework
- * calls this function as part of its close down.
- *
- * A work around is to call putenv("OPENSSL_DEBUG_MEMORY=0"); before exiting
- * but that is worse than avoiding the test framework's main.
+ * test framework to avoid CRYPTO_mem_leaks stuff.
  */
 
 int main(int argc, char *argv[])
 {
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-    char *p;
+#if __SANITIZE_ADDRESS__
+    int exitcode = EXIT_SUCCESS;
+#else
+    /*
+     * When we don't sanitize, we set the exit code to what we would expect
+     * to get when we are sanitizing.  This makes it easy for wrapper scripts
+     * to detect that we get the result we expect.
+     */
+    int exitcode = EXIT_FAILURE;
+#endif
     char *lost;
-    int noleak;
-
-    p = getenv("OPENSSL_DEBUG_MEMORY");
-    if (p != NULL && strcmp(p, "on") == 0)
-        CRYPTO_set_mem_debug(1);
-    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 
     lost = OPENSSL_malloc(3);
     if (!TEST_ptr(lost))
         return EXIT_FAILURE;
 
+    strcpy(lost, "ab");
+
     if (argv[1] && strcmp(argv[1], "freeit") == 0) {
         OPENSSL_free(lost);
-        lost = NULL;
+        exitcode = EXIT_SUCCESS;
     }
 
-    noleak = CRYPTO_mem_leaks_fp(stderr);
-    /* If -1 return value something bad happened */
-    if (!TEST_int_ne(noleak, -1))
-        return EXIT_FAILURE;
-
-    return TEST_int_eq(lost != NULL, noleak == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
-#else
-    return EXIT_SUCCESS;
-#endif
+    lost = NULL;
+    return exitcode;
 }
diff --git a/test/recipes/90-test_memleak.t b/test/recipes/90-test_memleak.t
index 312087cc80..76f1dcb06d 100644
--- a/test/recipes/90-test_memleak.t
+++ b/test/recipes/90-test_memleak.t
@@ -10,6 +10,10 @@
 use OpenSSL::Test;
 
 setup("test_memleak");
+
+plan skip_all => "MacOS currently doesn't support leak sanitizer"
+    if $^O eq 'darwin';
+
 plan tests => 2;
-ok(run(test(["memleaktest"])), "running leak test");
+ok(!run(test(["memleaktest"])), "running leak test");
 ok(run(test(["memleaktest", "freeit"])), "running no leak test");