Deprecate most of debug-memory

Fixes #8322

The leak-checking (and backtrace option, on some platforms) provided
by crypto-mdebug and crypto-mdebug-backtrace have been mostly neutered;
only the "make malloc fail" capability remains.  OpenSSL recommends using
the compiler's leak-detection instead.

The OPENSSL_DEBUG_MEMORY environment variable is no longer used.
CRYPTO_mem_ctrl(), CRYPTO_set_mem_debug(), CRYPTO_mem_leaks(),
CRYPTO_mem_leaks_fp() and CRYPTO_mem_leaks_cb() return a failure code.
CRYPTO_mem_debug_{malloc,realloc,free}() have been removed.  All of the
above are now deprecated.

Merge (now really small) mem_dbg.c into mem.c

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10572)

6 files changed, 4 insertions, 62 deletions

diff --git a/test/asynctest.c b/test/asynctest.c
index 3ce190fb1b..dc357da8b7 100644
--- a/test/asynctest.c
+++ b/test/asynctest.c
@@ -312,10 +312,7 @@ int main(int argc, char **argv)
         fprintf(stderr,
                 "OpenSSL build is not ASYNC capable - skipping async tests\n");
     } else {
-        CRYPTO_set_mem_debug(1);
-        CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-
-        if (       !test_ASYNC_init_thread()
+        if (!test_ASYNC_init_thread()
                 || !test_ASYNC_callback_status()
                 || !test_ASYNC_start_job()
                 || !test_ASYNC_get_current_job()
diff --git a/test/bio_memleak_test.c b/test/bio_memleak_test.c
index dab61f6f9a..49ee712a49 100644
--- a/test/bio_memleak_test.c
+++ b/test/bio_memleak_test.c
@@ -221,13 +221,6 @@ finish:
     return ok;
 }
 
-int global_init(void)
-{
-    CRYPTO_set_mem_debug(1);
-    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-    return 1;
-}
-
 int setup_tests(void)
 {
     ADD_TEST(test_bio_memleak);
diff --git a/test/sslapitest.c b/test/sslapitest.c
index 2442b6967d..46e490a417 100644
--- a/test/sslapitest.c
+++ b/test/sslapitest.c
@@ -691,7 +691,7 @@ static int execute_test_large_message(const SSL_METHOD *smeth,
 
     /*
      * Calling SSL_clear() first is not required but this tests that SSL_clear()
-     * doesn't leak (when using enable-crypto-mdebug).
+     * doesn't leak.
      */
     if (!TEST_true(SSL_clear(serverssl)))
         goto end;
@@ -2116,8 +2116,7 @@ static int test_ssl_set_bio(int idx)
     /*
      * This test is checking that the ref counting for SSL_set_bio is correct.
      * If we get here and we did too many frees then we will fail in the above
-     * functions. If we haven't done enough then this will only be detected in
-     * a crypto-mdebug build
+     * functions.
      */
     SSL_free(serverssl);
     SSL_free(clientssl);
@@ -2144,8 +2143,7 @@ static int execute_test_ssl_bio(int pop_ssl, bio_change_t change_bio)
     BIO_set_ssl(sslbio, ssl, BIO_CLOSE);
 
     /*
-     * If anything goes wrong here then we could leak memory, so this will
-     * be caught in a crypto-mdebug build
+     * If anything goes wrong here then we could leak memory.
      */
     BIO_push(sslbio, membio1);
 
diff --git a/test/sslbuffertest.c b/test/sslbuffertest.c
index e8293f7def..163beafb2f 100644
--- a/test/sslbuffertest.c
+++ b/test/sslbuffertest.c
@@ -150,13 +150,6 @@ static int test_func(int test)
     return result;
 }
 
-int global_init(void)
-{
-    CRYPTO_set_mem_debug(1);
-    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-    return 1;
-}
-
 OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
 
 int setup_tests(void)
diff --git a/test/ssltest_old.c b/test/ssltest_old.c
index 050470d33e..0c0aa6046a 100644
--- a/test/ssltest_old.c
+++ b/test/ssltest_old.c
@@ -884,7 +884,6 @@ int main(int argc, char *argv[])
     int server_auth = 0, i;
     struct app_verify_arg app_verify_arg =
         { APP_CALLBACK_STRING, 0 };
-    char *p;
     SSL_CTX *c_ctx = NULL;
     const SSL_METHOD *meth = NULL;
     SSL *c_ssl, *s_ssl;
@@ -922,12 +921,6 @@ int main(int argc, char *argv[])
     debug = 0;
 
     bio_err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
-
-    p = getenv("OPENSSL_DEBUG_MEMORY");
-    if (p != NULL && strcmp(p, "on") == 0)
-        CRYPTO_set_mem_debug(1);
-    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-
     bio_stdout = BIO_new_fp(stdout, BIO_NOCLOSE | BIO_FP_TEXT);
 
     s_cctx = SSL_CONF_CTX_new();
@@ -1860,10 +1853,6 @@ int main(int argc, char *argv[])
     SSL_SESSION_free(server_sess);
     SSL_SESSION_free(client_sess);
 
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-    if (CRYPTO_mem_leaks(bio_err) <= 0)
-        ret = EXIT_FAILURE;
-#endif
     BIO_free(bio_err);
     EXIT(ret);
 }
diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index 2ced5634bc..030fb1544c 100644
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -81,22 +81,6 @@ int subtest_level(void)
     return level;
 }
 
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-static int should_report_leaks(void)
-{
-    /*
-     * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
-     * can be used to disable leak checking at runtime.
-     * Note this only works when running the test binary manually;
-     * the test harness always enables OPENSSL_DEBUG_MEMORY.
-     */
-    char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
-
-    return mem_debug_env == NULL
-        || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
-}
-#endif
-
 static int gcd(int a, int b)
 {
     while (b != 0) {
@@ -128,13 +112,6 @@ int setup_test_framework(int argc, char *argv[])
     if (test_seed != NULL)
         set_seed(atoi(test_seed));
 
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-    if (should_report_leaks()) {
-        CRYPTO_set_mem_debug(1);
-        CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-    }
-#endif
-
 #if defined(OPENSSL_SYS_VMS) && defined(__DECC)
     argv = copy_argv(&argc, argv);
 #elif defined(_WIN32)
@@ -256,11 +233,6 @@ end:
 int pulldown_test_framework(int ret)
 {
     set_test_title(NULL);
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-    if (should_report_leaks()
-        && CRYPTO_mem_leaks_cb(openssl_error_cb, NULL) <= 0)
-        return EXIT_FAILURE;
-#endif
     return ret;
 }