Add the SSL_OP_NO_RENEGOTIATION option to 1.1.0

This is based on a heavily modified version of commit db0f35dda by Todd
Short from the master branch.

We are adding this because it used to be possible to disable reneg using
the flag SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS in 1.0.2. This is no longer
possible because of the opacity work.

A point to note about this is that if an application built against new
1.1.0 headers (that know about the new option SSL_OP_NO_RENEGOTIATION
option) is run using an older version of 1.1.0 (that doesn't know about
the option) then the option will be accepted but nothing will happen, i.e.
renegotiation will not be prevented. There's probably not much we can do
about that.

Fixes #4739

Reviewed-by: Ben Kaduk <kaduk@mit.edu>
(Merged from https://github.com/openssl/openssl/pull/4901)

1 files changed, 18 insertions, 11 deletions

diff --git a/test/handshake_helper.c b/test/handshake_helper.c
index 0a19f59ce4..98a8f43dde 100644
--- a/test/handshake_helper.c
+++ b/test/handshake_helper.c
@@ -505,7 +505,11 @@ static void do_handshake_step(PEER *peer)
 {
     int ret;
 
-    TEST_check(peer->status == PEER_RETRY);
+    if (peer->status != PEER_RETRY) {
+        peer->status = PEER_ERROR;
+        return;
+    }
+
     ret = SSL_do_handshake(peer->ssl);
 
     if (ret == 1) {
@@ -588,6 +592,17 @@ static void do_reneg_setup_step(const SSL_TEST_CTX *test_ctx, PEER *peer)
     int ret;
     char buf;
 
+    if (peer->status == PEER_SUCCESS) {
+        /*
+         * We are a client that succeeded this step previously, but the server
+         * wanted to retry. Probably there is a no_renegotiation warning alert
+         * waiting for us. Attempt to continue the handshake.
+         */
+        peer->status = PEER_RETRY;
+        do_handshake_step(peer);
+        return;
+    }
+
     TEST_check(peer->status == PEER_RETRY);
     TEST_check(test_ctx->handshake_mode == SSL_TEST_HANDSHAKE_RENEG_SERVER
                 || test_ctx->handshake_mode == SSL_TEST_HANDSHAKE_RENEG_CLIENT);
@@ -807,16 +822,8 @@ static handshake_status_t handshake_status(peer_status_t last_status,
         break;
 
     case PEER_RETRY:
-        if (previous_status == PEER_RETRY) {
-            /* Neither peer is done. */
-            return HANDSHAKE_RETRY;
-        } else {
-            /*
-             * Deadlock: second peer is waiting for more input while first
-             * peer thinks they're done (no more input is coming).
-             */
-            return INTERNAL_ERROR;
-        }
+        return HANDSHAKE_RETRY;
+
     case PEER_ERROR:
         switch (previous_status) {
         case PEER_SUCCESS: