Protocol version selection and negotiation rewrite

The protocol selection code is now consolidated in a few consecutive
short functions in a single file and is table driven.  Protocol-specific
constraints that influence negotiation are moved into the flags
field of the method structure.  The same protocol version constraints
are now applied in all code paths.  It is now much easier to add
new protocol versions without reworking the protocol selection
logic.

In the presence of "holes" in the list of enabled client protocols
we no longer select client protocols below the hole based on a
subset of the constraints and then fail shortly after when it is
found that these don't meet the remaining constraints (suiteb, FIPS,
security level, ...).  Ideally, with the new min/max controls users
will be less likely to create "holes" in the first place.

Reviewed-by: Kurt Roeckx <kurt@openssl.org>

1 files changed, 3 insertions, 35 deletions

diff --git a/ssl/d1_lib.c b/ssl/d1_lib.c
index a510b5bebe..3cd4b786e4 100644
--- a/ssl/d1_lib.c
+++ b/ssl/d1_lib.c
@@ -235,7 +235,7 @@ void dtls1_clear(SSL *s)
     if (s->options & SSL_OP_CISCO_ANYCONNECT)
         s->client_version = s->version = DTLS1_BAD_VER;
     else if (s->method->version == DTLS_ANY_VERSION)
-        s->version = DTLS1_2_VERSION;
+        s->version = DTLS_MAX_VERSION;
     else
         s->version = s->method->version;
 }
@@ -256,38 +256,6 @@ long dtls1_ctrl(SSL *s, int cmd, long larg, void *parg)
     case DTLS_CTRL_LISTEN:
         ret = dtls1_listen(s, parg);
         break;
-    case SSL_CTRL_CHECK_PROTO_VERSION:
-        /*
-         * For library-internal use; checks that the current protocol is the
-         * is the highest enabled version.
-         */
-        if (s->max_proto_version == 0 && s->version == DTLS_MAX_VERSION)
-            return 1;
-        if (s->max_proto_version != 0 && s->version == s->max_proto_version)
-            return 1;
-        /* We're not limited by the max_proto_version but might still have
-         * other reasons why we use an older version like not using a
-         * version-flexible SSL_METHOD.  Check s->ctx->method as version
-         * negotiation may have changed s->method.
-         * This check can be removed when we only have version-flexible
-         * SSL_METHODs
-         */
-        if (s->version == s->ctx->method->version)
-            return 1;
-        /*
-         * Apparently we're using a version-flexible SSL_METHOD (not at its
-         * highest protocol version, not limited by max_proto_version).
-         */
-        if (s->ctx->method->version == DTLS_method()->version) {
-#if DTLS_MAX_VERSION != DTLS1_2_VERSION
-# error Code needs update for DTLS_method() support beyond DTLS1_2_VERSION.
-#endif
-            if (!(s->options & SSL_OP_NO_DTLSv1_2))
-                return s->version == DTLS1_2_VERSION;
-            if (!(s->options & SSL_OP_NO_DTLSv1))
-                return s->version == DTLS1_VERSION;
-        }
-        return 0;               /* Unexpected state; fail closed. */
     case DTLS_CTRL_SET_LINK_MTU:
         if (larg < (long)dtls1_link_min_mtu())
             return 0;
@@ -708,8 +676,8 @@ int dtls1_listen(SSL *s, struct sockaddr *client)
         /*
          * Verify client version is supported
          */
-        if ((clientvers > (unsigned int)s->method->version &&
-                              s->method->version != DTLS_ANY_VERSION)) {
+        if (DTLS_VERSION_LT(clientvers, (unsigned int)s->method->version) &&
+            s->method->version != DTLS_ANY_VERSION) {
             SSLerr(SSL_F_DTLS1_LISTEN, SSL_R_WRONG_VERSION_NUMBER);
             goto end;
         }