Move state machine knowledge out of the record layer

The record layer was making decisions that should really be left to the
state machine around unexpected handshake messages that are received after
the initial handshake (i.e. renegotiation related messages). This commit
removes that code from the record layer and updates the state machine
accordingly. This simplifies the state machine and paves the way for
handling other messages post-handshake such as the NewSessionTicket in
TLSv1.3.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2259)

1 files changed, 5 insertions, 2 deletions

diff --git a/ssl/statem/statem.h b/ssl/statem/statem.h
index 6765c304a9..021d2d06ce 100644
--- a/ssl/statem/statem.h
+++ b/ssl/statem/statem.h
@@ -46,8 +46,6 @@ typedef enum {
     MSG_FLOW_UNINITED,
     /* A permanent error with this connection */
     MSG_FLOW_ERROR,
-    /* We are about to renegotiate */
-    MSG_FLOW_RENEGOTIATE,
     /* We are reading messages */
     MSG_FLOW_READING,
     /* We are writing messages */
@@ -92,6 +90,11 @@ struct ossl_statem_st {
     int read_state_first_init;
     /* true when we are actually in SSL_accept() or SSL_connect() */
     int in_handshake;
+    /*
+     * True when are processing a "real" handshake that needs cleaning up (not
+     * just a HelloRequest or similar).
+     */
+    int cleanuphand;
     /* Should we skip the CertificateVerify message? */
     unsigned int no_cert_verify;
     int use_timer;