GH787: Fix ALPN

* Perform ALPN after the SNI callback; the SSL_CTX may change due to
  that processing
* Add flags to indicate that we actually sent ALPN, to properly error
  out if unexpectedly received.
* clean up ssl3_free() no need to explicitly clear when doing memset
* document ALPN functions

Signed-off-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Emilia Käsper <emilia@openssl.org>

4 files changed, 8 insertions, 8 deletions

diff --git a/apps/apps.c b/apps/apps.c
index 19523d68fc..4e2322d7a7 100644
--- a/apps/apps.c
+++ b/apps/apps.c
@@ -1960,7 +1960,7 @@ void policies_print(X509_STORE_CTX *ctx)
  *
  *   returns: a malloced buffer or NULL on failure.
  */
-unsigned char *next_protos_parse(unsigned short *outlen, const char *in)
+unsigned char *next_protos_parse(size_t *outlen, const char *in)
 {
     size_t len;
     unsigned char *out;
diff --git a/apps/apps.h b/apps/apps.h
index 5450def13d..ebf696b81b 100644
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -565,7 +565,7 @@ int do_X509_CRL_sign(X509_CRL *x, EVP_PKEY *pkey, const EVP_MD *md,
 extern char *psk_key;
 # endif
 
-unsigned char *next_protos_parse(unsigned short *outlen, const char *in);
+unsigned char *next_protos_parse(size_t *outlen, const char *in);
 
 void print_cert_checks(BIO *bio, X509 *x,
                        const char *checkhost,
diff --git a/apps/s_client.c b/apps/s_client.c
index a1ef64b13f..725dcd3a83 100644
--- a/apps/s_client.c
+++ b/apps/s_client.c
@@ -445,7 +445,7 @@ static char *srtp_profiles = NULL;
 /* This the context that we pass to next_proto_cb */
 typedef struct tlsextnextprotoctx_st {
     unsigned char *data;
-    unsigned short len;
+    size_t len;
     int status;
 } tlsextnextprotoctx;
 
@@ -1634,7 +1634,7 @@ int s_client_main(int argc, char **argv)
         SSL_CTX_set_next_proto_select_cb(ctx, next_proto_cb, &next_proto);
 #endif
     if (alpn_in) {
-        unsigned short alpn_len;
+        size_t alpn_len;
         unsigned char *alpn = next_protos_parse(&alpn_len, alpn_in);
 
         if (alpn == NULL) {
diff --git a/apps/s_server.c b/apps/s_server.c
index 35a22f7900..69102d9e56 100644
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -743,7 +743,7 @@ static int next_proto_cb(SSL *s, const unsigned char **data,
 /* This the context that we pass to alpn_cb */
 typedef struct tlsextalpnctx_st {
     unsigned char *data;
-    unsigned short len;
+    size_t len;
 } tlsextalpnctx;
 
 static int alpn_cb(SSL *s, const unsigned char **out, unsigned char *outlen,
@@ -753,7 +753,7 @@ static int alpn_cb(SSL *s, const unsigned char **out, unsigned char *outlen,
 
     if (!s_quiet) {
         /* We can assume that |in| is syntactically valid. */
-        unsigned i;
+        unsigned int i;
         BIO_printf(bio_s_out, "ALPN protocols advertised by the client: ");
         for (i = 0; i < inlen;) {
             if (i)
@@ -1620,7 +1620,7 @@ int s_server_main(int argc, char *argv[])
     }
 #if !defined(OPENSSL_NO_NEXTPROTONEG)
     if (next_proto_neg_in) {
-        unsigned short len;
+        size_t len;
         next_proto.data = next_protos_parse(&len, next_proto_neg_in);
         if (next_proto.data == NULL)
             goto end;
@@ -1631,7 +1631,7 @@ int s_server_main(int argc, char *argv[])
 #endif
     alpn_ctx.data = NULL;
     if (alpn_in) {
-        unsigned short len;
+        size_t len;
         alpn_ctx.data = next_protos_parse(&len, alpn_in);
         if (alpn_ctx.data == NULL)
             goto end;