Make the "ticket" function return codes clearer

Remove "magic" return values and use an enum instead.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2259)

3 files changed, 42 insertions, 40 deletions

diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h
index a59683bdff..e74c0f480a 100644
--- a/ssl/ssl_locl.h
+++ b/ssl/ssl_locl.h
@@ -2191,18 +2191,24 @@ __owur  int tls1_get_curvelist(SSL *s, int sess, const unsigned char **pcurves,
 
 void ssl_set_default_md(SSL *s);
 __owur int tls1_set_server_sigalgs(SSL *s);
-__owur int tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
-                                      SSL_SESSION **ret);
-
-/* Return codes for tls_decrypt_ticket */
-#define TICKET_FATAL_ERR_MALLOC     -2
-#define TICKET_FATAL_ERR_OTHER      -1
-#define TICKET_NO_DECRYPT            2
-#define TICKET_SUCCESS               3
-#define TICKET_SUCCESS_RENEW         4
-__owur int tls_decrypt_ticket(SSL *s, const unsigned char *etick,
-                              size_t eticklen, const unsigned char *sess_id,
-                              size_t sesslen, SSL_SESSION **psess);
+
+/* Return codes for tls_get_ticket_from_client() and tls_decrypt_ticket() */
+typedef enum ticket_en {
+    TICKET_FATAL_ERR_MALLOC,
+    TICKET_FATAL_ERR_OTHER,
+    TICKET_NONE,
+    TICKET_EMPTY,
+    TICKET_NO_DECRYPT,
+    TICKET_SUCCESS,
+    TICKET_SUCCESS_RENEW
+} TICKET_RETURN;
+
+__owur TICKET_RETURN tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
+                                                SSL_SESSION **ret);
+__owur TICKET_RETURN tls_decrypt_ticket(SSL *s, const unsigned char *etick,
+                                        size_t eticklen,
+                                        const unsigned char *sess_id,
+                                        size_t sesslen, SSL_SESSION **psess);
 
 __owur int tls_use_ticket(SSL *s);
 
diff --git a/ssl/ssl_sess.c b/ssl/ssl_sess.c
index 2ef0006649..c0fc8b356c 100644
--- a/ssl/ssl_sess.c
+++ b/ssl/ssl_sess.c
@@ -465,7 +465,7 @@ int ssl_get_prev_session(SSL *s, CLIENTHELLO_MSG *hello)
     SSL_SESSION *ret = NULL;
     int fatal = 0;
     int try_session_cache = 0;
-    int r;
+    TICKET_RETURN r;
 
     if (SSL_IS_TLS13(s)) {
         int al;
@@ -479,18 +479,18 @@ int ssl_get_prev_session(SSL *s, CLIENTHELLO_MSG *hello)
         /* sets s->ext.ticket_expected */
         r = tls_get_ticket_from_client(s, hello, &ret);
         switch (r) {
-        case -1:                   /* Error during processing */
+        case TICKET_FATAL_ERR_MALLOC:
+        case TICKET_FATAL_ERR_OTHER: /* Error during processing */
             fatal = 1;
             goto err;
-        case 0:                    /* No ticket found */
-        case 1:                    /* Zero length ticket found */
+        case TICKET_NONE:            /* No ticket found */
+        case TICKET_EMPTY:           /* Zero length ticket found */
             try_session_cache = 1;
-            break;                  /* Ok to carry on processing session id. */
-        case 2:                    /* Ticket found but not decrypted. */
-        case 3:                    /* Ticket decrypted, *ret has been set. */
+            break;                   /* Ok to carry on processing session id. */
+        case TICKET_NO_DECRYPT:      /* Ticket found but not decrypted. */
+        case TICKET_SUCCESS:         /* Ticket decrypted, *ret has been set. */
+        case TICKET_SUCCESS_RENEW:
             break;
-        default:
-            abort();
         }
     }
 
diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index b0df32b406..ffde4eeec3 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -1049,8 +1049,8 @@ int tls1_set_server_sigalgs(SSL *s)
  *   s->ctx->ext.ticket_key_cb asked to renew the client's ticket.
  *   Otherwise, s->ext.ticket_expected is set to 0.
  */
-int tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
-                               SSL_SESSION **ret)
+TICKET_RETURN tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
+                                         SSL_SESSION **ret)
 {
     int retv;
     size_t size;
@@ -1065,11 +1065,11 @@ int tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
      * resumption.
      */
     if (s->version <= SSL3_VERSION || !tls_use_ticket(s))
-        return 0;
+        return TICKET_NONE;
 
     ticketext = &hello->pre_proc_exts[TLSEXT_IDX_session_ticket];
     if (!ticketext->present)
-        return 0;
+        return TICKET_NONE;
 
     size = PACKET_remaining(&ticketext->data);
     if (size == 0) {
@@ -1078,7 +1078,7 @@ int tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
          * one.
          */
         s->ext.ticket_expected = 1;
-        return 1;
+        return TICKET_EMPTY;
     }
     if (s->ext.session_secret_cb) {
         /*
@@ -1087,7 +1087,7 @@ int tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
          * abbreviated handshake based on external mechanism to
          * calculate the master secret later.
          */
-        return 2;
+        return TICKET_NO_DECRYPT;
     }
 
     retv = tls_decrypt_ticket(s, PACKET_data(&ticketext->data), size,
@@ -1095,17 +1095,17 @@ int tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
     switch (retv) {
     case TICKET_NO_DECRYPT: /* ticket couldn't be decrypted */
         s->ext.ticket_expected = 1;
-        return 2;
+        return TICKET_NO_DECRYPT;
 
     case TICKET_SUCCESS: /* ticket was decrypted */
-        return 3;
+        return TICKET_SUCCESS;
 
     case TICKET_SUCCESS_RENEW: /* ticket decrypted but need to renew */
         s->ext.ticket_expected = 1;
-        return 3;
+        return TICKET_SUCCESS;
 
     default:           /* fatal error */
-        return -1;
+        return TICKET_FATAL_ERR_OTHER;
     }
 }
 
@@ -1128,19 +1128,15 @@ int tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello,
  *                            set.
  *   TICKET_SUCCESS_RENEW:    same as 3, but the ticket needs to be renewed
  */
-#define TICKET_FATAL_ERR_MALLOC     -2
-#define TICKET_FATAL_ERR_OTHER      -1
-#define TICKET_NO_DECRYPT            2
-#define TICKET_SUCCESS               3
-#define TICKET_SUCCESS_RENEW         4
-int tls_decrypt_ticket(SSL *s, const unsigned char *etick, size_t eticklen,
-                       const unsigned char *sess_id, size_t sesslen,
-                       SSL_SESSION **psess)
+TICKET_RETURN tls_decrypt_ticket(SSL *s, const unsigned char *etick,
+                                 size_t eticklen, const unsigned char *sess_id,
+                                 size_t sesslen, SSL_SESSION **psess)
 {
     SSL_SESSION *sess;
     unsigned char *sdec;
     const unsigned char *p;
-    int slen, renew_ticket = 0, ret = TICKET_FATAL_ERR_OTHER, declen;
+    int slen, renew_ticket = 0, declen;
+    TICKET_RETURN ret = TICKET_FATAL_ERR_OTHER;
     size_t mlen;
     unsigned char tick_hmac[EVP_MAX_MD_SIZE];
     HMAC_CTX *hctx = NULL;