Improve some error messages if a digest is not available

If a digest is not available we just get an "internal error" error
message - which isn't very helpful for diagnosing problems. Instead we
explicitly state that we couldn't find a suitable digest.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12733)

5 files changed, 12 insertions, 3 deletions

diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index 43114dc545..643bf6b278 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -2710,8 +2710,8 @@ OCSP_R_UNKNOWN_MESSAGE_DIGEST:119:unknown message digest
 OCSP_R_UNKNOWN_NID:120:unknown nid
 OCSP_R_UNSUPPORTED_REQUESTORNAME_TYPE:129:unsupported requestorname type
 OSSL_DECODER_R_MISSING_GET_PARAMS:100:missing get params
-OSSL_ENCODER_R_INCORRECT_PROPERTY_QUERY:100:incorrect property query
 OSSL_ENCODER_R_ENCODER_NOT_FOUND:101:encoder not found
+OSSL_ENCODER_R_INCORRECT_PROPERTY_QUERY:100:incorrect property query
 OSSL_STORE_R_AMBIGUOUS_CONTENT_TYPE:107:ambiguous content type
 OSSL_STORE_R_BAD_PASSWORD_READ:115:bad password read
 OSSL_STORE_R_ERROR_VERIFYING_PKCS12_MAC:113:error verifying pkcs12 mac
@@ -3297,6 +3297,7 @@ SSL_R_NO_SHARED_CIPHER:193:no shared cipher
 SSL_R_NO_SHARED_GROUPS:410:no shared groups
 SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS:376:no shared signature algorithms
 SSL_R_NO_SRTP_PROFILES:359:no srtp profiles
+SSL_R_NO_SUITABLE_DIGEST_ALGORITHM:297:no suitable digest algorithm
 SSL_R_NO_SUITABLE_KEY_SHARE:101:no suitable key share
 SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM:118:no suitable signature algorithm
 SSL_R_NO_VALID_SCTS:216:no valid scts
diff --git a/include/openssl/sslerr.h b/include/openssl/sslerr.h
index c15a17f96f..d4ee837a1e 100644
--- a/include/openssl/sslerr.h
+++ b/include/openssl/sslerr.h
@@ -634,6 +634,7 @@ int ERR_load_SSL_strings(void);
 # define SSL_R_NO_SHARED_GROUPS                           410
 # define SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS             376
 # define SSL_R_NO_SRTP_PROFILES                           359
+# define SSL_R_NO_SUITABLE_DIGEST_ALGORITHM               297
 # define SSL_R_NO_SUITABLE_KEY_SHARE                      101
 # define SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM            118
 # define SSL_R_NO_VALID_SCTS                              216
diff --git a/ssl/s3_enc.c b/ssl/s3_enc.c
index bd668f317e..bd90e059b5 100644
--- a/ssl/s3_enc.c
+++ b/ssl/s3_enc.c
@@ -408,7 +408,12 @@ int ssl3_digest_cached_records(SSL *s, int keep)
         }
 
         md = ssl_handshake_md(s);
-        if (md == NULL || !EVP_DigestInit_ex(s->s3.handshake_dgst, md, NULL)
+        if (md == NULL) {
+            SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_SSL3_DIGEST_CACHED_RECORDS,
+                     SSL_R_NO_SUITABLE_DIGEST_ALGORITHM);
+            return 0;
+        }
+        if (!EVP_DigestInit_ex(s->s3.handshake_dgst, md, NULL)
             || !EVP_DigestUpdate(s->s3.handshake_dgst, hdata, hdatalen)) {
             SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_SSL3_DIGEST_CACHED_RECORDS,
                      ERR_R_INTERNAL_ERROR);
diff --git a/ssl/ssl_err.c b/ssl/ssl_err.c
index f84b3f94d8..9f47a924f0 100644
--- a/ssl/ssl_err.c
+++ b/ssl/ssl_err.c
@@ -300,6 +300,8 @@ static const ERR_STRING_DATA SSL_str_reasons[] = {
     {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS),
     "no shared signature algorithms"},
     {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SRTP_PROFILES), "no srtp profiles"},
+    {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_DIGEST_ALGORITHM),
+    "no suitable digest algorithm"},
     {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_KEY_SHARE),
     "no suitable key share"},
     {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM),
diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index ff48759436..4c994dd389 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -2356,7 +2356,7 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt)
 
         if (!tls1_lookup_md(s->ctx, s->s3.tmp.peer_sigalg, &md)) {
             SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS_PROCESS_KEY_EXCHANGE,
-                     ERR_R_INTERNAL_ERROR);
+                     SSL_R_NO_SUITABLE_DIGEST_ALGORITHM);
             goto err;
         }
         if (SSL_USE_SIGALGS(s))