EVP_*Update: ensure that input NULL with length 0 isn't passed

Even with custome ciphers, the combination in == NULL && inl == 0
should not be passed down to the backend cipher function.  The reason
is that these are the values passed by EVP_*Final, and some of the
backend cipher functions do check for these to see if a "final" call
is made.

Fixes #8675

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8676)

1 files changed, 10 insertions, 9 deletions

diff --git a/crypto/evp/evp_enc.c b/crypto/evp/evp_enc.c
index 7fdf7598d2..641ad19744 100644
--- a/crypto/evp/evp_enc.c
+++ b/crypto/evp/evp_enc.c
@@ -305,6 +305,11 @@ static int evp_EncryptDecryptUpdate(EVP_CIPHER_CTX *ctx,
 
     bl = ctx->cipher->block_size;
 
+    if (inl <= 0) {
+        *outl = 0;
+        return inl == 0;
+    }
+
     if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
         /* If block size > 1 then the cipher will have to do this check */
         if (bl == 1 && is_partially_overlapping(out, in, cmpl)) {
@@ -320,10 +325,6 @@ static int evp_EncryptDecryptUpdate(EVP_CIPHER_CTX *ctx,
         return 1;
     }
 
-    if (inl <= 0) {
-        *outl = 0;
-        return inl == 0;
-    }
     if (is_partially_overlapping(out + ctx->buf_len, in, cmpl)) {
         EVPerr(EVP_F_EVP_ENCRYPTDECRYPTUPDATE, EVP_R_PARTIALLY_OVERLAPPING);
         return 0;
@@ -457,6 +458,11 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS))
         cmpl = (cmpl + 7) / 8;
 
+    if (inl <= 0) {
+        *outl = 0;
+        return inl == 0;
+    }
+
     if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
         if (b == 1 && is_partially_overlapping(out, in, cmpl)) {
             EVPerr(EVP_F_EVP_DECRYPTUPDATE, EVP_R_PARTIALLY_OVERLAPPING);
@@ -472,11 +478,6 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
         return 1;
     }
 
-    if (inl <= 0) {
-        *outl = 0;
-        return inl == 0;
-    }
-
     if (ctx->flags & EVP_CIPH_NO_PADDING)
         return evp_EncryptDecryptUpdate(ctx, out, outl, in, inl);