Coverity fix in some crypto/asn1 code

Call to i2d method returns an int value.

Fix:
CID 1338183 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)
CID 1371691 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)
CID 1371692 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)

[extended tests]

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7359)

(cherry picked from commit da84249be6492ccfc5ecad32ac367fd06e9bdbef)

3 files changed, 43 insertions, 19 deletions

diff --git a/crypto/asn1/a_digest.c b/crypto/asn1/a_digest.c
index f4cc1f2e0e..cc3532ea7d 100644
--- a/crypto/asn1/a_digest.c
+++ b/crypto/asn1/a_digest.c
@@ -23,18 +23,22 @@
 int ASN1_digest(i2d_of_void *i2d, const EVP_MD *type, char *data,
                 unsigned char *md, unsigned int *len)
 {
-    int i;
+    int inl;
     unsigned char *str, *p;
 
-    i = i2d(data, NULL);
-    if ((str = OPENSSL_malloc(i)) == NULL) {
+    inl = i2d(data, NULL);
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_DIGEST, ERR_R_INTERNAL_ERROR);
+        return 0;
+    }
+    if ((str = OPENSSL_malloc(inl)) == NULL) {
         ASN1err(ASN1_F_ASN1_DIGEST, ERR_R_MALLOC_FAILURE);
         return 0;
     }
     p = str;
     i2d(data, &p);
 
-    if (!EVP_Digest(str, i, md, len, type, NULL)) {
+    if (!EVP_Digest(str, inl, md, len, type, NULL)) {
         OPENSSL_free(str);
         return 0;
     }
diff --git a/crypto/asn1/a_sign.c b/crypto/asn1/a_sign.c
index 130e23eaaa..146fdb9626 100644
--- a/crypto/asn1/a_sign.c
+++ b/crypto/asn1/a_sign.c
@@ -29,7 +29,8 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
 {
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char *p, *buf_in = NULL, *buf_out = NULL;
-    int i, inl = 0, outl = 0, outll = 0;
+    int i, inl = 0, outl = 0;
+    size_t inll = 0, outll = 0;
     X509_ALGOR *a;
 
     if (ctx == NULL) {
@@ -70,10 +71,15 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
         }
     }
     inl = i2d(data, NULL);
-    buf_in = OPENSSL_malloc((unsigned int)inl);
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_SIGN, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+    inll = (size_t)inl;
+    buf_in = OPENSSL_malloc(inll);
     outll = outl = EVP_PKEY_size(pkey);
-    buf_out = OPENSSL_malloc((unsigned int)outl);
-    if ((buf_in == NULL) || (buf_out == NULL)) {
+    buf_out = OPENSSL_malloc(outll);
+    if (buf_in == NULL || buf_out == NULL) {
         outl = 0;
         ASN1err(ASN1_F_ASN1_SIGN, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -101,7 +107,7 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:
     EVP_MD_CTX_free(ctx);
-    OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
+    OPENSSL_clear_free((char *)buf_in, inll);
     OPENSSL_clear_free((char *)buf_out, outll);
     return outl;
 }
@@ -138,7 +144,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
     EVP_PKEY *pkey;
     unsigned char *buf_in = NULL, *buf_out = NULL;
     size_t inl = 0, outl = 0, outll = 0;
-    int signid, paramtype;
+    int signid, paramtype, buf_len = 0;
     int rv;
 
     type = EVP_MD_CTX_md(ctx);
@@ -198,10 +204,16 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
 
     }
 
-    inl = ASN1_item_i2d(asn, &buf_in, it);
+    buf_len = ASN1_item_i2d(asn, &buf_in, it);
+    if (buf_len <= 0) {
+        outl = 0;
+        ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+    inl = buf_len;
     outll = outl = EVP_PKEY_size(pkey);
-    buf_out = OPENSSL_malloc((unsigned int)outl);
-    if ((buf_in == NULL) || (buf_out == NULL)) {
+    buf_out = OPENSSL_malloc(outll);
+    if (buf_in == NULL || buf_out == NULL) {
         outl = 0;
         ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -223,7 +235,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
     signature->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07);
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:
-    OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
+    OPENSSL_clear_free((char *)buf_in, inl);
     OPENSSL_clear_free((char *)buf_out, outll);
     return outl;
 }
diff --git a/crypto/asn1/a_verify.c b/crypto/asn1/a_verify.c
index 973d50d24d..cdaf17c3cb 100644
--- a/crypto/asn1/a_verify.c
+++ b/crypto/asn1/a_verify.c
@@ -48,6 +48,10 @@ int ASN1_verify(i2d_of_void *i2d, X509_ALGOR *a, ASN1_BIT_STRING *signature,
     }
 
     inl = i2d(data, NULL);
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_VERIFY, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
     buf_in = OPENSSL_malloc((unsigned int)inl);
     if (buf_in == NULL) {
         ASN1err(ASN1_F_ASN1_VERIFY, ERR_R_MALLOC_FAILURE);
@@ -87,8 +91,8 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
     EVP_MD_CTX *ctx = NULL;
     unsigned char *buf_in = NULL;
     int ret = -1, inl = 0;
-
     int mdnid, pknid;
+    size_t inll = 0;
 
     if (!pkey) {
         ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_PASSED_NULL_PARAMETER);
@@ -127,8 +131,8 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
             goto err;
         ret = -1;
     } else {
-        const EVP_MD *type;
-        type = EVP_get_digestbynid(mdnid);
+        const EVP_MD *type = EVP_get_digestbynid(mdnid);
+
         if (type == NULL) {
             ASN1err(ASN1_F_ASN1_ITEM_VERIFY,
                     ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM);
@@ -150,11 +154,15 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
     }
 
     inl = ASN1_item_i2d(asn, &buf_in, it);
-
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
     if (buf_in == NULL) {
         ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_MALLOC_FAILURE);
         goto err;
     }
+    inll = inl;
 
     ret = EVP_DigestVerify(ctx, signature->data, (size_t)signature->length,
                            buf_in, inl);
@@ -164,7 +172,7 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
     }
     ret = 1;
  err:
-    OPENSSL_clear_free(buf_in, (unsigned int)inl);
+    OPENSSL_clear_free(buf_in, inll);
     EVP_MD_CTX_free(ctx);
     return ret;
 }