Change the return type of EVP_EncodeUpdate

Previously EVP_EncodeUpdate returned a void. However there are a couple
of error conditions that can occur. Therefore the return type has been
changed to an int, with 0 indicating error and 1 indicating success.

Reviewed-by: Rich Salz <rsalz@openssl.org>

6 files changed, 25 insertions, 13 deletions

diff --git a/CHANGES b/CHANGES
index 792f602d76..ef01b27e48 100644
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,11 @@
 
  Changes between 1.0.2h and 1.1.0  [xx XXX 2016]
 
+  *) The EVP_EncryptUpdate() function has had its return type changed from void
+     to int. A return of 0 indicates and error while a return of 1 indicates
+     success.
+     [Matt Caswell]
+
   *) The flags RSA_FLAG_NO_CONSTTIME, DSA_FLAG_NO_EXP_CONSTTIME and
      DH_FLAG_NO_EXP_CONSTTIME which previously provided the ability to switch
      off the constant time implementation for RSA, DSA and DH have been made
diff --git a/crypto/evp/bio_b64.c b/crypto/evp/bio_b64.c
index 9067848578..32a884a711 100644
--- a/crypto/evp/bio_b64.c
+++ b/crypto/evp/bio_b64.c
@@ -403,9 +403,10 @@ static int b64_write(BIO *b, const char *in, int inl)
                 ret += n;
             }
         } else {
-            EVP_EncodeUpdate(ctx->base64,
-                             (unsigned char *)ctx->buf, &ctx->buf_len,
-                             (unsigned char *)in, n);
+            if (!EVP_EncodeUpdate(ctx->base64,
+                                 (unsigned char *)ctx->buf, &ctx->buf_len,
+                                 (unsigned char *)in, n))
+                return ((ret == 0) ? -1 : ret);
             OPENSSL_assert(ctx->buf_len <= (int)sizeof(ctx->buf));
             OPENSSL_assert(ctx->buf_len >= ctx->buf_off);
             ret += n;
diff --git a/crypto/evp/encode.c b/crypto/evp/encode.c
index bd2bbc0b09..e026a8f0ca 100644
--- a/crypto/evp/encode.c
+++ b/crypto/evp/encode.c
@@ -114,7 +114,7 @@ void EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
     ctx->line_num = 0;
 }
 
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
     int i, j;
@@ -122,12 +122,12 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 
     *outl = 0;
     if (inl <= 0)
-        return;
+        return 0;
     OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
     if (ctx->length - ctx->num > inl) {
         memcpy(&(ctx->enc_data[ctx->num]), in, inl);
         ctx->num += inl;
-        return;
+        return 1;
     }
     if (ctx->num != 0) {
         i = ctx->length - ctx->num;
@@ -153,12 +153,14 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     if (total > INT_MAX) {
         /* Too much output data! */
         *outl = 0;
-        return;
+        return 0;
     }
     if (inl != 0)
         memcpy(&(ctx->enc_data[0]), in, inl);
     ctx->num = inl;
     *outl = total;
+
+    return 1;
 }
 
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl)
diff --git a/crypto/pem/pem_lib.c b/crypto/pem/pem_lib.c
index 90893f1954..8965fda8d0 100644
--- a/crypto/pem/pem_lib.c
+++ b/crypto/pem/pem_lib.c
@@ -618,7 +618,8 @@ int PEM_write_bio(BIO *bp, const char *name, const char *header,
     i = j = 0;
     while (len > 0) {
         n = (int)((len > (PEM_BUFSIZE * 5)) ? (PEM_BUFSIZE * 5) : len);
-        EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n);
+        if (!EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n))
+            goto err;
         if ((outl) && (BIO_write(bp, (char *)buf, outl) != outl))
             goto err;
         i += outl;
diff --git a/doc/crypto/EVP_EncodeInit.pod b/doc/crypto/EVP_EncodeInit.pod
index f65322617d..4f62e71ec6 100644
--- a/doc/crypto/EVP_EncodeInit.pod
+++ b/doc/crypto/EVP_EncodeInit.pod
@@ -15,8 +15,8 @@ routines
  void EVP_ENCODE_CTX_free(EVP_ENCODE_CTX *ctx);
  int EVP_ENCODE_CTX_num(EVP_ENCODE_CTX *ctx);
  void EVP_EncodeInit(EVP_ENCODE_CTX *ctx);
- void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
-                       const unsigned char *in, int inl);
+ int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+                      const unsigned char *in, int inl);
  void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl);
  int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int n);
 
@@ -66,7 +66,8 @@ any remainder). This gives the number of blocks of data that will be processed.
 Ensure the output buffer contains 65 bytes of storage for each block, plus an
 additional byte for a NUL terminator. EVP_EncodeUpdate() may be called
 repeatedly to process large amounts of input data. In the event of an error
-EVP_EncodeUpdate() will set B<*outl> to 0.
+EVP_EncodeUpdate() will set B<*outl> to 0 and return 0. On success 1 wil be
+returned.
 
 EVP_EncodeFinal() must be called at the end of an encoding operation. It will
 process any partial block of data remaining in the B<ctx> object. The output
@@ -129,6 +130,8 @@ object or NULL on error.
 EVP_ENCODE_CTX_num() returns the number of bytes pending encoding or decoding in
 B<ctx>.
 
+EVP_EncodeUpdate() returns 0 on error or 1 on success.
+
 EVP_EncodeBlock() returns the number of bytes encoded excluding the NUL
 terminator.
 
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 796f4ccb3c..343cd9fd17 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -610,8 +610,8 @@ EVP_ENCODE_CTX *EVP_ENCODE_CTX_new(void);
 void EVP_ENCODE_CTX_free(EVP_ENCODE_CTX *ctx);
 int EVP_ENCODE_CTX_num(EVP_ENCODE_CTX *ctx);
 void EVP_EncodeInit(EVP_ENCODE_CTX *ctx);
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
-                      const unsigned char *in, int inl);
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+                     const unsigned char *in, int inl);
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl);
 int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int n);