Make SMIME_read_CMS_ex() and SMIME_read_ASN1_ex() support binary input

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12959)

7 files changed, 78 insertions, 41 deletions

diff --git a/crypto/asn1/asn_mime.c b/crypto/asn1/asn_mime.c
index 8ee0970dc6..68e0c5affd 100644
--- a/crypto/asn1/asn_mime.c
+++ b/crypto/asn1/asn_mime.c
@@ -54,7 +54,7 @@ static int mime_param_cmp(const MIME_PARAM *const *a,
                           const MIME_PARAM *const *b);
 static void mime_param_free(MIME_PARAM *param);
 static int mime_bound_check(char *line, int linelen, const char *bound, int blen);
-static int multi_split(BIO *bio, const char *bound, STACK_OF(BIO) **ret);
+static int multi_split(BIO *bio, int flags, const char *bound, STACK_OF(BIO) **ret);
 static int strip_eol(char *linebuf, int *plen, int flags);
 static MIME_HEADER *mime_hdr_find(STACK_OF(MIME_HEADER) *hdrs, const char *name);
 static MIME_PARAM *mime_param_find(MIME_HEADER *hdr, const char *name);
@@ -209,9 +209,9 @@ static int asn1_write_micalg(BIO *out, STACK_OF(X509_ALGOR) *mdalgs)
             goto err;
 
         default:
-            if (have_unknown)
+            if (have_unknown) {
                 write_comma = 0;
-            else {
+            } else {
                 BIO_puts(out, "unknown");
                 have_unknown = 1;
             }
@@ -292,9 +292,9 @@ int SMIME_write_ASN1_ex(BIO *bio, ASN1_VALUE *val, BIO *data, int flags,
 
     /* Determine smime-type header */
 
-    if (ctype_nid == NID_pkcs7_enveloped)
+    if (ctype_nid == NID_pkcs7_enveloped) {
         msg_type = "enveloped-data";
-    else if (ctype_nid == NID_pkcs7_signed) {
+    } else if (ctype_nid == NID_pkcs7_signed) {
         if (econt_nid == NID_id_smime_ct_receipt)
             msg_type = "signed-receipt";
         else if (sk_X509_ALGOR_num(mdalgs) >= 0)
@@ -388,7 +388,7 @@ static int asn1_output_data(BIO *out, BIO *data, ASN1_VALUE *val, int flags,
  * opaque this is set to NULL
  */
 
-ASN1_VALUE *SMIME_read_ASN1_ex(BIO *bio, BIO **bcont, const ASN1_ITEM *it,
+ASN1_VALUE *SMIME_read_ASN1_ex(BIO *bio, int flags, BIO **bcont, const ASN1_ITEM *it,
                                ASN1_VALUE **x)
 {
     BIO *asnin;
@@ -424,7 +424,7 @@ ASN1_VALUE *SMIME_read_ASN1_ex(BIO *bio, BIO **bcont, const ASN1_ITEM *it,
             ERR_raise(ERR_LIB_ASN1, ASN1_R_NO_MULTIPART_BOUNDARY);
             return NULL;
         }
-        ret = multi_split(bio, prm->param_value, &parts);
+        ret = multi_split(bio, flags, prm->param_value, &parts);
         sk_MIME_HEADER_pop_free(headers, mime_hdr_free);
         if (!ret || (sk_BIO_num(parts) != 2)) {
             ERR_raise(ERR_LIB_ASN1, ASN1_R_NO_MULTIPART_BODY_FAILURE);
@@ -471,8 +471,9 @@ ASN1_VALUE *SMIME_read_ASN1_ex(BIO *bio, BIO **bcont, const ASN1_ITEM *it,
             *bcont = sk_BIO_value(parts, 0);
             BIO_free(asnin);
             sk_BIO_free(parts);
-        } else
+        } else {
             sk_BIO_pop_free(parts, BIO_vfree);
+        }
         return val;
     }
 
@@ -497,7 +498,7 @@ ASN1_VALUE *SMIME_read_ASN1_ex(BIO *bio, BIO **bcont, const ASN1_ITEM *it,
 
 ASN1_VALUE *SMIME_read_ASN1(BIO *bio, BIO **bcont, const ASN1_ITEM *it)
 {
-    return SMIME_read_ASN1_ex(bio, bcont, it, NULL);
+    return SMIME_read_ASN1_ex(bio, 0, bcont, it, NULL);
 }
 
 /* Copy text from one BIO to another making the output CRLF at EOL */
@@ -524,7 +525,7 @@ int SMIME_crlf_copy(BIO *in, BIO *out, int flags)
             BIO_printf(out, "Content-Type: text/plain\r\n\r\n");
         while ((len = BIO_gets(in, linebuf, MAX_SMLEN)) > 0) {
             eol = strip_eol(linebuf, &len, flags);
-            if (len) {
+            if (len > 0) {
                 /* Not EOF: write out all CRLF */
                 if (flags & SMIME_ASCIICRLF) {
                     int i;
@@ -535,10 +536,11 @@ int SMIME_crlf_copy(BIO *in, BIO *out, int flags)
                 BIO_write(out, linebuf, len);
                 if (eol)
                     BIO_write(out, "\r\n", 2);
-            } else if (flags & SMIME_ASCIICRLF)
+            } else if (flags & SMIME_ASCIICRLF) {
                 eolcnt++;
-            else if (eol)
+            } else if (eol) {
                 BIO_write(out, "\r\n", 2);
+            }
         }
     }
     (void)BIO_flush(out);
@@ -584,7 +586,7 @@ int SMIME_text(BIO *in, BIO *out)
  * canonical parts in a STACK of bios
  */
 
-static int multi_split(BIO *bio, const char *bound, STACK_OF(BIO) **ret)
+static int multi_split(BIO *bio, int flags, const char *bound, STACK_OF(BIO) **ret)
 {
     char linebuf[MAX_SMLEN];
     int len, blen;
@@ -601,7 +603,7 @@ static int multi_split(BIO *bio, const char *bound, STACK_OF(BIO) **ret)
     *ret = parts;
     if (*ret == NULL)
         return 0;
-    while ((len = BIO_gets(bio, linebuf, MAX_SMLEN)) > 0) {
+    while ((len = BIO_get_line(bio, linebuf, MAX_SMLEN)) > 0) {
         state = mime_bound_check(linebuf, len, bound, blen);
         if (state == 1) {
             first = 1;
@@ -612,9 +614,9 @@ static int multi_split(BIO *bio, const char *bound, STACK_OF(BIO) **ret)
                 return 0;
             }
             return 1;
-        } else if (part) {
-            /* Strip CR+LF from linebuf */
-            next_eol = strip_eol(linebuf, &len, 0);
+        } else if (part != 0) {
+            /* Strip (possibly CR +) LF from linebuf */
+            next_eol = strip_eol(linebuf, &len, flags);
             if (first) {
                 first = 0;
                 if (bpart)
@@ -626,10 +628,20 @@ static int multi_split(BIO *bio, const char *bound, STACK_OF(BIO) **ret)
                 if (bpart == NULL)
                     return 0;
                 BIO_set_mem_eof_return(bpart, 0);
-            } else if (eol)
-                BIO_write(bpart, "\r\n", 2);
+            } else if (eol) {
+                if (
+#ifndef OPENSSL_NO_CMS
+                    (flags & CMS_BINARY) == 0
+#else
+                    1
+#endif
+                        || (flags & SMIME_CRLFEOL) != 0)
+                    BIO_write(bpart, "\r\n", 2);
+                else
+                    BIO_write(bpart, "\n", 1);
+            }
             eol = next_eol;
-            if (len)
+            if (len > 0)
                 BIO_write(bpart, linebuf, len);
         }
     }
@@ -753,15 +765,16 @@ static STACK_OF(MIME_HEADER) *mime_parse_hdr(BIO *bio)
                 goto err;
             mhdr = new_hdr;
             new_hdr = NULL;
-        } else if (state == MIME_VALUE)
+        } else if (state == MIME_VALUE) {
             mime_hdr_addparam(mhdr, ntmp, strip_ends(q));
+        }
         if (p == linebuf)
             break;              /* Blank line means end of headers */
     }
 
     return headers;
 
-err:
+ err:
     mime_hdr_free(new_hdr);
     sk_MIME_HEADER_pop_free(headers, mime_hdr_free);
     return NULL;
@@ -883,8 +896,8 @@ static int mime_hdr_addparam(MIME_HEADER *mhdr, const char *name, const char *va
 static int mime_hdr_cmp(const MIME_HEADER *const *a,
                         const MIME_HEADER *const *b)
 {
-    if (!(*a)->name || !(*b)->name)
-        return ! !(*a)->name - ! !(*b)->name;
+    if ((*a)->name == NULL || (*b)->name == NULL)
+        return ((*a)->name != NULL) - ((*b)->name != NULL);
 
     return strcmp((*a)->name, (*b)->name);
 }
@@ -892,8 +905,8 @@ static int mime_hdr_cmp(const MIME_HEADER *const *a,
 static int mime_param_cmp(const MIME_PARAM *const *a,
                           const MIME_PARAM *const *b)
 {
-    if (!(*a)->param_name || !(*b)->param_name)
-        return ! !(*a)->param_name - ! !(*b)->param_name;
+    if ((*a)->param_name == NULL || (*b)->param_name == NULL)
+        return ((*a)->param_name != NULL) - ((*b)->param_name != NULL);
     return strcmp((*a)->param_name, (*b)->param_name);
 }
 
@@ -973,11 +986,26 @@ static int strip_eol(char *linebuf, int *plen, int flags)
     char *p, c;
     int is_eol = 0;
 
+#ifndef OPENSSL_NO_CMS
+    if ((flags & CMS_BINARY) != 0) {
+        if (len <= 0 || linebuf[len - 1] != '\n')
+            return 0;
+        if ((flags & SMIME_CRLFEOL) != 0) {
+            if (len <= 1 || linebuf[len - 2] != '\r')
+                return 0;
+            len--;
+        }
+        len--;
+        *plen = len;
+        return 1;
+    }
+#endif
+
     for (p = linebuf + len - 1; len > 0; len--, p--) {
         c = *p;
         if (c == '\n') {
             is_eol = 1;
-        } else if (is_eol && flags & SMIME_ASCIICRLF && c == 32) {
+        } else if (is_eol && (flags & SMIME_ASCIICRLF) != 0 && c == 32) {
             /* Strip trailing space on a line; 32 == ASCII for ' ' */
             continue;
         } else if (c != '\r') {
diff --git a/crypto/cms/cms_io.c b/crypto/cms/cms_io.c
index 6b71ddfa90..9c260d0904 100644
--- a/crypto/cms/cms_io.c
+++ b/crypto/cms/cms_io.c
@@ -90,11 +90,11 @@ int SMIME_write_CMS(BIO *bio, CMS_ContentInfo *cms, BIO *data, int flags)
                                ossl_cms_ctx_get0_propq(ctx));
 }
 
-CMS_ContentInfo *SMIME_read_CMS_ex(BIO *bio, BIO **bcont, CMS_ContentInfo **cms)
+CMS_ContentInfo *SMIME_read_CMS_ex(BIO *bio, int flags, BIO **bcont, CMS_ContentInfo **cms)
 {
     CMS_ContentInfo *ci;
 
-    ci = (CMS_ContentInfo *)SMIME_read_ASN1_ex(bio, bcont,
+    ci = (CMS_ContentInfo *)SMIME_read_ASN1_ex(bio, flags, bcont,
                                                ASN1_ITEM_rptr(CMS_ContentInfo),
                                                (ASN1_VALUE **)cms);
     if (ci != NULL)
@@ -104,5 +104,5 @@ CMS_ContentInfo *SMIME_read_CMS_ex(BIO *bio, BIO **bcont, CMS_ContentInfo **cms)
 
 CMS_ContentInfo *SMIME_read_CMS(BIO *bio, BIO **bcont)
 {
-    return SMIME_read_CMS_ex(bio, bcont, NULL);
+    return SMIME_read_CMS_ex(bio, 0, bcont, NULL);
 }
diff --git a/crypto/pkcs7/pk7_mime.c b/crypto/pkcs7/pk7_mime.c
index e191e4e3b8..b446423384 100644
--- a/crypto/pkcs7/pk7_mime.c
+++ b/crypto/pkcs7/pk7_mime.c
@@ -50,7 +50,7 @@ PKCS7 *SMIME_read_PKCS7_ex(BIO *bio, BIO **bcont, PKCS7 **p7)
 {
     PKCS7 *ret;
 
-    ret = (PKCS7 *)SMIME_read_ASN1_ex(bio, bcont, ASN1_ITEM_rptr(PKCS7),
+    ret = (PKCS7 *)SMIME_read_ASN1_ex(bio, 0, bcont, ASN1_ITEM_rptr(PKCS7),
                                       (ASN1_VALUE **)p7);
     if (ret != NULL)
         ossl_pkcs7_resolve_libctx(ret);
diff --git a/doc/man3/SMIME_read_ASN1.pod b/doc/man3/SMIME_read_ASN1.pod
index 189c1ef5e4..cb4a2ac9f9 100644
--- a/doc/man3/SMIME_read_ASN1.pod
+++ b/doc/man3/SMIME_read_ASN1.pod
@@ -9,15 +9,20 @@ SMIME_read_ASN1_ex, SMIME_read_ASN1
 
  #include <openssl/asn1.h>
 
- ASN1_VALUE *SMIME_read_ASN1_ex(BIO *in, BIO **bcont, const ASN1_ITEM *it,
-                                ASN1_VALUE **x);
+ ASN1_VALUE *SMIME_read_ASN1_ex(BIO *in, int flags, BIO **bcont,
+                                const ASN1_ITEM *it, ASN1_VALUE **x);
  ASN1_VALUE *SMIME_read_ASN1(BIO *in, BIO **bcont, const ASN1_ITEM *it);
 
 =head1 DESCRIPTION
 
 SMIME_read_ASN1_ex() parses a message in S/MIME format.
 
-I<in> is a BIO to read the message from. I<x> can be used to optionally supply
+I<in> is a BIO to read the message from.
+If the I<flags> argument contains B<CMS_BINARY> then the input is assumed to be
+in binary format and is not translated to canonical form.
+If in addition B<SMIME_ASCIICRLF> is set then the binary input is assumed
+to be followed by B<CR> and B<LF> characters, else only by an B<LF> character.
+I<x> can be used to optionally supply
 a previously created I<it> ASN1_VALUE object (such as CMS_ContentInfo or PKCS7),
 it can be set to NULL. Valid values that can be used by ASN.1 structure I<it>
 are ASN1_ITEM_rptr(PKCS7) or ASN1_ITEM_rptr(CMS_ContentInfo).
@@ -28,7 +33,7 @@ written to I<*bcont>, otherwise I<*bcont> is set to NULL.
 The parsed ASN1_VALUE structure is returned or NULL if an error occurred.
 
 SMIME_read_ASN1() is similar to SMIME_read_ASN1_ex() but sets the value of I<x>
-to NULL.
+to NULL and the value of I<flags> to 0.
 
 =head1 NOTES
 
diff --git a/doc/man3/SMIME_read_CMS.pod b/doc/man3/SMIME_read_CMS.pod
index 36ef6dc846..9f0c855263 100644
--- a/doc/man3/SMIME_read_CMS.pod
+++ b/doc/man3/SMIME_read_CMS.pod
@@ -8,7 +8,7 @@ SMIME_read_CMS_ex, SMIME_read_CMS - parse S/MIME message
 
  #include <openssl/cms.h>
 
- CMS_ContentInfo *SMIME_read_CMS_ex(BIO *bio, BIO **bcont,
+ CMS_ContentInfo *SMIME_read_CMS_ex(BIO *bio, int flags, BIO **bcont,
                                     CMS_ContentInfo **cms);
  CMS_ContentInfo *SMIME_read_CMS(BIO *in, BIO **bcont);
 
@@ -24,10 +24,14 @@ written to B<*bcont>, otherwise B<*bcont> is set to NULL.
 The parsed CMS_ContentInfo structure is returned or NULL if an
 error occurred.
 
-SMIME_read_CMS_ex() is similar to SMIME_read_CMS() but can optionally supply a
-previously created I<cms> CMS_ContentInfo object. If I<cms> is NULL then it is
-identical to SMIME_read_CMS().
+SMIME_read_CMS_ex() is similar to SMIME_read_CMS() but optionally a previously
+created I<cms> CMS_ContentInfo object can be supplied as well as some I<flags>.
 To create a I<cms> object use L<CMS_ContentInfo_new_ex(3)>.
+If the I<flags> argument contains B<CMS_BINARY> then the input is assumed to be
+in binary format and is not translated to canonical form.
+If in addition B<SMIME_ASCIICRLF> is set then the binary input is assumed
+to be followed by B<CR> and B<LF> characters, else only by an B<LF> character.
+If I<flags> is 0 and I<cms> is NULL then it is identical to SMIME_read_CMS().
 
 =head1 NOTES
 
diff --git a/include/openssl/asn1.h.in b/include/openssl/asn1.h.in
index 0ee82e7d58..36abcff28c 100644
--- a/include/openssl/asn1.h.in
+++ b/include/openssl/asn1.h.in
@@ -919,7 +919,7 @@ int SMIME_write_ASN1_ex(BIO *bio, ASN1_VALUE *val, BIO *data, int flags,
                         STACK_OF(X509_ALGOR) *mdalgs, const ASN1_ITEM *it,
                         OSSL_LIB_CTX *libctx, const char *propq);
 ASN1_VALUE *SMIME_read_ASN1(BIO *bio, BIO **bcont, const ASN1_ITEM *it);
-ASN1_VALUE *SMIME_read_ASN1_ex(BIO *bio, BIO **bcont, const ASN1_ITEM *it,
+ASN1_VALUE *SMIME_read_ASN1_ex(BIO *bio, int flags, BIO **bcont, const ASN1_ITEM *it,
                                ASN1_VALUE **x);
 int SMIME_crlf_copy(BIO *in, BIO *out, int flags);
 int SMIME_text(BIO *in, BIO *out);
diff --git a/include/openssl/cms.h.in b/include/openssl/cms.h.in
index 451191b796..da20ddf2f4 100644
--- a/include/openssl/cms.h.in
+++ b/include/openssl/cms.h.in
@@ -114,7 +114,7 @@ int i2d_CMS_bio_stream(BIO *out, CMS_ContentInfo *cms, BIO *in, int flags);
 int PEM_write_bio_CMS_stream(BIO *out, CMS_ContentInfo *cms, BIO *in,
                              int flags);
 CMS_ContentInfo *SMIME_read_CMS(BIO *bio, BIO **bcont);
-CMS_ContentInfo *SMIME_read_CMS_ex(BIO *bio, BIO **bcont, CMS_ContentInfo **ci);
+CMS_ContentInfo *SMIME_read_CMS_ex(BIO *bio, int flags, BIO **bcont, CMS_ContentInfo **ci);
 int SMIME_write_CMS(BIO *bio, CMS_ContentInfo *cms, BIO *data, int flags);
 
 int CMS_final(CMS_ContentInfo *cms, BIO *data, BIO *dcont,