ENCODER & DECODER: Allow decoder implementations to specify "carry on"

So far, decoder implementations would return true (1) for a successful
decode all the way, including what the callback it called returned,
and false (0) in all other cases.

This construction didn't allow to stop to decoding process on fatal
errors, nor to choose what to report in the provider code.

This is now changed so that decoders implementations are made to
return false only on errors that should stop the decoding process from
carrying on with other implementations, and return true for all other
cases, even if that didn't result in a constructed object (EVP_PKEY
for example), essentially making it OK to return "empty handed".

The success of the decoding process is now all about successfully
constructing the final object, rather than about the return value of
the decoding chain.  If no construction is attempted, the central
decoding processing code concludes that whatever the input consisted
of, it's not supported by the available decoder implementations.

Fixes #14423

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14834)

6 files changed, 100 insertions, 26 deletions

diff --git a/crypto/encode_decode/decoder_err.c b/crypto/encode_decode/decoder_err.c
index cf68a4c7c5..1880c8f409 100644
--- a/crypto/encode_decode/decoder_err.c
+++ b/crypto/encode_decode/decoder_err.c
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -15,6 +15,8 @@
 #ifndef OPENSSL_NO_ERR
 
 static const ERR_STRING_DATA OSSL_DECODER_str_reasons[] = {
+    {ERR_PACK(ERR_LIB_OSSL_DECODER, 0, OSSL_DECODER_R_COULD_NOT_DECODE_OBJECT),
+    "could not decode object"},
     {ERR_PACK(ERR_LIB_OSSL_DECODER, 0, OSSL_DECODER_R_MISSING_GET_PARAMS),
     "missing get params"},
     {0, NULL}
diff --git a/crypto/encode_decode/decoder_lib.c b/crypto/encode_decode/decoder_lib.c
index a644924aeb..e37989fec4 100644
--- a/crypto/encode_decode/decoder_lib.c
+++ b/crypto/encode_decode/decoder_lib.c
@@ -32,6 +32,12 @@ struct decoder_process_data_st {
     size_t current_decoder_inst_index;
     /* For tracing, count recursion level */
     size_t recursion;
+
+    /*-
+     * Flags
+     */
+    unsigned int flag_next_level_called : 1;
+    unsigned int flag_construct_called : 1;
 };
 
 static int decoder_process(const OSSL_PARAM params[], void *arg);
@@ -57,6 +63,29 @@ int OSSL_DECODER_from_bio(OSSL_DECODER_CTX *ctx, BIO *in)
 
     ok = decoder_process(NULL, &data);
 
+    if (!data.flag_construct_called) {
+        const char *spaces
+            = ctx->start_input_type != NULL && ctx->input_structure != NULL
+            ? " " : "";
+        const char *input_type_label
+            = ctx->start_input_type != NULL ? "Input type: " : "";
+        const char *input_structure_label
+            = ctx->input_structure != NULL ? "Input structure: " : "";
+        const char *comma
+            = ctx->start_input_type != NULL && ctx->input_structure != NULL
+            ? ", " : "";
+        const char *input_type
+            = ctx->start_input_type != NULL ? ctx->start_input_type : "";
+        const char *input_structure
+            = ctx->input_structure != NULL ? ctx->input_structure : "";
+
+        ERR_raise_data(ERR_LIB_OSSL_DECODER, ERR_R_UNSUPPORTED,
+                       "No supported for the data to decode.%s%s%s%s%s%s",
+                       spaces, input_type_label, input_type, comma,
+                       input_structure_label, input_structure);
+        ok = 0;
+    }
+
     /* Clear any internally cached passphrase */
     (void)ossl_pw_clear_passphrase_cache(&ctx->pwdata);
 
@@ -525,12 +554,18 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
     BIO *bio = data->bio;
     long loc;
     size_t i;
-    int err, lib, reason, ok = 0;
+    int ok = 0;
     /* For recursions */
     struct decoder_process_data_st new_data;
     const char *data_type = NULL;
     const char *data_structure = NULL;
 
+    /*
+     * This is an indicator up the call stack that something was indeed
+     * decoded, leading to a recursive call of this function.
+     */
+    data->flag_next_level_called = 1;
+
     memset(&new_data, 0, sizeof(new_data));
     new_data.ctx = data->ctx;
     new_data.recursion = data->recursion + 1;
@@ -562,10 +597,14 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
                                            data->current_decoder_inst_index);
         decoder = OSSL_DECODER_INSTANCE_get_decoder(decoder_inst);
 
-        if (ctx->construct != NULL
-            && ctx->construct(decoder_inst, params, ctx->construct_data)) {
-            ok = 1;
-            goto end;
+        data->flag_construct_called = 0;
+        if (ctx->construct != NULL) {
+            int rv = ctx->construct(decoder_inst, params, ctx->construct_data);
+
+            data->flag_construct_called = 1;
+            ok = (rv > 0);
+            if (ok)
+                goto end;
         }
 
         /* The constructor didn't return success */
@@ -746,6 +785,12 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
                        (void *)new_decoder_inst);
         } OSSL_TRACE_END(DECODER);
 
+        /*
+         * We only care about errors reported from decoder implementations
+         * if it returns false (i.e. there was a fatal error).
+         */
+        ERR_set_mark();
+
         new_data.current_decoder_inst_index = i;
         ok = new_decoder->decode(new_decoderctx, cbio,
                                  new_data.ctx->selection,
@@ -755,31 +800,29 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
 
         OSSL_TRACE_BEGIN(DECODER) {
             BIO_printf(trc_out,
-                       "(ctx %p) %s [%u] Running decoder instance %p => %d\n",
+                       "(ctx %p) %s [%u] Running decoder instance %p => %d"
+                       " (recursed further: %s, construct called: %s)\n",
                        (void *)new_data.ctx, LEVEL, (unsigned int)i,
-                       (void *)new_decoder_inst, ok);
+                       (void *)new_decoder_inst, ok,
+                       new_data.flag_next_level_called ? "yes" : "no",
+                       new_data.flag_construct_called ? "yes" : "no");
         } OSSL_TRACE_END(DECODER);
 
-        if (ok)
+        data->flag_construct_called = new_data.flag_construct_called;
+
+        /* Break on error or if we tried to construct an object already */
+        if (!ok || data->flag_construct_called) {
+            ERR_clear_last_mark();
             break;
+        }
+        ERR_pop_to_mark();
 
         /*
-         * These errors are assumed to come from ossl_store_handle_load_result()
-         * in crypto/store/store_result.c.  They are currently considered fatal
-         * errors, so we preserve them in the error queue and stop.
+         * Break if the decoder implementation that we called recursed, since
+         * that indicates that it successfully decoded something.
          */
-        err = ERR_peek_last_error();
-        lib = ERR_GET_LIB(err);
-        reason = ERR_GET_REASON(err);
-        if ((lib == ERR_LIB_EVP
-             && reason == EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM)
-#ifndef OPENSSL_NO_EC
-            || (lib == ERR_LIB_EC && reason == EC_R_UNKNOWN_GROUP)
-#endif
-            || (lib == ERR_LIB_X509 && reason == X509_R_UNSUPPORTED_ALGORITHM)
-            || (lib == ERR_LIB_PKCS12
-                && reason == PKCS12_R_PKCS12_CIPHERFINAL_ERROR))
-            goto end;
+        if (new_data.flag_next_level_called)
+            break;
     }
 
  end:
diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index eed0b71ada..81f9f1ef49 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -811,6 +811,7 @@ OCSP_R_STATUS_TOO_OLD:127:status too old
 OCSP_R_UNKNOWN_MESSAGE_DIGEST:119:unknown message digest
 OCSP_R_UNKNOWN_NID:120:unknown nid
 OCSP_R_UNSUPPORTED_REQUESTORNAME_TYPE:129:unsupported requestorname type
+OSSL_DECODER_R_COULD_NOT_DECODE_OBJECT:101:could not decode object
 OSSL_DECODER_R_MISSING_GET_PARAMS:100:missing get params
 OSSL_ENCODER_R_ENCODER_NOT_FOUND:101:encoder not found
 OSSL_ENCODER_R_INCORRECT_PROPERTY_QUERY:100:incorrect property query
diff --git a/doc/man7/provider-decoder.pod b/doc/man7/provider-decoder.pod
index 73f653e063..23b4fbc9df 100644
--- a/doc/man7/provider-decoder.pod
+++ b/doc/man7/provider-decoder.pod
@@ -210,6 +210,32 @@ The decoding functions also take an B<OSSL_PASSPHRASE_CALLBACK> function
 pointer along with a pointer to application data I<cbarg>, which should be
 used when a pass phrase prompt is needed.
 
+It's important to understand that the return value from this function is
+interpreted as follows:
+
+=over 4
+
+=item True (1)
+
+This means "carry on the decoding process", and is meaningful even though
+this function couldn't decode the input into anything, because there may be
+another decoder implementation that can decode it into something.
+
+The I<data_cb> callback should never be called when this function can't
+decode the input into anything.
+
+=item False (0)
+
+This means "stop the decoding process", and is meaningful when the input
+could be decoded into some sort of object that this function understands,
+but further treatment of that object results into errors that won't be
+possible for some other decoder implementation to get a different result.
+
+=back
+
+The conditions to stop the decoding process are at the discretion of the
+implementation.
+
 =head2 Decoder parameters
 
 The decoder implementation itself has parameters that can be used to
@@ -315,7 +341,8 @@ constant B<OSSL_PARAM> elements.
 OSSL_FUNC_decoder_does_selection() returns 1 if the decoder implementation
 supports any of the I<selection> bits, otherwise 0.
 
-OSSL_FUNC_decoder_decode() returns 1 on success, or 0 on failure.
+OSSL_FUNC_decoder_decode() returns 1 to signal that the decoding process
+should continue, or 0 to signal that it should stop.
 
 =head1 SEE ALSO
 
diff --git a/include/crypto/decodererr.h b/include/crypto/decodererr.h
index c19f70c1c6..edf826798d 100644
--- a/include/crypto/decodererr.h
+++ b/include/crypto/decodererr.h
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
diff --git a/include/openssl/decodererr.h b/include/openssl/decodererr.h
index 886c3750fe..824a0a9253 100644
--- a/include/openssl/decodererr.h
+++ b/include/openssl/decodererr.h
@@ -21,6 +21,7 @@
 /*
  * OSSL_DECODER reason codes.
  */
+# define OSSL_DECODER_R_COULD_NOT_DECODE_OBJECT           101
 # define OSSL_DECODER_R_MISSING_GET_PARAMS                100
 
 #endif