eng_dyn: Avoid spurious errors when checking for 1.1.x engine

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17901)

(cherry picked from commit bd5c91c82cdc4b6ffe4a2970f9512fc5ec7d2d06)

1 files changed, 22 insertions, 11 deletions

diff --git a/crypto/engine/eng_dyn.c b/crypto/engine/eng_dyn.c
index 3c0f960fc5..6d402927c5 100644
--- a/crypto/engine/eng_dyn.c
+++ b/crypto/engine/eng_dyn.c
@@ -401,6 +401,26 @@ static int int_load(dynamic_data_ctx *ctx)
     return 0;
 }
 
+/*
+ * Unfortunately the version checker does not distinguish between
+ * engines built for openssl 1.1.x and openssl 3.x, but loading
+ * an engine that is built for openssl 1.1.x will cause a fatal
+ * error.  Detect such engines, since EVP_PKEY_base_id is exported
+ * as a function in openssl 1.1.x, while it is named EVP_PKEY_get_base_id
+ * in openssl 3.x.  Therefore we take the presence of that symbol
+ * as an indication that the engine will be incompatible.
+ */
+static int using_libcrypto_11(dynamic_data_ctx *ctx)
+{
+    int ret;
+
+    ERR_set_mark();
+    ret = DSO_bind_func(ctx->dynamic_dso, "EVP_PKEY_base_id") != NULL;
+    ERR_pop_to_mark();
+
+    return ret;
+}
+
 static int dynamic_load(ENGINE *e, dynamic_data_ctx *ctx)
 {
     ENGINE cpy;
@@ -450,18 +470,9 @@ static int dynamic_load(ENGINE *e, dynamic_data_ctx *ctx)
         /*
          * We fail if the version checker veto'd the load *or* if it is
          * deferring to us (by returning its version) and we think it is too
-         * old.
-         * Unfortunately the version checker does not distinguish between
-         * engines built for openssl 1.1.x and openssl 3.x, but loading
-         * an engine that is built for openssl 1.1.x will cause a fatal
-         * error.  Detect such engines, since EVP_PKEY_base_id is exported
-         * as a function in openssl 1.1.x, while it is a macro in openssl 3.x,
-         * and therefore only the symbol EVP_PKEY_get_base_id is available
-         * in openssl 3.x.
+         * old. Also fail if this is engine for openssl 1.1.x.
          */
-        if (vcheck_res < OSSL_DYNAMIC_OLDEST
-                || DSO_bind_func(ctx->dynamic_dso,
-                                 "EVP_PKEY_base_id") != NULL) {
+        if (vcheck_res < OSSL_DYNAMIC_OLDEST || using_libcrypto_11(ctx)) {
             /* Fail */
             ctx->bind_engine = NULL;
             ctx->v_check = NULL;