[ec] Do not default to OPENSSL_EC_NAMED_CURVE for curves without OID

Some curves don't have an associated OID: for those we should not
default to `OPENSSL_EC_NAMED_CURVE` encoding of parameters and instead
set the ASN1 flag to `OPENSSL_EC_EXPLICIT_CURVE`.

This is a follow-up to https://github.com/openssl/openssl/pull/12312

(cherry picked from commit 7aa3dfc42104588f65301d20324388ac2c9a6b11)

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12457)

2 files changed, 28 insertions, 1 deletions

diff --git a/crypto/ec/ec_asn1.c b/crypto/ec/ec_asn1.c
index c8ee1e6f17..4335b3da1a 100644
--- a/crypto/ec/ec_asn1.c
+++ b/crypto/ec/ec_asn1.c
@@ -548,7 +548,7 @@ ECPKPARAMETERS *EC_GROUP_get_ecpkparameters(const EC_GROUP *group,
             ECPARAMETERS_free(ret->value.parameters);
     }
 
-    if (EC_GROUP_get_asn1_flag(group)) {
+    if (EC_GROUP_get_asn1_flag(group) == OPENSSL_EC_NAMED_CURVE) {
         /*
          * use the asn1 OID to describe the elliptic curve parameters
          */
diff --git a/crypto/ec/ec_curve.c b/crypto/ec/ec_curve.c
index 8de486cbd7..dfe5263f59 100644
--- a/crypto/ec/ec_curve.c
+++ b/crypto/ec/ec_curve.c
@@ -12,6 +12,7 @@
 #include "ec_local.h"
 #include <openssl/err.h>
 #include <openssl/obj_mac.h>
+#include <openssl/objects.h>
 #include <openssl/opensslconf.h>
 #include "internal/nelem.h"
 
@@ -3097,6 +3098,32 @@ static EC_GROUP *ec_group_new_from_data(const ec_list_element curve)
             goto err;
         }
     }
+
+    if (EC_GROUP_get_asn1_flag(group) == OPENSSL_EC_NAMED_CURVE) {
+        /*
+         * Some curves don't have an associated OID: for those we should not
+         * default to `OPENSSL_EC_NAMED_CURVE` encoding of parameters and
+         * instead set the ASN1 flag to `OPENSSL_EC_EXPLICIT_CURVE`.
+         *
+         * Note that `OPENSSL_EC_NAMED_CURVE` is set as the default ASN1 flag on
+         * `EC_GROUP_new()`, when we don't have enough elements to determine if
+         * an OID for the curve name actually exists.
+         * We could implement this check on `EC_GROUP_set_curve_name()` but
+         * overloading the simple setter with this lookup could have a negative
+         * performance impact and unexpected consequences.
+         */
+        ASN1_OBJECT *asn1obj = OBJ_nid2obj(curve.nid);
+
+        if (asn1obj == NULL) {
+            ECerr(EC_F_EC_GROUP_NEW_FROM_DATA, ERR_R_OBJ_LIB);
+            goto err;
+        }
+        if (OBJ_length(asn1obj) == 0)
+            EC_GROUP_set_asn1_flag(group, OPENSSL_EC_EXPLICIT_CURVE);
+
+        ASN1_OBJECT_free(asn1obj);
+    }
+
     ok = 1;
  err:
     if (!ok) {