RFC7250 (RPK) support

Add support for the RFC7250 certificate-type extensions.
Alows the use of only private keys for connection (i.e. certs not needed).

Add APIs
Add unit tests
Add documentation
Add s_client/s_server support

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Viktor Dukhovni <viktor@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18185)

3 files changed, 204 insertions, 20 deletions

diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index 3f7f4bf109..330aba84c7 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -1280,6 +1280,7 @@ SSL_R_ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT:272:\
 	attempt to reuse session in different context
 SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE:158:\
 	at least (D)TLS 1.2 needed in Suite B mode
+SSL_R_BAD_CERTIFICATE:348:bad certificate
 SSL_R_BAD_CHANGE_CIPHER_SPEC:103:bad change cipher spec
 SSL_R_BAD_CIPHER:186:bad cipher
 SSL_R_BAD_COMPRESSION_ALGORITHM:326:bad compression algorithm
@@ -1369,6 +1370,7 @@ SSL_R_DUPLICATE_COMPRESSION_ID:309:duplicate compression id
 SSL_R_ECC_CERT_NOT_FOR_SIGNING:318:ecc cert not for signing
 SSL_R_ECDH_REQUIRED_FOR_SUITEB_MODE:374:ecdh required for suiteb mode
 SSL_R_EE_KEY_TOO_SMALL:399:ee key too small
+SSL_R_EMPTY_RAW_PUBLIC_KEY:349:empty raw public key
 SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST:354:empty srtp protection profile list
 SSL_R_ENCRYPTED_LENGTH_TOO_LONG:150:encrypted length too long
 SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST:151:error in received cipher list
@@ -1404,6 +1406,7 @@ SSL_R_INVALID_CT_VALIDATION_TYPE:212:invalid ct validation type
 SSL_R_INVALID_KEY_UPDATE_TYPE:120:invalid key update type
 SSL_R_INVALID_MAX_EARLY_DATA:174:invalid max early data
 SSL_R_INVALID_NULL_CMD_NAME:385:invalid null cmd name
+SSL_R_INVALID_RAW_PUBLIC_KEY:350:invalid raw public key
 SSL_R_INVALID_RECORD:317:invalid record
 SSL_R_INVALID_SEQUENCE_NUMBER:402:invalid sequence number
 SSL_R_INVALID_SERVERINFO_DATA:388:invalid serverinfo data
@@ -1612,6 +1615,7 @@ SSL_R_VERSION_TOO_LOW:396:version too low
 SSL_R_WRONG_CERTIFICATE_TYPE:383:wrong certificate type
 SSL_R_WRONG_CIPHER_RETURNED:261:wrong cipher returned
 SSL_R_WRONG_CURVE:378:wrong curve
+SSL_R_WRONG_RPK_TYPE:351:wrong rpk type
 SSL_R_WRONG_SIGNATURE_LENGTH:264:wrong signature length
 SSL_R_WRONG_SIGNATURE_SIZE:265:wrong signature size
 SSL_R_WRONG_SIGNATURE_TYPE:370:wrong signature type
diff --git a/crypto/x509/x509_txt.c b/crypto/x509/x509_txt.c
index 8fb38cf3e9..e825ce2db8 100644
--- a/crypto/x509/x509_txt.c
+++ b/crypto/x509/x509_txt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -212,6 +212,8 @@ const char *X509_verify_cert_error_string(long n)
         return "Using cert extension requires at least X509v3";
     case X509_V_ERR_EC_KEY_EXPLICIT_PARAMS:
         return "Certificate public key has explicit ECC parameters";
+    case X509_V_ERR_RPK_UNTRUSTED:
+        return "Raw public key untrusted, no trusted keys configured";
 
         /*
          * Entries must be kept consistent with include/openssl/x509_vfy.h.in
diff --git a/crypto/x509/x509_vfy.c b/crypto/x509/x509_vfy.c
index 951beafb3e..cc02c1ccc4 100644
--- a/crypto/x509/x509_vfy.c
+++ b/crypto/x509/x509_vfy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -41,9 +41,13 @@
 #define CRL_SCORE_AKID          0x004 /* CRL issuer matches CRL AKID */
 #define CRL_SCORE_TIME_DELTA    0x002 /* Have a delta CRL with valid times */
 
+static int x509_verify_x509(X509_STORE_CTX *ctx);
+static int x509_verify_rpk(X509_STORE_CTX *ctx);
 static int build_chain(X509_STORE_CTX *ctx);
 static int verify_chain(X509_STORE_CTX *ctx);
+static int verify_rpk(X509_STORE_CTX *ctx);
 static int dane_verify(X509_STORE_CTX *ctx);
+static int dane_verify_rpk(X509_STORE_CTX *ctx);
 static int null_callback(int ok, X509_STORE_CTX *e);
 static int check_issued(X509_STORE_CTX *ctx, X509 *x, X509 *issuer);
 static X509 *find_issuer(X509_STORE_CTX *ctx, STACK_OF(X509) *sk, X509 *x);
@@ -56,7 +60,8 @@ static int check_cert(X509_STORE_CTX *ctx);
 static int check_policy(X509_STORE_CTX *ctx);
 static int get_issuer_sk(X509 **issuer, X509_STORE_CTX *ctx, X509 *x);
 static int check_dane_issuer(X509_STORE_CTX *ctx, int depth);
-static int check_key_level(X509_STORE_CTX *ctx, X509 *cert);
+static int check_cert_key_level(X509_STORE_CTX *ctx, X509 *cert);
+static int check_key_level(X509_STORE_CTX *ctx, EVP_PKEY *pkey);
 static int check_sig_level(X509_STORE_CTX *ctx, X509 *cert);
 static int check_curve(X509 *cert);
 
@@ -197,7 +202,7 @@ static int check_auth_level(X509_STORE_CTX *ctx)
          * We've already checked the security of the leaf key, so here we only
          * check the security of issuer keys.
          */
-        CB_FAIL_IF(i > 0 && !check_key_level(ctx, cert),
+        CB_FAIL_IF(i > 0 && !check_cert_key_level(ctx, cert),
                    ctx, cert, i, X509_V_ERR_CA_KEY_TOO_SMALL);
         /*
          * We also check the signature algorithm security of all certificates
@@ -213,6 +218,20 @@ static int check_auth_level(X509_STORE_CTX *ctx)
  * Returns -1 on internal error.
  * Sadly, returns 0 also on internal error in ctx->verify_cb().
  */
+static int verify_rpk(X509_STORE_CTX *ctx)
+{
+    /* Not much to verify on a RPK */
+    if (ctx->verify != NULL)
+        return ctx->verify(ctx);
+
+    return !!ctx->verify_cb(ctx->error == X509_V_OK, ctx);
+}
+
+
+/*-
+ * Returns -1 on internal error.
+ * Sadly, returns 0 also on internal error in ctx->verify_cb().
+ */
 static int verify_chain(X509_STORE_CTX *ctx)
 {
     int err;
@@ -258,23 +277,58 @@ int X509_STORE_CTX_verify(X509_STORE_CTX *ctx)
         ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER);
         return -1;
     }
+    if (ctx->rpk != NULL)
+        return x509_verify_rpk(ctx);
     if (ctx->cert == NULL && sk_X509_num(ctx->untrusted) >= 1)
         ctx->cert = sk_X509_value(ctx->untrusted, 0);
-    return X509_verify_cert(ctx);
+    return x509_verify_x509(ctx);
+}
+
+int X509_verify_cert(X509_STORE_CTX *ctx)
+{
+    if (ctx == NULL) {
+        ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER);
+        return -1;
+    }
+    return (ctx->rpk != NULL) ? x509_verify_rpk(ctx) : x509_verify_x509(ctx);
 }
 
 /*-
  * Returns -1 on internal error.
  * Sadly, returns 0 also on internal error in ctx->verify_cb().
  */
-int X509_verify_cert(X509_STORE_CTX *ctx)
+static int x509_verify_rpk(X509_STORE_CTX *ctx)
+{
+    int ret;
+
+    /* If the peer's public key is too weak, we can stop early. */
+    if (!check_key_level(ctx, ctx->rpk)
+        && verify_cb_cert(ctx, NULL, 0, X509_V_ERR_EE_KEY_TOO_SMALL) == 0)
+        return 0;
+
+    /* Barring any data to verify the RPK, simply report it as untrusted */
+    ctx->error = X509_V_ERR_RPK_UNTRUSTED;
+
+    ret = DANETLS_ENABLED(ctx->dane) ? dane_verify_rpk(ctx) : verify_rpk(ctx);
+
+    /*
+     * Safety-net.  If we are returning an error, we must also set ctx->error,
+     * so that the chain is not considered verified should the error be ignored
+     * (e.g. TLS with SSL_VERIFY_NONE).
+     */
+    if (ret <= 0 && ctx->error == X509_V_OK)
+        ctx->error = X509_V_ERR_UNSPECIFIED;
+    return ret;
+}
+
+/*-
+ * Returns -1 on internal error.
+ * Sadly, returns 0 also on internal error in ctx->verify_cb().
+ */
+static int x509_verify_x509(X509_STORE_CTX *ctx)
 {
     int ret;
 
-    if (ctx == NULL) {
-        ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER);
-        return -1;
-    }
     if (ctx->cert == NULL) {
         ERR_raise(ERR_LIB_X509, X509_R_NO_CERT_SET_FOR_US_TO_VERIFY);
         ctx->error = X509_V_ERR_INVALID_CALL;
@@ -298,7 +352,7 @@ int X509_verify_cert(X509_STORE_CTX *ctx)
     ctx->num_untrusted = 1;
 
     /* If the peer's public key is too weak, we can stop early. */
-    CB_FAIL_IF(!check_key_level(ctx, ctx->cert),
+    CB_FAIL_IF(!check_cert_key_level(ctx, ctx->cert),
                ctx, ctx->cert, 0, X509_V_ERR_EE_KEY_TOO_SMALL);
 
     ret = DANETLS_ENABLED(ctx->dane) ? dane_verify(ctx) : verify_chain(ctx);
@@ -1758,9 +1812,19 @@ int ossl_x509_check_cert_time(X509_STORE_CTX *ctx, X509 *x, int depth)
  */
 static int internal_verify(X509_STORE_CTX *ctx)
 {
-    int n = sk_X509_num(ctx->chain) - 1;
-    X509 *xi = sk_X509_value(ctx->chain, n);
-    X509 *xs = xi;
+    int n;
+    X509 *xi;
+    X509 *xs;
+
+    /* For RPK: just do the verify callback */
+    if (ctx->rpk != NULL) {
+        if (!ctx->verify_cb(ctx->error == X509_V_OK, ctx))
+            return 0;
+        return 1;
+    }
+    n = sk_X509_num(ctx->chain) - 1;
+    xi = sk_X509_value(ctx->chain, n);
+    xs = xi;
 
     ctx->error_depth = n;
     if (ctx->bare_ta_signed) {
@@ -2227,6 +2291,11 @@ void X509_STORE_CTX_set_cert(X509_STORE_CTX *ctx, X509 *x)
     ctx->cert = x;
 }
 
+void X509_STORE_CTX_set0_rpk(X509_STORE_CTX *ctx, EVP_PKEY *rpk)
+{
+    ctx->rpk = rpk;
+}
+
 void X509_STORE_CTX_set0_crls(X509_STORE_CTX *ctx, STACK_OF(X509_CRL) *sk)
 {
     ctx->crls = sk;
@@ -2348,6 +2417,15 @@ void X509_STORE_CTX_free(X509_STORE_CTX *ctx)
     OPENSSL_free(ctx);
 }
 
+
+int X509_STORE_CTX_init_rpk(X509_STORE_CTX *ctx, X509_STORE *store, EVP_PKEY *rpk)
+{
+    if (!X509_STORE_CTX_init(ctx, store, NULL, NULL))
+        return 0;
+    ctx->rpk = rpk;
+    return 1;
+}
+
 int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *store, X509 *x509,
                         STACK_OF(X509) *chain)
 {
@@ -2377,6 +2455,7 @@ int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *store, X509 *x509,
     ctx->parent = NULL;
     ctx->dane = NULL;
     ctx->bare_ta_signed = 0;
+    ctx->rpk = NULL;
     /* Zero ex_data to make sure we're cleanup-safe */
     memset(&ctx->ex_data, 0, sizeof(ctx->ex_data));
 
@@ -2540,6 +2619,11 @@ X509 *X509_STORE_CTX_get0_cert(const X509_STORE_CTX *ctx)
     return ctx->cert;
 }
 
+EVP_PKEY *X509_STORE_CTX_get0_rpk(const X509_STORE_CTX *ctx)
+{
+    return ctx->rpk;
+}
+
 STACK_OF(X509) *X509_STORE_CTX_get0_untrusted(const X509_STORE_CTX *ctx)
 {
     return ctx->untrusted;
@@ -2712,7 +2796,7 @@ static unsigned char *dane_i2d(X509 *cert, uint8_t selector,
 #define DANETLS_NONE 256 /* impossible uint8_t */
 
 /* Returns -1 on internal error */
-static int dane_match(X509_STORE_CTX *ctx, X509 *cert, int depth)
+static int dane_match_cert(X509_STORE_CTX *ctx, X509 *cert, int depth)
 {
     SSL_DANE *dane = ctx->dane;
     unsigned usage = DANETLS_NONE;
@@ -2870,7 +2954,7 @@ static int check_dane_issuer(X509_STORE_CTX *ctx, int depth)
      * for an exact match for the leaf certificate).
      */
     cert = sk_X509_value(ctx->chain, depth);
-    if (cert != NULL && (matched = dane_match(ctx, cert, depth)) < 0)
+    if (cert != NULL && (matched = dane_match_cert(ctx, cert, depth)) < 0)
         return matched;
     if (matched > 0) {
         ctx->num_untrusted = depth - 1;
@@ -2917,6 +3001,62 @@ static int check_dane_pkeys(X509_STORE_CTX *ctx)
     return X509_TRUST_UNTRUSTED;
 }
 
+/*
+ * Only DANE-EE and SPKI are supported
+ * Returns -1 on internal error
+ */
+static int dane_match_rpk(X509_STORE_CTX *ctx, EVP_PKEY *rpk)
+{
+    SSL_DANE *dane = ctx->dane;
+    danetls_record *t = NULL;
+    int mtype = DANETLS_MATCHING_FULL;
+    unsigned char *i2dbuf = NULL;
+    unsigned int i2dlen = 0;
+    unsigned char mdbuf[EVP_MAX_MD_SIZE];
+    unsigned char *cmpbuf;
+    unsigned int cmplen = 0;
+    int len;
+    int recnum = sk_danetls_record_num(dane->trecs);
+    int i;
+    int matched = 0;
+
+    /* Calculate ASN.1 DER of RPK */
+    if ((len = i2d_PUBKEY(rpk, &i2dbuf)) <= 0)
+        return -1;
+    cmplen = i2dlen = (unsigned int)len;
+    cmpbuf = i2dbuf;
+
+    for (i = 0; i < recnum; i++) {
+        t = sk_danetls_record_value(dane->trecs, i);
+        if (t->usage != DANETLS_USAGE_DANE_EE || t->selector != DANETLS_SELECTOR_SPKI)
+            continue;
+
+        /* Calculate hash - keep only one around */
+        if (t->mtype != mtype) {
+            const EVP_MD *md = dane->dctx->mdevp[mtype = t->mtype];
+
+            cmpbuf = i2dbuf;
+            cmplen = i2dlen;
+
+            if (md != NULL) {
+                cmpbuf = mdbuf;
+                if (!EVP_Digest(i2dbuf, i2dlen, cmpbuf, &cmplen, md, 0)) {
+                    matched = -1;
+                    break;
+                }
+            }
+        }
+        if (cmplen == t->dlen && memcmp(cmpbuf, t->data, cmplen) == 0) {
+            matched = 1;
+            dane->mdpth = 0;
+            dane->mtlsa = t;
+            break;
+        }
+    }
+    OPENSSL_free(i2dbuf);
+    return matched;
+}
+
 static void dane_reset(SSL_DANE *dane)
 {
     /* Reset state to verify another chain, or clear after failure. */
@@ -2937,6 +3077,36 @@ static int check_leaf_suiteb(X509_STORE_CTX *ctx, X509 *cert)
 }
 
 /* Returns -1 on internal error */
+static int dane_verify_rpk(X509_STORE_CTX *ctx)
+{
+    SSL_DANE *dane = ctx->dane;
+    int matched;
+
+    dane_reset(dane);
+
+    /*
+     * Look for a DANE record for RPK
+     * If error, return -1
+     * If found, call ctx->verify_cb(1, ctx)
+     * If not found call ctx->verify_cb(0, ctx)
+     */
+    matched = dane_match_rpk(ctx, ctx->rpk);
+    ctx->error_depth = 0;
+
+    if (matched < 0) {
+        ctx->error = X509_V_ERR_UNSPECIFIED;
+        return -1;
+    }
+
+    if (matched > 0)
+        ctx->error = X509_V_OK;
+    else
+        ctx->error = X509_V_ERR_DANE_NO_MATCH;
+
+    return verify_rpk(ctx);
+}
+
+/* Returns -1 on internal error */
 static int dane_verify(X509_STORE_CTX *ctx)
 {
     X509 *cert = ctx->cert;
@@ -2958,7 +3128,7 @@ static int dane_verify(X509_STORE_CTX *ctx)
      *   + matched == 0, mdepth < 0 (no PKIX-EE match) and there are no
      *     DANE-TA(2) or PKIX-TA(0) to test.
      */
-    matched = dane_match(ctx, ctx->cert, 0);
+    matched = dane_match_cert(ctx, ctx->cert, 0);
     done = matched != 0 || (!DANETLS_HAS_TA(dane) && dane->mdpth < 0);
 
     if (done && !X509_get_pubkey_parameters(NULL, ctx->chain))
@@ -3416,12 +3586,11 @@ static const int minbits_table[] = { 80, 112, 128, 192, 256 };
 static const int NUM_AUTH_LEVELS = OSSL_NELEM(minbits_table);
 
 /*-
- * Check whether the public key of `cert` meets the security level of `ctx`.
+ * Check whether the given public key meets the security level of `ctx`.
  * Returns 1 on success, 0 otherwise.
  */
-static int check_key_level(X509_STORE_CTX *ctx, X509 *cert)
+static int check_key_level(X509_STORE_CTX *ctx, EVP_PKEY *pkey)
 {
-    EVP_PKEY *pkey = X509_get0_pubkey(cert);
     int level = ctx->param->auth_level;
 
     /*
@@ -3444,6 +3613,15 @@ static int check_key_level(X509_STORE_CTX *ctx, X509 *cert)
 }
 
 /*-
+ * Check whether the public key of `cert` meets the security level of `ctx`.
+ * Returns 1 on success, 0 otherwise.
+ */
+static int check_cert_key_level(X509_STORE_CTX *ctx, X509 *cert)
+{
+    return check_key_level(ctx, X509_get0_pubkey(cert));
+}
+
+/*-
  * Check whether the public key of ``cert`` does not use explicit params
  * for an elliptic curve.
  *