S390x: Support ME and CRT offloading

S390x has to ability to offload modular exponentiation and CRT operations to
Crypto Express Adapters.  This possible performance optimization was not yet
used by OpenSSL.  Add support for offloading and implement an optimized
version of RSA and DH with it.

The environment variable OPENSSL_s390xcap now recognizes the token "nocex" to
prevent offloading.

Signed-off-by: Juergen Christ <jchrist@linux.ibm.com>

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/20113)

6 files changed, 276 insertions, 4 deletions

diff --git a/crypto/bn/bn_s390x.c b/crypto/bn/bn_s390x.c
new file mode 100644
index 0000000000..da69b02d90
--- /dev/null
+++ b/crypto/bn/bn_s390x.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2023-2023 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "crypto/bn.h"
+#include "crypto/s390x_arch.h"
+
+#ifdef S390X_MOD_EXP
+
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <asm/zcrypt.h>
+# include <sys/ioctl.h>
+# include <unistd.h>
+# include <errno.h>
+
+static int s390x_mod_exp_hw(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+                            const BIGNUM *m)
+{
+    struct ica_rsa_modexpo me;
+    unsigned char *buffer;
+    size_t size;
+    int res = 0;
+
+    if (OPENSSL_s390xcex == -1)
+        return 0;
+    size = BN_num_bytes(m);
+    buffer = OPENSSL_zalloc(4 * size);
+    if (buffer == NULL)
+        return 0;
+    me.inputdata = buffer;
+    me.inputdatalength = size;
+    me.outputdata = buffer + size;
+    me.outputdatalength = size;
+    me.b_key = buffer + 2 * size;
+    me.n_modulus = buffer + 3 * size;
+    if (BN_bn2binpad(a, me.inputdata, size) == -1
+        || BN_bn2binpad(p, me.b_key, size) == -1
+        || BN_bn2binpad(m, me.n_modulus, size) == -1)
+        goto dealloc;
+    if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) != -1) {
+        if (BN_bin2bn(me.outputdata, size, r) != NULL)
+            res = 1;
+    } else if (errno == EBADF) {
+        /*-
+         * In this cases, someone (e.g. a sandbox) closed the fd.
+         * Make sure to not further use this hardware acceleration.
+         */
+        OPENSSL_s390xcex = -1;
+    }
+ dealloc:
+    OPENSSL_clear_free(buffer, 4 * size);
+    return res;
+}
+
+int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+                  const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
+{
+    if (s390x_mod_exp_hw(r, a, p, m) == 1)
+        return 1;
+    return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+}
+
+int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
+              const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp)
+{
+    struct ica_rsa_modexpo_crt crt;
+    unsigned char *buffer, *part;
+    size_t size, plen, qlen;
+    int res = 0;
+
+    if (OPENSSL_s390xcex == -1)
+        return 0;
+    /*-
+     * Hardware-accelerated CRT can only deal with p>q.  Fall back to
+     * software in the (hopefully rare) other cases.
+     */
+    if (BN_ucmp(p, q) != 1)
+        return 0;
+    plen = BN_num_bytes(p);
+    qlen = BN_num_bytes(q);
+    size = (plen > qlen ? plen : qlen);
+    buffer = OPENSSL_zalloc(9 * size + 24);
+    if (buffer == NULL)
+        return 0;
+    part = buffer;
+    crt.inputdata = part;
+    crt.inputdatalength = 2 * size;
+    part += 2 * size;
+    crt.outputdata = part;
+    crt.outputdatalength = 2 * size;
+    part += 2 * size;
+    crt.bp_key = part;
+    part += size + 8;
+    crt.bq_key = part;
+    part += size;
+    crt.np_prime = part;
+    part += size + 8;
+    crt.nq_prime = part;
+    part += size;
+    crt.u_mult_inv = part;
+    if (BN_bn2binpad(i, crt.inputdata, crt.inputdatalength) == -1
+        || BN_bn2binpad(p, crt.np_prime, size + 8) == -1
+        || BN_bn2binpad(q, crt.nq_prime, size) == -1
+        || BN_bn2binpad(dmp, crt.bp_key, size + 8) == -1
+        || BN_bn2binpad(dmq, crt.bq_key, size) == -1
+        || BN_bn2binpad(iqmp, crt.u_mult_inv, size + 8) == -1)
+        goto dealloc;
+    if (ioctl(OPENSSL_s390xcex, ICARSACRT, &crt) != -1) {
+        if (BN_bin2bn(crt.outputdata, crt.outputdatalength, r) != NULL)
+            res = 1;
+    } else if (errno == EBADF) {
+        /*-
+         * In this cases, someone (e.g. a sandbox) closed the fd.
+         * Make sure to not further use this hardware acceleration.
+         */
+        OPENSSL_s390xcex = -1;
+    }
+ dealloc:
+    OPENSSL_clear_free(buffer, 9 * size + 24);
+    return res;
+}
+
+#else
+int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+                  const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
+{
+    return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+}
+
+int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
+              const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp)
+{
+    return 0;
+}
+
+#endif
diff --git a/crypto/bn/build.info b/crypto/bn/build.info
index 0732519a11..77ec6d4721 100644
--- a/crypto/bn/build.info
+++ b/crypto/bn/build.info
@@ -58,7 +58,7 @@ IF[{- !$disabled{asm} -}]
   IF[{- ($target{perlasm_scheme} // '') eq '31' -}]
     $BNASM_s390x=bn_asm.c s390x-mont.S
   ELSE
-    $BNASM_s390x=asm/s390x.S s390x-mont.S
+    $BNASM_s390x=asm/s390x.S s390x-mont.S bn_s390x.c
   ENDIF
   $BNDEF_s390x=OPENSSL_BN_ASM_MONT
   $BNASM_s390x_ec2m=s390x-gf2m.s
diff --git a/crypto/dh/dh_key.c b/crypto/dh/dh_key.c
index 0854395fc5..328780b9a9 100644
--- a/crypto/dh/dh_key.c
+++ b/crypto/dh/dh_key.c
@@ -184,7 +184,11 @@ static int dh_bn_mod_exp(const DH *dh, BIGNUM *r,
                          const BIGNUM *a, const BIGNUM *p,
                          const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
 {
+#ifdef S390X_MOD_EXP
+    return s390x_mod_exp(r, a, p, m, ctx, m_ctx);
+#else
     return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+#endif
 }
 
 static int dh_init(DH *dh)
diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index 10d08ebc35..ec69076e63 100644
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -33,6 +33,27 @@ static int rsa_ossl_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
                            BN_CTX *ctx);
 static int rsa_ossl_init(RSA *rsa);
 static int rsa_ossl_finish(RSA *rsa);
+#ifdef S390X_MOD_EXP
+static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
+                                BN_CTX *ctx);
+static RSA_METHOD rsa_pkcs1_ossl_meth = {
+    "OpenSSL PKCS#1 RSA",
+    rsa_ossl_public_encrypt,
+    rsa_ossl_public_decrypt,     /* signature verification */
+    rsa_ossl_private_encrypt,    /* signing */
+    rsa_ossl_private_decrypt,
+    rsa_ossl_s390x_mod_exp,
+    s390x_mod_exp,
+    rsa_ossl_init,
+    rsa_ossl_finish,
+    RSA_FLAG_FIPS_METHOD,       /* flags */
+    NULL,
+    0,                          /* rsa_sign */
+    0,                          /* rsa_verify */
+    NULL,                       /* rsa_keygen */
+    NULL                        /* rsa_multi_prime_keygen */
+};
+#else
 static RSA_METHOD rsa_pkcs1_ossl_meth = {
     "OpenSSL PKCS#1 RSA",
     rsa_ossl_public_encrypt,
@@ -51,6 +72,7 @@ static RSA_METHOD rsa_pkcs1_ossl_meth = {
     NULL,                       /* rsa_keygen */
     NULL                        /* rsa_multi_prime_keygen */
 };
+#endif
 
 static const RSA_METHOD *default_RSA_meth = &rsa_pkcs1_ossl_meth;
 
@@ -1118,3 +1140,16 @@ static int rsa_ossl_finish(RSA *rsa)
     BN_MONT_CTX_free(rsa->_method_mod_q);
     return 1;
 }
+
+#ifdef S390X_MOD_EXP
+static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
+                                BN_CTX *ctx)
+{
+    if (rsa->version != RSA_ASN1_VERSION_MULTI) {
+        if (s390x_crt(r0, i, rsa->p, rsa->q, rsa->dmp1, rsa->dmq1, rsa->iqmp) == 1)
+            return 1;
+    }
+    return rsa_ossl_mod_exp(r0, i, rsa, ctx);
+}
+
+#endif
diff --git a/crypto/s390x_arch.h b/crypto/s390x_arch.h
index c48cde8d9d..cfb3460d5b 100644
--- a/crypto/s390x_arch.h
+++ b/crypto/s390x_arch.h
@@ -12,6 +12,8 @@
 
 # ifndef __ASSEMBLER__
 
+#include "crypto/bn.h"
+
 void s390x_kimd(const unsigned char *in, size_t len, unsigned int fc,
                 void *param);
 void s390x_klmd(const unsigned char *in, size_t inlen, unsigned char *out,
@@ -77,6 +79,13 @@ __attribute__ ((visibility("hidden")))
 #endif
 extern struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P;
 
+#ifdef S390X_MOD_EXP
+# if defined(__GNUC__) && defined(__linux)
+__attribute__ ((visibility("hidden")))
+# endif
+extern int OPENSSL_s390xcex;
+#endif
+
 /* Max number of 64-bit words currently returned by STFLE */
 #  define S390X_STFLE_MAX       3
 
diff --git a/crypto/s390xcap.c b/crypto/s390xcap.c
index 59f0f3ef45..5d2e96805c 100644
--- a/crypto/s390xcap.c
+++ b/crypto/s390xcap.c
@@ -16,6 +16,15 @@
 #include "crypto/ctype.h"
 #include "s390x_arch.h"
 
+#if defined(OPENSSL_SYS_LINUX) && !defined(FIPS_MODULE)
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <asm/zcrypt.h>
+# include <sys/ioctl.h>
+# include <unistd.h>
+#endif
+
 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
 # if __GLIBC_PREREQ(2, 16)
 #  include <sys/auxv.h>
@@ -67,19 +76,41 @@ void OPENSSL_vx_probe(void);
 #endif
 
 static const char *env;
-static int parse_env(struct OPENSSL_s390xcap_st *cap);
+static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex);
 
 void OPENSSL_s390x_facilities(void);
 void OPENSSL_s390x_functions(void);
 
 struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P;
 
+#ifdef S390X_MOD_EXP
+static int probe_cex(void);
+int OPENSSL_s390xcex;
+
+#if defined(__GNUC__)
+__attribute__ ((visibility("hidden")))
+#endif
+void OPENSSL_s390x_cleanup(void);
+
+#if defined(__GNUC__)
+__attribute__ ((visibility("hidden")))
+#endif
+void OPENSSL_s390x_cleanup(void)
+{
+    if (OPENSSL_s390xcex != -1) {
+        (void)close(OPENSSL_s390xcex);
+        OPENSSL_s390xcex = -1;
+    }
+}
+#endif
+
 #if defined(__GNUC__) && defined(__linux)
 __attribute__ ((visibility("hidden")))
 #endif
 void OPENSSL_cpuid_setup(void)
 {
     struct OPENSSL_s390xcap_st cap;
+    int cex = 1;
 
     if (OPENSSL_s390xcap_P.stfle[0])
         return;
@@ -140,7 +171,7 @@ void OPENSSL_cpuid_setup(void)
 
     env = getenv("OPENSSL_s390xcap");
     if (env != NULL) {
-        if (!parse_env(&cap))
+        if (!parse_env(&cap, &cex))
             env = NULL;
     }
 
@@ -178,9 +209,52 @@ void OPENSSL_cpuid_setup(void)
         OPENSSL_s390xcap_P.kdsa[0] &= cap.kdsa[0];
         OPENSSL_s390xcap_P.kdsa[1] &= cap.kdsa[1];
     }
+
+#ifdef S390X_MOD_EXP
+    if (cex == 0) {
+        OPENSSL_s390xcex = -1;
+    } else {
+        OPENSSL_s390xcex = open("/dev/z90crypt", O_RDWR | O_CLOEXEC);
+        if (probe_cex() == 1)
+            OPENSSL_atexit(OPENSSL_s390x_cleanup);
+    }
+#endif
 }
 
-static int parse_env(struct OPENSSL_s390xcap_st *cap)
+#ifdef S390X_MOD_EXP
+static int probe_cex(void)
+{
+    struct ica_rsa_modexpo me;
+    const unsigned char inval[16] = {
+        0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,2
+    };
+    const unsigned char modulus[16] = {
+        0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,3
+    };
+    unsigned char res[16];
+    int olderrno;
+    int rc = 1;
+
+    me.inputdata = (unsigned char *)inval;
+    me.inputdatalength = sizeof(inval);
+    me.outputdata = (unsigned char *)res;
+    me.outputdatalength = sizeof(res);
+    me.b_key = (unsigned char *)inval;
+    me.n_modulus = (unsigned char *)modulus;
+    olderrno = errno;
+    if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) == -1) {
+        (void)close(OPENSSL_s390xcex);
+        OPENSSL_s390xcex = -1;
+        rc = 0;
+    }
+    errno = olderrno;
+    return rc;
+}
+#endif
+
+static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex)
 {
     /*-
      * CPU model data
@@ -732,6 +806,13 @@ static int parse_env(struct OPENSSL_s390xcap_st *cap)
         else if TOK_CPU(z15)
         else if TOK_CPU(z16)
 
+        /* nocex to deactivate cex support */
+        else if (sscanf(tok_begin, " %" STR(LEN) "s %" STR(LEN) "s ",
+                        tok[0], tok[1]) == 1
+                && !strcmp(tok[0], "nocex")) {
+            *cex = 0;
+        }
+
         /* whitespace(ignored) or invalid tokens */
         else {
             while (*tok_begin != '\0') {