ec: Add PPC64 vector assembly version of p521 field operations

Only field multiplication and squaring (but not reduction) show a
significant improvement.  This is enabled on Power ISA >= 3.0.

On a Power 9 CPU an average 10% performance improvement is seen (ECHDE:
14%, ECDSA sign: 6%, ECDSA verify 10%), compared to existing code.

On an upcoming Power 10 CPU we see an average performance improvement
of 26% (ECHDE: 38%, ECDSA sign: 16%, ECDSA verify 25%), compared to
existing code.

Signed-off-by: Amitay Isaacs <amitay@ozlabs.org>
Signed-off-by: Martin Schwenke <martin@meltin.net>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15401)

1 files changed, 2 insertions, 217 deletions

diff --git a/providers/fips.module.sources b/providers/fips.module.sources
index 7dc0061919..78febf7714 100644
--- a/providers/fips.module.sources
+++ b/providers/fips.module.sources
@@ -1,7 +1,6 @@
 crypto/aes/aes_cbc.c
 crypto/aes/aes_core.c
 crypto/aes/aes_ecb.c
-crypto/aes/aes_local.h
 crypto/aes/aes_misc.c
 crypto/aes/asm/aes-586.pl
 crypto/aes/asm/aes-armv4.pl
@@ -80,14 +79,12 @@ crypto/bn/bn_gf2m.c
 crypto/bn/bn_intern.c
 crypto/bn/bn_kron.c
 crypto/bn/bn_lib.c
-crypto/bn/bn_local.h
 crypto/bn/bn_mod.c
 crypto/bn/bn_mont.c
 crypto/bn/bn_mpi.c
 crypto/bn/bn_mul.c
 crypto/bn/bn_nist.c
 crypto/bn/bn_prime.c
-crypto/bn/bn_prime.h
 crypto/bn/bn_rand.c
 crypto/bn/bn_recp.c
 crypto/bn/bn_rsa_fips186_4.c
@@ -96,7 +93,6 @@ crypto/bn/bn_sqr.c
 crypto/bn/bn_sqrt.c
 crypto/bn/bn_word.c
 crypto/bn/rsaz_exp.c
-crypto/bn/rsaz_exp.h
 crypto/bn/rsaz_exp_x2.c
 crypto/bsearch.c
 crypto/buffer/buffer.c
@@ -110,12 +106,9 @@ crypto/cryptlib.c
 crypto/ctype.c
 crypto/der_writer.c
 crypto/des/des_enc.c
-crypto/des/des_local.h
 crypto/des/ecb3_enc.c
 crypto/des/fcrypt_b.c
-crypto/des/ncbc_enc.c
 crypto/des/set_key.c
-crypto/des/spr.h
 crypto/dh/dh_backend.c
 crypto/dh/dh_check.c
 crypto/dh/dh_gen.c
@@ -123,16 +116,15 @@ crypto/dh/dh_group_params.c
 crypto/dh/dh_kdf.c
 crypto/dh/dh_key.c
 crypto/dh/dh_lib.c
-crypto/dh/dh_local.h
 crypto/dsa/dsa_backend.c
 crypto/dsa/dsa_check.c
 crypto/dsa/dsa_gen.c
 crypto/dsa/dsa_key.c
 crypto/dsa/dsa_lib.c
-crypto/dsa/dsa_local.h
 crypto/dsa/dsa_ossl.c
 crypto/dsa/dsa_sign.c
 crypto/dsa/dsa_vrf.c
+crypto/ec/asm/ecp_nistp521-ppc64.pl
 crypto/ec/asm/ecp_nistz256-armv4.pl
 crypto/ec/asm/ecp_nistz256-armv8.pl
 crypto/ec/asm/ecp_nistz256-ppc64.pl
@@ -143,20 +135,12 @@ crypto/ec/asm/x25519-ppc64.pl
 crypto/ec/asm/x25519-x86_64.pl
 crypto/ec/curve25519.c
 crypto/ec/curve448/arch_32/f_impl32.c
-crypto/ec/curve448/arch_64/arch_intrinsics.h
-crypto/ec/curve448/arch_64/f_impl.h
 crypto/ec/curve448/arch_64/f_impl64.c
 crypto/ec/curve448/curve448.c
-crypto/ec/curve448/curve448_local.h
 crypto/ec/curve448/curve448_tables.c
-crypto/ec/curve448/curve448utils.h
-crypto/ec/curve448/ed448.h
 crypto/ec/curve448/eddsa.c
 crypto/ec/curve448/f_generic.c
-crypto/ec/curve448/field.h
-crypto/ec/curve448/point_448.h
 crypto/ec/curve448/scalar.c
-crypto/ec/curve448/word.h
 crypto/ec/ec2_oct.c
 crypto/ec/ec2_smpl.c
 crypto/ec/ec_asn1.c
@@ -167,7 +151,6 @@ crypto/ec/ec_cvt.c
 crypto/ec/ec_key.c
 crypto/ec/ec_kmeth.c
 crypto/ec/ec_lib.c
-crypto/ec/ec_local.h
 crypto/ec/ec_mult.c
 crypto/ec/ec_oct.c
 crypto/ec/ecdh_kdf.c
@@ -181,7 +164,6 @@ crypto/ec/ecp_nistz256.c
 crypto/ec/ecp_oct.c
 crypto/ec/ecp_smpl.c
 crypto/ec/ecx_backend.c
-crypto/ec/ecx_backend.h
 crypto/ec/ecx_key.c
 crypto/evp/asymcipher.c
 crypto/evp/dh_support.c
@@ -190,7 +172,6 @@ crypto/evp/ec_support.c
 crypto/evp/evp_enc.c
 crypto/evp/evp_fetch.c
 crypto/evp/evp_lib.c
-crypto/evp/evp_local.h
 crypto/evp/evp_rand.c
 crypto/evp/evp_utils.c
 crypto/evp/exchange.c
@@ -216,10 +197,8 @@ crypto/ffc/ffc_params.c
 crypto/ffc/ffc_params_generate.c
 crypto/ffc/ffc_params_validate.c
 crypto/hmac/hmac.c
-crypto/hmac/hmac_local.h
 crypto/initthread.c
 crypto/lhash/lhash.c
-crypto/lhash/lhash_local.h
 crypto/mem_clr.c
 crypto/modes/asm/aes-gcm-armv8_64.pl
 crypto/modes/asm/aesni-gcm-x86_64.pl
@@ -252,21 +231,17 @@ crypto/params_from_text.c
 crypto/passphrase.c
 crypto/property/defn_cache.c
 crypto/property/property.c
-crypto/property/property_local.h
 crypto/property/property_parse.c
 crypto/property/property_string.c
 crypto/provider_core.c
-crypto/provider_local.h
 crypto/provider_predefined.c
 crypto/rand/rand_lib.c
-crypto/rand/rand_local.h
 crypto/rand/rand_meth.c
 crypto/rsa/rsa_backend.c
 crypto/rsa/rsa_chk.c
 crypto/rsa/rsa_crpt.c
 crypto/rsa/rsa_gen.c
 crypto/rsa/rsa_lib.c
-crypto/rsa/rsa_local.h
 crypto/rsa/rsa_mp_names.c
 crypto/rsa/rsa_none.c
 crypto/rsa/rsa_oaep.c
@@ -326,7 +301,6 @@ crypto/sha/sha1dgst.c
 crypto/sha/sha256.c
 crypto/sha/sha3.c
 crypto/sha/sha512.c
-crypto/sha/sha_local.h
 crypto/sparse_array.c
 crypto/stack/stack.c
 crypto/threads_lib.c
@@ -334,184 +308,22 @@ crypto/threads_none.c
 crypto/threads_pthread.c
 crypto/threads_win.c
 crypto/x86_64cpuid.pl
-e_os.h
-include/crypto/aes_platform.h
-include/crypto/asn1.h
-include/crypto/asn1_dsa.h
-include/crypto/bn.h
-include/crypto/bn_conf.h.in
-include/crypto/bn_dh.h
-include/crypto/cryptlib.h
-include/crypto/ctype.h
-include/crypto/des_platform.h
-include/crypto/dh.h
-include/crypto/dsa.h
-include/crypto/ec.h
-include/crypto/ecx.h
-include/crypto/evp.h
-include/crypto/lhash.h
-include/crypto/md32_common.h
-include/crypto/modes.h
-include/crypto/rand.h
-include/crypto/rand_pool.h
-include/crypto/rsa.h
-include/crypto/security_bits.h
-include/crypto/sha.h
-include/crypto/sm2.h
-include/crypto/sparse_array.h
-include/crypto/types.h
-include/crypto/x509.h
-include/internal/bio.h
-include/internal/constant_time.h
-include/internal/core.h
-include/internal/cryptlib.h
-include/internal/deprecated.h
-include/internal/der.h
-include/internal/dso.h
-include/internal/dsoerr.h
-include/internal/endian.h
-include/internal/ffc.h
-include/internal/namemap.h
-include/internal/nelem.h
-include/internal/numbers.h
-include/internal/packet.h
-include/internal/param_build_set.h
-include/internal/passphrase.h
-include/internal/property.h
-include/internal/propertyerr.h
-include/internal/provider.h
-include/internal/refcount.h
-include/internal/sha3.h
-include/internal/sizes.h
-include/internal/symhacks.h
-include/internal/thread_once.h
-include/internal/tlsgroups.h
-include/internal/tsan_assist.h
-include/openssl/aes.h
-include/openssl/asn1.h.in
-include/openssl/asn1err.h
-include/openssl/asn1t.h.in
-include/openssl/async.h
-include/openssl/asyncerr.h
-include/openssl/bio.h.in
-include/openssl/bioerr.h
-include/openssl/bn.h
-include/openssl/bnerr.h
-include/openssl/buffer.h
-include/openssl/buffererr.h
-include/openssl/cmac.h
-include/openssl/comp.h
-include/openssl/comperr.h
-include/openssl/conf.h.in
-include/openssl/conferr.h
-include/openssl/configuration.h.in
-include/openssl/core.h
-include/openssl/core_dispatch.h
-include/openssl/core_names.h
-include/openssl/crypto.h.in
-include/openssl/cryptoerr.h
-include/openssl/cryptoerr_legacy.h
-include/openssl/ct.h.in
-include/openssl/cterr.h
-include/openssl/des.h
-include/openssl/dh.h
-include/openssl/dherr.h
-include/openssl/dsa.h
-include/openssl/dsaerr.h
-include/openssl/dtls1.h
-include/openssl/e_os2.h
-include/openssl/ebcdic.h
-include/openssl/ec.h
-include/openssl/ecerr.h
-include/openssl/encoder.h
-include/openssl/encodererr.h
-include/openssl/engine.h
-include/openssl/engineerr.h
-include/openssl/err.h.in
-include/openssl/evp.h
-include/openssl/evperr.h
-include/openssl/fips_names.h
-include/openssl/fipskey.h.in
-include/openssl/hmac.h
-include/openssl/http.h
-include/openssl/kdf.h
-include/openssl/lhash.h.in
-include/openssl/macros.h
-include/openssl/md4.h
-include/openssl/md5.h
-include/openssl/mdc2.h
-include/openssl/modes.h
-include/openssl/obj_mac.h
-include/openssl/objects.h
-include/openssl/objectserr.h
-include/openssl/ocsp.h.in
-include/openssl/ocsperr.h
-include/openssl/opensslconf.h
-include/openssl/opensslv.h.in
-include/openssl/param_build.h
-include/openssl/params.h
-include/openssl/pem.h
-include/openssl/pemerr.h
-include/openssl/pkcs7.h.in
-include/openssl/pkcs7err.h
-include/openssl/proverr.h
-include/openssl/provider.h
-include/openssl/rand.h
-include/openssl/randerr.h
-include/openssl/ripemd.h
-include/openssl/rsa.h
-include/openssl/rsaerr.h
-include/openssl/safestack.h.in
-include/openssl/self_test.h
-include/openssl/sha.h
-include/openssl/srtp.h
-include/openssl/ssl.h.in
-include/openssl/ssl2.h
-include/openssl/ssl3.h
-include/openssl/sslerr.h
-include/openssl/sslerr_legacy.h
-include/openssl/stack.h
-include/openssl/symhacks.h
-include/openssl/tls1.h
-include/openssl/trace.h
-include/openssl/types.h
-include/openssl/ui.h.in
-include/openssl/uierr.h
-include/openssl/x509.h.in
-include/openssl/x509_vfy.h.in
-include/openssl/x509err.h
-include/openssl/x509v3.h.in
-include/openssl/x509v3err.h
 providers/common/bio_prov.c
 providers/common/capabilities.c
-providers/common/der/der_digests.h.in
 providers/common/der/der_digests_gen.c.in
-providers/common/der/der_dsa.h.in
 providers/common/der/der_dsa_gen.c.in
 providers/common/der/der_dsa_key.c
 providers/common/der/der_dsa_sig.c
-providers/common/der/der_ec.h.in
 providers/common/der/der_ec_gen.c.in
 providers/common/der/der_ec_key.c
 providers/common/der/der_ec_sig.c
-providers/common/der/der_ecx.h.in
 providers/common/der/der_ecx_gen.c.in
 providers/common/der/der_ecx_key.c
-providers/common/der/der_rsa.h.in
 providers/common/der/der_rsa_gen.c.in
 providers/common/der/der_rsa_key.c
 providers/common/der/der_rsa_sig.c
-providers/common/der/der_sm2.h.in
-providers/common/der/der_sm2_gen.c.in
-providers/common/der/der_wrap.h.in
 providers/common/der/der_wrap_gen.c.in
 providers/common/digest_to_nid.c
-providers/common/include/prov/bio.h
-providers/common/include/prov/proverr.h
-providers/common/include/prov/provider_ctx.h
-providers/common/include/prov/provider_util.h
-providers/common/include/prov/providercommon.h
-providers/common/include/prov/securitycheck.h
 providers/common/provider_ctx.c
 providers/common/provider_err.c
 providers/common/provider_seeding.c
@@ -521,39 +333,25 @@ providers/common/securitycheck_fips.c
 providers/fips/fips_entry.c
 providers/fips/fipsprov.c
 providers/fips/self_test.c
-providers/fips/self_test.h
-providers/fips/self_test_data.inc
 providers/fips/self_test_kats.c
 providers/implementations/asymciphers/rsa_enc.c
 providers/implementations/ciphers/cipher_aes.c
-providers/implementations/ciphers/cipher_aes.h
 providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.c
-providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.h
 providers/implementations/ciphers/cipher_aes_cbc_hmac_sha1_hw.c
 providers/implementations/ciphers/cipher_aes_cbc_hmac_sha256_hw.c
 providers/implementations/ciphers/cipher_aes_ccm.c
-providers/implementations/ciphers/cipher_aes_ccm.h
 providers/implementations/ciphers/cipher_aes_ccm_hw.c
-providers/implementations/ciphers/cipher_aes_ccm_hw_aesni.inc
 providers/implementations/ciphers/cipher_aes_cts.c
-providers/implementations/ciphers/cipher_aes_cts.h
-providers/implementations/ciphers/cipher_aes_cts.inc
 providers/implementations/ciphers/cipher_aes_gcm.c
-providers/implementations/ciphers/cipher_aes_gcm.h
 providers/implementations/ciphers/cipher_aes_gcm_hw.c
-providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.inc
 providers/implementations/ciphers/cipher_aes_hw.c
-providers/implementations/ciphers/cipher_aes_hw_aesni.inc
 providers/implementations/ciphers/cipher_aes_ocb.c
-providers/implementations/ciphers/cipher_aes_ocb.h
 providers/implementations/ciphers/cipher_aes_ocb_hw.c
 providers/implementations/ciphers/cipher_aes_wrp.c
 providers/implementations/ciphers/cipher_aes_xts.c
-providers/implementations/ciphers/cipher_aes_xts.h
 providers/implementations/ciphers/cipher_aes_xts_fips.c
 providers/implementations/ciphers/cipher_aes_xts_hw.c
 providers/implementations/ciphers/cipher_tdes.c
-providers/implementations/ciphers/cipher_tdes.h
 providers/implementations/ciphers/cipher_tdes_common.c
 providers/implementations/ciphers/cipher_tdes_hw.c
 providers/implementations/ciphers/ciphercommon.c
@@ -563,7 +361,6 @@ providers/implementations/ciphers/ciphercommon_ccm_hw.c
 providers/implementations/ciphers/ciphercommon_gcm.c
 providers/implementations/ciphers/ciphercommon_gcm_hw.c
 providers/implementations/ciphers/ciphercommon_hw.c
-providers/implementations/ciphers/ciphercommon_local.h
 providers/implementations/digests/digestcommon.c
 providers/implementations/digests/sha2_prov.c
 providers/implementations/digests/sha3_prov.c
@@ -571,20 +368,9 @@ providers/implementations/exchange/dh_exch.c
 providers/implementations/exchange/ecdh_exch.c
 providers/implementations/exchange/ecx_exch.c
 providers/implementations/exchange/kdf_exch.c
-providers/implementations/include/prov/ciphercommon.h
-providers/implementations/include/prov/ciphercommon_aead.h
-providers/implementations/include/prov/ciphercommon_ccm.h
-providers/implementations/include/prov/ciphercommon_gcm.h
-providers/implementations/include/prov/digestcommon.h
-providers/implementations/include/prov/implementations.h
-providers/implementations/include/prov/kdfexchange.h
-providers/implementations/include/prov/macsignature.h
-providers/implementations/include/prov/names.h
-providers/implementations/include/prov/seeding.h
 providers/implementations/kdfs/hkdf.c
 providers/implementations/kdfs/kbkdf.c
 providers/implementations/kdfs/pbkdf2.c
-providers/implementations/kdfs/pbkdf2.h
 providers/implementations/kdfs/pbkdf2_fips.c
 providers/implementations/kdfs/sshkdf.c
 providers/implementations/kdfs/sskdf.c
@@ -594,7 +380,6 @@ providers/implementations/kem/rsa_kem.c
 providers/implementations/keymgmt/dh_kmgmt.c
 providers/implementations/keymgmt/dsa_kmgmt.c
 providers/implementations/keymgmt/ec_kmgmt.c
-providers/implementations/keymgmt/ec_kmgmt_imexport.inc
 providers/implementations/keymgmt/ecx_kmgmt.c
 providers/implementations/keymgmt/kdf_legacy_kmgmt.c
 providers/implementations/keymgmt/mac_legacy_kmgmt.c
@@ -608,7 +393,6 @@ providers/implementations/rands/drbg.c
 providers/implementations/rands/drbg_ctr.c
 providers/implementations/rands/drbg_hash.c
 providers/implementations/rands/drbg_hmac.c
-providers/implementations/rands/drbg_local.h
 providers/implementations/rands/test_rng.c
 providers/implementations/signature/dsa_sig.c
 providers/implementations/signature/ecdsa_sig.c
@@ -617,3 +401,4 @@ providers/implementations/signature/mac_legacy_sig.c
 providers/implementations/signature/rsa_sig.c
 ssl/record/tls_pad.c
 ssl/s3_cbc.c
+util/providers.num