diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2018-08-15 16:01:47 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2018-08-15 16:01:47 -0700 |
commit | dafa5f6577a9eecd2941add553d1672c30b02364 (patch) | |
tree | ff9d3d2dffafd6eba1b6ac21ba50623812041b70 /arch | |
parent | 9a76aba02a37718242d7cdc294f0a3901928aa57 (diff) | |
parent | 22240df7ac6d76a271197571a7be45addef2ba15 (diff) |
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto updates from Herbert Xu:
"API:
- Fix dcache flushing crash in skcipher.
- Add hash finup self-tests.
- Reschedule during speed tests.
Algorithms:
- Remove insecure vmac and replace it with vmac64.
- Add public key verification for DH/ECDH.
Drivers:
- Decrease priority of sha-mb on x86.
- Improve NEON latency/throughput on ARM64.
- Add md5/sha384/sha512/des/3des to inside-secure.
- Support eip197d in inside-secure.
- Only register algorithms supported by the host in virtio.
- Add cts and remove incompatible cts1 from ccree.
- Add hisilicon SEC security accelerator driver.
- Replace msm hwrng driver with qcom pseudo rng driver.
Misc:
- Centralize CRC polynomials"
* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (121 commits)
crypto: arm64/ghash-ce - implement 4-way aggregation
crypto: arm64/ghash-ce - replace NEON yield check with block limit
crypto: hisilicon - sec_send_request() can be static
lib/mpi: remove redundant variable esign
crypto: arm64/aes-ce-gcm - don't reload key schedule if avoidable
crypto: arm64/aes-ce-gcm - implement 2-way aggregation
crypto: arm64/aes-ce-gcm - operate on two input blocks at a time
crypto: dh - make crypto_dh_encode_key() make robust
crypto: dh - fix calculating encoded key size
crypto: ccp - Check for NULL PSP pointer at module unload
crypto: arm/chacha20 - always use vrev for 16-bit rotates
crypto: ccree - allow bigger than sector XTS op
crypto: ccree - zero all of request ctx before use
crypto: ccree - remove cipher ivgen left overs
crypto: ccree - drop useless type flag during reg
crypto: ablkcipher - fix crash flushing dcache in error path
crypto: blkcipher - fix crash flushing dcache in error path
crypto: skcipher - fix crash flushing dcache in error path
crypto: skcipher - remove unnecessary setting of walk->nbytes
crypto: scatterwalk - remove scatterwalk_samebuf()
...
Diffstat (limited to 'arch')
47 files changed, 648 insertions, 266 deletions
diff --git a/arch/arm/crypto/chacha20-neon-core.S b/arch/arm/crypto/chacha20-neon-core.S index 3fecb2124c35..451a849ad518 100644 --- a/arch/arm/crypto/chacha20-neon-core.S +++ b/arch/arm/crypto/chacha20-neon-core.S @@ -51,9 +51,8 @@ ENTRY(chacha20_block_xor_neon) .Ldoubleround: // x0 += x1, x3 = rotl32(x3 ^ x0, 16) vadd.i32 q0, q0, q1 - veor q4, q3, q0 - vshl.u32 q3, q4, #16 - vsri.u32 q3, q4, #16 + veor q3, q3, q0 + vrev32.16 q3, q3 // x2 += x3, x1 = rotl32(x1 ^ x2, 12) vadd.i32 q2, q2, q3 @@ -82,9 +81,8 @@ ENTRY(chacha20_block_xor_neon) // x0 += x1, x3 = rotl32(x3 ^ x0, 16) vadd.i32 q0, q0, q1 - veor q4, q3, q0 - vshl.u32 q3, q4, #16 - vsri.u32 q3, q4, #16 + veor q3, q3, q0 + vrev32.16 q3, q3 // x2 += x3, x1 = rotl32(x1 ^ x2, 12) vadd.i32 q2, q2, q3 diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index d9bb52cae2ac..8930fc4e7c22 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -152,7 +152,7 @@ static struct shash_alg ghash_alg = { .cra_name = "__ghash", .cra_driver_name = "__driver-ghash-ce", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_INTERNAL, + .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct ghash_key), .cra_module = THIS_MODULE, @@ -308,9 +308,8 @@ static struct ahash_alg ghash_async_alg = { .cra_name = "ghash", .cra_driver_name = "ghash-ce", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_type = &crypto_ahash_type, .cra_ctxsize = sizeof(struct ghash_async_ctx), .cra_module = THIS_MODULE, .cra_init = ghash_async_init_tfm, diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index 555f72b5e659..b732522e20f8 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -75,7 +75,6 @@ static struct shash_alg alg = { .cra_name = "sha1", .cra_driver_name = "sha1-ce", .cra_priority = 200, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 6fc73bf8766d..98ab8239f919 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -67,7 +67,6 @@ static struct shash_alg alg = { .cra_name = "sha1", .cra_driver_name= "sha1-asm", .cra_priority = 150, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 4e22f122f966..d15e0ea2c95e 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -83,7 +83,6 @@ static struct shash_alg alg = { .cra_name = "sha1", .cra_driver_name = "sha1-neon", .cra_priority = 250, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index df4dcef054ae..1211a5c129fc 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -78,7 +78,6 @@ static struct shash_alg algs[] = { { .cra_name = "sha224", .cra_driver_name = "sha224-ce", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -93,7 +92,6 @@ static struct shash_alg algs[] = { { .cra_name = "sha256", .cra_driver_name = "sha256-ce", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c index a84e869ef900..bf8ccff2c9d0 100644 --- a/arch/arm/crypto/sha256_glue.c +++ b/arch/arm/crypto/sha256_glue.c @@ -71,7 +71,6 @@ static struct shash_alg algs[] = { { .cra_name = "sha256", .cra_driver_name = "sha256-asm", .cra_priority = 150, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -86,7 +85,6 @@ static struct shash_alg algs[] = { { .cra_name = "sha224", .cra_driver_name = "sha224-asm", .cra_priority = 150, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index 39ccd658817e..9bbee56fbdc8 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -79,7 +79,6 @@ struct shash_alg sha256_neon_algs[] = { { .cra_name = "sha256", .cra_driver_name = "sha256-neon", .cra_priority = 250, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -94,7 +93,6 @@ struct shash_alg sha256_neon_algs[] = { { .cra_name = "sha224", .cra_driver_name = "sha224-neon", .cra_priority = 250, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 269a394e4a53..86540cd4a6fa 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -63,7 +63,6 @@ static struct shash_alg sha512_arm_algs[] = { { .cra_name = "sha384", .cra_driver_name = "sha384-arm", .cra_priority = 250, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -78,7 +77,6 @@ static struct shash_alg sha512_arm_algs[] = { { .cra_name = "sha512", .cra_driver_name = "sha512-arm", .cra_priority = 250, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index 32693684a3ab..8a5642b41fd6 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -75,7 +75,6 @@ struct shash_alg sha512_neon_algs[] = { { .cra_name = "sha384", .cra_driver_name = "sha384-neon", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, @@ -91,7 +90,6 @@ struct shash_alg sha512_neon_algs[] = { { .cra_name = "sha512", .cra_driver_name = "sha512-neon", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/arm64/boot/dts/hisilicon/hip07.dtsi b/arch/arm64/boot/dts/hisilicon/hip07.dtsi index 9c10030a07f8..c33adefc3061 100644 --- a/arch/arm64/boot/dts/hisilicon/hip07.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hip07.dtsi @@ -1049,7 +1049,74 @@ num-pins = <2>; }; }; + p0_mbigen_alg_a:interrupt-controller@d0080000 { + compatible = "hisilicon,mbigen-v2"; + reg = <0x0 0xd0080000 0x0 0x10000>; + p0_mbigen_sec_a: intc_sec { + msi-parent = <&p0_its_dsa_a 0x40400>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <33>; + }; + p0_mbigen_smmu_alg_a: intc_smmu_alg { + msi-parent = <&p0_its_dsa_a 0x40b1b>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <3>; + }; + }; + p0_mbigen_alg_b:interrupt-controller@8,d0080000 { + compatible = "hisilicon,mbigen-v2"; + reg = <0x8 0xd0080000 0x0 0x10000>; + + p0_mbigen_sec_b: intc_sec { + msi-parent = <&p0_its_dsa_b 0x42400>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <33>; + }; + p0_mbigen_smmu_alg_b: intc_smmu_alg { + msi-parent = <&p0_its_dsa_b 0x42b1b>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <3>; + }; + }; + p1_mbigen_alg_a:interrupt-controller@400,d0080000 { + compatible = "hisilicon,mbigen-v2"; + reg = <0x400 0xd0080000 0x0 0x10000>; + + p1_mbigen_sec_a: intc_sec { + msi-parent = <&p1_its_dsa_a 0x44400>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <33>; + }; + p1_mbigen_smmu_alg_a: intc_smmu_alg { + msi-parent = <&p1_its_dsa_a 0x44b1b>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <3>; + }; + }; + p1_mbigen_alg_b:interrupt-controller@408,d0080000 { + compatible = "hisilicon,mbigen-v2"; + reg = <0x408 0xd0080000 0x0 0x10000>; + + p1_mbigen_sec_b: intc_sec { + msi-parent = <&p1_its_dsa_b 0x46400>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <33>; + }; + p1_mbigen_smmu_alg_b: intc_smmu_alg { + msi-parent = <&p1_its_dsa_b 0x46b1b>; + interrupt-controller; + #interrupt-cells = <2>; + num-pins = <3>; + }; + }; p0_mbigen_dsa_a: interrupt-controller@c0080000 { compatible = "hisilicon,mbigen-v2"; reg = <0x0 0xc0080000 0x0 0x10000>; @@ -1107,6 +1174,58 @@ hisilicon,broken-prefetch-cmd; status = "disabled"; }; + p0_smmu_alg_a: smmu_alg@d0040000 { + compatible = "arm,smmu-v3"; + reg = <0x0 0xd0040000 0x0 0x20000>; + interrupt-parent = <&p0_mbigen_smmu_alg_a>; + interrupts = <733 1>, + <734 1>, + <735 1>; + interrupt-names = "eventq", "gerror", "priq"; + #iommu-cells = <1>; + dma-coherent; + hisilicon,broken-prefetch-cmd; + /* smmu-cb-memtype = <0x0 0x1>;*/ + }; + p0_smmu_alg_b: smmu_alg@8,d0040000 { + compatible = "arm,smmu-v3"; + reg = <0x8 0xd0040000 0x0 0x20000>; + interrupt-parent = <&p0_mbigen_smmu_alg_b>; + interrupts = <733 1>, + <734 1>, + <735 1>; + interrupt-names = "eventq", "gerror", "priq"; + #iommu-cells = <1>; + dma-coherent; + hisilicon,broken-prefetch-cmd; + /* smmu-cb-memtype = <0x0 0x1>;*/ + }; + p1_smmu_alg_a: smmu_alg@400,d0040000 { + compatible = "arm,smmu-v3"; + reg = <0x400 0xd0040000 0x0 0x20000>; + interrupt-parent = <&p1_mbigen_smmu_alg_a>; + interrupts = <733 1>, + <734 1>, + <735 1>; + interrupt-names = "eventq", "gerror", "priq"; + #iommu-cells = <1>; + dma-coherent; + hisilicon,broken-prefetch-cmd; + /* smmu-cb-memtype = <0x0 0x1>;*/ + }; + p1_smmu_alg_b: smmu_alg@408,d0040000 { + compatible = "arm,smmu-v3"; + reg = <0x408 0xd0040000 0x0 0x20000>; + interrupt-parent = <&p1_mbigen_smmu_alg_b>; + interrupts = <733 1>, + <734 1>, + <735 1>; + interrupt-names = "eventq", "gerror", "priq"; + #iommu-cells = <1>; + dma-coherent; + hisilicon,broken-prefetch-cmd; + /* smmu-cb-memtype = <0x0 0x1>;*/ + }; soc { compatible = "simple-bus"; @@ -1603,5 +1722,170 @@ 0x0 0 0 4 &mbigen_pcie2_a 671 4>; status = "disabled"; }; + p0_sec_a: crypto@d2000000 { + compatible = "hisilicon,hip07-sec"; + reg = <0x0 0xd0000000 0x0 0x10000 + 0x0 0xd2000000 0x0 0x10000 + 0x0 0xd2010000 0x0 0x10000 + 0x0 0xd2020000 0x0 0x10000 + 0x0 0xd2030000 0x0 0x10000 + 0x0 0xd2040000 0x0 0x10000 + 0x0 0xd2050000 0x0 0x10000 + 0x0 0xd2060000 0x0 0x10000 + 0x0 0xd2070000 0x0 0x10000 + 0x0 0xd2080000 0x0 0x10000 + 0x0 0xd2090000 0x0 0x10000 + 0x0 0xd20a0000 0x0 0x10000 + 0x0 0xd20b0000 0x0 0x10000 + 0x0 0xd20c0000 0x0 0x10000 + 0x0 0xd20d0000 0x0 0x10000 + 0x0 0xd20e0000 0x0 0x10000 + 0x0 0xd20f0000 0x0 0x10000 + 0x0 0xd2100000 0x0 0x10000>; + interrupt-parent = <&p0_mbigen_sec_a>; + iommus = <&p0_smmu_alg_a 0x600>; + dma-coherent; + interrupts = <576 4>, + <577 1>, <578 4>, + <579 1>, <580 4>, + <581 1>, <582 4>, + <583 1>, <584 4>, + <585 1>, <586 4>, + <587 1>, <588 4>, + <589 1>, <590 4>, + <591 1>, <592 4>, + <593 1>, <594 4>, + <595 1>, <596 4>, + <597 1>, <598 4>, + <599 1>, <600 4>, + <601 1>, <602 4>, + <603 1>, <604 4>, + <605 1>, <606 4>, + <607 1>, <608 4>; + }; + p0_sec_b: crypto@8,d2000000 { + compatible = "hisilicon,hip07-sec"; + reg = <0x8 0xd0000000 0x0 0x10000 + 0x8 0xd2000000 0x0 0x10000 + 0x8 0xd2010000 0x0 0x10000 + 0x8 0xd2020000 0x0 0x10000 + 0x8 0xd2030000 0x0 0x10000 + 0x8 0xd2040000 0x0 0x10000 + 0x8 0xd2050000 0x0 0x10000 + 0x8 0xd2060000 0x0 0x10000 + 0x8 0xd2070000 0x0 0x10000 + 0x8 0xd2080000 0x0 0x10000 + 0x8 0xd2090000 0x0 0x10000 + 0x8 0xd20a0000 0x0 0x10000 + 0x8 0xd20b0000 0x0 0x10000 + 0x8 0xd20c0000 0x0 0x10000 + 0x8 0xd20d0000 0x0 0x10000 + 0x8 0xd20e0000 0x0 0x10000 + 0x8 0xd20f0000 0x0 0x10000 + 0x8 0xd2100000 0x0 0x10000>; + interrupt-parent = <&p0_mbigen_sec_b>; + iommus = <&p0_smmu_alg_b 0x600>; + dma-coherent; + interrupts = <576 4>, + <577 1>, <578 4>, + <579 1>, <580 4>, + <581 1>, <582 4>, + <583 1>, <584 4>, + <585 1>, <586 4>, + <587 1>, <588 4>, + <589 1>, <590 4>, + <591 1>, <592 4>, + <593 1>, <594 4>, + <595 1>, <596 4>, + <597 1>, <598 4>, + <599 1>, <600 4>, + <601 1>, <602 4>, + <603 1>, <604 4>, + <605 1>, <606 4>, + <607 1>, <608 4>; + }; + p1_sec_a: crypto@400,d2000000 { + compatible = "hisilicon,hip07-sec"; + reg = <0x400 0xd0000000 0x0 0x10000 + 0x400 0xd2000000 0x0 0x10000 + 0x400 0xd2010000 0x0 0x10000 + 0x400 0xd2020000 0x0 0x10000 + 0x400 0xd2030000 0x0 0x10000 + 0x400 0xd2040000 0x0 0x10000 + 0x400 0xd2050000 0x0 0x10000 + 0x400 0xd2060000 0x0 0x10000 + 0x400 0xd2070000 0x0 0x10000 + 0x400 0xd2080000 0x0 0x10000 + 0x400 0xd2090000 0x0 0x10000 + 0x400 0xd20a0000 0x0 0x10000 + 0x400 0xd20b0000 0x0 0x10000 + 0x400 0xd20c0000 0x0 0x10000 + 0x400 0xd20d0000 0x0 0x10000 + 0x400 0xd20e0000 0x0 0x10000 + 0x400 0xd20f0000 0x0 0x10000 + 0x400 0xd2100000 0x0 0x10000>; + interrupt-parent = <&p1_mbigen_sec_a>; + iommus = <&p1_smmu_alg_a 0x600>; + dma-coherent; + interrupts = <576 4>, + <577 1>, <578 4>, + <579 1>, <580 4>, + <581 1>, <582 4>, + <583 1>, <584 4>, + <585 1>, <586 4>, + <587 1>, <588 4>, + <589 1>, <590 4>, + <591 1>, <592 4>, + <593 1>, <594 4>, + <595 1>, <596 4>, + <597 1>, <598 4>, + <599 1>, <600 4>, + <601 1>, <602 4>, + <603 1>, <604 4>, + <605 1>, <606 4>, + <607 1>, <608 4>; + }; + p1_sec_b: crypto@408,d2000000 { + compatible = "hisilicon,hip07-sec"; + reg = <0x408 0xd0000000 0x0 0x10000 + 0x408 0xd2000000 0x0 0x10000 + 0x408 0xd2010000 0x0 0x10000 + 0x408 0xd2020000 0x0 0x10000 + 0x408 0xd2030000 0x0 0x10000 + 0x408 0xd2040000 0x0 0x10000 + 0x408 0xd2050000 0x0 0x10000 + 0x408 0xd2060000 0x0 0x10000 + 0x408 0xd2070000 0x0 0x10000 + 0x408 0xd2080000 0x0 0x10000 + 0x408 0xd2090000 0x0 0x10000 + 0x408 0xd20a0000 0x0 0x10000 + 0x408 0xd20b0000 0x0 0x10000 + 0x408 0xd20c0000 0x0 0x10000 + 0x408 0xd20d0000 0x0 0x10000 + 0x408 0xd20e0000 0x0 0x10000 + 0x408 0xd20f0000 0x0 0x10000 + 0x408 0xd2100000 0x0 0x10000>; + interrupt-parent = <&p1_mbigen_sec_b>; + iommus = <&p1_smmu_alg_b 0x600>; + dma-coherent; + interrupts = <576 4>, + <577 1>, <578 4>, + <579 1>, <580 4>, + <581 1>, <582 4>, + <583 1>, <584 4>, + <585 1>, <586 4>, + <587 1>, <588 4>, + <589 1>, <590 4>, + <591 1>, <592 4>, + <593 1>, <594 4>, + <595 1>, <596 4>, + <597 1>, <598 4>, + <599 1>, <600 4>, + <601 1>, <602 4>, + <603 1>, <604 4>, + <605 1>, <606 4>, + <607 1>, <608 4>; + }; + }; }; diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index e3e50950a863..adcb83eb683c 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -567,7 +567,6 @@ static struct shash_alg mac_algs[] = { { .base.cra_name = "cmac(aes)", .base.cra_driver_name = "cmac-aes-" MODE, .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 2 * AES_BLOCK_SIZE, @@ -583,7 +582,6 @@ static struct shash_alg mac_algs[] = { { .base.cra_name = "xcbc(aes)", .base.cra_driver_name = "xcbc-aes-" MODE, .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 2 * AES_BLOCK_SIZE, @@ -599,7 +597,6 @@ static struct shash_alg mac_algs[] = { { .base.cra_name = "cbcmac(aes)", .base.cra_driver_name = "cbcmac-aes-" MODE, .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), .base.cra_module = THIS_MODULE, diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index c723647b37db..1b319b716d5e 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -1,7 +1,7 @@ /* * Accelerated GHASH implementation with ARMv8 PMULL instructions. * - * Copyright (C) 2014 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> + * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -46,6 +46,19 @@ ss3 .req v26 ss4 .req v27 + XL2 .req v8 + XM2 .req v9 + XH2 .req v10 + XL3 .req v11 + XM3 .req v12 + XH3 .req v13 + TT3 .req v14 + TT4 .req v15 + HH .req v16 + HH3 .req v17 + HH4 .req v18 + HH34 .req v19 + .text .arch armv8-a+crypto @@ -134,11 +147,25 @@ .endm .macro __pmull_pre_p64 + add x8, x3, #16 + ld1 {HH.2d-HH4.2d}, [x8] + + trn1 SHASH2.2d, SHASH.2d, HH.2d + trn2 T1.2d, SHASH.2d, HH.2d + eor SHASH2.16b, SHASH2.16b, T1.16b + + trn1 HH34.2d, HH3.2d, HH4.2d + trn2 T1.2d, HH3.2d, HH4.2d + eor HH34.16b, HH34.16b, T1.16b + movi MASK.16b, #0xe1 shl MASK.2d, MASK.2d, #57 .endm .macro __pmull_pre_p8 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + // k00_16 := 0x0000000000000000_000000000000ffff // k32_48 := 0x00000000ffffffff_0000ffffffffffff movi k32_48.2d, #0xffffffff @@ -213,31 +240,88 @@ .endm .macro __pmull_ghash, pn - frame_push 5 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - -0: ld1 {SHASH.2d}, [x22] - ld1 {XL.2d}, [x20] - ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 - eor SHASH2.16b, SHASH2.16b, SHASH.16b + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] __pmull_pre_\pn /* do the head block first, if supplied */ - cbz x23, 1f - ld1 {T1.2d}, [x23] - mov x23, xzr - b 2f + cbz x4, 0f + ld1 {T1.2d}, [x4] + mov x4, xzr + b 3f + +0: .ifc \pn, p64 + tbnz w0, #0, 2f // skip until #blocks is a + tbnz w0, #1, 2f // round multiple of 4 + +1: ld1 {XM3.16b-TT4.16b}, [x2], #64 + + sub w0, w0, #4 + + rev64 T1.16b, XM3.16b + rev64 T2.16b, XH3.16b + rev64 TT4.16b, TT4.16b + rev64 TT3.16b, TT3.16b + + ext IN1.16b, TT4.16b, TT4.16b, #8 + ext XL3.16b, TT3.16b, TT3.16b, #8 + + eor TT4.16b, TT4.16b, IN1.16b + pmull2 XH2.1q, SHASH.2d, IN1.2d // a1 * b1 + pmull XL2.1q, SHASH.1d, IN1.1d // a0 * b0 + pmull XM2.1q, SHASH2.1d, TT4.1d // (a1 + a0)(b1 + b0) + + eor TT3.16b, TT3.16b, XL3.16b + pmull2 XH3.1q, HH.2d, XL3.2d // a1 * b1 + pmull XL3.1q, HH.1d, XL3.1d // a0 * b0 + pmull2 XM3.1q, SHASH2.2d, TT3.2d // (a1 + a0)(b1 + b0) -1: ld1 {T1.2d}, [x21], #16 - sub w19, w19, #1 + ext IN1.16b, T2.16b, T2.16b, #8 + eor XL2.16b, XL2. |