diff options
author | Andy Polyakov <appro@openssl.org> | 2011-04-01 20:58:34 +0000 |
---|---|---|
committer | Andy Polyakov <appro@openssl.org> | 2011-04-01 20:58:34 +0000 |
commit | 1e86318091817459351a19e72f7d12959e9ec570 (patch) | |
tree | c8f8c679d4d8251a2e33c215e2970af0129abf7d | |
parent | d8d958323bb116bf9f88137ba46948dcb1691a77 (diff) |
ARM assembler pack: profiler-assisted optimizations and NEON support.
-rwxr-xr-x | Configure | 5 | ||||
-rw-r--r-- | TABLE | 32 | ||||
-rwxr-xr-x | config | 1 | ||||
-rw-r--r-- | crypto/aes/Makefile | 3 | ||||
-rw-r--r-- | crypto/aes/asm/aes-armv4.pl | 161 | ||||
-rw-r--r-- | crypto/modes/Makefile | 4 | ||||
-rw-r--r-- | crypto/modes/asm/ghash-armv4.pl | 193 | ||||
-rw-r--r-- | crypto/modes/gcm128.c | 32 | ||||
-rw-r--r-- | crypto/sha/Makefile | 14 | ||||
-rw-r--r-- | crypto/sha/asm/sha1-armv4-large.pl | 26 | ||||
-rw-r--r-- | crypto/sha/asm/sha256-armv4.pl | 55 | ||||
-rw-r--r-- | crypto/sha/asm/sha512-armv4.pl | 357 | ||||
-rw-r--r-- | crypto/sha/sha512.c | 50 |
13 files changed, 712 insertions, 221 deletions
@@ -398,8 +398,9 @@ my %table=( "linux-alpha-ccc","ccc:-fast -readonly_strings -DL_ENDIAN -DTERMIO::-D_REENTRANT:::SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_PTR DES_RISC1 DES_UNROLL:${alpha_asm}", "linux-alpha+bwx-ccc","ccc:-fast -readonly_strings -DL_ENDIAN -DTERMIO::-D_REENTRANT:::SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_INT DES_PTR DES_RISC1 DES_UNROLL:${alpha_asm}", -# Android: linux-armv4 but without -DTERMIO and pointers to headers and libs. -"android","gcc:-mandroid -I\$(ANDROID_DEV)/include -B\$(ANDROID_DEV)/lib -O3 -fomit-frame-pointer -Wall::-D_REENTRANT::-ldl:BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR:${armv4_asm}:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)", +# Android: linux-* but without -DTERMIO and pointers to headers and libs. +"android","gcc:-mandroid -I\$(ANDROID_DEV)/include -B\$(ANDROID_DEV)/lib -O3 -fomit-frame-pointer -Wall::-D_REENTRANT::-ldl:BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR:${no_asm}:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)", +"android-armv7","gcc:-march=armv7-a -mandroid -I\$(ANDROID_DEV)/include -B\$(ANDROID_DEV)/lib -O3 -fomit-frame-pointer -Wall::-D_REENTRANT::-ldl:BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR:${armv4_asm}:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)", #### *BSD [do see comment about ${BSDthreads} above!] "BSD-generic32","gcc:-DTERMIOS -O3 -fomit-frame-pointer -Wall::${BSDthreads}:::BN_LLONG RC2_CHAR RC4_INDEX DES_INT DES_UNROLL:${no_asm}:dlfcn:bsd-gcc-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)", @@ -1001,6 +1001,38 @@ $sys_id = $lflags = -ldl $bn_ops = BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR $cpuid_obj = +$bn_obj = +$des_obj = +$aes_obj = +$bf_obj = +$md5_obj = +$sha1_obj = +$cast_obj = +$rc4_obj = +$rmd160_obj = +$rc5_obj = +$wp_obj = +$cmll_obj = +$modes_obj = +$perlasm_scheme = void +$dso_scheme = dlfcn +$shared_target= linux-shared +$shared_cflag = -fPIC +$shared_ldflag = +$shared_extension = .so.$(SHLIB_MAJOR).$(SHLIB_MINOR) +$ranlib = +$arflags = +$multilib = + +*** android-armv7 +$cc = gcc +$cflags = -march=armv7-a -mandroid -I$(ANDROID_DEV)/include -B$(ANDROID_DEV)/lib -O3 -fomit-frame-pointer -Wall +$unistd = +$thread_cflag = -D_REENTRANT +$sys_id = +$lflags = -ldl +$bn_ops = BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR +$cpuid_obj = $bn_obj = bn_asm.o armv4-mont.o $des_obj = $aes_obj = aes_cbc.o aes-armv4.o @@ -821,6 +821,7 @@ case "$GUESSOS" in beos-*) OUT="$GUESSOS" ;; x86pc-*-qnx6) OUT="QNX6-i386" ;; *-*-qnx6) OUT="QNX6" ;; + armv[7-9]*-*-android) OUT="android-armv7" ;; *) OUT=`echo $GUESSOS | awk -F- '{print $3}'`;; esac diff --git a/crypto/aes/Makefile b/crypto/aes/Makefile index 5b52b9c43c..1c6e8c6d5e 100644 --- a/crypto/aes/Makefile +++ b/crypto/aes/Makefile @@ -68,7 +68,8 @@ aes-parisc.s: asm/aes-parisc.pl $(PERL) asm/aes-parisc.pl $(PERLASM_SCHEME) $@ # GNU make "catch all" -aes-%.s: asm/aes-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +aes-%.S: asm/aes-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +aes-armv4.o: aes-armv4.S files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/aes/asm/aes-armv4.pl b/crypto/aes/asm/aes-armv4.pl index c51ee1fbf6..55b6e04b67 100644 --- a/crypto/aes/asm/aes-armv4.pl +++ b/crypto/aes/asm/aes-armv4.pl @@ -27,6 +27,11 @@ # Rescheduling for dual-issue pipeline resulted in 12% improvement on # Cortex A8 core and ~25 cycles per byte processed with 128-bit key. +# February 2011. +# +# Profiler-assisted and platform-specific optimization resulted in 16% +# improvement on Cortex A8 core and ~21.5 cycles per byte. + while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; @@ -46,6 +51,7 @@ $key="r11"; $rounds="r12"; $code=<<___; +#include "arm_arch.h" .text .code 32 @@ -166,7 +172,7 @@ AES_encrypt: mov $rounds,r0 @ inp mov $key,r2 sub $tbl,r3,#AES_encrypt-AES_Te @ Te - +#if __ARM_ARCH__<7 ldrb $s0,[$rounds,#3] @ load input data in endian-neutral ldrb $t1,[$rounds,#2] @ manner... ldrb $t2,[$rounds,#1] @@ -195,10 +201,33 @@ AES_encrypt: orr $s3,$s3,$t1,lsl#8 orr $s3,$s3,$t2,lsl#16 orr $s3,$s3,$t3,lsl#24 - +#else + ldr $s0,[$rounds,#0] + ldr $s1,[$rounds,#4] + ldr $s2,[$rounds,#8] + ldr $s3,[$rounds,#12] +#ifdef __ARMEL__ + rev $s0,$s0 + rev $s1,$s1 + rev $s2,$s2 + rev $s3,$s3 +#endif +#endif bl _armv4_AES_encrypt ldr $rounds,[sp],#4 @ pop out +#if __ARM_ARCH__>=7 +#ifdef __ARMEL__ + rev $s0,$s0 + rev $s1,$s1 + rev $s2,$s2 + rev $s3,$s3 +#endif + str $s0,[$rounds,#0] + str $s1,[$rounds,#4] + str $s2,[$rounds,#8] + str $s3,[$rounds,#12] +#else mov $t1,$s0,lsr#24 @ write output in endian-neutral mov $t2,$s0,lsr#16 @ manner... mov $t3,$s0,lsr#8 @@ -227,11 +256,15 @@ AES_encrypt: strb $t2,[$rounds,#13] strb $t3,[$rounds,#14] strb $s3,[$rounds,#15] - +#endif +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else ldmia sp!,{r4-r12,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .size AES_encrypt,.-AES_encrypt .type _armv4_AES_encrypt,%function @@ -271,11 +304,11 @@ _armv4_AES_encrypt: and $i2,lr,$s2,lsr#16 @ i1 eor $t3,$t3,$i3,ror#8 and $i3,lr,$s2 - eor $s1,$s1,$t1,ror#24 ldr $i1,[$tbl,$i1,lsl#2] @ Te2[s2>>8] + eor $s1,$s1,$t1,ror#24 + ldr $i2,[$tbl,$i2,lsl#2] @ Te1[s2>>16] mov $s2,$s2,lsr#24 - ldr $i2,[$tbl,$i2,lsl#2] @ Te1[s2>>16] ldr $i3,[$tbl,$i3,lsl#2] @ Te3[s2>>0] eor $s0,$s0,$i1,ror#16 ldr $s2,[$tbl,$s2,lsl#2] @ Te0[s2>>24] @@ -284,16 +317,16 @@ _armv4_AES_encrypt: and $i2,lr,$s3,lsr#8 @ i1 eor $t3,$t3,$i3,ror#16 and $i3,lr,$s3,lsr#16 @ i2 - eor $s2,$s2,$t2,ror#16 ldr $i1,[$tbl,$i1,lsl#2] @ Te3[s3>>0] + eor $s2,$s2,$t2,ror#16 + ldr $i2,[$tbl,$i2,lsl#2] @ Te2[s3>>8] mov $s3,$s3,lsr#24 - ldr $i2,[$tbl,$i2,lsl#2] @ Te2[s3>>8] ldr $i3,[$tbl,$i3,lsl#2] @ Te1[s3>>16] eor $s0,$s0,$i1,ror#24 - ldr $s3,[$tbl,$s3,lsl#2] @ Te0[s3>>24] - eor $s1,$s1,$i2,ror#16 ldr $i1,[$key],#16 + eor $s1,$s1,$i2,ror#16 + ldr $s3,[$tbl,$s3,lsl#2] @ Te0[s3>>24] eor $s2,$s2,$i3,ror#8 ldr $t1,[$key,#-12] eor $s3,$s3,$t3,ror#8 @@ -333,11 +366,11 @@ _armv4_AES_encrypt: and $i2,lr,$s2,lsr#16 @ i1 eor $t3,$i3,$t3,lsl#8 and $i3,lr,$s2 - eor $s1,$t1,$s1,lsl#24 ldrb $i1,[$tbl,$i1,lsl#2] @ Te4[s2>>8] + eor $s1,$t1,$s1,lsl#24 + ldrb $i2,[$tbl,$i2,lsl#2] @ Te4[s2>>16] mov $s2,$s2,lsr#24 - ldrb $i2,[$tbl,$i2,lsl#2] @ Te4[s2>>16] ldrb $i3,[$tbl,$i3,lsl#2] @ Te4[s2>>0] eor $s0,$i1,$s0,lsl#8 ldrb $s2,[$tbl,$s2,lsl#2] @ Te4[s2>>24] @@ -346,15 +379,15 @@ _armv4_AES_encrypt: and $i2,lr,$s3,lsr#8 @ i1 eor $t3,$i3,$t3,lsl#8 and $i3,lr,$s3,lsr#16 @ i2 - eor $s2,$t2,$s2,lsl#24 ldrb $i1,[$tbl,$i1,lsl#2] @ Te4[s3>>0] + eor $s2,$t2,$s2,lsl#24 + ldrb $i2,[$tbl,$i2,lsl#2] @ Te4[s3>>8] mov $s3,$s3,lsr#24 - ldrb $i2,[$tbl,$i2,lsl#2] @ Te4[s3>>8] ldrb $i3,[$tbl,$i3,lsl#2] @ Te4[s3>>16] eor $s0,$i1,$s0,lsl#8 - ldrb $s3,[$tbl,$s3,lsl#2] @ Te4[s3>>24] ldr $i1,[$key,#0] + ldrb $s3,[$tbl,$s3,lsl#2] @ Te4[s3>>24] eor $s1,$s1,$i2,lsl#8 ldr $t1,[$key,#4] eor $s2,$s2,$i3,lsl#16 @@ -398,6 +431,7 @@ AES_set_encrypt_key: mov lr,r1 @ bits mov $key,r2 @ key +#if __ARM_ARCH__<7 ldrb $s0,[$rounds,#3] @ load input data in endian-neutral ldrb $t1,[$rounds,#2] @ manner... ldrb $t2,[$rounds,#1] @@ -430,6 +464,22 @@ AES_set_encrypt_key: orr $s3,$s3,$t3,lsl#24 str $s2,[$key,#-8] str $s3,[$key,#-4] +#else + ldr $s0,[$rounds,#0] + ldr $s1,[$rounds,#4] + ldr $s2,[$rounds,#8] + ldr $s3,[$rounds,#12] +#ifdef __ARMEL__ + rev $s0,$s0 + rev $s1,$s1 + rev $s2,$s2 + rev $s3,$s3 +#endif + str $s0,[$key],#16 + str $s1,[$key,#-12] + str $s2,[$key,#-8] + str $s3,[$key,#-4] +#endif teq lr,#128 bne .Lnot128 @@ -466,6 +516,7 @@ AES_set_encrypt_key: b .Ldone .Lnot128: +#if __ARM_ARCH__<7 ldrb $i2,[$rounds,#19] ldrb $t1,[$rounds,#18] ldrb $t2,[$rounds,#17] @@ -482,6 +533,16 @@ AES_set_encrypt_key: str $i2,[$key],#8 orr $i3,$i3,$t3,lsl#24 str $i3,[$key,#-4] +#else + ldr $i2,[$rounds,#16] + ldr $i3,[$rounds,#20] +#ifdef __ARMEL__ + rev $i2,$i2 + rev $i3,$i3 +#endif + str $i2,[$key],#8 + str $i3,[$key,#-4] +#endif teq lr,#192 bne .Lnot192 @@ -526,6 +587,7 @@ AES_set_encrypt_key: b .L192_loop .Lnot192: +#if __ARM_ARCH__<7 ldrb $i2,[$rounds,#27] ldrb $t1,[$rounds,#26] ldrb $t2,[$rounds,#25] @@ -542,6 +604,16 @@ AES_set_encrypt_key: str $i2,[$key],#8 orr $i3,$i3,$t3,lsl#24 str $i3,[$key,#-4] +#else + ldr $i2,[$rounds,#24] + ldr $i3,[$rounds,#28] +#ifdef __ARMEL__ + rev $i2,$i2 + rev $i3,$i3 +#endif + str $i2,[$key],#8 + str $i3,[$key,#-4] +#endif mov $rounds,#14 str $rounds,[$key,#240-32] @@ -692,10 +764,14 @@ $code.=<<___; bne .Lmix mov r0,#0 +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else ldmia sp!,{r4-r12,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .size AES_set_decrypt_key,.-AES_set_decrypt_key .type AES_Td,%object @@ -811,7 +887,7 @@ AES_decrypt: mov $rounds,r0 @ inp mov $key,r2 sub $tbl,r3,#AES_decrypt-AES_Td @ Td - +#if __ARM_ARCH__<7 ldrb $s0,[$rounds,#3] @ load input data in endian-neutral ldrb $t1,[$rounds,#2] @ manner... ldrb $t2,[$rounds,#1] @@ -840,10 +916,33 @@ AES_decrypt: orr $s3,$s3,$t1,lsl#8 orr $s3,$s3,$t2,lsl#16 orr $s3,$s3,$t3,lsl#24 - +#else + ldr $s0,[$rounds,#0] + ldr $s1,[$rounds,#4] + ldr $s2,[$rounds,#8] + ldr $s3,[$rounds,#12] +#ifdef __ARMEL__ + rev $s0,$s0 + rev $s1,$s1 + rev $s2,$s2 + rev $s3,$s3 +#endif +#endif bl _armv4_AES_decrypt ldr $rounds,[sp],#4 @ pop out +#if __ARM_ARCH__>=7 +#ifdef __ARMEL__ + rev $s0,$s0 + rev $s1,$s1 + rev $s2,$s2 + rev $s3,$s3 +#endif + str $s0,[$rounds,#0] + str $s1,[$rounds,#4] + str $s2,[$rounds,#8] + str $s3,[$rounds,#12] +#else mov $t1,$s0,lsr#24 @ write output in endian-neutral mov $t2,$s0,lsr#16 @ manner... mov $t3,$s0,lsr#8 @@ -872,11 +971,15 @@ AES_decrypt: strb $t2,[$rounds,#13] strb $t3,[$rounds,#14] strb $s3,[$rounds,#15] - +#endif +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else ldmia sp!,{r4-r12,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .size AES_decrypt,.-AES_decrypt .type _armv4_AES_decrypt,%function @@ -916,11 +1019,11 @@ _armv4_AES_decrypt: and $i2,lr,$s2 @ i1 eor $t3,$i3,$t3,ror#8 and $i3,lr,$s2,lsr#16 - eor $s1,$s1,$t1,ror#8 ldr $i1,[$tbl,$i1,lsl#2] @ Td2[s2>>8] + eor $s1,$s1,$t1,ror#8 + ldr $i2,[$tbl,$i2,lsl#2] @ Td3[s2>>0] mov $s2,$s2,lsr#24 - ldr $i2,[$tbl,$i2,lsl#2] @ Td3[s2>>0] ldr $i3,[$tbl,$i3,lsl#2] @ Td1[s2>>16] eor $s0,$s0,$i1,ror#16 ldr $s2,[$tbl,$s2,lsl#2] @ Td0[s2>>24] @@ -929,22 +1032,22 @@ _armv4_AES_decrypt: and $i2,lr,$s3,lsr#8 @ i1 eor $t3,$i3,$t3,ror#8 and $i3,lr,$s3 @ i2 - eor $s2,$s2,$t2,ror#8 ldr $i1,[$tbl,$i1,lsl#2] @ Td1[s3>>16] + eor $s2,$s2,$t2,ror#8 + ldr $i2,[$tbl,$i2,lsl#2] @ Td2[s3>>8] mov $s3,$s3,lsr#24 - ldr $i2,[$tbl,$i2,lsl#2] @ Td2[s3>>8] ldr $i3,[$tbl,$i3,lsl#2] @ Td3[s3>>0] eor $s0,$s0,$i1,ror#8 - ldr $s3,[$tbl,$s3,lsl#2] @ Td0[s3>>24] + ldr $i1,[$key],#16 eor $s1,$s1,$i2,ror#16 + ldr $s3,[$tbl,$s3,lsl#2] @ Td0[s3>>24] eor $s2,$s2,$i3,ror#24 - ldr $i1,[$key],#16 - eor $s3,$s3,$t3,ror#8 ldr $t1,[$key,#-12] - ldr $t2,[$key,#-8] eor $s0,$s0,$i1 + ldr $t2,[$key,#-8] + eor $s3,$s3,$t3,ror#8 ldr $t3,[$key,#-4] and $i1,lr,$s0,lsr#16 eor $s1,$s1,$t1 @@ -985,11 +1088,11 @@ _armv4_AES_decrypt: and $i1,lr,$s2,lsr#8 @ i0 eor $t2,$t2,$i2,lsl#8 and $i2,lr,$s2 @ i1 - eor $t3,$t3,$i3,lsl#8 ldrb $i1,[$tbl,$i1] @ Td4[s2>>8] + eor $t3,$t3,$i3,lsl#8 + ldrb $i2,[$tbl,$i2] @ Td4[s2>>0] and $i3,lr,$s2,lsr#16 - ldrb $i2,[$tbl,$i2] @ Td4[s2>>0] ldrb $s2,[$tbl,$s2,lsr#24] @ Td4[s2>>24] eor $s0,$s0,$i1,lsl#8 ldrb $i3,[$tbl,$i3] @ Td4[s2>>16] @@ -997,11 +1100,11 @@ _armv4_AES_decrypt: and $i1,lr,$s3,lsr#16 @ i0 eor $s2,$t2,$s2,lsl#16 and $i2,lr,$s3,lsr#8 @ i1 - eor $t3,$t3,$i3,lsl#16 ldrb $i1,[$tbl,$i1] @ Td4[s3>>16] + eor $t3,$t3,$i3,lsl#16 + ldrb $i2,[$tbl,$i2] @ Td4[s3>>8] and $i3,lr,$s3 @ i2 - ldrb $i2,[$tbl,$i2] @ Td4[s3>>8] ldrb $i3,[$tbl,$i3] @ Td4[s3>>0] ldrb $s3,[$tbl,$s3,lsr#24] @ Td4[s3>>24] eor $s0,$s0,$i1,lsl#16 diff --git a/crypto/modes/Makefile b/crypto/modes/Makefile index c3221be753..28ee07c3fa 100644 --- a/crypto/modes/Makefile +++ b/crypto/modes/Makefile @@ -57,7 +57,9 @@ ghash-parisc.s: asm/ghash-parisc.pl $(PERL) asm/ghash-parisc.pl $(PERLASM_SCHEME) $@ # GNU make "catch all" -ghash-%.s: asm/ghash-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +ghash-%.S: asm/ghash-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ + +ghash-armv4.o: ghash-armv4.S files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/modes/asm/ghash-armv4.pl b/crypto/modes/asm/ghash-armv4.pl index 2036f46f40..d91586ee29 100644 --- a/crypto/modes/asm/ghash-armv4.pl +++ b/crypto/modes/asm/ghash-armv4.pl @@ -25,6 +25,18 @@ # Cortex A8 core and ~25 cycles per processed byte (which was observed # to be ~3 times faster than gcc-generated code:-) # +# February 2011 +# +# Profiler-assisted and platform-specific optimization resulted in 7% +# improvement on Cortex A8 core and ~23.5 cycles per byte. +# +# March 2011 +# +# Add NEON implementation featuring polynomial multiplication, i.e. no +# lookup tables involved. On Cortex A8 it was measured to process one +# byte in 15 cycles or 55% faster than integer-only code. + +# ==================================================================== # Note about "528B" variant. In ARM case it makes lesser sense to # implement it for following reasons: # @@ -52,6 +64,7 @@ $Xi="r0"; # argument block $Htbl="r1"; $inp="r2"; $len="r3"; + $Zll="r4"; # variables $Zlh="r5"; $Zhl="r6"; @@ -72,8 +85,13 @@ sub Zsmash() { my $i=12; my @args=@_; for ($Zll,$Zlh,$Zhl,$Zhh) { - # can be reduced to single "str $_,[$Xi,$i]" on big-endian platforms $code.=<<___; +#if __ARM_ARCH__>=7 && defined(__ARMEL__) + rev $_,$_ + str $_,[$Xi,#$i] +#elif defined(__ARMEB__) + str $_,[$Xi,#$i] +#else mov $Tlh,$_,lsr#8 strb $_,[$Xi,#$i+3] mov $Thl,$_,lsr#16 @@ -81,6 +99,7 @@ sub Zsmash() { mov $Thh,$_,lsr#24 strb $Thl,[$Xi,#$i+1] strb $Thh,[$Xi,#$i] +#endif ___ $code.="\t".shift(@args)."\n"; $i-=4; @@ -88,6 +107,8 @@ ___ } $code=<<___; +#include "arm_arch.h" + .text .code 32 @@ -149,41 +170,41 @@ gcm_ghash_4bit: and $nlo,$nlo,#0x0f eor $Zhh,$Zhh,$Tll,lsl#16 -.Loop: +.Linner: add $Thh,$Htbl,$nlo,lsl#4 - subs $cnt,$cnt,#1 and $nlo,$Zll,#0xf @ rem - ldmia $Thh,{$Tll-$Thh} @ load Htbl[nlo] + subs $cnt,$cnt,#1 add $nlo,$nlo,$nlo + ldmia $Thh,{$Tll-$Thh} @ load Htbl[nlo] eor $Zll,$Tll,$Zll,lsr#4 - ldrh $Tll,[sp,$nlo] @ rem_4bit[rem] eor $Zll,$Zll,$Zlh,lsl#28 eor $Zlh,$Tlh,$Zlh,lsr#4 eor $Zlh,$Zlh,$Zhl,lsl#28 + ldrh $Tll,[sp,$nlo] @ rem_4bit[rem] eor $Zhl,$Thl,$Zhl,lsr#4 + ldrplb $nlo,[$inp,$cnt] eor $Zhl,$Zhl,$Zhh,lsl#28 eor $Zhh,$Thh,$Zhh,lsr#4 - ldrplb $nlo,[$inp,$cnt] add $Thh,$Htbl,$nhi - eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem] and $nhi,$Zll,#0xf @ rem - ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi] + eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem] add $nhi,$nhi,$nhi + ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi] eor $Zll,$Tll,$Zll,lsr#4 - ldrh $Tll,[sp,$nhi] @ rem_4bit[rem] + ldrplb $Tll,[$Xi,$cnt] eor $Zll,$Zll,$Zlh,lsl#28 eor $Zlh,$Tlh,$Zlh,lsr#4 - ldrplb $nhi,[$Xi,$cnt] + ldrh $Tlh,[sp,$nhi] eor $Zlh,$Zlh,$Zhl,lsl#28 eor $Zhl,$Thl,$Zhl,lsr#4 eor $Zhl,$Zhl,$Zhh,lsl#28 - eorpl $nlo,$nlo,$nhi + eorpl $nlo,$nlo,$Tll eor $Zhh,$Thh,$Zhh,lsr#4 andpl $nhi,$nlo,#0xf0 andpl $nlo,$nlo,#0x0f - eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem] - bpl .Loop + eor $Zhh,$Zhh,$Tlh,lsl#16 @ ^= rem_4bit[rem] + bpl .Linner ldr $len,[sp,#32] @ re-load $len/end add $inp,$inp,#16 @@ -194,10 +215,14 @@ $code.=<<___; bne .Louter add sp,sp,#36 +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else ldmia sp!,{r4-r11,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .size gcm_ghash_4bit,.-gcm_ghash_4bit .global gcm_gmult_4bit @@ -231,31 +256,31 @@ gcm_gmult_4bit: eor $Zhh,$Zhh,$Tll,lsl#16 and $nlo,$nlo,#0x0f -.Loop2: +.Loop: add $Thh,$Htbl,$nlo,lsl#4 - subs $cnt,$cnt,#1 and $nlo,$Zll,#0xf @ rem - ldmia $Thh,{$Tll-$Thh} @ load Htbl[nlo] + subs $cnt,$cnt,#1 add $nlo,$nlo,$nlo + ldmia $Thh,{$Tll-$Thh} @ load Htbl[nlo] eor $Zll,$Tll,$Zll,lsr#4 - ldrh $Tll,[$rem_4bit,$nlo] @ rem_4bit[rem] eor $Zll,$Zll,$Zlh,lsl#28 eor $Zlh,$Tlh,$Zlh,lsr#4 eor $Zlh,$Zlh,$Zhl,lsl#28 + ldrh $Tll,[$rem_4bit,$nlo] @ rem_4bit[rem] eor $Zhl,$Thl,$Zhl,lsr#4 + ldrplb $nlo,[$Xi,$cnt] eor $Zhl,$Zhl,$Zhh,lsl#28 eor $Zhh,$Thh,$Zhh,lsr#4 - ldrplb $nlo,[$Xi,$cnt] add $Thh,$Htbl,$nhi - eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem] and $nhi,$Zll,#0xf @ rem - ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi] + eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem] add $nhi,$nhi,$nhi + ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi] eor $Zll,$Tll,$Zll,lsr#4 - ldrh $Tll,[$rem_4bit,$nhi] @ rem_4bit[rem] eor $Zll,$Zll,$Zlh,lsl#28 eor $Zlh,$Tlh,$Zlh,lsr#4 + ldrh $Tll,[$rem_4bit,$nhi] @ rem_4bit[rem] eor $Zlh,$Zlh,$Zhl,lsl#28 eor $Zhl,$Thl,$Zhl,lsr#4 eor $Zhl,$Zhl,$Zhh,lsl#28 @@ -263,16 +288,138 @@ gcm_gmult_4bit: andpl $nhi,$nlo,#0xf0 andpl $nlo,$nlo,#0x0f eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem] - bpl .Loop2 + bpl .Loop ___ &Zsmash(); $code.=<<___; +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else ldmia sp!,{r4-r11,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .size gcm_gmult_4bit,.-gcm_gmult_4bit -.asciz "GHASH for ARMv4, CRYPTOGAMS by <appro\@openssl.org>" +___ +{ +my $cnt=$Htbl; # $Htbl is used once in the very beginning + +my ($Hhi, $Hlo, $Zo, $T, $xi, $mod) = map("d$_",(0..7)); +my ($Qhi, $Qlo, $Z, $R, $zero, $Qpost, $IN) = map("q$_",(8..15)); + +# Z:Zo keeps 128-bit result shifted by 1 to the right, with bottom bit +# in Zo. Or should I say "top bit", because GHASH is specified in +# reverse bit order? Otherwise straightforward 128-bt H by one input +# byte multiplication and modulo-reduction, times 16. + +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } +sub Q() { shift=~m|d([1-3]?[02468])|?"q".($1/2):""; } + +$code.=<<___; +#if __ARM_ARCH__>=7 +.fpu neon + +.global gcm_gmult_neon +.type gcm_gmult_neon,%function +.align 4 +gcm_gmult_neon: + sub $Htbl,#16 @ point at H in GCM128_CTX + vld1.64 `&Dhi("$IN")`,[$Xi,:64]!@ load Xi + vmov.i32 $mod,#0xe1 @ our irreducible polynomial + vld1.64 `&Dlo("$IN")`,[$Xi,:64]! + vshr.u64 $mod,#32 + vldmia $Htbl,{$Hhi-$Hlo} @ load H + veor $zero,$zero +#ifdef __ARMEL__ + vrev64.8 $IN,$IN +#endif + veor $Qpost,$Qpost + veor $R,$R + mov $cnt,#16 + veor $Z,$Z + mov $len,#16 + veor $Zo,$Zo + vdup.8 $xi,`&Dlo("$IN")`[0] @ broadcast lowest byte + b .Linner_neon +.size gcm_gmult_neon,.-gcm_gmult_neon + +.global gcm_ghash_neon +.type gcm_ghash_neon,%function +.align 4 +gcm_ghash_neon: + vld1.64 `&Dhi("$Z")`,[$Xi,:64]! @ load Xi + vmov.i32 $mod,#0xe1 @ our irreducible polynomial + vld1.64 `&Dlo("$Z")`,[$Xi,:64]! + vshr.u64 $mod,#32 + vldmia $Xi,{$Hhi-$Hlo} @ load H + veor $zero,$zero + nop +#ifdef __ARMEL__ + vrev64.8 $Z,$Z +#endif +.Louter_neon: + vld1.64 `&Dhi($IN)`,[$inp]! @ load inp + veor $Qpost,$Qpost + vld1.64 `&Dlo($IN)`,[$inp]! + veor $R,$R + mov $cnt,#16 +#ifdef __ARMEL__ + vrev64.8 $IN,$IN +#endif + veor $Zo,$Zo + veor $IN,$Z @ inp^=Xi + veor $Z,$Z + vdup.8 $xi,`&Dlo("$IN")`[0] @ broadcast lowest byte +.Linner_neon: + subs $cnt,$cnt,#1 + vmull.p8 $Qlo,$Hlo,$xi @ H.lo·Xi[i] + vmull.p8 $Qhi,$Hhi,$xi @ H.hi·Xi[i] + vext.8 $IN,$zero,#1 @ IN>>=8 + + veor $Z,$Qpost @ modulo-scheduled part + vshl.i64 `&Dlo("$R")`,#48 + vdup.8 $xi,`&Dlo("$IN")`[0] @ broadcast lowest byte + veor $T,`&Dlo("$Qlo")`,`&Dlo("$Z")` + + veor `&Dhi("$Z")`,`&Dlo("$R")` + vuzp.8 $Qlo,$Qhi + vsli.8 $Zo,$T,#1 @ compose the "carry" byte + vext.8 $Z,$zero,#1 @ Z>>=8 + + vmull.p8 $R,$Zo,$mod @ "carry"·0xe1 + vshr.u8 $Zo,$T,#7 @ save Z's bottom bit + vext.8 $Qpost,$Qlo,$zero,#1 @ Qlo>>=8 + veor $Z,$Qhi + bne .Linner_neon + + veor $Z,$Qpost @ modulo-scheduled artefact + vshl.i64 `&Dlo("$R")`,#48 + veor `&Dhi("$Z")`,`&Dlo("$R")` + + @ finalization, normalize Z:Zo + vand $Zo,$mod @ suffices to mask the bit + vshr.u64 `&Dhi(&Q("$Zo"))`,`&Dlo("$Z")`,#63 + vshl.i64 $Z,#1 + subs $len,#16 + vorr $Z,`&Q("$Zo")` @ Z=Z:Zo<<1 + bne .Louter_neon + +#ifdef __ARMEL__ + vrev64.8 $Z,$Z +#endif + sub $Xi,#16 + vst1.64 `&Dhi("$Z")`,[$Xi,:64]! @ write out Xi + vst1.64 `&Dlo("$Z")`,[$Xi,:64] + + bx lr +.size gcm_ghash_neon,.-gcm_ghash_neon +#endif +___ +} +$code.=<<___; +.asciz "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>" .align 2 ___ diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c index c2a2d5e6d9..875f6cab7d 100644 --- a/crypto/modes/gcm128.c +++ b/crypto/modes/gcm128.c @@ -642,27 +642,38 @@ static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2]) #endif -#if TABLE_BITS==4 && defined(GHASH_ASM) && !defined(I386_ONLY) && \ +#if TABLE_BITS==4 && defined(GHASH_ASM) +# if !defined(I386_ONLY) && \ (defined(__i386) || defined(__i386__) || \ defined(__x86_64) || defined(__x86_64__) || \ defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64)) -# define GHASH_ASM_X86_OR_64 +# define GHASH_ASM_X86_OR_64 +# define GCM_FUNCREF_4BIT extern unsigned int OPENSSL_ia32cap_P[2]; void gcm_init_clmul(u128 Htable[16],const u64 Xi[2]); void gcm_gmult_clmul(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_clmul(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); -# if defined(__i386) || defined(__i386__) || defined(_M_IX86) -# define GHASH_ASM_X86 +# if defined(__i386) || defined(__i386__) || defined(_M_IX86) +# define GHASH_ASM_X86 void gcm_gmult_4bit_mmx(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_4bit_mmx(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); void gcm_gmult_4bit_x86(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); +# endif +# elif defined(__arm__) || defined(__arm) +# include "arm_arch.h" +# if __ARM_ARCH__>=7 +# define GHASH_ASM_ARM +# define GCM_FUNCREF_4BIT +extern unsigned int OPENSSL_armcap; + +void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]); +void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); +# endif # endif - -# define GCM_FUNCREF_4BIT #endif void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block) @@ -715,6 +726,15 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block) ctx->gmult = gcm_gmult_4bit; ctx->ghash = gcm_ghash_4bit; # endif +# elif defined(GHASH_ASM_ARM) + if (OPENSSL_armcap & 1) { + ctx->gmult = gcm_gmult_neon; + ctx->ghash = gcm_ghash_neon; + } else { + gcm_init_4bit(ctx->Htable,ctx->H.u); + ctx->gmult = gcm_gmult_4bit; + ctx->ghash = gcm_ghash_4bit; + } # else gcm_init_4bit(ctx->Htable,ctx->H.u); # endif diff --git a/crypto/sha/Makefile b/crypto/sha/Makefile index 2119e33a91..561a0ade5f 100644 --- a/crypto/sha/Makefile +++ b/crypto/sha/Makefile @@ -56,8 +56,8 @@ sha256-ia64.s: asm/sha512-ia64.pl sha512-ia64.s: asm/sha512-ia64.pl (cd asm; $(PERL) sha512-ia64.pl ../$@ $(CFLAGS)) -sha256-armv4.s: asm/sha256-armv4.pl - $(PERL) $< $@ +sha256-armv4.S: asm/sha256-armv4.pl + $(PERL) $< $(PERLASM_SCHEME) $@ sha1-alpha.s: asm/sha1-alpha.pl $(PERL) $< | $(CC) -E - | tee $@ > /dev/null @@ -83,9 +83,13 @@ sha256-mips.s: asm/sha512-mips.pl; $(PERL) asm/sha512-mips.pl $(PERLASM_SCHEME) sha512-mips.s: asm/sha512-mips.pl; $(PERL) asm/sha512-mips.pl $(PERLASM_SCHEME) $@ # GNU make "catch all" -sha1-%.s: asm/sha1-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ -sha256-%.s: asm/sha512-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ -sha512-%.s: asm/sha512-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +sha1-%.S: asm/sha1-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +sha256-%.S: asm/sha512-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +sha512-%.S: asm/sha512-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ + +sha1-armv4-large.o: sha1-armv4-large.S +sha256-armv4.o: sha256-armv4.S +sha512-armv4.o: sha512-armv4.S files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/sha/asm/sha1-armv4-large.pl b/crypto/sha/asm/sha1-armv4-large.pl index afe506be30..fe8207f77f 100644 --- a/crypto/sha/asm/sha1-armv4-large.pl +++ b/crypto/sha/asm/sha1-armv4-large.pl @@ -47,6 +47,10 @@ # Cortex A8 core and in absolute terms ~870 cycles per input block # [or 13.6 cycles per byte]. +# February 2011. +# +# Profiler-assisted and platform-specific optimization resulted in 10% +# improvement on Cortex A8 core and 12.2 cycles per byte. while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; @@ -76,31 +80,41 @@ $code.=<<___; add $e,$K,$e,ror#2 @ E+=K_xx_xx ldr $t3,[$Xi,#2*4] eor $t0,$t0,$t1 - eor $t2,$t2,$t3 + eor $t2,$t2,$t3 @ 1 cycle stall eor $t1,$c,$d @ F_xx_xx mov $t0,$t0,ror#31 add $e,$e,$a,ror#27 @ E+=ROR(A,27) eor $t0,$t0,$t2,ror#31 + str $t0,[$Xi,#-4]! $opt1 @ F_xx_xx $opt2 @ F_xx_xx add $e,$e,$t0 @ E+=X[i] - str $t0,[$Xi,#-4]! ___ } sub BODY_00_15 { my ($a,$b,$c,$d,$e)=@_; $code.=<<___; +#if __ARM_ARCH__<7 ldrb $t1,[$inp,#2] ldrb $t0,[$inp,#3] ldrb $t2,[$inp,#1] add $e,$K,$e,ror#2 @ E+=K_00_19 ldrb $t3,[$inp],#4 - add $e,$e,$a,ror#27 @ E+=ROR(A,27) orr $t0,$t0,$t1,lsl#8 eor $t1,$c,$d @ F_xx_xx orr $t0,$t0,$t2,lsl#16 + add $e,$e,$a,ror#27 @ E+=ROR(A,27) orr $t0,$t0,$t3,lsl#24 +#else + ldr $t0,[$inp],#4 @ handles unaligned + add $e,$K,$e,ror#2 @ E+=K_00_19 + eor $t1,$c,$d @ F_xx_xx + add $e,$e,$a,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev $t0,$t0 @ byte swap +#endif +#endif and $t1,$b,$t1,ror#2 add $e,$e,$t0 @ E+=X[i] eor $t1,$t1,$d,ror#2 @ F_00_19(B,C,D) @@ -136,6 +150,8 @@ ___ } $code=<<___; +#include "arm_arch.h" + .text .global sha1_block_data_order @@ -209,10 +225,14 @@ $code.=<<___; teq $inp,$len bne .Lloop @ [+18], total 1307 +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else ldmia sp!,{r4-r12,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .align 2 .LK_00_19: .word 0x5a827999 .LK_20_39: .word 0x6ed9eba1 diff --git a/crypto/sha/asm/sha256-armv4.pl b/crypto/sha/asm/sha256-armv4.pl index 492cb62bc0..9c84e8d93c 100644 --- a/crypto/sha/asm/sha256-armv4.pl +++ b/crypto/sha/asm/sha256-armv4.pl @@ -18,11 +18,16 @@ # Rescheduling for dual-issue pipeline resulted in 22% improvement on # Cortex A8 core and ~20 cycles per processed byte. +# February 2011. +# +# Profiler-assisted and platform-specific optimization resulted in 16% +# improvement on Cortex A8 core and ~17 cycles per processed byte. + while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; $ctx="r0"; $t0="r0"; -$inp="r1"; +$inp="r1"; $t3="r1"; $len="r2"; $t1="r2"; $T1="r3"; $A="r4"; @@ -46,6 +51,9 @@ sub BODY_00_15 { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; $code.=<<___ if ($i<16); |