summaryrefslogtreecommitdiffstats
path: root/crypto/sha/asm/sha256-586.pl
diff options
context:
space:
mode:
Diffstat (limited to 'crypto/sha/asm/sha256-586.pl')
-rw-r--r--crypto/sha/asm/sha256-586.pl447
1 files changed, 344 insertions, 103 deletions
diff --git a/crypto/sha/asm/sha256-586.pl b/crypto/sha/asm/sha256-586.pl
index 447c50b0cc..bd48b638c5 100644
--- a/crypto/sha/asm/sha256-586.pl
+++ b/crypto/sha/asm/sha256-586.pl
@@ -28,18 +28,31 @@
# May version, >60% over original. Add AVX+shrd code path, >25%
# improvement on Sandy Bridge over May version, 60% over original.
#
+# May 2013.
+#
+# Replace AMD XOP code path with SSSE3 to cover more processors.
+# (Biggest improvement coefficient is on upcoming Atom Silvermont,
+# not shown.) Add AVX+BMI code path.
+#
# Performance in clock cycles per processed byte (less is better):
#
-# PIII P4 AMD K8 Core2 SB Atom Bldzr
-# gcc 36 41 27 26 25 50 36
-# icc 33 38 25 23 - - -
-# x86 asm(*) 27/24 28 19/15.5 18/15.6 12.3 30/25 16.6
-# x86_64 asm(**) 17.5 15.1 13.9 11.6 22 13.7
+# gcc icc x86 asm(*) SIMD x86_64 asm(**)
+# Pentium 46 57 40/38 - -
+# PIII 36 33 27/24 - -
+# P4 41 38 28 - 17.3
+# AMD K8 27 25 19/15.5 - 14.9
+# Core2 26 23 18/15.6 14.3 13.8
+# Westmere 27 - 19/15.7 13.4 12.3
+# Sandy Bridge 25 - 15.9 12.4 11.6
+# Ivy Bridge 24 - 15.0 11.4 10.3
+# Haswell 22 - 13.9 9.46 7.80
+# Bulldozer 36 - 27/22 17.0 13.6
+# VIA Nano 36 - 25/22 16.8 16.5
+# Atom 50 - 30/25 21.9 18.9
#
-# (*) numbers after slash are for unrolled loop, where available,
-# otherwise best applicable such as AVX/XOP;
+# (*) numbers after slash are for unrolled loop, where applicable;
# (**) x86_64 assembly performance is presented for reference
-# purposes.
+# purposes, results are best-available;
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
@@ -47,21 +60,23 @@ require "x86asm.pl";
&asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386");
-$xmm=$ymm=0;
+$xmm=$avx=0;
for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
-$ymm=1 if ($xmm &&
- `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
- =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
- $1>=2.19); # first version supporting AVX
+if ($xmm && `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
+ $avx = ($1>=2.19) + ($1>=2.22);
+}
-$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
- `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
- $1>=2.03); # first version supporting AVX
+if ($xmm && !$avx && $ARGV[0] eq "win32n" &&
+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
+ $avx = ($1>=2.03) + ($1>=2.10);
+}
-$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" &&
- `ml 2>&1` =~ /Version ([0-9]+)\./ &&
- $1>=10); # first version supporting AVX
+if ($xmm && !$avx && $ARGV[0] eq "win32" &&
+ `ml 2>&1` =~ /Version ([0-9]+)\./) {
+ $avx = ($1>=10) + ($1>=11);
+}
$unroll_after = 64*4; # If pre-evicted from L1P cache first spin of
# fully unrolled loop was measured to run about
@@ -178,17 +193,21 @@ sub BODY_00_15() {
if (!$i386) {
&picmeup("edx","OPENSSL_ia32cap_P",$K256,&label("K256"));
&mov ("ecx",&DWP(0,"edx"));
- &mov ("edx",&DWP(4,"edx"));
+ &mov ("ebx",&DWP(4,"edx"));
&test ("ecx",1<<20); # check for P4
&jnz (&label("loop"));
- &test ("edx",1<<11); # check for XOP
- &jnz (&label("XOP")) if ($ymm);
&and ("ecx",1<<30); # mask "Intel CPU" bit
- &and ("edx",1<<28); # mask AVX bit
- &or ("ecx","edx");
+ &and ("ebx",1<<28|1<<9); # mask AVX and SSSE3 bits
+ &or ("ecx","ebx");
+ &and ("ecx",1<<28|1<<30);
&cmp ("ecx",1<<28|1<<30);
- &je (&label("AVX")) if ($ymm);
- &je (&label("loop_shrd")) if (!$ymm);
+ if ($xmm) {
+ &je (&label("AVX")) if ($avx);
+ &test ("ebx",1<<9); # check for SSSE3
+ &jnz (&label("SSSE3"));
+ } else {
+ &je (&label("loop_shrd"));
+ }
if ($unroll_after) {
&sub ("eax","edi");
&cmp ("eax",$unroll_after);
@@ -199,7 +218,7 @@ sub BODY_00_15() {
sub COMPACT_LOOP() {
my $suffix=shift;
-&set_label("loop$suffix",16);
+&set_label("loop$suffix",$suffix?32:16);
# copy input block to stack reversing byte and dword order
for($i=0;$i<4;$i++) {
&mov ("eax",&DWP($i*16+0,"edi"));
@@ -292,7 +311,7 @@ my $suffix=shift;
&COMPACT_LOOP();
&mov ("esp",&DWP(12,"esp")); # restore sp
&function_end_A();
- if (!$i386 && !$ymm) {
+ if (!$i386 && !$xmm) {
# ~20% improvement on Sandy Bridge
local *ror = sub { &shrd(@_[0],@_) };
&COMPACT_LOOP("_shrd");
@@ -318,7 +337,11 @@ my $suffix=shift;
0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,
0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 );
&data_word(@K256);
-&data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f);
+&data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # byte swap mask
+&asciz("SHA256 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>");
+
+($a,$b,$c,$d,$e,$f,$g,$h)=(0..7); # offsets
+sub off { &DWP(4*(((shift)-$i)&7),"esp"); }
if (!$i386 && $unroll_after) {
my @AH=($A,$K256);
@@ -365,8 +388,6 @@ my @AH=($A,$K256);
&mov (&DWP(32+12*$i,"esp"),"ebx");
my ($t1,$t2) = ("ecx","esi");
- my ($a,$b,$c,$d,$e,$f,$g,$h)=(0..7); # offsets
- sub off { &DWP(4*(((shift)-$i)&7),"esp"); }
for ($i=0;$i<64;$i++) {
@@ -472,15 +493,14 @@ my @AH=($A,$K256);
&mov ("esp",&DWP(96+12,"esp")); # restore sp
&function_end_A();
-
- if ($ymm) {{{
+}
+ if (!$i386 && $xmm) {{{
my @X = map("xmm$_",(0..3));
my ($t0,$t1,$t2,$t3) = map("xmm$_",(4..7));
my @AH = ($A,$T);
-&set_label("XOP",16);
+&set_label("SSSE3",32);
&lea ("esp",&DWP(-96,"esp"));
- &vzeroall ();
# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
&mov ($AH[0],&DWP(0,"esi"));
&mov ($AH[1],&DWP(4,"esi"));
@@ -500,135 +520,208 @@ my @AH = ($A,$T);
&mov ("edi",&DWP(96+4,"esp")); # inp
&mov (&DWP(24,"esp"),"ecx");
&mov (&DWP(28,"esp"),"esi");
- &vmovdqa ($t3,&QWP(256,$K256));
- &jmp (&label("grand_xop"));
+ &movdqa ($t3,&QWP(256,$K256));
+ &jmp (&label("grand_ssse3"));
-&set_label("grand_xop",16);
+&set_label("grand_ssse3",16);
# load input, reverse byte order, add K256[0..15], save to stack
- &vmovdqu (@X[0],&QWP(0,"edi"));
- &vmovdqu (@X[1],&QWP(16,"edi"));
- &vmovdqu (@X[2],&QWP(32,"edi"));
- &vmovdqu (@X[3],&QWP(48,"edi"));
- &add ("edi",64);
- &vpshufb (@X[0],@X[0],$t3);
- &mov (&DWP(96+4,"esp"),"edi");
- &vpshufb (@X[1],@X[1],$t3);
- &vpshufb (@X[2],@X[2],$t3);
- &vpaddd ($t0,@X[0],&QWP(0,$K256));
- &vpshufb (@X[3],@X[3],$t3);
- &vpaddd ($t1,@X[1],&QWP(16,$K256));
- &vpaddd ($t2,@X[2],&QWP(32,$K256));
- &vpaddd ($t3,@X[3],&QWP(48,$K256));
- &vmovdqa (&QWP(32+0,"esp"),$t0);
- &vmovdqa (&QWP(32+16,"esp"),$t1);
- &vmovdqa (&QWP(32+32,"esp"),$t2);
- &vmovdqa (&QWP(32+48,"esp"),$t3);
- &jmp (&label("xop_00_47"));
-
-&set_label("xop_00_47",16);
+ &movdqu (@X[0],&QWP(0,"edi"));
+ &movdqu (@X[1],&QWP(16,"edi"));
+ &movdqu (@X[2],&QWP(32,"edi"));
+ &movdqu (@X[3],&QWP(48,"edi"));
+ &add ("edi",64);
+ &pshufb (@X[0],$t3);
+ &mov (&DWP(96+4,"esp"),"edi");
+ &pshufb (@X[1],$t3);
+ &movdqa ($t0,&QWP(0,$K256));
+ &pshufb (@X[2],$t3);
+ &movdqa ($t1,&QWP(16,$K256));
+ &paddd ($t0,@X[0]);
+ &pshufb (@X[3],$t3);
+ &movdqa ($t2,&QWP(32,$K256));
+ &paddd ($t1,@X[1]);
+ &movdqa ($t3,&QWP(48,$K256));
+ &movdqa (&QWP(32+0,"esp"),$t0);
+ &paddd ($t2,@X[2]);
+ &movdqa (&QWP(32+16,"esp"),$t1);
+ &paddd ($t3,@X[3]);
+ &movdqa (&QWP(32+32,"esp"),$t2);
+ &movdqa (&QWP(32+48,"esp"),$t3);
+ &jmp (&label("ssse3_00_47"));
+
+&set_label("ssse3_00_47",16);
&add ($K256,64);
-sub XOP_00_47 () {
+sub SSSE3_00_47 () {
my $j = shift;
my $body = shift;
my @X = @_;
my @insns = (&$body,&$body,&$body,&$body); # 120 instructions
- &vpalignr ($t0,@X[1],@X[0],4); # X[1..4]
eval(shift(@insns));
+ &movdqa ($t0,@X[1]);
+ eval(shift(@insns)); # @
eval(shift(@insns));
- &vpalignr ($t3,@X[3],@X[2],4); # X[9..12]
+ &movdqa ($t3,@X[3]);
eval(shift(@insns));
eval(shift(@insns));
- &vprotd ($t1,$t0,14);
+ &palignr ($t0,@X[0],4); # X[1..4]
eval(shift(@insns));
+ eval(shift(@insns)); # @
eval(shift(@insns));
- &vpsrld ($t0,$t0,3);
- &vpaddd (@X[0],@X[0],$t3); # X[0..3] += X[9..12]
+ &palignr ($t3,@X[2],4); # X[9..12]
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
+ &movdqa ($t1,$t0);
+ eval(shift(@insns)); # @
eval(shift(@insns));
- &vprotd ($t2,$t1,25-14);
- &vpxor ($t0,$t0,$t1);
+ &movdqa ($t2,$t0);
eval(shift(@insns));
eval(shift(@insns));
+ &psrld ($t0,3);
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &paddd (@X[0],$t3); # X[0..3] += X[9..12]
eval(shift(@insns));
- &vprotd ($t3,@X[3],13);
- &vpxor ($t0,$t0,$t2); # sigma0(X[1..4])
eval(shift(@insns));
+ &psrld ($t2,7);
eval(shift(@insns));
- &vpsrld ($t2,@X[3],10);
eval(shift(@insns));
+ eval(shift(@insns)); # @
eval(shift(@insns));
- &vpaddd (@X[0],@X[0],$t0); # X[0..3] += sigma0(X[1..4])
+ &pshufd ($t3,@X[3],0b11111010); # X[14..15]
eval(shift(@insns));
eval(shift(@insns));
- &vprotd ($t1,$t3,15-13);
- &vpxor ($t3,$t3,$t2);
+ &pslld ($t1,32-18);
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t0,$t2);
eval(shift(@insns));
eval(shift(@insns));
+ &psrld ($t2,18-7);
eval(shift(@insns));
- &vpxor ($t3,$t3,$t1); # sigma1(X[14..15])
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t0,$t1);
eval(shift(@insns));
eval(shift(@insns));
+ &pslld ($t1,18-7);
eval(shift(@insns));
- &vpsrldq ($t3,$t3,8);
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t0,$t2);
eval(shift(@insns));
eval(shift(@insns));
+ &movdqa ($t2,$t3);
eval(shift(@insns));
- &vpaddd (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15])
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t0,$t1); # sigma0(X[1..4])
eval(shift(@insns));
eval(shift(@insns));
+ &psrld ($t3,10);
eval(shift(@insns));
- &vprotd ($t3,@X[0],13);
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &paddd (@X[0],$t0); # X[0..3] += sigma0(X[1..4])
eval(shift(@insns));
- &vpsrld ($t2,@X[0],10);
eval(shift(@insns));
+ &psrlq ($t2,17);
eval(shift(@insns));
- &vprotd ($t1,$t3,15-13);
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t3,$t2);
eval(shift(@insns));
- &vpxor ($t3,$t3,$t2);
eval(shift(@insns));
+ &psrlq ($t2,19-17);
eval(shift(@insns));
eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t3,$t2);
eval(shift(@insns));
- &vpxor ($t3,$t3,$t1); # sigma1(X[16..17])
eval(shift(@insns));
+ &pshufd ($t3,$t3,0b10000000);
eval(shift(@insns));
eval(shift(@insns));
+ eval(shift(@insns)); # @
eval(shift(@insns));
- &vpslldq ($t3,$t3,8); # 22 instructions
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
+ eval(shift(@insns)); # @
eval(shift(@insns));
- &vpaddd (@X[0],@X[0],$t3); # X[2..3] += sigma1(X[16..17])
+ &psrldq ($t3,8);
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
+ &paddd (@X[0],$t3); # X[0..1] += sigma1(X[14..15])
+ eval(shift(@insns)); # @
eval(shift(@insns));
- &vpaddd ($t2,@X[0],&QWP(16*$j,$K256));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns)); # @
+ eval(shift(@insns));
+ &pshufd ($t3,@X[0],0b01010000); # X[16..17]
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ &movdqa ($t2,$t3);
+ eval(shift(@insns)); # @
+ &psrld ($t3,10);
+ eval(shift(@insns));
+ &psrlq ($t2,17);
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t3,$t2);
+ eval(shift(@insns));
+ eval(shift(@insns));
+ &psrlq ($t2,19-17);
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &pxor ($t3,$t2);
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ &pshufd ($t3,$t3,0b00001000);
+ eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &movdqa ($t2,&QWP(16*$j,$K256));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ &pslldq ($t3,8);
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns)); # @
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns)); # @
+ &paddd (@X[0],$t3); # X[2..3] += sigma1(X[16..17])
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ eval(shift(@insns));
+ &paddd ($t2,@X[0]);
+ eval(shift(@insns)); # @
foreach (@insns) { eval; } # remaining instructions
- &vmovdqa (&QWP(32+16*$j,"esp"),$t2);
+ &movdqa (&QWP(32+16*$j,"esp"),$t2);
}
sub body_00_15 () {
(
'&mov ("ecx",$E);',
- '&mov ("esi",&off($f));',
'&ror ($E,25-11);',
- '&mov ("edi",&off($g));',
+ '&mov ("esi",&off($f));',
'&xor ($E,"ecx");',
+ '&mov ("edi",&off($g));',
'&xor ("esi","edi");',
'&ror ($E,11-6);',
'&and ("esi","ecx");',
@@ -637,19 +730,19 @@ sub body_00_15 () {
'&xor ("edi","esi");', # Ch(e,f,g)
'&ror ($E,6);', # T = Sigma1(e)
'&mov ("ecx",$AH[0]);',
+ '&add ($E,"edi");', # T += Ch(e,f,g)
+ '&mov ("edi",&off($b));',
'&mov ("esi",$AH[0]);',
- '&add ($E,&off($h));', # T += h
'&ror ("ecx",22-13);',
- '&add ($E,"edi");', # T += Ch(e,f,g)
- '&mov ("edi",&off($b));',
- '&xor ("ecx",$AH[0]);',
'&mov (&off($a),$AH[0]);', # save $A, modulo-scheduled
+ '&xor ("ecx",$AH[0]);',
'&xor ($AH[0],"edi");', # a ^= b, (b^c) in next round
+ '&add ($E,&off($h));', # T += h
'&ror ("ecx",13-2);',
'&and ($AH[1],$AH[0]);', # (b^c) &= (a^b)
- '&add ($E,&DWP(32+4*($i&15),"esp"));', # T += K[i]+X[i]
'&xor ("ecx","esi");',
+ '&add ($E,&DWP(32+4*($i&15),"esp"));', # T += K[i]+X[i]
'&xor ($AH[1],"edi");', # h = Maj(a,b,c) = Ch(a^b,c,b)
'&ror ("ecx",2);', # Sigma0(a)
@@ -662,11 +755,11 @@ sub body_00_15 () {
}
for ($i=0,$j=0; $j<4; $j++) {
- &XOP_00_47($j,\&body_00_15,@X);
+ &SSSE3_00_47($j,\&body_00_15,@X);
push(@X,shift(@X)); # rotate(@X)
}
&cmp (&DWP(16*$j,$K256),0x00010203);
- &jne (&label("xop_00_47"));
+ &jne (&label("ssse3_00_47"));
for ($i=0; $i<16; ) {
foreach(body_00_15()) { eval; }
@@ -708,16 +801,21 @@ sub body_00_15 () {
&mov (&DWP(28,"esp"),"edi");
&mov ("edi",&DWP(96+4,"esp")); # inp
- &vmovdqa ($t3,&QWP(64,$K256));
+ &movdqa ($t3,&QWP(64,$K256));
&sub ($K256,3*64); # rewind K
&cmp ("edi",&DWP(96+8,"esp")); # are we done yet?
- &jb (&label("grand_xop"));
+ &jb (&label("grand_ssse3"));
&mov ("esp",&DWP(96+12,"esp")); # restore sp
- &vzeroall ();
&function_end_A();
-
-&set_label("AVX",16);
+ if ($avx) {
+&set_label("AVX",32);
+ if ($avx>1) {
+ &mov ("edx",&DWP(8,"edx"));
+ &and ("edx",1<<8|1<<3); # check for BMI2+BMI1
+ &cmp ("edx",1<<8|1<<3);
+ &je (&label("AVX_BMI"));
+ }
&lea ("esp",&DWP(-96,"esp"));
&vzeroall ();
# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
@@ -742,7 +840,7 @@ sub body_00_15 () {
&vmovdqa ($t3,&QWP(256,$K256));
&jmp (&label("grand_avx"));
-&set_label("grand_avx",16);
+&set_label("grand_avx",32);
# load input, reverse byte order, add K256[0..15], save to stack
&vmovdqu (@X[0],&QWP(0,"edi"));
&vmovdqu (@X[1],&QWP(16,"edi"));
@@ -809,12 +907,14 @@ my $j = shift;
my $body = shift;
my @X = @_;
my @insns = (&$body,&$body,&$body,&$body); # 120 instructions
+my $insn;
foreach (Xupdate_AVX()) { # 31 instructions
eval;
eval(shift(@insns));
eval(shift(@insns));
- eval(shift(@insns));
+ eval($insn = shift(@insns));
+ eval(shift(@insns)) if ($insn =~ /rorx/ && @insns[0] =~ /rorx/);
}
&vpaddd ($t2,@X[0],&QWP(16*$j,$K256));
foreach (@insns) { eval; } # remaining instructions
@@ -876,9 +976,150 @@ my @insns = (&$body,&$body,&$body,&$body); # 120 instructions
&mov ("esp",&DWP(96+12,"esp")); # restore sp
&vzeroall ();
&function_end_A();
- }}}
+ if ($avx>1) {
+sub bodyx_00_15 () { # +10%
+ (
+ '&rorx ("ecx",$E,6)',
+ '&rorx ("esi",$E,11)',
+ '&mov (&off($e),$E)', # save $E, modulo-scheduled
+ '&rorx ("edi",$E,25)',
+ '&xor ("ecx","esi")',
+ '&andn ("esi",$E,&off($g))',
+ '&xor ("ecx","edi")', # Sigma1(e)
+ '&and ($E,&off($f))',
+ '&mov (&off($a),$AH[0]);', # save $A, modulo-scheduled
+ '&or ($E,"esi")', # T = Ch(e,f,g)
+
+ '&rorx ("edi",$AH[0],2)',
+ '&rorx ("esi",$AH[0],13)',
+ '&lea ($E,&DWP(0,$E,"ecx"))', # T += Sigma1(e)
+ '&rorx ("ecx",$AH[0],22)',
+ '&xor ("esi","edi")',
+ '&mov ("edi",&off($b))',
+ '&xor ("ecx","esi")', # Sigma0(a)
+
+ '&xor ($AH[0],"edi")', # a ^= b, (b^c) in next round
+ '&add ($E,&off($h))', # T += h
+ '&and ($AH[1],$AH[0])', # (b^c) &= (a^b)
+ '&add ($E,&DWP(32+4*($i&15),"esp"))', # T += K[i]+X[i]
+ '&xor ($AH[1],"edi")', # h = Maj(a,b,c) = Ch(a^b,c,b)
+
+ '&add ("ecx",$E)', # h += T
+ '&add ($E,&off($d))', # d += T
+ '&lea ($AH[1],&DWP(0,$AH[1],"ecx"));'. # h += Sigma0(a)
+
+ '@AH = reverse(@AH); $i++;' # rotate(a,h)
+ );
}
+
+&set_label("AVX_BMI",32);
+ &lea ("esp",&DWP(-96,"esp"));
+ &vzeroall ();
+ # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
+ &mov ($AH[0],&DWP(0,"esi"));
+ &mov ($AH[1],&DWP(4,"esi"));
+ &mov ("ecx",&DWP(8,"esi"));
+ &mov ("edi",&DWP(12,"esi"));
+ #&mov (&DWP(0,"esp"),$AH[0]);
+ &mov (&DWP(4,"esp"),$AH[1]);
+ &xor ($AH[1],"ecx"); # magic
+ &mov (&DWP(8,"esp"),"ecx");
+ &mov (&DWP(12,"esp"),"edi");
+ &mov ($E,&DWP(16,"esi"));
+ &mov ("edi",&DWP(20,"esi"));
+ &mov ("ecx",&DWP(24,"esi"));
+ &mov ("esi",&DWP(28,"esi"));
+ #&mov (&DWP(16,"esp"),$E);
+ &mov (&DWP(20,"esp"),"edi");
+ &mov ("edi",&DWP(96+4,"esp")); # inp
+ &mov (&DWP(24,"esp"),"ecx");
+ &mov (&DWP(28,"esp"),"esi");
+ &vmovdqa ($t3,&QWP(256,$K256));
+ &jmp (&label("grand_avx_bmi"));
+
+&set_label("grand_avx_bmi",32);
+ # load input, reverse byte order, add K256[0..15], save to stack
+ &vmovdqu (@X[0],&QWP(0,"edi"));
+ &vmovdqu (@X[1],&QWP(16,"edi"));
+ &vmovdqu (@X[2],&QWP(32,"edi"));
+ &vmovdqu (@X[3],&QWP(48,"edi"));
+ &add ("edi",64);
+ &vpshufb (@X[0],@X[0],$t3);
+ &mov (&DWP(96+4,"esp"),"edi");
+ &vpshufb (@X[1],@X[1],$t3);
+ &vpshufb (@X[2],@X[2],$t3);
+ &vpaddd ($t0,@X[0],&QWP(0,$K256));
+ &vpshufb (@X[3],@X[3],$t3);
+ &vpaddd ($t1,@X[1],&QWP(16,$K256));
+ &vpaddd ($t2,@X[2],&QWP(32,$K256));
+ &vpaddd ($t3,@X[3],&QWP(48,$K256));
+ &vmovdqa (&QWP(32+0,"esp"),$t0);
+ &vmovdqa (&QWP(32+16,"esp"),$t1);
+ &vmovdqa (&QWP(32+32,"esp"),$t2);
+ &vmovdqa (&QWP(32+48,"esp"),$t3);
+ &jmp (&label("avx_bmi_00_47"));
+
+&set_label("avx_bmi_00_47",16);
+ &add ($K256,64);
+
+ for ($i=0,$j=0; $j<4; $j++) {
+ &AVX_00_47($j,\&bodyx_00_15,@X);
+ push(@X,shift(@X)); # rotate(@X)
+ }
+ &cmp (&DWP(16*$j,$K256),0x00010203);
+ &jne (&label("avx_bmi_00_47"));
+
+ for ($i=0; $i<16; ) {
+ foreach(bodyx_00_15()) { eval; }
+ }
+
+ &mov ("esi",&DWP(96,"esp")); #ctx
+ #&mov ($AH[0],&DWP(0,"esp"));
+ &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp"));
+ #&mov ("edi", &DWP(8,"esp"));
+ &mov ("ecx",&DWP(12,"esp"));
+ &add ($AH[0],&DWP(0,"esi"));
+ &add ($AH[1],&DWP(4,"esi"));
+ &add ("edi",&DWP(8,"esi"));
+ &add ("ecx",&DWP(12,"esi"));
+ &mov (&DWP(0,"esi"),$AH[0]);
+ &mov (&DWP(4,"esi"),$AH[1]);
+ &mov (&DWP(8,"esi"),"edi");
+ &mov (&DWP(12,"esi"),"ecx");
+ #&mov (&DWP(0,"esp"),$AH[0]);
+ &mov (&DWP(4,"esp"),$AH[1]);
+ &xor ($AH[1],"edi"); # magic
+ &mov (&DWP(8,"esp"),"edi");
+ &mov (&DWP(12,"esp"),"ecx");
+ #&mov ($E,&DWP(16,"esp"));
+ &mov ("edi",&DWP(20,"esp"));
+ &mov ("ecx",&DWP(24,"esp"));
+ &add ($E,&DWP(16,"esi"));
+ &add ("edi",&DWP(20,"esi"));
+ &add ("ecx",&DWP(24,"esi"));
+ &mov (&DWP(16,"esi"),$E);
+ &mov (&DWP(20,"esi"),"edi");
+ &mov (&DWP(20,"esp"),"edi");
+ &mov ("edi",&DWP(28,"esp"));
+ &mov (&DWP(24,"esi"),"ecx");
+ #&mov (&DWP(16,"esp"),$E);
+ &add ("edi",&DWP(28,"esi"));
+ &mov (&DWP(24,"esp"),"ecx");
+ &mov (&DWP(28,"esi"),"edi");
+ &mov (&DWP(28,"esp"),"edi");
+ &mov ("edi",&DWP(96+4,"esp")); # inp
+
+ &vmovdqa ($t3,&QWP(64,$K256));
+ &sub ($K256,3*64); # rewind K
+ &cmp ("edi",&DWP(96+8,"esp")); # are we done yet?
+ &jb (&label("grand_avx_bmi"));
+
+ &mov ("esp",&DWP(96+12,"esp")); # restore sp
+ &vzeroall ();
+&function_end_A();
+ }
+ }
+ }}}
&function_end_B("sha256_block_data_order");
-&asciz("SHA256 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>");
&asm_finish();
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-23 22:47:39 +0000