diff options
| -rw-r--r-- | crypto/aes/asm/aesni-sha1-x86_64.pl | 250 | ||||
| -rw-r--r-- | crypto/aes/asm/aesni-sha256-x86_64.pl | 324 | ||||
| -rw-r--r-- | crypto/sha/asm/sha1-586.pl | 120 | ||||
| -rw-r--r-- | crypto/sha/asm/sha1-mb-x86_64.pl | 511 | ||||
| -rwxr-xr-x | crypto/sha/asm/sha1-x86_64.pl | 202 | ||||
| -rw-r--r-- | crypto/sha/asm/sha256-586.pl | 151 | ||||
| -rw-r--r-- | crypto/sha/asm/sha256-mb-x86_64.pl | 641 | ||||
| -rwxr-xr-x | crypto/sha/asm/sha512-x86_64.pl | 240 |
8 files changed, 2387 insertions, 52 deletions
diff --git a/crypto/aes/asm/aesni-sha1-x86_64.pl b/crypto/aes/asm/aesni-sha1-x86_64.pl index afd6067ec3..ff0b068229 100644 --- a/crypto/aes/asm/aesni-sha1-x86_64.pl +++ b/crypto/aes/asm/aesni-sha1-x86_64.pl @@ -118,7 +118,9 @@ $code.=<<___; aesni_cbc_sha1_enc: # caller should check for SSSE3 and AES-NI bits mov OPENSSL_ia32cap_P+0(%rip),%r10d - mov OPENSSL_ia32cap_P+4(%rip),%r11d + mov OPENSSL_ia32cap_P+4(%rip),%r11 + bt \$61,%r11 # check SHA bit + jc aesni_cbc_sha1_enc_shaext ___ $code.=<<___ if ($avx); and \$`1<<28`,%r11d # mask AVX bit @@ -200,7 +202,7 @@ $code.=<<___; mov $in0,%r12 # reassign arguments mov $out,%r13 mov $len,%r14 - mov $key,%r15 + lea 112($key),%r15 # size optimization movdqu ($ivp),$iv # load IV mov $ivp,88(%rsp) # save $ivp ___ @@ -209,7 +211,7 @@ my $rounds="${ivp}d"; $code.=<<___; shl \$6,$len sub $in0,$out - mov 240($key),$rounds + mov 240-112($key),$rounds add $inp,$len # end of input lea K_XX_XX(%rip),$K_XX_XX @@ -243,8 +245,8 @@ $code.=<<___; psubd @Tx[1],@X[-3&7] movdqa @X[-2&7],32(%rsp) psubd @Tx[1],@X[-2&7] - movups ($key),$rndkey0 # $key[0] - movups 16($key),$rndkey[0] # forward reference + movups -112($key),$rndkey0 # $key[0] + movups 16-112($key),$rndkey[0] # forward reference jmp .Loop_ssse3 ___ @@ -261,31 +263,31 @@ ___ ___ $code.=<<___; xorps $in,$iv + movups `32+16*$k-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*$k`($key),$rndkey[1] ___ } elsif ($k==9) { $sn++; $code.=<<___; cmp \$11,$rounds jb .Laesenclast$sn - movups `32+16*($k+0)`($key),$rndkey[1] + movups `32+16*($k+0)-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*($k+1)`($key),$rndkey[0] + movups `32+16*($k+1)-112`($key),$rndkey[0] aesenc $rndkey[1],$iv je .Laesenclast$sn - movups `32+16*($k+2)`($key),$rndkey[1] + movups `32+16*($k+2)-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*($k+3)`($key),$rndkey[0] + movups `32+16*($k+3)-112`($key),$rndkey[0] aesenc $rndkey[1],$iv .Laesenclast$sn: aesenclast $rndkey[0],$iv - movups 16($key),$rndkey[1] # forward reference + movups 16-112($key),$rndkey[1] # forward reference ___ } else { $code.=<<___; + movups `32+16*$k-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*$k`($key),$rndkey[1] ___ } $r++; unshift(@rndkey,pop(@rndkey)); @@ -1041,7 +1043,7 @@ $code.=<<___; mov $in0,%r12 # reassign arguments mov $out,%r13 mov $len,%r14 - mov $key,%r15 + lea 112($key),%r15 # size optimization vmovdqu ($ivp),$iv # load IV mov $ivp,88(%rsp) # save $ivp ___ @@ -1050,8 +1052,7 @@ my $rounds="${ivp}d"; $code.=<<___; shl \$6,$len sub $in0,$out - mov 240($key),$rounds - add \$112,$key # size optimization + mov 240-112($key),$rounds add $inp,$len # end of input lea K_XX_XX(%rip),$K_XX_XX @@ -1651,11 +1652,180 @@ K_XX_XX: .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask +.byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 .asciz "AESNI-CBC+SHA1 stitch for x86_64, CRYPTOGAMS by <appro\@openssl.org>" .align 64 ___ + {{{ +($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); + +$rounds="%r11d"; + +($iv,$in,$rndkey0)=map("%xmm$_",(2,14,15)); +@rndkey=("%xmm0","%xmm1"); +$r=0; +my ($BSWAP,$ABCD,$E,$E_,$ABCD_SAVE,$E_SAVE)=map("%xmm$_",(7..12)); +my @MSG=map("%xmm$_",(3..6)); + +$code.=<<___; +.type aesni_cbc_sha1_enc_shaext,\@function,6 +.align 32 +aesni_cbc_sha1_enc_shaext: + mov `($win64?56:8)`(%rsp),$inp # load 7th argument +___ +$code.=<<___ if ($win64); + lea `-8-4*16`(%rsp),%rsp + movaps %xmm6,-8-10*16(%rax) + movaps %xmm7,-8-9*16(%rax) + movaps %xmm8,-8-8*16(%rax) + movaps %xmm9,-8-7*16(%rax) + movaps %xmm10,-8-6*16(%rax) + movaps %xmm11,-8-5*16(%rax) + movaps %xmm12,-8-4*16(%rax) + movaps %xmm13,-8-3*16(%rax) + movaps %xmm14,-8-2*16(%rax) + movaps %xmm15,-8-1*16(%rax) +.Lprologue_shaext: +___ +$code.=<<___; + movdqu ($ctx),$ABCD + movd 16($ctx),$E + movdqa K_XX_XX+0x50(%rip),$BSWAP # byte-n-word swap + + mov 240($key),$rounds + sub $in0,$out + movups ($key),$rndkey0 # $key[0] + movups 16($key),$rndkey[0] # forward reference + lea 112($key),$key # size optimization + + pshufd \$0b00011011,$ABCD,$ABCD # flip word order + pshufd \$0b00011011,$E,$E # flip word order + jmp .Loop_shaext + +.align 16 +.Loop_shaext: +___ + &$aesenc(); +$code.=<<___; + movdqu ($inp),@MSG[0] + movdqa $E,$E_SAVE # offload $E + pshufb $BSWAP,@MSG[0] + movdqu 0x10($inp),@MSG[1] + movdqa $ABCD,$ABCD_SAVE # offload $ABCD +___ + &$aesenc(); +$code.=<<___; + pshufb $BSWAP,@MSG[1] + + paddd @MSG[0],$E + movdqu 0x20($inp),@MSG[2] + lea 0x40($inp),$inp + pxor $E_SAVE,@MSG[0] # black magic +___ + &$aesenc(); +$code.=<<___; + pxor $E_SAVE,@MSG[0] # black magic + movdqa $ABCD,$E_ + pshufb $BSWAP,@MSG[2] + sha1rnds4 \$0,$E,$ABCD # 0-3 + sha1nexte @MSG[1],$E_ +___ + &$aesenc(); +$code.=<<___; + sha1msg1 @MSG[1],@MSG[0] + movdqu -0x10($inp),@MSG[3] + movdqa $ABCD,$E + pshufb $BSWAP,@MSG[3] +___ + &$aesenc(); +$code.=<<___; + sha1rnds4 \$0,$E_,$ABCD # 4-7 + sha1nexte @MSG[2],$E + pxor @MSG[2],@MSG[0] + sha1msg1 @MSG[2],@MSG[1] +___ + &$aesenc(); + +for($i=2;$i<20-4;$i++) { +$code.=<<___; + movdqa $ABCD,$E_ + sha1rnds4 \$`int($i/5)`,$E,$ABCD # 8-11 + sha1nexte @MSG[3],$E_ +___ + &$aesenc(); +$code.=<<___; + sha1msg2 @MSG[3],@MSG[0] + pxor @MSG[3],@MSG[1] + sha1msg1 @MSG[3],@MSG[2] +___ + ($E,$E_)=($E_,$E); + push(@MSG,shift(@MSG)); + + &$aesenc(); +} +$code.=<<___; + movdqa $ABCD,$E_ + sha1rnds4 \$3,$E,$ABCD # 64-67 + sha1nexte @MSG[3],$E_ + sha1msg2 @MSG[3],@MSG[0] + pxor @MSG[3],@MSG[1] +___ + &$aesenc(); +$code.=<<___; + movdqa $ABCD,$E + sha1rnds4 \$3,$E_,$ABCD # 68-71 + sha1nexte @MSG[0],$E + sha1msg2 @MSG[0],@MSG[1] +___ + &$aesenc(); +$code.=<<___; + movdqa $E_SAVE,@MSG[0] + movdqa $ABCD,$E_ + sha1rnds4 \$3,$E,$ABCD # 72-75 + sha1nexte @MSG[1],$E_ +___ + &$aesenc(); +$code.=<<___; + movdqa $ABCD,$E + sha1rnds4 \$3,$E_,$ABCD # 76-79 + sha1nexte $MSG[0],$E +___ + while($r<40) { &$aesenc(); } # remaining aesenc's +$code.=<<___; + dec $len + + paddd $ABCD_SAVE,$ABCD + movups $iv,48($out,$in0) # write output + lea 64($in0),$in0 + jnz .Loop_shaext + + pshufd \$0b00011011,$ABCD,$ABCD + pshufd \$0b00011011,$E,$E + movups $iv,($ivp) # write IV + movdqu $ABCD,($ctx) + movd $E,16($ctx) +___ +$code.=<<___ if ($win64); + movaps -8-10*16(%rax),%xmm6 + movaps -8-9*16(%rax),%xmm7 + movaps -8-8*16(%rax),%xmm8 + movaps -8-7*16(%rax),%xmm9 + movaps -8-6*16(%rax),%xmm10 + movaps -8-5*16(%rax),%xmm11 + movaps -8-4*16(%rax),%xmm12 + movaps -8-3*16(%rax),%xmm13 + movaps -8-2*16(%rax),%xmm14 + movaps -8-1*16(%rax),%xmm15 + mov %rax,%rsp +.Lepilogue_shaext: +___ +$code.=<<___; + ret +.size aesni_cbc_sha1_enc_shaext,.-aesni_cbc_sha1_enc_shaext +___ + }}} # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, # CONTEXT *context,DISPATCHER_CONTEXT *disp) if ($win64) { @@ -1793,12 +1963,43 @@ sub rex { $rex|=0x04 if($dst>=8); $rex|=0x01 if($src>=8); - push @opcode,$rex|0x40 if($rex); + unshift @opcode,$rex|0x40 if($rex); +} + +sub sha1rnds4 { + if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x3a,0xcc); + rex(\@opcode,$3,$2); + push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + my $c=$1; + push @opcode,$c=~/^0/?oct($c):$c; + return ".byte\t".join(',',@opcode); + } else { + return "sha1rnds4\t".@_[0]; + } +} + +sub sha1op38 { + my $instr = shift; + my %opcodelet = ( + "sha1nexte" => 0xc8, + "sha1msg1" => 0xc9, + "sha1msg2" => 0xca ); + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x38); + rex(\@opcode,$2,$1); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } } sub aesni { my $line=shift; - my @opcode=(0x66); + my @opcode=(0x0f,0x38); if ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { my %opcodelet = ( @@ -1807,15 +2008,20 @@ sub aesni { ); return undef if (!defined($opcodelet{$1})); rex(\@opcode,$3,$2); - push @opcode,0x0f,0x38,$opcodelet{$1}; - push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + push @opcode,$opcodelet{$1},0xc0|($2&7)|(($3&7)<<3); # ModR/M + unshift @opcode,0x66; return ".byte\t".join(',',@opcode); } return $line; } -$code =~ s/\`([^\`]*)\`/eval($1)/gem; -$code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; -print $code; + s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or + s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo or + s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/geo; + + print $_,"\n"; +} close STDOUT; diff --git a/crypto/aes/asm/aesni-sha256-x86_64.pl b/crypto/aes/asm/aesni-sha256-x86_64.pl index d19439e4be..b6ad7b29ed 100644 --- a/crypto/aes/asm/aesni-sha256-x86_64.pl +++ b/crypto/aes/asm/aesni-sha256-x86_64.pl @@ -112,8 +112,13 @@ $code.=<<___ if ($avx); cmp \$0,`$win64?"%rcx":"%rdi"` je .Lprobe mov 0(%r11),%eax - mov 4(%r11),%r10d - mov 8(%r11),%r11d + mov 4(%r11),%r10 + + bt \$61,%r10 # check for SHA + jc ${func}_shaext + + mov %r10,%r11 + shr \$32,%r11 test \$`1<<11`,%r10d # check for XOP jnz ${func}_xop @@ -1196,6 +1201,288 @@ $code.=<<___; .size ${func}_avx2,.-${func}_avx2 ___ }} +}} +{{ +my ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); + +my ($rounds,$Tbl)=("%r11d","%rbx"); + +my ($iv,$in,$rndkey0)=map("%xmm$_",(6,14,15)); +my @rndkey=("%xmm4","%xmm5"); +my $r=0; +my $sn=0; + +my ($Wi,$ABEF,$CDGH,$TMP,$BSWAP,$ABEF_SAVE,$CDGH_SAVE)=map("%xmm$_",(0..3,7..9)); +my @MSG=map("%xmm$_",(10..13)); + +my $aesenc=sub { + use integer; + my ($n,$k)=($r/10,$r%10); + if ($k==0) { + $code.=<<___; + movups `16*$n`($in0),$in # load input + xorps $rndkey0,$in +___ + $code.=<<___ if ($n); + movups $iv,`16*($n-1)`($out,$in0) # write output +___ + $code.=<<___; + xorps $in,$iv + movups `32+16*$k-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv +___ + } elsif ($k==9) { + $sn++; + $code.=<<___; + cmp \$11,$rounds + jb .Laesenclast$sn + movups `32+16*($k+0)-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv + movups `32+16*($k+1)-112`($key),$rndkey[0] + aesenc $rndkey[1],$iv + je .Laesenclast$sn + movups `32+16*($k+2)-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv + movups `32+16*($k+3)-112`($key),$rndkey[0] + aesenc $rndkey[1],$iv +.Laesenclast$sn: + aesenclast $rndkey[0],$iv + movups 16-112($key),$rndkey[1] # forward reference + nop +___ + } else { + $code.=<<___; + movups `32+16*$k-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv +___ + } + $r++; unshift(@rndkey,pop(@rndkey)); +}; + +$code.=<<___; +.type ${func}_shaext,\@function,6 +.align 32 +${func}_shaext: + mov %rsp,%rax + mov `($win64?56:8)`(%rsp),$inp # load 7th argument + push %rbx +___ +$code.=<<___ if ($win64); + lea `-4*16`(%rsp),%rsp + movaps %xmm6,-8-10*16(%rax) + movaps %xmm7,-8-9*16(%rax) + movaps %xmm8,-8-8*16(%rax) + movaps %xmm9,-8-7*16(%rax) + movaps %xmm10,-8-6*16(%rax) + movaps %xmm11,-8-5*16(%rax) + movaps %xmm12,-8-4*16(%rax) + movaps %xmm13,-8-3*16(%rax) + movaps %xmm14,-8-2*16(%rax) + movaps %xmm15,-8-1*16(%rax) +.Lprologue_shaext: +___ +$code.=<<___; + lea K256+0x80(%rip),$Tbl + movdqu ($ctx),$ABEF # DCBA + movdqu 16($ctx),$CDGH # HGFE + movdqa 0x200-0x80($Tbl),$TMP # byte swap mask + + mov 240($key),$rounds + sub $in0,$out + movups ($key),$rndkey0 # $key[0] + movups 16($key),$rndkey[0] # forward reference + lea 112($key),$key # size optimization + + pshufd \$0x1b,$ABEF,$Wi # ABCD + pshufd \$0xb1,$ABEF,$ABEF # CDAB + pshufd \$0x1b,$CDGH,$CDGH # EFGH + movdqa $TMP,$BSWAP # offload + palignr \$8,$CDGH,$ABEF # ABEF + punpcklqdq $Wi,$CDGH # CDGH + + jmp .Loop_shaext + +.align 16 +.Loop_shaext: + movdqu ($inp),@MSG[0] + movdqu 0x10($inp),@MSG[1] + movdqu 0x20($inp),@MSG[2] + pshufb $TMP,@MSG[0] + movdqu 0x30($inp),@MSG[3] + + movdqa 0*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + pshufb $TMP,@MSG[1] + movdqa $CDGH,$CDGH_SAVE # offload + movdqa $ABEF,$ABEF_SAVE # offload +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 0-3 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 1*32-0x80($Tbl),$Wi + paddd @MSG[1],$Wi + pshufb $TMP,@MSG[2] + lea 0x40($inp),$inp +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 4-7 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 2*32-0x80($Tbl),$Wi + paddd @MSG[2],$Wi + pshufb $TMP,@MSG[3] + sha256msg1 @MSG[1],@MSG[0] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 8-11 + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[3],$TMP + palignr \$4,@MSG[2],$TMP + paddd $TMP,@MSG[0] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 3*32-0x80($Tbl),$Wi + paddd @MSG[3],$Wi + sha256msg2 @MSG[3],@MSG[0] + sha256msg1 @MSG[2],@MSG[1] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 12-15 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + movdqa @MSG[0],$TMP + palignr \$4,@MSG[3],$TMP + paddd $TMP,@MSG[1] + sha256rnds2 $CDGH,$ABEF +___ +for($i=4;$i<16-3;$i++) { + &$aesenc() if (($r%10)==0); +$code.=<<___; + movdqa $i*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + sha256msg2 @MSG[0],@MSG[1] + sha256msg1 @MSG[3],@MSG[2] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 16-19... + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[1],$TMP + palignr \$4,@MSG[0],$TMP + paddd $TMP,@MSG[2] +___ + &$aesenc(); + &$aesenc() if ($r==19); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF +___ + push(@MSG,shift(@MSG)); +} +$code.=<<___; + movdqa 13*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + sha256msg2 @MSG[0],@MSG[1] + sha256msg1 @MSG[3],@MSG[2] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 52-55 + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[1],$TMP + palignr \$4,@MSG[0],$TMP + paddd $TMP,@MSG[2] +___ + &$aesenc(); + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 14*32-0x80($Tbl),$Wi + paddd @MSG[1],$Wi + sha256msg2 @MSG[1],@MSG[2] + movdqa $BSWAP,$TMP +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 56-59 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 15*32-0x80($Tbl),$Wi + paddd @MSG[2],$Wi +___ + &$aesenc(); + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 60-63 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + #pxor $CDGH,$rndkey0 # black magic +___ + while ($r<40) { &$aesenc(); } # remaining aesenc's +$code.=<<___; + #xorps $CDGH,$rndkey0 # black magic + paddd $CDGH_SAVE,$CDGH + paddd $ABEF_SAVE,$ABEF + + dec $len + movups $iv,48($out,$in0) # write output + lea 64($in0),$in0 + jnz .Loop_shaext + + pshufd \$0xb1,$CDGH,$CDGH # DCHG + pshufd \$0x1b,$ABEF,$TMP # FEBA + pshufd \$0xb1,$ABEF,$ABEF # BAFE + punpckhqdq $CDGH,$ABEF # DCBA + palignr \$8,$TMP,$CDGH # HGFE + + movups $iv,($ivp) # write IV + movdqu $ABEF,($ctx) + movdqu $CDGH,16($ctx) +___ +$code.=<<___ if ($win64); + movaps -8-10*16(%rax),%xmm6 + movaps -8-9*16(%rax),%xmm7 + movaps -8-8*16(%rax),%xmm8 + movaps -8-7*16(%rax),%xmm9 + movaps -8-6*16(%rax),%xmm10 + movaps -8-5*16(%rax),%xmm11 + movaps -8-4*16(%rax),%xmm12 + movaps -8-3*16(%rax),%xmm13 + movaps -8-2*16(%rax),%xmm14 + movaps -8-1*16(%rax),%xmm15 +.Lepilogue_shaext: +___ +$code.=<<___; + mov -8(%rax),%rbx + mov %rax,%rsp + ret +.size ${func}_shaext,.-${func}_shaext +___ }}}}} # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, @@ -1347,6 +1634,39 @@ $code.=<<___ if ($avx>1); ___ } +#################################################################### +sub rex { + local *opcode=shift; + my ($dst,$src)=@_; + my $rex=0; + + $rex|=0x04 if($dst>=8); + $rex|=0x01 if($src>=8); + unshift @opcode,$rex|0x40 if($rex); +} + +{ + my %opcodelet = ( + "sha256rnds2" => 0xcb, + "sha256msg1" => 0xcc, + "sha256msg2" => 0xcd ); + + sub sha256op38 { + my $instr = shift; + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-7]),\s*%xmm([0-7])/) { + my @opcode=(0x0f,0x38); + rex(\@opcode,$2,$1); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } + } +} + $code =~ s/\`([^\`]*)\`/eval $1/gem; +$code =~ s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/gem; print $code; close STDOUT; diff --git a/crypto/sha/asm/sha1-586.pl b/crypto/sha/asm/sha1-586.pl index 17b84c8bfe..acc4f639a7 100644 --- a/crypto/sha/asm/sha1-586.pl +++ b/crypto/sha/asm/sha1-586.pl @@ -79,6 +79,10 @@ # strongly, it's probably more appropriate to discuss possibility of # using vector rotate XOP on AMD... +# March 2014. +# +# Add support for Intel SHA Extensions. + ###################################################################### # Current performance is summarized in following table. Numbers are # CPU clock cycles spent to process single byte (less is better). @@ -303,6 +307,7 @@ if ($alt) { &function_begin("sha1_block_data_order"); if ($xmm) { + &static_label("shaext_shortcut"); &static_label("ssse3_shortcut"); &static_label("avx_shortcut") if ($ymm); &static_label("K_XX_XX"); @@ -317,8 +322,11 @@ if ($xmm) { &mov ($D,&DWP(4,$T)); &test ($D,1<<9); # check SSSE3 bit &jz (&label("x86")); + &mov ($C,&DWP(8,$T)); &test ($A,1<<24); # check FXSR bit &jz (&label("x86")); + &test ($C,1<<29); # check SHA bit + &jnz (&label("shaext_shortcut")); if ($ymm) { &and ($D,1<<28); # mask AVX bit &and ($A,1<<30); # mask "Intel CPU" bit @@ -397,6 +405,117 @@ if ($xmm) { &function_end("sha1_block_data_order"); if ($xmm) { +{ +###################################################################### +# Intel SHA Extensions implementation of SHA1 update function. +# +my ($ctx,$inp,$num)=("edi","esi","ecx"); +my ($ABCD,$E,$E_,$BSWAP)=map("xmm$_",(0..3)); +my @MSG=map("xmm$_",(4..7)); + +sub sha1rnds4 { + my ($dst,$src,$imm)=@_; + if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) + { &data_byte(0x0f,0x3a,0xcc,0xc0|($1<<3)|$2,$imm); } +} +sub sha1op38 { + my ($opcodelet,$dst,$src)=@_; + if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) + { &data_byte(0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2); } +} +sub sha1nexte { sha1op38(0xc8,@_); } +sub sha1msg1 { sha1op38(0xc9,@_); } +sub sha1msg2 { sha1op38(0xca,@_); } + +&function_begin("_sha1_block_data_order_shaext"); + &call (&label("pic_point")); # make it PIC! + &set_label("pic_point"); + &blindpop($tmp1); + &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); +&set_label("shaext_shortcut"); + &mov ($ctx,&wparam(0)); + &mov ("ebx","esp"); + &mov ($inp,&wparam(1)); + &mov ($num,&wparam(2)); + &sub ("esp",32); + + &movdqu ($ABCD,&QWP(0,$ctx)); + &movd ($E,&QWP(16,$ctx)); + &and ("esp",-32); + &movdqa ($BSWAP,&QWP(0x50,$tmp1)); # byte-n-word swap + + &movdqu (@MSG[0],&QWP(0,$inp)); + &pshufd ($ABCD,$ABCD,0b00011011); # flip word order + &movdqu (@MSG[1],&QWP(0x10,$inp)); + &pshufd ($E,$E,0b00011011); # flip word order + &movdqu (@MSG[2],&QWP(0x20,$inp)); + &pshufb (@MSG[0],$BSWAP); + &movdqu (@MSG[3],&QWP(0x30,$inp)); + &pshufb (@MSG[1],$BSWAP); + &pshufb (@MSG[2],$BSWAP); + &pshufb (@MSG[3],$BSWAP); + &jmp (&label("loop_shaext")); + +&set_label("loop_shaext",16); + &dec ($num); + &lea ("eax",&DWP(0x40,$inp)); + &movdqa (&QWP(0,"esp"),$E); # offload $E + &paddd ($E,@MSG[0]); + &cmovne ($inp,"eax"); + &movdqa (&QWP(16,"esp"),$ABCD); # offload $ABCD + +for($i=0;$i<20-4;$i+=2) { + &sha1msg1 (@MSG[0],@MSG[1]); + &movdqa ($E_,$ABCD); + &sha1rnds4 ($ABCD,$E,int($i/5)); # 0-3... + &sha1nexte ($E_,@MSG[1]); + &pxor (@MSG[0],@MSG[2]); + &sha1msg1 (@MSG[1],@MSG[2]); + &sha1msg2 (@MSG[0],@MSG[3]); + + &movdqa ($E,$ABCD); + &sha1rnds4 ($ABCD,$E_,int(($i+1)/5)); + &sha1nexte ($E,@MSG[2]); + &pxor (@MSG[1],@MSG[3]); + &sha1msg2 (@MSG[1],@MSG[0]); + + push(@MSG,shift(@MSG)); push(@MSG,shift(@MSG)); +} + &movdqu (@MSG[0],&QWP(0,$inp)); + &movdqa ($E_,$ABCD); + &sha1rnds4 ($ABCD,$E,3); # 64-67 + &sha1nexte ($E_,@MSG[1]); + &movdqu (@MSG[1],&QWP(0x10,$inp)); + &pshufb (@MSG[0],$BSWAP); + + &movdqa ($E,$ABCD); + &sha1rnds4 ($ABCD,$E_,3); # 68-71 + &sha1nexte ($E,@MSG[2]); + &movdqu (@MSG[2],&QWP(0x20,$inp)); + &pshufb (@MSG[1],$BSWAP); + + &movdqa ($E_,$ABCD); + &sha1rnds4 ($ABCD,$E,3); # 72-75 + &sha1nexte ($E_,@MSG[3]); + &movdqu (@MSG[3],&QWP(0x30,$inp)); + &pshufb (@MSG[2],$BSWAP); + + &movdqa ($E,$ABCD); + &sha1rnds4 ($ABCD,$E_,3); # 76-79 + &movdqa ($E_,&QWP(0,"esp")); + &pshufb (@MSG[3],$BSWAP); + &sha1nexte ($E,$E_); + &paddd ($ABCD,&QWP(16,"esp")); + + &jnz (&label("loop_shaext")); + + &pshufd ($ABCD,$ABCD,0b00011011); + &pshufd ($E,$E,0b00011011); + &movdqu (&QWP(0,$ctx),$ABCD) + &movd (&DWP(16,$ctx),$E); + &mov ("esp","ebx"); +&function_end("_sha1_block_data_order_shaext"); +} ###################################################################### # The SSSE3 implementation. # @@ -1340,6 +1459,7 @@ sub Xtail_avx() &data_word(0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc); # K_40_59 &data_word(0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6); # K_60_79 &data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # pbswap mask +&data_byte(0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0); } &asciz("SHA1 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>"); diff --git a/crypto/sha/asm/sha1-mb-x86_64.pl b/crypto/sha/asm/sha1-mb-x86_64.pl index 33e6620c68..3a19029e5e 100644 --- a/crypto/sha/asm/sha1-mb-x86_64.pl +++ b/crypto/sha/asm/sha1-mb-x86_64.pl @@ -15,7 +15,7 @@ # this +aesni(i) sha1 aesni-sha1 gain(iv) # ------------------------------------------------------------------- # Westmere(ii) 10.7/n +1.28=3.96(n=4) 5.30 6.66 +68% -# Atom(ii) 18.9?/n +3.93=8.66(n=4) 10.0 14.0 +62% +# Atom(ii) 18.1/n +3.93=8.46(n=4) 9.37 12.8 +51% # Sandy Bridge (8.16 +5.15=13.3)/n 4.99 5.98 +80% # Ivy Bridge (8.08 +5.14=13.2)/n 4.60 5.54 +68% # Haswell(iii) (8.96 +5.00=14.0)/n 3.57 4.55 +160% @@ -338,9 +338,11 @@ $code.=<<___; .type sha1_multi_block,\@function,3 .align 32 sha1_multi_block: + mov OPENSSL_ia32cap_P+4(%rip),%rcx + bt \$61,%rcx # check SHA bit + jc _shaext_shortcut ___ $code.=<<___ if ($avx); - mov OPENSSL_ia32cap_P+4(%rip),%rcx test \$`1<<28`,%ecx jnz _avx_shortcut ___ @@ -366,6 +368,7 @@ $code.=<<___; sub \$`$REG_SZ*18`,%rsp and \$-256,%rsp mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody: lea K_XX_XX(%rip),$Tbl lea `$REG_SZ*16`(%rsp),%rbx @@ -476,9 +479,265 @@ $code.=<<___; mov -16(%rax),%rbp mov -8(%rax),%rbx lea (%rax),%rsp +.Lepilogue: ret .size sha1_multi_block,.-sha1_multi_block ___ + {{{ +my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10)); +my @MSG0=map("%xmm$_",(4..7)); +my @MSG1=map("%xmm$_",(11..14)); + +$code.=<<___; +.type sha1_multi_block_shaext,\@function,3 +.align 32 +sha1_multi_block_shaext: +_shaext_shortcut: + mov %rsp,%rax + push %rbx + push %rbp +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,-0x78(%rax) + movaps %xmm11,-0x68(%rax) + movaps %xmm12,-0x58(%rax) + movaps %xmm13,-0x48(%rax) + movaps %xmm14,-0x38(%rax) + movaps %xmm15,-0x28(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`,%rsp + shl \$1,$num # we process pair at a time + and \$-256,%rsp + lea 0x40($ctx),$ctx # size optimization + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody_shaext: + lea `$REG_SZ*16`(%rsp),%rbx + movdqa K_XX_XX+0x80(%rip),$BSWAP # byte-n-word swap + +.Loop_grande_shaext: + mov $num,`$REG_SZ*17+8`(%rsp) # orignal $num + xor $num,$num +___ +for($i=0;$i<2;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle %rsp,@ptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldone_shaext + + movq 0x00-0x40($ctx),$ABCD0 # a1.a0 + movq 0x20-0x40($ctx),@MSG0[0]# b1.b0 + movq 0x40-0x40($ctx),@MSG0[1]# c1.c0 + movq 0x60-0x40($ctx),@MSG0[2]# d1.d0 + movq 0x80-0x40($ctx),@MSG0[3]# e1.e0 + + punpckldq @MSG0[0],$ABCD0 # b1.a1.b0.a0 + punpckldq @MSG0[2],@MSG0[1] # d1.c1.d0.c0 + + movdqa $ABCD0,$ABCD1 + punpcklqdq @MSG0[1],$ABCD0 # d0.c0.b0.a0 + punpckhqdq @MSG0[1],$ABCD1 # d1.c1.b1.a1 + + pshufd \$0b00111111,@MSG0[3],$E0 + pshufd \$0b01111111,@MSG0[3],$E1 + pshufd \$0b00011011,$ABCD0,$ABCD0 + pshufd \$0b00011011,$ABCD1,$ABCD1 + jmp .Loop_shaext + +.align 32 +.Loop_shaext: + movdqu 0x00(@ptr[0]),@MSG0[0] + movdqu 0x00(@ptr[1]),@MSG1[0] + movdqu 0x10(@ptr[0]),@MSG0[1] + movdqu 0x10(@ptr[1]),@MSG1[1] + movdqu 0x20(@ptr[0]),@MSG0[2] + pshufb $BSWAP,@MSG0[0] + movdqu 0x20(@ptr[1]),@MSG1[2] + pshufb $BSWAP,@MSG1[0] + movdqu 0x30(@ptr[0]),@MSG0[3] + lea 0x40(@ptr[0]),@ptr[0] + pshufb $BSWAP,@MSG0[1] + movdqu 0x30(@ptr[1]),@MSG1[3] + lea 0x40(@ptr[1]),@ptr[1] + pshufb $BSWAP,@MSG1[1] + + movdqa $E0,0x50(%rsp) # offload + paddd @MSG0[0],$E0 + movdqa $E1,0x70(%rsp) + paddd @MSG1[0],$E1 + movdqa $ABCD0,0x40(%rsp) # offload + movdqa $ABCD0,$E0_ + movdqa $ABCD1,0x60(%rsp) + movdqa $ABCD1,$E1_ + sha1rnds4 \$0,$E0,$ABCD0 # 0-3 + sha1nexte @MSG0[1],$E0_ + sha1rnds4 \$0,$E1,$ABCD1 # 0-3 + sha1nexte @MSG1[1],$E1_ + pshufb $BSWAP,@MSG0[2] + prefetcht0 127(@ptr[0]) + sha1msg1 @MSG0[1],@MSG0[0] + pshufb $BSWAP,@MSG1[2] + prefetcht0 127(@ptr[1]) + sha1msg1 @MSG1[1],@MSG1[0] + + pshufb $BSWAP,@MSG0[3] + movdqa $ABCD0,$E0 + pshufb $BSWAP,@MSG1[3] + movdqa $ABCD1,$E1 + sha1rnds4 \$0,$E0_,$ABCD0 # 4-7 + sha1nexte @MSG0[2],$E0 + sha1rnds4 \$0,$E1_,$ABCD1 # 4-7 + sha1nexte @MSG1[2],$E1 + pxor @MSG0[2],@MSG0[0] + sha1msg1 @MSG0[2],@MSG0[1] + pxor @MSG1[2],@MSG1[0] + sha1msg1 @MSG1[2],@MSG1[1] +___ +for($i=2;$i<20-4;$i++) { +$code.=<<___; + movdqa $ABCD0,$E0_ + movdqa $ABCD1,$E1_ + sha1rnds4 \$`int($i/5)`,$E0,$ABCD0 # 8-11 + sha1nexte @MSG0[3],$E0_ + sha1rnds4 \$`int($i/5)`,$E1,$ABCD1 # 8-11 + sha1nexte @MSG1[3],$E1_ + sha1msg2 @MSG0[3],@MSG0[0] + sha1msg2 @MSG1[3],@MSG1[0] + pxor @MSG0[3],@MSG0[1] + sha1msg1 @MSG0[3],@MSG0[2] + pxor @MSG1[3],@MSG1[1] + sha1msg1 @MSG1[3],@MSG1[2] +___ + ($E0,$E0_)=($E0_,$E0); ($E1,$E1_)=($E1_,$E1); + push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1)); +} +$code.=<<___; + movdqa $ABCD0,$E0_ + movdqa $ABCD1,$E1_ + sha1rnds4 \$3,$E0,$ABCD0 # 64-67 + sha1nexte @MSG0[3],$E0_ + sha1rnds4 \$3,$E1,$ABCD1 # 64-67 + sha1nexte @MSG1[3],$E1_ + sha1msg2 @MSG0[3],@MSG0[0] + sha1msg2 @MSG1[3],@MSG1[0] + pxor @MSG0[3],@MSG0[1] + pxor @MSG1[3],@MSG1[1] + + mov \$1,%ecx + pxor @MSG0[2],@MSG0[2] # zero + cmp 4*0(%rbx),%ecx # examine counters + cmovge %rsp,@ptr[0] # cancel input + + movdqa $ABCD0,$E0 + movdqa $ABCD1,$E1 + sha1rnds4 \$3,$E0_,$ABCD0 # 68-71 + sha1nexte @MSG0[0],$E0 + sha1rnds4 \$3,$E1_,$ABCD1 # 68-71 + sha1nexte @MSG1[0],$E1 + sha1msg2 @MSG0[0],@MSG0[1] + sha1msg2 @MSG1[0],@MSG1[1] + + cmp 4*1(%rbx),%ecx + cmovge %rsp,@ptr[1] + movq (%rbx),@MSG0[0] # pull counters + + movdqa $ABCD0,$E0_ + movdqa $ABCD1,$E1_ + sha1rnds4 \$3,$E0,$ABCD0 # 72-75 + sha1nexte @MSG0[1],$E0_ + sha1rnds4 \$3,$E1,$ABCD1 # 72-75 + sha1nexte @MSG1[1],$E1_ + + pshufd \$0x00,@MSG0[0],@MSG1[2] + pshufd \$0x55,@MSG0[0],@MSG1[3] + movdqa @MSG0[0],@MSG0[1] + pcmpgtd @MSG0[2],@MSG1[2] + pcmpgtd @MSG0[2],@MSG1[3] + + movdqa $ABCD0,$E0 + movdqa $ABCD1,$E1 + sha1rnds4 \$3,$E0_,$ABCD0 # 76-79 + sha1nexte $MSG0[2],$E0 + sha1rnds4 \$3,$E1_,$ABCD1 # 76-79 + sha1nexte $MSG0[2],$E1 + + pcmpgtd @MSG0[2],@MSG0[1] # counter mask + pand @MSG1[2],$ABCD0 + pand @MSG1[2],$E0 + pand @MSG1[3],$ABCD1 + pand @MSG1[3],$E1 + paddd @MSG0[1],@MSG0[0] # counters-- + + paddd 0x40(%rsp),$ABCD0 + paddd 0x50(%rsp),$E0 + paddd 0x60(%rsp),$ABCD1 + paddd 0x70(%rsp),$E1 + + movq @MSG0[0],(%rbx) # save counters + dec $num + jnz .Loop_shaext + + mov `$REG_SZ*17+8`(%rsp),$num + + pshufd \$0b00011011,$ABCD0,$ABCD0 + pshufd \$0b00011011,$ABCD1,$ABCD1 + + movdqa $ABCD0,@MSG0[0] + punpckldq $ABCD1,$ABCD0 # b1.b0.a1.a0 + punpckhdq $ABCD1,@MSG0[0] # d1.d0.c1.c0 + punpckhdq $E1,$E0 # e1.e0.xx.xx + movq $ABCD0,0x00-0x40($ctx) # a1.a0 + psrldq \$8,$ABCD0 + movq @MSG0[0],0x40-0x40($ctx)# c1.c0 + psrldq \$8,@MSG0[0] + movq $ABC |