aesni-sha1-x86_64.pl: update from master.

1 files changed, 92 insertions, 59 deletions

diff --git a/crypto/aes/asm/aesni-sha1-x86_64.pl b/crypto/aes/asm/aesni-sha1-x86_64.pl
index 3c8f6c19e7..ad17f35b4c 100644
--- a/crypto/aes/asm/aesni-sha1-x86_64.pl
+++ b/crypto/aes/asm/aesni-sha1-x86_64.pl
@@ -21,16 +21,22 @@
 # subroutine:
 #
 #		AES-128-CBC	+SHA1		stitch      gain
-# Westmere	3.77[+5.6]	9.37		6.65	    +41%
-# Sandy Bridge	5.05[+5.2(6.3)]	10.25(11.35)	6.16(7.08)  +67%(+60%)
+# Westmere	3.77[+5.5]	9.26		6.58	    +41%
+# Sandy Bridge	5.05[+5.0(6.2)]	10.06(11.21)	6.09(7.05)  +65%(+59%)
+# Ivy Bridge	5.05[+4.6]	9.65		5.54        +74%
+# Bulldozer	5.77[+6.0]	11.72		6.37        +84%
 #
 #		AES-192-CBC
-# Westmere	4.51		10.11		6.97	    +45%
-# Sandy Bridge	6.05		11.25(12.35)	6.34(7.27)  +77%(+70%)
+# Westmere	4.51		10.00		6.87	    +46%
+# Sandy Bridge	6.05		11.06(12.21)	6.11(7.20)  +81%(+70%)
+# Ivy Bridge	6.05		10.65		6.07        +75%
+# Bulldozer	6.89		12.84		6.96        +84%
 #
 #		AES-256-CBC
-# Westmere	5.25		10.85		7.25	    +50%
-# Sandy Bridge	7.05		12.25(13.35)	7.06(7.70)  +74%(+73%)
+# Westmere	5.25		10.74		7.19	    +49%
+# Sandy Bridge	7.05		12.06(13.21)	7.12(7.68)  +69%(+72%)
+# Ivy Bridge	7.05		11.65		7.12        +64%
+# Bulldozer	8.00		13.95		8.25        +69%
 #
 # (*)	There are two code paths: SSSE3 and AVX. See sha1-568.pl for
 #	background information. Above numbers in parentheses are SSSE3
@@ -47,6 +53,8 @@
 #		AES-128-CBC	AES-192-CBC	AES-256-CBC
 # Westmere	1.31		1.55		1.80
 # Sandy Bridge	0.93		1.06		1.22
+# Ivy Bridge	0.92		1.06		1.21
+# Bulldozer	0.76		0.90		1.04
 
 $flavour = shift;
 $output  = shift;
@@ -112,7 +120,7 @@ my @X=map("%xmm$_",(4..7,0..3));
 my @Tx=map("%xmm$_",(8..10));
 my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp");	# size optimization
 my @T=("%esi","%edi");
-my $j=0; my $jj=0; my $r=0; my $sn=0;
+my $j=0; my $jj=0; my $r=0; my $sn=0; my $rx=0;
 my $K_XX_XX="%r11";
 my ($iv,$in,$rndkey0)=map("%xmm$_",(11..13));
 my @rndkey=("%xmm14","%xmm15");
@@ -180,6 +188,9 @@ $code.=<<___;
 	mov	12($ctx),$D
 	mov	$B,@T[0]		# magic seed
 	mov	16($ctx),$E
+	mov	$C,@T[1]
+	xor	$D,@T[1]
+	and	@T[1],@T[0]
 
 	movdqa	64($K_XX_XX),@X[2]	# pbswap mask
 	movdqa	0($K_XX_XX),@Tx[1]	# K_00_19
@@ -465,21 +476,27 @@ sub Xtail_ssse3()
 	foreach (@insns) { eval; }
 }
 
-sub body_00_19 () {
+sub body_00_19 () {	# ((c^d)&b)^d
+  # on start @T[0]=(c^d)&b
+  return &body_20_39() if ($rx==19); $rx++;
+
   use integer;
   my ($k,$n);
   my @r=(
 	'($a,$b,$c,$d,$e)=@V;'.
-	'&add	($e,eval(4*($j&15))."(%rsp)");',	# X[]+K xfer
-	'&xor	($c,$d);',
-	'&mov	(@T[1],$a);',	# $b in next round
-	'&$_rol	($a,5);',
-	'&and	(@T[0],$c);',	# ($b&($c^$d))
-	'&xor	($c,$d);',	# restore $c
-	'&xor	(@T[0],$d);',
-	'&add	($e,$a);',
 	'&$_ror	($b,$j?7:2);',	# $b>>>2
-	'&add	($e,@T[0]);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
+	'&xor	(@T[0],$d);',
+	'&mov	(@T[1],$a);',	# $b for next round
+
+	'&add	($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer
+	'&xor	($b,$c);',	# $c^$d for next round
+
+	'&$_rol	($a,5);',
+	'&add	($e,@T[0]);',
+	'&and	(@T[1],$b);',	# ($b&($c^$d)) for next round
+
+	'&xor	($b,$c);',	# restore $b
+	'&add	($e,$a);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
 	);
 	$n = scalar(@r);
 	$k = (($jj+1)*12/20)*20*$n/12;	# 12 aesencs per these 20 rounds
@@ -488,48 +505,60 @@ sub body_00_19 () {
     return @r;
 }
 
-sub body_20_39 () {
+sub body_20_39 () {	# b^d^c
+  # on entry @T[0]=b^d
+  return &body_40_59() if ($rx==39); $rx++;
+
   use integer;
   my ($k,$n);
   my @r=(
 	'($a,$b,$c,$d,$e)=@V;'.
-	'&add	($e,eval(4*($j++&15))."(%rsp)");',	# X[]+K xfer
-	'&xor	(@T[0],$d);',	# ($b^$d)
-	'&mov	(@T[1],$a);',	# $b in next round
+	'&add	($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer
+	'&xor	(@T[0],$d)	if($j==19);'.
+	'&xor	(@T[0],$c)	if($j> 19);',	# ($b^$d^$c)
+	'&mov	(@T[1],$a);',	# $b for next round
+
 	'&$_rol	($a,5);',
-	'&xor	(@T[0],$c);',	# ($b^$d^$c)
-	'&add	($e,$a);',
+	'&add	($e,@T[0]);',
+	'&xor	(@T[1],$c)	if ($j< 79);',	# $b^$d for next round
+
 	'&$_ror	($b,7);',	# $b>>>2
-	'&add	($e,@T[0]);'	.'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
+	'&add	($e,$a);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
 	);
 	$n = scalar(@r);
 	$k = (($jj+1)*8/20)*20*$n/8;	# 8 aesencs per these 20 rounds
-	@r[$k%$n].='&$aesenc();'	if ($jj==$k/$n);
+	@r[$k%$n].='&$aesenc();'	if ($jj==$k/$n && $rx!=20);
 	$jj++;
     return @r;
 }
 
-sub body_40_59 () {
+sub body_40_59 () {	# ((b^c)&(c^d))^c
+  # on entry @T[0]=(b^c), (c^=d)
+  $rx++;
+
   use integer;
   my ($k,$n);
   my @r=(
 	'($a,$b,$c,$d,$e)=@V;'.
-	'&mov	(@T[1],$c);',
-	'&xor	($c,$d);',
-	'&add	($e,eval(4*($j++&15))."(%rsp)");',	# X[]+K xfer
-	'&and	(@T[1],$d);',
-	'&and	(@T[0],$c);',	# ($b&($c^$d))
+	'&add	($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer
+	'&and	(@T[0],$c)	if ($j>=40);',	# (b^c)&(c^d)
+	'&xor	($c,$d)		if ($j>=40);',	# restore $c
+
 	'&$_ror	($b,7);',	# $b>>>2
-	'&add	($e,@T[1]);',
-	'&mov	(@T[1],$a);',	# $b in next round
+	'&mov	(@T[1],$a);',	# $b for next round
+	'&xor	(@T[0],$c);',
+
 	'&$_rol	($a,5);',
 	'&add	($e,@T[0]);',
-	'&xor	($c,$d);',	# restore $c
-	'&add	($e,$a);'	.'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
+	'&xor	(@T[1],$c)	if ($j==59);'.
+	'&xor	(@T[1],$b)	if ($j< 59);',	# b^c for next round
+
+	'&xor	($b,$c)		if ($j< 59);',	# c^d for next round
+	'&add	($e,$a);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
 	);
 	$n = scalar(@r);
 	$k=(($jj+1)*12/20)*20*$n/12;	# 12 aesencs per these 20 rounds
-	@r[$k%$n].='&$aesenc();'	if ($jj==$k/$n);
+	@r[$k%$n].='&$aesenc();'	if ($jj==$k/$n && $rx!=40);
 	$jj++;
     return @r;
 }
@@ -575,8 +604,11 @@ $code.=<<___;
 	mov	@T[0],4($ctx)
 	mov	@T[0],$B			# magic seed
 	mov	$C,8($ctx)
+	mov	$C,@T[1]
 	mov	$D,12($ctx)
+	xor	$D,@T[1]
 	mov	$E,16($ctx)
+	and	@T[1],@T[0]
 	jmp	.Loop_ssse3
 
 .align	16
@@ -631,7 +663,7 @@ $code.=<<___;
 .size	aesni_cbc_sha1_enc_ssse3,.-aesni_cbc_sha1_enc_ssse3
 ___
 
-$j=$jj=$r=$sn=0;
+$j=$jj=$r=$sn=$rx=0;
 
 if ($avx) {
 my ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10");
@@ -639,6 +671,7 @@ my ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9"
 my $Xi=4;
 my @X=map("%xmm$_",(4..7,0..3));
 my @Tx=map("%xmm$_",(8..10));
+my $Kx=$rndkey0;
 my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp");	# size optimization
 my @T=("%esi","%edi");
 
@@ -700,9 +733,12 @@ $code.=<<___;
 	mov	12($ctx),$D
 	mov	$B,@T[0]		# magic seed
 	mov	16($ctx),$E
+	mov	$C,@T[1]
+	xor	$D,@T[1]
+	and	@T[1],@T[0]
 
 	vmovdqa	64($K_XX_XX),@X[2]	# pbswap mask
-	vmovdqa	0($K_XX_XX),@Tx[1]	# K_00_19
+	vmovdqa	0($K_XX_XX),$Kx		# K_00_19
 	vmovdqu	0($inp),@X[-4&7]	# load input to %xmm[0-3]
 	vmovdqu	16($inp),@X[-3&7]
 	vmovdqu	32($inp),@X[-2&7]
@@ -712,13 +748,13 @@ $code.=<<___;
 	vpshufb	@X[2],@X[-3&7],@X[-3&7]
 	vpshufb	@X[2],@X[-2&7],@X[-2&7]
 	vpshufb	@X[2],@X[-1&7],@X[-1&7]
-	vpaddd	@Tx[1],@X[-4&7],@X[0]	# add K_00_19
-	vpaddd	@Tx[1],@X[-3&7],@X[1]
-	vpaddd	@Tx[1],@X[-2&7],@X[2]
+	vpaddd	$Kx,@X[-4&7],@X[0]	# add K_00_19
+	vpaddd	$Kx,@X[-3&7],@X[1]
+	vpaddd	$Kx,@X[-2&7],@X[2]
 	vmovdqa	@X[0],0(%rsp)		# X[]+K xfer to IALU
 	vmovdqa	@X[1],16(%rsp)
 	vmovdqa	@X[2],32(%rsp)
-	vmovups	-112($key),$rndkey0	# $key[0]
+	vmovups	-112($key),$rndkey[1]	# $key[0]
 	vmovups	16-112($key),$rndkey[0]	# forward reference
 	jmp	.Loop_avx
 ___
@@ -729,7 +765,7 @@ my $aesenc=sub {
     if ($k==0) {
       $code.=<<___;
 	vmovups		`16*$n`($in0),$in		# load input
-	vxorps		$rndkey0,$in,$in
+	vxorps		$rndkey[1],$in,$in
 ___
       $code.=<<___ if ($n);
 	vmovups		$iv,`16*($n-1)`($out,$in0)	# write output
@@ -755,6 +791,7 @@ ___
 	vmovups		`32+16*($k+3)-112`($key),$rndkey[0]
 .Lvaesenclast$sn:
 	vaesenclast	$rndkey[0],$iv,$iv
+	vmovups		-112($key),$rndkey[0]
 	vmovups		16-112($key),$rndkey[1]		# forward reference
 ___
     } else {
@@ -778,10 +815,10 @@ sub Xupdate_avx_16_31()		# recall that $Xi starts wtih 4
 	 eval(shift(@insns));
 	 eval(shift(@insns));
 
-	  &vpaddd	(@Tx[1],@Tx[1],@X[-1&7]);
+	  &vpaddd	(@Tx[1],$Kx,@X[-1&7]);
 	 eval(shift(@insns));
 	 eval(shift(@insns));
-	&vpsrldq(@Tx[0],@X[-1&7],4);	# "X[-3]", 3 dwords
+	&vpsrldq(@Tx[0],@X[-1&7],4);		# "X[-3]", 3 dwords
 	 eval(shift(@insns));
 	 eval(shift(@insns));
 	&vpxor	(@X[0],@X[0],@X[-4&7]);		# "X[0]"^="X[-16]"
@@ -831,7 +868,7 @@ sub Xupdate_avx_16_31()		# recall that $Xi starts wtih 4
 	&vpxor	(@X[0],@X[0],@Tx[2]);		# "X[0]"^=("X[0]">>96)<<<2
 	 eval(shift(@insns));
 	 eval(shift(@insns));
-	  &vmovdqa	(@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)");	# K_XX_XX
+	  &vmovdqa	($Kx,eval(16*(($Xi)/5))."($K_XX_XX)")	if ($Xi%5==0);	# K_XX_XX
 	 eval(shift(@insns));
 	 eval(shift(@insns));
 
@@ -839,7 +876,6 @@ sub Xupdate_avx_16_31()		# recall that $Xi starts wtih 4
 	 foreach (@insns) { eval; }	# remaining instructions [if any]
 
   $Xi++;	push(@X,shift(@X));	# "rotate" X[]
-		push(@Tx,shift(@Tx));
 }
 
 sub Xupdate_avx_32_79()
@@ -858,12 +894,8 @@ sub Xupdate_avx_32_79()
 	&vpxor	(@X[0],@X[0],@X[-7&7]);		# "X[0]"^="X[-28]"
 	 eval(shift(@insns));
 	 eval(shift(@insns))	if (@insns[0] !~ /&ro[rl]/);
-	if ($Xi%5) {
-	  &vmovdqa	(@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX...
-	} else {			# ... or load next one
-	  &vmovdqa	(@Tx[2],eval(16*($Xi/5))."($K_XX_XX)");
-	}
-	  &vpaddd	(@Tx[1],@Tx[1],@X[-1&7]);
+	  &vpaddd	(@Tx[1],$Kx,@X[-1&7]);
+	  &vmovdqa	($Kx,eval(16*($Xi/5))."($K_XX_XX)")	if ($Xi%5==0);
 	 eval(shift(@insns));		# ror
 	 eval(shift(@insns));
 
@@ -893,7 +925,6 @@ sub Xupdate_avx_32_79()
 	&vpor	(@X[0],@X[0],@Tx[0]);		# "X[0]"<<<=2
 	 eval(shift(@insns));		# body_20_39
 	 eval(shift(@insns));
-	  &vmovdqa	(@Tx[1],@X[0])	if ($Xi<19);
 	 eval(shift(@insns));
 	 eval(shift(@insns));		# rol
 	 eval(shift(@insns));
@@ -904,7 +935,6 @@ sub Xupdate_avx_32_79()
 	 foreach (@insns) { eval; }	# remaining instructions
 
   $Xi++;	push(@X,shift(@X));	# "rotate" X[]
-		push(@Tx,shift(@Tx));
 }
 
 sub Xuplast_avx_80()
@@ -914,13 +944,13 @@ sub Xuplast_avx_80()
   my ($a,$b,$c,$d,$e);
 
 	 eval(shift(@insns));
-	  &vpaddd	(@Tx[1],@Tx[1],@X[-1&7]);
+	  &vpaddd	(@Tx[1],$Kx,@X[-1&7]);
 	 eval(shift(@insns));
 	 eval(shift(@insns));
 	 eval(shift(@insns));
 	 eval(shift(@insns));
 
-	  &movdqa	(eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]);	# X[]+K xfer IALU
+	  &vmovdqa	(eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]);	# X[]+K xfer IALU
 
 	 foreach (@insns) { eval; }		# remaining instructions
 
@@ -930,7 +960,7 @@ sub Xuplast_avx_80()
 	unshift(@Tx,pop(@Tx));
 
 	&vmovdqa(@X[2],"64($K_XX_XX)");		# pbswap mask
-	&vmovdqa(@Tx[1],"0($K_XX_XX)");		# K_00_19
+	&vmovdqa($Kx,"0($K_XX_XX)");		# K_00_19
 	&vmovdqu(@X[-4&7],"0($inp)");		# load input
 	&vmovdqu(@X[-3&7],"16($inp)");
 	&vmovdqu(@X[-2&7],"32($inp)");
@@ -952,7 +982,7 @@ sub Xloop_avx()
 	&vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]);
 	 eval(shift(@insns));
 	 eval(shift(@insns));
-	&vpaddd	(@X[$Xi&7],@X[($Xi-4)&7],@Tx[1]);
+	&vpaddd	(@X[$Xi&7],@X[($Xi-4)&7],$Kx);
 	 eval(shift(@insns));
 	 eval(shift(@insns));
 	 eval(shift(@insns));
@@ -1016,8 +1046,11 @@ $code.=<<___;
 	mov	@T[0],4($ctx)
 	mov	@T[0],$B			# magic seed
 	mov	$C,8($ctx)
+	mov	$C,@T[1]
 	mov	$D,12($ctx)
+	xor	$D,@T[1]
 	mov	$E,16($ctx)
+	and	@T[1],@T[0]
 	jmp	.Loop_avx
 
 .align	16