diff options
author | Andy Polyakov <appro@openssl.org> | 2013-10-10 23:06:43 +0200 |
---|---|---|
committer | Andy Polyakov <appro@openssl.org> | 2013-10-10 23:09:54 +0200 |
commit | df5c435c0b45f92dfb82c0c6088de91352ee474c (patch) | |
tree | 25f1218bd7264b974268ed12897cb0a014972082 | |
parent | 1ebaf97c4402a58fc8c9d5feff9fcc06cfaf3d5f (diff) |
bn/asm/rsax-avx2.pl: minor optimization [for Decoded ICache].
(cherry picked from commit fa104be35e24f3fea895d55bb7042d6f4b2963e9)
-rwxr-xr-x | crypto/bn/asm/rsaz-avx2.pl | 151 |
1 files changed, 72 insertions, 79 deletions
diff --git a/crypto/bn/asm/rsaz-avx2.pl b/crypto/bn/asm/rsaz-avx2.pl index 2d9e239912..09c45b0ec2 100755 --- a/crypto/bn/asm/rsaz-avx2.pl +++ b/crypto/bn/asm/rsaz-avx2.pl @@ -48,7 +48,7 @@ # +10% improvement by <appro@openssl.org> # # rsa2048 sign/sec OpenSSL 1.0.1 scalar(*) this -# 2GHz Haswell 544 632/+16% 947/+74% +# 2.3GHz Haswell 621 732/+18% 1112/+79% # # (*) if system doesn't support AVX2, for reference purposes; @@ -125,6 +125,7 @@ my $FrameSize=32*18+32*8; # place for A^2 and 2*A my $aap=$r0; my $tp0="%rbx"; my $tp1=$r3; +my $tpa=$tmp; $np="%r13"; # reassigned argument @@ -263,7 +264,7 @@ $code.=<<___; vpbroadcastq 32*2-128($ap), $B1 vmovdqu $ACC9, 32*17-448($tp1) - xor $tmp, $tmp + mov $ap, $tpa mov \$4, $i jmp .Lsqr_entry_1024 ___ @@ -272,38 +273,29 @@ $TEMP2=$Y2; $code.=<<___; .align 32 .LOOP_SQR_1024: - vmovdqu 32*0(%rsp,$tmp), $TEMP0 # 32*0-192($tp0,$tmp) - vmovdqu 32*1(%rsp,$tmp), $TEMP1 # 32*1-192($tp0,$tmp) - vpbroadcastq 32*1-128($ap,$tmp), $B2 + vpbroadcastq 32*1-128($tpa), $B2 vpmuludq 32*0-128($ap), $B1, $ACC0 - vmovdqu 32*2-192($tp0,$tmp), $TEMP2 - vpaddq $TEMP0, $ACC0, $ACC0 + vpaddq 32*0-192($tp0), $ACC0, $ACC0 vpmuludq 32*0-128($aap), $B1, $ACC1 - vmovdqu 32*3-192($tp0,$tmp), $TEMP0 - vpaddq $TEMP1, $ACC1, $ACC1 + vpaddq 32*1-192($tp0), $ACC1, $ACC1 vpmuludq 32*1-128($aap), $B1, $ACC2 - vmovdqu 32*4-192($tp0,$tmp), $TEMP1 - vpaddq $TEMP2, $ACC2, $ACC2 + vpaddq 32*2-192($tp0), $ACC2, $ACC2 vpmuludq 32*2-128($aap), $B1, $ACC3 - vmovdqu 32*5-192($tp0,$tmp), $TEMP2 - vpaddq $TEMP0, $ACC3, $ACC3 + vpaddq 32*3-192($tp0), $ACC3, $ACC3 vpmuludq 32*3-128($aap), $B1, $ACC4 - vmovdqu 32*6-192($tp0,$tmp), $TEMP0 - vpaddq $TEMP1, $ACC4, $ACC4 + vpaddq 32*4-192($tp0), $ACC4, $ACC4 vpmuludq 32*4-128($aap), $B1, $ACC5 - vmovdqu 32*7-192($tp0,$tmp), $TEMP1 - vpaddq $TEMP2, $ACC5, $ACC5 + vpaddq 32*5-192($tp0), $ACC5, $ACC5 vpmuludq 32*5-128($aap), $B1, $ACC6 - vmovdqu 32*8-192($tp0,$tmp), $TEMP2 - vpaddq $TEMP0, $ACC6, $ACC6 + vpaddq 32*6-192($tp0), $ACC6, $ACC6 vpmuludq 32*6-128($aap), $B1, $ACC7 - vpaddq $TEMP1, $ACC7, $ACC7 + vpaddq 32*7-192($tp0), $ACC7, $ACC7 vpmuludq 32*7-128($aap), $B1, $ACC8 - vpbroadcastq 32*2-128($ap,$tmp), $B1 - vpaddq $TEMP2, $ACC8, $ACC8 + vpbroadcastq 32*2-128($tpa), $B1 + vpaddq 32*8-192($tp0), $ACC8, $ACC8 .Lsqr_entry_1024: - vmovdqu $ACC0, 32*0(%rsp,$tmp) # 32*0-192($tp0,$tmp) - vmovdqu $ACC1, 32*1(%rsp,$tmp) # 32*1-192($tp0,$tmp) + vmovdqu $ACC0, 32*0-192($tp0) + vmovdqu $ACC1, 32*1-192($tp0) vpmuludq 32*1-128($ap), $B2, $TEMP0 vpaddq $TEMP0, $ACC2, $ACC2 @@ -316,16 +308,15 @@ $code.=<<___; vpmuludq 32*4-128($aap), $B2, $TEMP1 vpaddq $TEMP1, $ACC6, $ACC6 vpmuludq 32*5-128($aap), $B2, $TEMP2 - vmovdqu 32*9-192($tp0,$tmp), $TEMP1 vpaddq $TEMP2, $ACC7, $ACC7 vpmuludq 32*6-128($aap), $B2, $TEMP0 vpaddq $TEMP0, $ACC8, $ACC8 vpmuludq 32*7-128($aap), $B2, $ACC0 - vpbroadcastq 32*3-128($ap,$tmp), $B2 - vpaddq $TEMP1, $ACC0, $ACC0 + vpbroadcastq 32*3-128($tpa), $B2 + vpaddq 32*9-192($tp0), $ACC0, $ACC0 - vmovdqu $ACC2, 32*2-192($tp0,$tmp) - vmovdqu $ACC3, 32*3-192($tp0,$tmp) + vmovdqu $ACC2, 32*2-192($tp0) + vmovdqu $ACC3, 32*3-192($tp0) vpmuludq 32*2-128($ap), $B1, $TEMP2 vpaddq $TEMP2, $ACC4, $ACC4 @@ -336,16 +327,15 @@ $code.=<<___; vpmuludq 32*4-128($aap), $B1, $TEMP2 vpaddq $TEMP2, $ACC7, $ACC7 vpmuludq 32*5-128($aap), $B1, $TEMP0 - vmovdqu 32*10-448($tp1,$tmp), $TEMP2 vpaddq $TEMP0, $ACC8, $ACC8 vpmuludq 32*6-128($aap), $B1, $TEMP1 vpaddq $TEMP1, $ACC0, $ACC0 vpmuludq 32*7-128($aap), $B1, $ACC1 - vpbroadcastq 32*4-128($ap,$tmp), $B1 - vpaddq $TEMP2, $ACC1, $ACC1 + vpbroadcastq 32*4-128($tpa), $B1 + vpaddq 32*10-448($tp1), $ACC1, $ACC1 - vmovdqu $ACC4, 32*4-192($tp0,$tmp) - vmovdqu $ACC5, 32*5-192($tp0,$tmp) + vmovdqu $ACC4, 32*4-192($tp0) + vmovdqu $ACC5, 32*5-192($tp0) vpmuludq 32*3-128($ap), $B2, $TEMP0 vpaddq $TEMP0, $ACC6, $ACC6 @@ -354,74 +344,70 @@ $code.=<<___; vpmuludq 32*4-128($aap), $B2, $TEMP2 vpaddq $TEMP2, $ACC8, $ACC8 vpmuludq 32*5-128($aap), $B2, $TEMP0 - vmovdqu 32*11-448($tp1,$tmp), $TEMP2 vpaddq $TEMP0, $ACC0, $ACC0 vpmuludq 32*6-128($aap), $B2, $TEMP1 vpaddq $TEMP1, $ACC1, $ACC1 vpmuludq 32*7-128($aap), $B2, $ACC2 - vpbroadcastq 32*5-128($ap,$tmp), $B2 - vpaddq $TEMP2, $ACC2, $ACC2 + vpbroadcastq 32*5-128($tpa), $B2 + vpaddq 32*11-448($tp1), $ACC2, $ACC2 - vmovdqu $ACC6, 32*6-192($tp0,$tmp) - vmovdqu $ACC7, 32*7-192($tp0,$tmp) + vmovdqu $ACC6, 32*6-192($tp0) + vmovdqu $ACC7, 32*7-192($tp0) vpmuludq 32*4-128($ap), $B1, $TEMP0 vpaddq $TEMP0, $ACC8, $ACC8 vpmuludq 32*4-128($aap), $B1, $TEMP1 vpaddq $TEMP1, $ACC0, $ACC0 vpmuludq 32*5-128($aap), $B1, $TEMP2 - vmovdqu 32*12-448($tp1,$tmp), $TEMP1 vpaddq $TEMP2, $ACC1, $ACC1 vpmuludq 32*6-128($aap), $B1, $TEMP0 vpaddq $TEMP0, $ACC2, $ACC2 vpmuludq 32*7-128($aap), $B1, $ACC3 - vpbroadcastq 32*6-128($ap,$tmp), $B1 - vpaddq $TEMP1, $ACC3, $ACC3 + vpbroadcastq 32*6-128($tpa), $B1 + vpaddq 32*12-448($tp1), $ACC3, $ACC3 - vmovdqu $ACC8, 32*8-192($tp0,$tmp) - vmovdqu $ACC0, 32*9-192($tp0,$tmp) + vmovdqu $ACC8, 32*8-192($tp0) + vmovdqu $ACC0, 32*9-192($tp0) + lea 8($tp0), $tp0 vpmuludq 32*5-128($ap), $B2, $TEMP2 vpaddq $TEMP2, $ACC1, $ACC1 vpmuludq 32*5-128($aap), $B2, $TEMP0 - vmovdqu 32*13-448($tp1,$tmp), $TEMP2 vpaddq $TEMP0, $ACC2, $ACC2 vpmuludq 32*6-128($aap), $B2, $TEMP1 vpaddq $TEMP1, $ACC3, $ACC3 vpmuludq 32*7-128($aap), $B2, $ACC4 - vpbroadcastq 32*7-128($ap,$tmp), $B2 - vpaddq $TEMP2, $ACC4, $ACC4 + vpbroadcastq 32*7-128($tpa), $B2 + vpaddq 32*13-448($tp1), $ACC4, $ACC4 - vmovdqu $ACC1, 32*10-448($tp1,$tmp) - vmovdqu $ACC2, 32*11-448($tp1,$tmp) + vmovdqu $ACC1, 32*10-448($tp1) + vmovdqu $ACC2, 32*11-448($tp1) vpmuludq 32*6-128($ap), $B1, $TEMP0 - vmovdqu 32*14-448($tp1,$tmp), $TEMP2 vpaddq $TEMP0, $ACC3, $ACC3 vpmuludq 32*6-128($aap), $B1, $TEMP1 - vpbroadcastq 32*8-128($ap,$tmp), $ACC0 # borrow $ACC0 for $B1 + vpbroadcastq 32*8-128($tpa), $ACC0 # borrow $ACC0 for $B1 vpaddq $TEMP1, $ACC4, $ACC4 vpmuludq 32*7-128($aap), $B1, $ACC5 - vpbroadcastq 32*0+8-128($ap,$tmp), $B1 # for next iteration - vpaddq $TEMP2, $ACC5, $ACC5 - vmovdqu 32*15-448($tp1,$tmp), $TEMP1 + vpbroadcastq 32*0+8-128($tpa), $B1 # for next iteration + vpaddq 32*14-448($tp1), $ACC5, $ACC5 - vmovdqu $ACC3, 32*12-448($tp1,$tmp) - vmovdqu $ACC4, 32*13-448($tp1,$tmp) + vmovdqu $ACC3, 32*12-448($tp1) + vmovdqu $ACC4, 32*13-448($tp1) + lea 8($tpa), $tpa vpmuludq 32*7-128($ap), $B2, $TEMP0 - vmovdqu 32*16-448($tp1,$tmp), $TEMP2 vpaddq $TEMP0, $ACC5, $ACC5 vpmuludq 32*7-128($aap), $B2, $ACC6 - vpaddq $TEMP1, $ACC6, $ACC6 + vpaddq 32*15-448($tp1), $ACC6, $ACC6 vpmuludq 32*8-128($ap), $ACC0, $ACC7 - vmovdqu $ACC5, 32*14-448($tp1,$tmp) - vpaddq $TEMP2, $ACC7, $ACC7 - vmovdqu $ACC6, 32*15-448($tp1,$tmp) - vmovdqu $ACC7, 32*16-448($tp1,$tmp) + vmovdqu $ACC5, 32*14-448($tp1) + vpaddq 32*16-448($tp1), $ACC7, $ACC7 + vmovdqu $ACC6, 32*15-448($tp1) + vmovdqu $ACC7, 32*16-448($tp1) + lea 8($tp1), $tp1 - lea 8($tmp), $tmp dec $i jnz .LOOP_SQR_1024 ___ @@ -432,9 +418,10 @@ $TEMP3 = $Y1; $TEMP4 = $Y2; $code.=<<___; #we need to fix indexes 32-39 to avoid overflow - vmovdqu 32*8-192($tp0), $ACC8 - vmovdqu 32*9-192($tp0), $ACC1 - vmovdqu 32*10-448($tp1), $ACC2 + vmovdqu 32*8(%rsp), $ACC8 # 32*8-192($tp0), + vmovdqu 32*9(%rsp), $ACC1 # 32*9-192($tp0) + vmovdqu 32*10(%rsp), $ACC2 # 32*10-192($tp0) + lea 192(%rsp), $tp0 # 64+128=192 vpsrlq \$29, $ACC8, $TEMP1 vpand $AND_MASK, $ACC8, $ACC8 @@ -452,7 +439,7 @@ $code.=<<___; vpaddq $TEMP1, $ACC1, $ACC1 vpaddq $TEMP2, $ACC2, $ACC2 vmovdqu $ACC1, 32*9-192($tp0) - vmovdqu $ACC2, 32*10-448($tp1) + vmovdqu $ACC2, 32*10-192($tp0) mov (%rsp), %rax mov 8(%rsp), $r1 @@ -502,13 +489,15 @@ $code.=<<___; mov %rax, %rdx imulq -128($np), %rax vpaddq $TEMP0, $ACC1, $ACC1 - vpmuludq 32*2-128($np), $Y1, $TEMP1 add %rax, $r1 + vpmuludq 32*2-128($np), $Y1, $TEMP1 mov %rdx, %rax imulq 8-128($np), %rax vpaddq $TEMP1, $ACC2, $ACC2 vpmuludq 32*3-128($np), $Y1, $TEMP2 + .byte 0x67 add %rax, $r2 + .byte 0x67 mov %rdx, %rax imulq 16-128($np), %rax shr \$29, $r1 @@ -528,17 +517,17 @@ $code.=<<___; vpaddq $TEMP0, $ACC7, $ACC7 vpmuludq 32*8-128($np), $Y1, $TEMP1 vmovd %eax, $Y1 - vmovdqu 32*1-8-128($np), $TEMP2 + #vmovdqu 32*1-8-128($np), $TEMP2 # moved below vpaddq $TEMP1, $ACC8, $ACC8 - vmovdqu 32*2-8-128($np), $TEMP0 + #vmovdqu 32*2-8-128($np), $TEMP0 # moved below vpbroadcastq $Y1, $Y1 - vpmuludq $Y2, $TEMP2, $TEMP2 + vpmuludq 32*1-8-128($np), $Y2, $TEMP2 # see above vmovdqu 32*3-8-128($np), $TEMP1 mov %rax, %rdx imulq -128($np), %rax vpaddq $TEMP2, $ACC1, $ACC1 - vpmuludq $Y2, $TEMP0, $TEMP0 + vpmuludq 32*2-8-128($np), $Y2, $TEMP0 # see above vmovdqu 32*4-8-128($np), $TEMP2 add %rax, $r2 mov %rdx, %rax @@ -552,11 +541,12 @@ $code.=<<___; vpaddq $TEMP1, $ACC3, $ACC3 vpmuludq $Y2, $TEMP2, $TEMP2 vmovdqu 32*6-8-128($np), $TEMP1 + .byte 0x67 mov %rax, $r3 imull $n0, %eax vpaddq $TEMP2, $ACC4, $ACC4 vpmuludq $Y2, $TEMP0, $TEMP0 - vmovdqu 32*7-8-128($np), $TEMP2 + .byte 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 # vmovdqu 32*7-8-128($np), $TEMP2 and \$0x1fffffff, %eax vpaddq $TEMP0, $ACC5, $ACC5 vpmuludq $Y2, $TEMP1, $TEMP1 @@ -584,11 +574,12 @@ $code.=<<___; vpaddq $TEMP1, $ACC1, $ACC1 vpmuludq $Y2, $ACC0, $ACC0 vpmuludq $Y1, $TEMP2, $TEMP2 - vmovdqu 32*4-16-128($np), $TEMP1 + .byte 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff # vmovdqu 32*4-16-128($np), $TEMP1 vpaddq $ACC1, $ACC0, $ACC0 vpaddq $TEMP2, $ACC2, $ACC2 vpmuludq $Y1, $TEMP0, $TEMP0 vmovdqu 32*5-16-128($np), $TEMP2 + .byte 0x67 vmovq $ACC0, %rax vmovdqu $ACC0, (%rsp) # transfer $r0-$r3 vpaddq $TEMP0, $ACC3, $ACC3 @@ -602,12 +593,12 @@ $code.=<<___; vmovdqu 32*8-16-128($np), $TEMP2 vpaddq $TEMP0, $ACC6, $ACC6 vpmuludq $Y1, $TEMP1, $TEMP1 - vmovdqu 32*9-16-128($np), $TEMP0 shr \$29, $r3 + vmovdqu 32*9-16-128($np), $TEMP0 + add $r3, %rax vpaddq $TEMP1, $ACC7, $ACC7 vpmuludq $Y1, $TEMP2, $TEMP2 - vmovdqu 32*2-24-128($np), $TEMP1 - add $r3, %rax + #vmovdqu 32*2-24-128($np), $TEMP1 # moved below mov %rax, $r0 imull $n0, %eax vpaddq $TEMP2, $ACC8, $ACC8 @@ -615,10 +606,11 @@ $code.=<<___; and \$0x1fffffff, %eax vmovd %eax, $Y1 vmovdqu 32*3-24-128($np), $TEMP2 + .byte 0x67 vpaddq $TEMP0, $ACC9, $ACC9 vpbroadcastq $Y1, $Y1 - vpmuludq $Y2, $TEMP1, $TEMP1 + vpmuludq 32*2-24-128($np), $Y2, $TEMP1 # see above vmovdqu 32*4-24-128($np), $TEMP0 mov %rax, %rdx imulq -128($np), %rax @@ -629,6 +621,7 @@ $code.=<<___; add %rax, $r0 mov %rdx, %rax imulq 8-128($np), %rax + .byte 0x67 shr \$29, $r0 mov 16(%rsp), $r2 vpaddq $TEMP2, $ACC3, $ACC2 @@ -648,8 +641,8 @@ $code.=<<___; vmovdqu 32*8-24-128($np), $TEMP1 mov %rax, $r1 imull $n0, %eax - vpaddq $TEMP2, $ACC6, $ACC5 vpmuludq $Y2, $TEMP0, $TEMP0 + vpaddq $TEMP2, $ACC6, $ACC5 vmovdqu 32*9-24-128($np), $TEMP2 and \$0x1fffffff, %eax vpaddq $TEMP0, $ACC7, $ACC6 |