diff options
author | Andy Polyakov <appro@openssl.org> | 2014-09-12 00:37:41 +0200 |
---|---|---|
committer | Andy Polyakov <appro@openssl.org> | 2014-09-22 00:07:44 +0200 |
commit | 3842a64d36bc47ef7cc1370147fd0e5b60a27a2b (patch) | |
tree | 8b37e4d2f08428c2be459d481bfea1150dce6dca | |
parent | 8aed2a7548362e88e84a7feb795a3a97e8395008 (diff) |
Add ECP_NISTZ256 by Shay Gueron, Intel Corp.
RT: 3149
Reviewed-by: Rich Salz <rsalz@openssl.org>
(cherry picked from commit 4d3fa06fce52682bfbc503c7ded2d0289e3f8cde)
-rw-r--r-- | crypto/ec/Makefile | 6 | ||||
-rwxr-xr-x | crypto/ec/asm/ecp_nistz256-avx2.pl | 2093 | ||||
-rwxr-xr-x | crypto/ec/asm/ecp_nistz256-x86_64.pl | 3092 | ||||
-rw-r--r-- | crypto/ec/ec.h | 5 | ||||
-rw-r--r-- | crypto/ec/ec_curve.c | 10 | ||||
-rw-r--r-- | crypto/ec/ec_err.c | 7 | ||||
-rw-r--r-- | crypto/ec/ecp_nistz256.c | 1458 | ||||
-rw-r--r-- | crypto/ec/ecp_nistz256_table.c | 9534 |
8 files changed, 16201 insertions, 4 deletions
diff --git a/crypto/ec/Makefile b/crypto/ec/Makefile index 8cca59e078..0d9f3ab256 100644 --- a/crypto/ec/Makefile +++ b/crypto/ec/Makefile @@ -48,6 +48,12 @@ lib: $(LIBOBJ) $(RANLIB) $(LIB) || echo Never mind. @touch lib +ecp_nistz256-x86_64.s: asm/ecp_nistz256-x86_64.pl + $(PERL) asm/ecp_nistz256-x86_64.pl $(PERLASM_SCHEME) > $@ + +ecp_nistz256-avx2.s: asm/ecp_nistz256-avx2.pl + $(PERL) asm/ecp_nistz256-avx2.pl $(PERLASM_SCHEME) > $@ + files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/ec/asm/ecp_nistz256-avx2.pl b/crypto/ec/asm/ecp_nistz256-avx2.pl new file mode 100755 index 0000000000..4c220aa645 --- /dev/null +++ b/crypto/ec/asm/ecp_nistz256-avx2.pl @@ -0,0 +1,2093 @@ +#!/usr/bin/env perl + +############################################################################## +# # +# Copyright 2014 Intel Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# # +############################################################################## +# # +# Developers and authors: # +# Shay Gueron (1, 2), and Vlad Krasnov (1) # +# (1) Intel Corporation, Israel Development Center # +# (2) University of Haifa # +# Reference: # +# S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with# +# 256 Bit Primes" # +# # +############################################################################## + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); + $addx = ($1>=2.23); +} + +if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); + $addx = ($1>=2.10); +} + +if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + $addx = ($1>=12); +} + +if (!$addx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { + my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 + $avx = ($ver>=3.0) + ($ver>=3.01); + $addx = ($ver>=3.03); +} + +if ($avx>=2) {{ +$digit_size = "\$29"; +$n_digits = "\$9"; + +$code.=<<___; +.text + +.align 64 +.LAVX2_AND_MASK: +.LAVX2_POLY: +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x000001ff, 0x000001ff, 0x000001ff, 0x000001ff +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00040000, 0x00040000, 0x00040000, 0x00040000 +.quad 0x1fe00000, 0x1fe00000, 0x1fe00000, 0x1fe00000 +.quad 0x00ffffff, 0x00ffffff, 0x00ffffff, 0x00ffffff + +.LAVX2_POLY_x2: +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x400007FC, 0x400007FC, 0x400007FC, 0x400007FC +.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE +.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE +.quad 0x400FFFFE, 0x400FFFFE, 0x400FFFFE, 0x400FFFFE +.quad 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE +.quad 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC + +.LAVX2_POLY_x8: +.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8 +.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8 +.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8 +.quad 0x80000FF8, 0x80000FF8, 0x80000FF8, 0x80000FF8 +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x801FFFFC, 0x801FFFFC, 0x801FFFFC, 0x801FFFFC +.quad 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC +.quad 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8 + +.LONE: +.quad 0x00000020, 0x00000020, 0x00000020, 0x00000020 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x1fffc000, 0x1fffc000, 0x1fffc000, 0x1fffc000 +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1f7fffff, 0x1f7fffff, 0x1f7fffff, 0x1f7fffff +.quad 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +# RR = 2^266 mod p in AVX2 format, to transform from the native OpenSSL +# Montgomery form (*2^256) to our format (*2^261) + +.LTO_MONT_AVX2: +.quad 0x00000400, 0x00000400, 0x00000400, 0x00000400 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x1ff80000, 0x1ff80000, 0x1ff80000, 0x1ff80000 +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x0fffffff, 0x0fffffff, 0x0fffffff, 0x0fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x00000003, 0x00000003, 0x00000003, 0x00000003 + +.LFROM_MONT_AVX2: +.quad 0x00000001, 0x00000001, 0x00000001, 0x00000001 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x1ffffe00, 0x1ffffe00, 0x1ffffe00, 0x1ffffe00 +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1ffbffff, 0x1ffbffff, 0x1ffbffff, 0x1ffbffff +.quad 0x001fffff, 0x001fffff, 0x001fffff, 0x001fffff +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +.LIntOne: +.long 1,1,1,1,1,1,1,1 +___ + +{ +# This function recieves a pointer to an array of four affine points +# (X, Y, <1>) and rearanges the data for AVX2 execution, while +# converting it to 2^29 radix redundant form + +my ($X0,$X1,$X2,$X3, $Y0,$Y1,$Y2,$Y3, + $T0,$T1,$T2,$T3, $T4,$T5,$T6,$T7)=map("%ymm$_",(0..15)); + +$code.=<<___; +.globl ecp_nistz256_avx2_transpose_convert +.type ecp_nistz256_avx2_transpose_convert,\@function,2 +.align 64 +ecp_nistz256_avx2_transpose_convert: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -8-16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + # Load the data + vmovdqa 32*0(%rsi), $X0 + lea 112(%rsi), %rax # size optimization + vmovdqa 32*1(%rsi), $Y0 + lea .LAVX2_AND_MASK(%rip), %rdx + vmovdqa 32*2(%rsi), $X1 + vmovdqa 32*3(%rsi), $Y1 + vmovdqa 32*4-112(%rax), $X2 + vmovdqa 32*5-112(%rax), $Y2 + vmovdqa 32*6-112(%rax), $X3 + vmovdqa 32*7-112(%rax), $Y3 + + # Transpose X and Y independently + vpunpcklqdq $X1, $X0, $T0 # T0 = [B2 A2 B0 A0] + vpunpcklqdq $X3, $X2, $T1 # T1 = [D2 C2 D0 C0] + vpunpckhqdq $X1, $X0, $T2 # T2 = [B3 A3 B1 A1] + vpunpckhqdq $X3, $X2, $T3 # T3 = [D3 C3 D1 C1] + + vpunpcklqdq $Y1, $Y0, $T4 + vpunpcklqdq $Y3, $Y2, $T5 + vpunpckhqdq $Y1, $Y0, $T6 + vpunpckhqdq $Y3, $Y2, $T7 + + vperm2i128 \$0x20, $T1, $T0, $X0 # X0 = [D0 C0 B0 A0] + vperm2i128 \$0x20, $T3, $T2, $X1 # X1 = [D1 C1 B1 A1] + vperm2i128 \$0x31, $T1, $T0, $X2 # X2 = [D2 C2 B2 A2] + vperm2i128 \$0x31, $T3, $T2, $X3 # X3 = [D3 C3 B3 A3] + + vperm2i128 \$0x20, $T5, $T4, $Y0 + vperm2i128 \$0x20, $T7, $T6, $Y1 + vperm2i128 \$0x31, $T5, $T4, $Y2 + vperm2i128 \$0x31, $T7, $T6, $Y3 + vmovdqa (%rdx), $T7 + + vpand (%rdx), $X0, $T0 # out[0] = in[0] & mask; + vpsrlq \$29, $X0, $X0 + vpand $T7, $X0, $T1 # out[1] = (in[0] >> shift) & mask; + vpsrlq \$29, $X0, $X0 + vpsllq \$6, $X1, $T2 + vpxor $X0, $T2, $T2 + vpand $T7, $T2, $T2 # out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask; + vpsrlq \$23, $X1, $X1 + vpand $T7, $X1, $T3 # out[3] = (in[1] >> ((shift*3)%64)) & mask; + vpsrlq \$29, $X1, $X1 + vpsllq \$12, $X2, $T4 + vpxor $X1, $T4, $T4 + vpand $T7, $T4, $T4 # out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask; + vpsrlq \$17, $X2, $X2 + vpand $T7, $X2, $T5 # out[5] = (in[2] >> ((shift*5)%64)) & mask; + vpsrlq \$29, $X2, $X2 + vpsllq \$18, $X3, $T6 + vpxor $X2, $T6, $T6 + vpand $T7, $T6, $T6 # out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask; + vpsrlq \$11, $X3, $X3 + vmovdqa $T0, 32*0(%rdi) + lea 112(%rdi), %rax # size optimization + vpand $T7, $X3, $T0 # out[7] = (in[3] >> ((shift*7)%64)) & mask; + vpsrlq \$29, $X3, $X3 # out[8] = (in[3] >> ((shift*8)%64)) & mask; + + vmovdqa $T1, 32*1(%rdi) + vmovdqa $T2, 32*2(%rdi) + vmovdqa $T3, 32*3(%rdi) + vmovdqa $T4, 32*4-112(%rax) + vmovdqa $T5, 32*5-112(%rax) + vmovdqa $T6, 32*6-112(%rax) + vmovdqa $T0, 32*7-112(%rax) + vmovdqa $X3, 32*8-112(%rax) + lea 448(%rdi), %rax # size optimization + + vpand $T7, $Y0, $T0 # out[0] = in[0] & mask; + vpsrlq \$29, $Y0, $Y0 + vpand $T7, $Y0, $T1 # out[1] = (in[0] >> shift) & mask; + vpsrlq \$29, $Y0, $Y0 + vpsllq \$6, $Y1, $T2 + vpxor $Y0, $T2, $T2 + vpand $T7, $T2, $T2 # out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask; + vpsrlq \$23, $Y1, $Y1 + vpand $T7, $Y1, $T3 # out[3] = (in[1] >> ((shift*3)%64)) & mask; + vpsrlq \$29, $Y1, $Y1 + vpsllq \$12, $Y2, $T4 + vpxor $Y1, $T4, $T4 + vpand $T7, $T4, $T4 # out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask; + vpsrlq \$17, $Y2, $Y2 + vpand $T7, $Y2, $T5 # out[5] = (in[2] >> ((shift*5)%64)) & mask; + vpsrlq \$29, $Y2, $Y2 + vpsllq \$18, $Y3, $T6 + vpxor $Y2, $T6, $T6 + vpand $T7, $T6, $T6 # out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask; + vpsrlq \$11, $Y3, $Y3 + vmovdqa $T0, 32*9-448(%rax) + vpand $T7, $Y3, $T0 # out[7] = (in[3] >> ((shift*7)%64)) & mask; + vpsrlq \$29, $Y3, $Y3 # out[8] = (in[3] >> ((shift*8)%64)) & mask; + + vmovdqa $T1, 32*10-448(%rax) + vmovdqa $T2, 32*11-448(%rax) + vmovdqa $T3, 32*12-448(%rax) + vmovdqa $T4, 32*13-448(%rax) + vmovdqa $T5, 32*14-448(%rax) + vmovdqa $T6, 32*15-448(%rax) + vmovdqa $T0, 32*16-448(%rax) + vmovdqa $Y3, 32*17-448(%rax) + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*0(%rsp), %xmm6 + movaps 16*1(%rsp), %xmm7 + movaps 16*2(%rsp), %xmm8 + movaps 16*3(%rsp), %xmm9 + movaps 16*4(%rsp), %xmm10 + movaps 16*5(%rsp), %xmm11 + movaps 16*6(%rsp), %xmm12 + movaps 16*7(%rsp), %xmm13 + movaps 16*8(%rsp), %xmm14 + movaps 16*9(%rsp), %xmm15 + lea 8+16*10(%rsp), %rsp +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_transpose_convert,.-ecp_nistz256_avx2_transpose_convert +___ +} +{ +################################################################################ +# This function recieves a pointer to an array of four AVX2 formatted points +# (X, Y, Z) convert the data to normal representation, and rearanges the data + +my ($D0,$D1,$D2,$D3, $D4,$D5,$D6,$D7, $D8)=map("%ymm$_",(0..8)); +my ($T0,$T1,$T2,$T3, $T4,$T5,$T6)=map("%ymm$_",(9..15)); + +$code.=<<___; + +.globl ecp_nistz256_avx2_convert_transpose_back +.type ecp_nistz256_avx2_convert_transpose_back,\@function,2 +.align 32 +ecp_nistz256_avx2_convert_transpose_back: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -8-16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + mov \$3, %ecx + +.Lconv_loop: + vmovdqa 32*0(%rsi), $D0 + lea 160(%rsi), %rax # size optimization + vmovdqa 32*1(%rsi), $D1 + vmovdqa 32*2(%rsi), $D2 + vmovdqa 32*3(%rsi), $D3 + vmovdqa 32*4-160(%rax), $D4 + vmovdqa 32*5-160(%rax), $D5 + vmovdqa 32*6-160(%rax), $D6 + vmovdqa 32*7-160(%rax), $D7 + vmovdqa 32*8-160(%rax), $D8 + + vpsllq \$29, $D1, $D1 + vpsllq \$58, $D2, $T0 + vpaddq $D1, $D0, $D0 + vpaddq $T0, $D0, $D0 # out[0] = (in[0]) ^ (in[1] << shift*1) ^ (in[2] << shift*2); + + vpsrlq \$6, $D2, $D2 + vpsllq \$23, $D3, $D3 + vpsllq \$52, $D4, $T1 + vpaddq $D2, $D3, $D3 + vpaddq $D3, $T1, $D1 # out[1] = (in[2] >> (64*1-shift*2)) ^ (in[3] << shift*3%64) ^ (in[4] << shift*4%64); + + vpsrlq \$12, $D4, $D4 + vpsllq \$17, $D5, $D5 + vpsllq \$46, $D6, $T2 + vpaddq $D4, $D5, $D5 + vpaddq $D5, $T2, $D2 # out[2] = (in[4] >> (64*2-shift*4)) ^ (in[5] << shift*5%64) ^ (in[6] << shift*6%64); + + vpsrlq \$18, $D6, $D6 + vpsllq \$11, $D7, $D7 + vpsllq \$40, $D8, $T3 + vpaddq $D6, $D7, $D7 + vpaddq $D7, $T3, $D3 # out[3] = (in[6] >> (64*3-shift*6)) ^ (in[7] << shift*7%64) ^ (in[8] << shift*8%64); + + vpunpcklqdq $D1, $D0, $T0 # T0 = [B2 A2 B0 A0] + vpunpcklqdq $D3, $D2, $T1 # T1 = [D2 C2 D0 C0] + vpunpckhqdq $D1, $D0, $T2 # T2 = [B3 A3 B1 A1] + vpunpckhqdq $D3, $D2, $T3 # T3 = [D3 C3 D1 C1] + + vperm2i128 \$0x20, $T1, $T0, $D0 # X0 = [D0 C0 B0 A0] + vperm2i128 \$0x20, $T3, $T2, $D1 # X1 = [D1 C1 B1 A1] + vperm2i128 \$0x31, $T1, $T0, $D2 # X2 = [D2 C2 B2 A2] + vperm2i128 \$0x31, $T3, $T2, $D3 # X3 = [D3 C3 B3 A3] + + vmovdqa $D0, 32*0(%rdi) + vmovdqa $D1, 32*3(%rdi) + vmovdqa $D2, 32*6(%rdi) + vmovdqa $D3, 32*9(%rdi) + + lea 32*9(%rsi), %rsi + lea 32*1(%rdi), %rdi + + dec %ecx + jnz .Lconv_loop + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*0(%rsp), %xmm6 + movaps 16*1(%rsp), %xmm7 + movaps 16*2(%rsp), %xmm8 + movaps 16*3(%rsp), %xmm9 + movaps 16*4(%rsp), %xmm10 + movaps 16*5(%rsp), %xmm11 + movaps 16*6(%rsp), %xmm12 + movaps 16*7(%rsp), %xmm13 + movaps 16*8(%rsp), %xmm14 + movaps 16*9(%rsp), %xmm15 + lea 8+16*10(%rsp), %rsp +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_convert_transpose_back,.-ecp_nistz256_avx2_convert_transpose_back +___ +} +{ +my ($r_ptr,$a_ptr,$b_ptr,$itr)=("%rdi","%rsi","%rdx","%ecx"); +my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4,$ACC5,$ACC6,$ACC7,$ACC8)=map("%ymm$_",(0..8)); +my ($B,$Y,$T0,$AND_MASK,$OVERFLOW)=map("%ymm$_",(9..13)); + +sub NORMALIZE { +my $ret=<<___; + vpsrlq $digit_size, $ACC0, $T0 + vpand $AND_MASK, $ACC0, $ACC0 + vpaddq $T0, $ACC1, $ACC1 + + vpsrlq $digit_size, $ACC1, $T0 + vpand $AND_MASK, $ACC1, $ACC1 + vpaddq $T0, $ACC2, $ACC2 + + vpsrlq $digit_size, $ACC2, $T0 + vpand $AND_MASK, $ACC2, $ACC2 + vpaddq $T0, $ACC3, $ACC3 + + vpsrlq $digit_size, $ACC3, $T0 + vpand $AND_MASK, $ACC3, $ACC3 + vpaddq $T0, $ACC4, $ACC4 + + vpsrlq $digit_size, $ACC4, $T0 + vpand $AND_MASK, $ACC4, $ACC4 + vpaddq $T0, $ACC5, $ACC5 + + vpsrlq $digit_size, $ACC5, $T0 + vpand $AND_MASK, $ACC5, $ACC5 + vpaddq $T0, $ACC6, $ACC6 + + vpsrlq $digit_size, $ACC6, $T0 + vpand $AND_MASK, $ACC6, $ACC6 + vpaddq $T0, $ACC7, $ACC7 + + vpsrlq $digit_size, $ACC7, $T0 + vpand $AND_MASK, $ACC7, $ACC7 + vpaddq $T0, $ACC8, $ACC8 + #vpand $AND_MASK, $ACC8, $ACC8 +___ + $ret; +} + +sub STORE { +my $ret=<<___; + vmovdqa $ACC0, 32*0(%rdi) + lea 160(%rdi), %rax # size optimization + vmovdqa $ACC1, 32*1(%rdi) + vmovdqa $ACC2, 32*2(%rdi) + vmovdqa $ACC3, 32*3(%rdi) + vmovdqa $ACC4, 32*4-160(%rax) + vmovdqa $ACC5, 32*5-160(%rax) + vmovdqa $ACC6, 32*6-160(%rax) + vmovdqa $ACC7, 32*7-160(%rax) + vmovdqa $ACC8, 32*8-160(%rax) +___ + $ret; +} + +$code.=<<___; +.type avx2_normalize,\@abi-omnipotent +.align 32 +avx2_normalize: + vpsrlq $digit_size, $ACC0, $T0 + vpand $AND_MASK, $ACC0, $ACC0 + vpaddq $T0, $ACC1, $ACC1 + + vpsrlq $digit_size, $ACC1, $T0 + vpand $AND_MASK, $ACC1, $ACC1 + vpaddq $T0, $ACC2, $ACC2 + + vpsrlq $digit_size, $ACC2, $T0 + vpand $AND_MASK, $ACC2, $ACC2 + vpaddq $T0, $ACC3, $ACC3 + + vpsrlq $digit_size, $ACC3, $T0 + vpand $AND_MASK, $ACC3, $ACC3 + vpaddq $T0, $ACC4, $ACC4 + + vpsrlq $digit_size, $ACC4, $T0 + vpand $AND_MASK, $ACC4, $ACC4 + vpaddq $T0, $ACC5, $ACC5 + + vpsrlq $digit_size, $ACC5, $T0 + vpand $AND_MASK, $ACC5, $ACC5 + vpaddq $T0, $ACC6, $ACC6 + + vpsrlq $digit_size, $ACC6, $T0 + vpand $AND_MASK, $ACC6, $ACC6 + vpaddq $T0, $ACC7, $ACC7 + + vpsrlq $digit_size, $ACC7, $T0 + vpand $AND_MASK, $ACC7, $ACC7 + vpaddq $T0, $ACC8, $ACC8 + #vpand $AND_MASK, $ACC8, $ACC8 + + ret +.size avx2_normalize,.-avx2_normalize + +.type avx2_normalize_n_store,\@abi-omnipotent +.align 32 +avx2_normalize_n_store: + vpsrlq $digit_size, $ACC0, $T0 + vpand $AND_MASK, $ACC0, $ACC0 + vpaddq $T0, $ACC1, $ACC1 + + vpsrlq $digit_size, $ACC1, $T0 + vpand $AND_MASK, $ACC1, $ACC1 + vmovdqa $ACC0, 32*0(%rdi) + lea 160(%rdi), %rax # size optimization + vpaddq $T0, $ACC2, $ACC2 + + vpsrlq $digit_size, $ACC2, $T0 + vpand $AND_MASK, $ACC2, $ACC2 + vmovdqa $ACC1, 32*1(%rdi) + vpaddq $T0, $ACC3, $ACC3 + + vpsrlq $digit_size, $ACC3, $T0 + vpand $AND_MASK, $ACC3, $ACC3 + vmovdqa $ACC2, 32*2(%rdi) + vpaddq $T0, $ACC4, $ACC4 + + vpsrlq $digit_size, $ACC4, $T0 + vpand $AND_MASK, $ACC4, $ACC4 + vmovdqa $ACC3, 32*3(%rdi) + vpaddq $T0, $ACC5, $ACC5 + + vpsrlq $digit_size, $ACC5, $T0 + vpand $AND_MASK, $ACC5, $ACC5 + vmovdqa $ACC4, 32*4-160(%rax) + vpaddq $T0, $ACC6, $ACC6 + + vpsrlq $digit_size, $ACC6, $T0 + vpand $AND_MASK, $ACC6, $ACC6 + vmovdqa $ACC5, 32*5-160(%rax) + vpaddq $T0, $ACC7, $ACC7 + + vpsrlq $digit_size, $ACC7, $T0 + vpand $AND_MASK, $ACC7, $ACC7 + vmovdqa $ACC6, 32*6-160(%rax) + vpaddq $T0, $ACC8, $ACC8 + #vpand $AND_MASK, $ACC8, $ACC8 + vmovdqa $ACC7, 32*7-160(%rax) + vmovdqa $ACC8, 32*8-160(%rax) + + ret +.size avx2_normalize_n_store,.-avx2_normalize_n_store + +################################################################################ +# void avx2_mul_x4(void* RESULTx4, void *Ax4, void *Bx4); +.type avx2_mul_x4,\@abi-omnipotent +.align 32 +avx2_mul_x4: + lea .LAVX2_POLY(%rip), %rax + + vpxor $ACC0, $ACC0, $ACC0 + vpxor $ACC1, $ACC1, $ACC1 + vpxor $ACC2, $ACC2, $ACC2 + vpxor $ACC3, $ACC3, $ACC3 + vpxor $ACC4, $ACC4, $ACC4 + vpxor $ACC5, $ACC5, $ACC5 + vpxor $ACC6, $ACC6, $ACC6 + vpxor $ACC7, $ACC7, $ACC7 + + vmovdqa 32*7(%rax), %ymm14 + vmovdqa 32*8(%rax), %ymm15 + + mov $n_digits, $itr + lea -512($a_ptr), $a_ptr # strategic bias to control u-op density + jmp .Lavx2_mul_x4_loop + +.align 32 +.Lavx2_mul_x4_loop: + vmovdqa 32*0($b_ptr), $B + lea 32*1($b_ptr), $b_ptr + + vpmuludq 32*0+512($a_ptr), $B, $T0 + vpmuludq 32*1+512($a_ptr), $B, $OVERFLOW # borrow $OVERFLOW + vpaddq $T0, $ACC0, $ACC0 + vpmuludq 32*2+512($a_ptr), $B, $T0 + vpaddq $OVERFLOW, $ACC1, $ACC1 + vpand $AND_MASK, $ACC0, $Y + vpmuludq 32*3+512($a_ptr), $B, $OVERFLOW + vpaddq $T0, $ACC2, $ACC2 + vpmuludq 32*4+512($a_ptr), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*5+512($a_ptr), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*6+512($a_ptr), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*7+512($a_ptr), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + # Skip some multiplications, optimizing for the constant poly + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*8+512($a_ptr), $B, $ACC8 + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + .byte 0x67 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $OVERFLOW + .byte 0x67 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $T0 + vpaddq $OVERFLOW, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $OVERFLOW + vpaddq $T0, $ACC7, $ACC6 + vpaddq $OVERFLOW, $ACC8, $ACC7 + + dec $itr + jnz .Lavx2_mul_x4_loop + + vpxor $ACC8, $ACC8, $ACC8 + + ret +.size avx2_mul_x4,.-avx2_mul_x4 + +# Function optimized for the constant 1 +################################################################################ +# void avx2_mul_by1_x4(void* RESULTx4, void *Ax4); +.type avx2_mul_by1_x4,\@abi-omnipotent +.align 32 +avx2_mul_by1_x4: + lea .LAVX2_POLY(%rip), %rax + + vpxor $ACC0, $ACC0, $ACC0 + vpxor $ACC1, $ACC1, $ACC1 + vpxor $ACC2, $ACC2, $ACC2 + vpxor $ACC3, $ACC3, $ACC3 + vpxor $ACC4, $ACC4, $ACC4 + vpxor $ACC5, $ACC5, $ACC5 + vpxor $ACC6, $ACC6, $ACC6 + vpxor $ACC7, $ACC7, $ACC7 + vpxor $ACC8, $ACC8, $ACC8 + + vmovdqa 32*3+.LONE(%rip), %ymm14 + vmovdqa 32*7+.LONE(%rip), %ymm15 + + mov $n_digits, $itr + jmp .Lavx2_mul_by1_x4_loop + +.align 32 +.Lavx2_mul_by1_x4_loop: + vmovdqa 32*0($a_ptr), $B + .byte 0x48,0x8d,0xb6,0x20,0,0,0 # lea 32*1($a_ptr), $a_ptr + + vpsllq \$5, $B, $OVERFLOW + vpmuludq %ymm14, $B, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC3 + .byte 0x67 + vpmuludq $AND_MASK, $B, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $T0, $ACC4, $ACC4 + vpaddq $T0, $ACC5, $ACC5 + vpaddq $T0, $ACC6, $ACC6 + vpsllq \$23, $B, $T0 + + .byte 0x67,0x67 + vpmuludq %ymm15, $B, $OVERFLOW + vpsubq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + .byte 0x67,0x67 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $OVERFLOW + vmovdqa $ACC5, $ACC4 + vpmuludq 32*7(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC6, $ACC5 + vpaddq $T0, $ACC7, $ACC6 + vpmuludq 32*8(%rax), $Y, $ACC7 + + dec $itr + jnz .Lavx2_mul_by1_x4_loop + + ret +.size avx2_mul_by1_x4,.-avx2_mul_by1_x4 + +################################################################################ +# void avx2_sqr_x4(void* RESULTx4, void *Ax4, void *Bx4); +.type avx2_sqr_x4,\@abi-omnipotent +.align 32 +avx2_sqr_x4: + lea .LAVX2_POLY(%rip), %rax + + vmovdqa 32*7(%rax), %ymm14 + vmovdqa 32*8(%rax), %ymm15 + + vmovdqa 32*0($a_ptr), $B + vmovdqa 32*1($a_ptr), $ACC1 + vmovdqa 32*2($a_ptr), $ACC2 + vmovdqa 32*3($a_ptr), $ACC3 + vmovdqa 32*4($a_ptr), $ACC4 + vmovdqa 32*5($a_ptr), $ACC5 + vmovdqa 32*6($a_ptr), $ACC6 + vmovdqa 32*7($a_ptr), $ACC7 + vpaddq $ACC1, $ACC1, $ACC1 # 2*$ACC0..7 + vmovdqa 32*8($a_ptr), $ACC8 + vpaddq $ACC2, $ACC2, $ACC2 + vmovdqa $ACC1, 32*0(%rcx) + vpaddq $ACC3, $ACC3, $ACC3 + vmovdqa $ACC2, 32*1(%rcx) + vpaddq $ACC4, $ACC4, $ACC4 + vmovdqa $ACC3, 32*2(%rcx) + vpaddq $ACC5, $ACC5, $ACC5 + vmovdqa $ACC4, 32*3(%rcx) + vpaddq $ACC6, $ACC6, $ACC6 + vmovdqa $ACC5, 32*4(%rcx) + vpaddq $ACC7, $ACC7, $ACC7 + vmovdqa $ACC6, 32*5(%rcx) + vpaddq $ACC8, $ACC8, $ACC8 + vmovdqa $ACC7, 32*6(%rcx) + vmovdqa $ACC8, 32*7(%rcx) + + #itr 1 + vpmuludq $B, $B, $ACC0 + vpmuludq $B, $ACC1, $ACC1 + vpand $AND_MASK, $ACC0, $Y + vpmuludq $B, $ACC2, $ACC2 + vpmuludq $B, $ACC3, $ACC3 + vpmuludq $B, $ACC4, $ACC4 + vpmuludq $B, $ACC5, $ACC5 + vpmuludq $B, $ACC6, $ACC6 + vpmuludq $AND_MASK, $Y, $T0 + vpmuludq $B, $ACC7, $ACC7 + vpmuludq $B, $ACC8, $ACC8 + vmovdqa 32*1($a_ptr), $B + + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 2 + vpmuludq $B, $B, $OVERFLOW + vpand $AND_MASK, $ACC0, $Y + vpmuludq 32*1(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC1, $ACC1 + vpmuludq 32*2(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC2, $ACC2 + vpmuludq 32*3(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*2($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 3 + vpmuludq $B, $B, $T0 + vpand $AND_MASK, $ACC0, $Y + vpmuludq 32*2(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC2, $ACC2 + vpmuludq 32*3(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*3($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 4 + vpmuludq $B, $B, $OVERFLOW + vpmuludq 32*3(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*4($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 5 + vpmuludq $B, $B, $T0 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*5($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3+.LAVX2_POLY(%rip), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 6 + vpmuludq $B, $B, $OVERFLOW + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*6($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 7 + vpmuludq $B, $B, $T0 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*7($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 8 + vpmuludq $B, $B, $OVERFLOW + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*8($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 9 + vpmuludq $B, $B, $ACC8 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 |