[crypto/ec] Remove unreachable AVX2 code in NISTZ256 implementation

`crypto/ec/ecp_nistz256.c` contained code sections guarded by a
`ECP_NISTZ256_AVX2` define.

The relevant comment read:

> /*
>  * Note that by default ECP_NISTZ256_AVX2 is undefined. While it's great
>  * code processing 4 points in parallel, corresponding serial operation
>  * is several times slower, because it uses 29x29=58-bit multiplication
>  * as opposite to 64x64=128-bit in integer-only scalar case. As result
>  * it doesn't provide *significant* performance improvement. Note that
>  * just defining ECP_NISTZ256_AVX2 is not sufficient to make it work,
>  * you'd need to compile even asm/ecp_nistz256-avx.pl module.
>  */

Without diminishing the quality of the original submission, it's evident
that this code has been basically unreachable without modifications to
the library source code and is under-tested.

This commit removes these sections from the codebase.

(cherry picked from commit 00da0f69890874feaa555fafb99b967b861e9118 ,
 backported from https://github.com/openssl/openssl/pull/12019 )

Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/12046)

2 files changed, 46 insertions, 2334 deletions

diff --git a/crypto/ec/asm/ecp_nistz256-avx2.pl b/crypto/ec/asm/ecp_nistz256-avx2.pl
deleted file mode 100755
index 5071d09ac2..0000000000
--- a/crypto/ec/asm/ecp_nistz256-avx2.pl
+++ /dev/null
@@ -1,2080 +0,0 @@
-#! /usr/bin/env perl
-# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved.
-# Copyright (c) 2014, Intel Corporation. All Rights Reserved.
-#
-# Licensed under the OpenSSL license (the "License").  You may not use
-# this file except in compliance with the License.  You can obtain a copy
-# in the file LICENSE in the source distribution or at
-# https://www.openssl.org/source/license.html
-#
-# Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1)
-# (1) Intel Corporation, Israel Development Center, Haifa, Israel
-# (2) University of Haifa, Israel
-#
-# Reference:
-# S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with
-#                          256 Bit Primes"
-
-$flavour = shift;
-$output  = shift;
-if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
-
-$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
-
-$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
-( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
-( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
-die "can't locate x86_64-xlate.pl";
-
-open OUT,"| \"$^X\" $xlate $flavour $output";
-*STDOUT=*OUT;
-
-if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
-		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
-	$avx = ($1>=2.19) + ($1>=2.22);
-	$addx = ($1>=2.23);
-}
-
-if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
-	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
-	$avx = ($1>=2.09) + ($1>=2.10);
-	$addx = ($1>=2.10);
-}
-
-if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
-	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
-	$avx = ($1>=10) + ($1>=11);
-	$addx = ($1>=12);
-}
-
-if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|based on LLVM) ([0-9]+)\.([0-9]+)/) {
-	my $ver = $2 + $3/100.0;	# 3.1->3.01, 3.10->3.10
-	$avx = ($ver>=3.0) + ($ver>=3.01);
-	$addx = ($ver>=3.03);
-}
-
-if ($avx>=2) {{
-$digit_size = "\$29";
-$n_digits = "\$9";
-
-$code.=<<___;
-.text
-
-.align 64
-.LAVX2_AND_MASK:
-.LAVX2_POLY:
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x000001ff, 0x000001ff, 0x000001ff, 0x000001ff
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x00040000, 0x00040000, 0x00040000, 0x00040000
-.quad 0x1fe00000, 0x1fe00000, 0x1fe00000, 0x1fe00000
-.quad 0x00ffffff, 0x00ffffff, 0x00ffffff, 0x00ffffff
-
-.LAVX2_POLY_x2:
-.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
-.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
-.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
-.quad 0x400007FC, 0x400007FC, 0x400007FC, 0x400007FC
-.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE
-.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE
-.quad 0x400FFFFE, 0x400FFFFE, 0x400FFFFE, 0x400FFFFE
-.quad 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE
-.quad 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC
-
-.LAVX2_POLY_x8:
-.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8
-.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8
-.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8
-.quad 0x80000FF8, 0x80000FF8, 0x80000FF8, 0x80000FF8
-.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
-.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
-.quad 0x801FFFFC, 0x801FFFFC, 0x801FFFFC, 0x801FFFFC
-.quad 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC
-.quad 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8
-
-.LONE:
-.quad 0x00000020, 0x00000020, 0x00000020, 0x00000020
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x1fffc000, 0x1fffc000, 0x1fffc000, 0x1fffc000
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x1f7fffff, 0x1f7fffff, 0x1f7fffff, 0x1f7fffff
-.quad 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-
-# RR = 2^266 mod p in AVX2 format, to transform from the native OpenSSL
-# Montgomery form (*2^256) to our format (*2^261)
-
-.LTO_MONT_AVX2:
-.quad 0x00000400, 0x00000400, 0x00000400, 0x00000400
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x1ff80000, 0x1ff80000, 0x1ff80000, 0x1ff80000
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x0fffffff, 0x0fffffff, 0x0fffffff, 0x0fffffff
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x00000003, 0x00000003, 0x00000003, 0x00000003
-
-.LFROM_MONT_AVX2:
-.quad 0x00000001, 0x00000001, 0x00000001, 0x00000001
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-.quad 0x1ffffe00, 0x1ffffe00, 0x1ffffe00, 0x1ffffe00
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
-.quad 0x1ffbffff, 0x1ffbffff, 0x1ffbffff, 0x1ffbffff
-.quad 0x001fffff, 0x001fffff, 0x001fffff, 0x001fffff
-.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
-
-.LIntOne:
-.long 1,1,1,1,1,1,1,1
-___
-
-{
-# This function receives a pointer to an array of four affine points
-# (X, Y, <1>) and rearranges the data for AVX2 execution, while
-# converting it to 2^29 radix redundant form
-
-my ($X0,$X1,$X2,$X3, $Y0,$Y1,$Y2,$Y3,
-    $T0,$T1,$T2,$T3, $T4,$T5,$T6,$T7)=map("%ymm$_",(0..15));
-
-$code.=<<___;
-.globl	ecp_nistz256_avx2_transpose_convert
-.type	ecp_nistz256_avx2_transpose_convert,\@function,2
-.align 64
-ecp_nistz256_avx2_transpose_convert:
-	vzeroupper
-___
-$code.=<<___	if ($win64);
-	lea	-8-16*10(%rsp), %rsp
-	vmovaps	%xmm6, -8-16*10(%rax)
-	vmovaps	%xmm7, -8-16*9(%rax)
-	vmovaps	%xmm8, -8-16*8(%rax)
-	vmovaps	%xmm9, -8-16*7(%rax)
-	vmovaps	%xmm10, -8-16*6(%rax)
-	vmovaps	%xmm11, -8-16*5(%rax)
-	vmovaps	%xmm12, -8-16*4(%rax)
-	vmovaps	%xmm13, -8-16*3(%rax)
-	vmovaps	%xmm14, -8-16*2(%rax)
-	vmovaps	%xmm15, -8-16*1(%rax)
-___
-$code.=<<___;
-	# Load the data
-	vmovdqa		32*0(%rsi), $X0
-	lea		112(%rsi), %rax		# size optimization
-	vmovdqa		32*1(%rsi), $Y0
-	lea		.LAVX2_AND_MASK(%rip), %rdx
-	vmovdqa		32*2(%rsi), $X1
-	vmovdqa		32*3(%rsi), $Y1
-	vmovdqa		32*4-112(%rax), $X2
-	vmovdqa		32*5-112(%rax), $Y2
-	vmovdqa		32*6-112(%rax), $X3
-	vmovdqa		32*7-112(%rax), $Y3
-
-	# Transpose X and Y independently
-	vpunpcklqdq	$X1, $X0, $T0		# T0 = [B2 A2 B0 A0]
-	vpunpcklqdq	$X3, $X2, $T1		# T1 = [D2 C2 D0 C0]
-	vpunpckhqdq	$X1, $X0, $T2		# T2 = [B3 A3 B1 A1]
-	vpunpckhqdq	$X3, $X2, $T3		# T3 = [D3 C3 D1 C1]
-
-	vpunpcklqdq	$Y1, $Y0, $T4
-	vpunpcklqdq	$Y3, $Y2, $T5
-	vpunpckhqdq	$Y1, $Y0, $T6
-	vpunpckhqdq	$Y3, $Y2, $T7
-
-	vperm2i128	\$0x20, $T1, $T0, $X0	# X0 = [D0 C0 B0 A0]
-	vperm2i128	\$0x20, $T3, $T2, $X1	# X1 = [D1 C1 B1 A1]
-	vperm2i128	\$0x31, $T1, $T0, $X2	# X2 = [D2 C2 B2 A2]
-	vperm2i128	\$0x31, $T3, $T2, $X3	# X3 = [D3 C3 B3 A3]
-
-	vperm2i128	\$0x20, $T5, $T4, $Y0
-	vperm2i128	\$0x20, $T7, $T6, $Y1
-	vperm2i128	\$0x31, $T5, $T4, $Y2
-	vperm2i128	\$0x31, $T7, $T6, $Y3
-	vmovdqa		(%rdx), $T7
-
-	vpand		(%rdx), $X0, $T0	# out[0] = in[0] & mask;
-	vpsrlq		\$29, $X0, $X0
-	vpand		$T7, $X0, $T1		# out[1] = (in[0] >> shift) & mask;
-	vpsrlq		\$29, $X0, $X0
-	vpsllq		\$6, $X1, $T2
-	vpxor		$X0, $T2, $T2
-	vpand		$T7, $T2, $T2		# out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask;
-	vpsrlq		\$23, $X1, $X1
-	vpand		$T7, $X1, $T3		# out[3] = (in[1] >> ((shift*3)%64)) & mask;
-	vpsrlq		\$29, $X1, $X1
-	vpsllq		\$12, $X2, $T4
-	vpxor		$X1, $T4, $T4
-	vpand		$T7, $T4, $T4		# out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask;
-	vpsrlq		\$17, $X2, $X2
-	vpand		$T7, $X2, $T5		# out[5] = (in[2] >> ((shift*5)%64)) & mask;
-	vpsrlq		\$29, $X2, $X2
-	vpsllq		\$18, $X3, $T6
-	vpxor		$X2, $T6, $T6
-	vpand		$T7, $T6, $T6		# out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask;
-	vpsrlq		\$11, $X3, $X3
-	 vmovdqa	$T0, 32*0(%rdi)
-	 lea		112(%rdi), %rax		# size optimization
-	vpand		$T7, $X3, $T0		# out[7] = (in[3] >> ((shift*7)%64)) & mask;
-	vpsrlq		\$29, $X3, $X3		# out[8] = (in[3] >> ((shift*8)%64)) & mask;
-
-	vmovdqa		$T1, 32*1(%rdi)
-	vmovdqa		$T2, 32*2(%rdi)
-	vmovdqa		$T3, 32*3(%rdi)
-	vmovdqa		$T4, 32*4-112(%rax)
-	vmovdqa		$T5, 32*5-112(%rax)
-	vmovdqa		$T6, 32*6-112(%rax)
-	vmovdqa		$T0, 32*7-112(%rax)
-	vmovdqa		$X3, 32*8-112(%rax)
-	lea		448(%rdi), %rax		# size optimization
-
-	vpand		$T7, $Y0, $T0		# out[0] = in[0] & mask;
-	vpsrlq		\$29, $Y0, $Y0
-	vpand		$T7, $Y0, $T1		# out[1] = (in[0] >> shift) & mask;
-	vpsrlq		\$29, $Y0, $Y0
-	vpsllq		\$6, $Y1, $T2
-	vpxor		$Y0, $T2, $T2
-	vpand		$T7, $T2, $T2		# out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask;
-	vpsrlq		\$23, $Y1, $Y1
-	vpand		$T7, $Y1, $T3		# out[3] = (in[1] >> ((shift*3)%64)) & mask;
-	vpsrlq		\$29, $Y1, $Y1
-	vpsllq		\$12, $Y2, $T4
-	vpxor		$Y1, $T4, $T4
-	vpand		$T7, $T4, $T4		# out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask;
-	vpsrlq		\$17, $Y2, $Y2
-	vpand		$T7, $Y2, $T5		# out[5] = (in[2] >> ((shift*5)%64)) & mask;
-	vpsrlq		\$29, $Y2, $Y2
-	vpsllq		\$18, $Y3, $T6
-	vpxor		$Y2, $T6, $T6
-	vpand		$T7, $T6, $T6		# out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask;
-	vpsrlq		\$11, $Y3, $Y3
-	 vmovdqa	$T0, 32*9-448(%rax)
-	vpand		$T7, $Y3, $T0		# out[7] = (in[3] >> ((shift*7)%64)) & mask;
-	vpsrlq		\$29, $Y3, $Y3		# out[8] = (in[3] >> ((shift*8)%64)) & mask;
-
-	vmovdqa		$T1, 32*10-448(%rax)
-	vmovdqa		$T2, 32*11-448(%rax)
-	vmovdqa		$T3, 32*12-448(%rax)
-	vmovdqa		$T4, 32*13-448(%rax)
-	vmovdqa		$T5, 32*14-448(%rax)
-	vmovdqa		$T6, 32*15-448(%rax)
-	vmovdqa		$T0, 32*16-448(%rax)
-	vmovdqa		$Y3, 32*17-448(%rax)
-
-	vzeroupper
-___
-$code.=<<___	if ($win64);
-	movaps	16*0(%rsp), %xmm6
-	movaps	16*1(%rsp), %xmm7
-	movaps	16*2(%rsp), %xmm8
-	movaps	16*3(%rsp), %xmm9
-	movaps	16*4(%rsp), %xmm10
-	movaps	16*5(%rsp), %xmm11
-	movaps	16*6(%rsp), %xmm12
-	movaps	16*7(%rsp), %xmm13
-	movaps	16*8(%rsp), %xmm14
-	movaps	16*9(%rsp), %xmm15
-	lea	8+16*10(%rsp), %rsp
-___
-$code.=<<___;
-	ret
-.size	ecp_nistz256_avx2_transpose_convert,.-ecp_nistz256_avx2_transpose_convert
-___
-}
-{
-################################################################################
-# This function receives a pointer to an array of four AVX2 formatted points
-# (X, Y, Z) convert the data to normal representation, and rearranges the data
-
-my ($D0,$D1,$D2,$D3, $D4,$D5,$D6,$D7, $D8)=map("%ymm$_",(0..8));
-my ($T0,$T1,$T2,$T3, $T4,$T5,$T6)=map("%ymm$_",(9..15));
-
-$code.=<<___;
-
-.globl	ecp_nistz256_avx2_convert_transpose_back
-.type	ecp_nistz256_avx2_convert_transpose_back,\@function,2
-.align	32
-ecp_nistz256_avx2_convert_transpose_back:
-	vzeroupper
-___
-$code.=<<___	if ($win64);
-	lea	-8-16*10(%rsp), %rsp
-	vmovaps	%xmm6, -8-16*10(%rax)
-	vmovaps	%xmm7, -8-16*9(%rax)
-	vmovaps	%xmm8, -8-16*8(%rax)
-	vmovaps	%xmm9, -8-16*7(%rax)
-	vmovaps	%xmm10, -8-16*6(%rax)
-	vmovaps	%xmm11, -8-16*5(%rax)
-	vmovaps	%xmm12, -8-16*4(%rax)
-	vmovaps	%xmm13, -8-16*3(%rax)
-	vmovaps	%xmm14, -8-16*2(%rax)
-	vmovaps	%xmm15, -8-16*1(%rax)
-___
-$code.=<<___;
-	mov	\$3, %ecx
-
-.Lconv_loop:
-	vmovdqa		32*0(%rsi), $D0
-	lea		160(%rsi), %rax		# size optimization
-	vmovdqa		32*1(%rsi), $D1
-	vmovdqa		32*2(%rsi), $D2
-	vmovdqa		32*3(%rsi), $D3
-	vmovdqa		32*4-160(%rax), $D4
-	vmovdqa		32*5-160(%rax), $D5
-	vmovdqa		32*6-160(%rax), $D6
-	vmovdqa		32*7-160(%rax), $D7
-	vmovdqa		32*8-160(%rax), $D8
-
-	vpsllq		\$29, $D1, $D1
-	vpsllq		\$58, $D2, $T0
-	vpaddq		$D1, $D0, $D0
-	vpaddq		$T0, $D0, $D0		# out[0] = (in[0]) ^ (in[1] << shift*1) ^ (in[2] << shift*2);
-
-	vpsrlq		\$6, $D2, $D2
-	vpsllq		\$23, $D3, $D3
-	vpsllq		\$52, $D4, $T1
-	vpaddq		$D2, $D3, $D3
-	vpaddq		$D3, $T1, $D1		# out[1] = (in[2] >> (64*1-shift*2)) ^ (in[3] << shift*3%64) ^ (in[4] << shift*4%64);
-
-	vpsrlq		\$12, $D4, $D4
-	vpsllq		\$17, $D5, $D5
-	vpsllq		\$46, $D6, $T2
-	vpaddq		$D4, $D5, $D5
-	vpaddq		$D5, $T2, $D2		# out[2] = (in[4] >> (64*2-shift*4)) ^ (in[5] << shift*5%64) ^ (in[6] << shift*6%64);
-
-	vpsrlq		\$18, $D6, $D6
-	vpsllq		\$11, $D7, $D7
-	vpsllq		\$40, $D8, $T3
-	vpaddq		$D6, $D7, $D7
-	vpaddq		$D7, $T3, $D3		# out[3] = (in[6] >> (64*3-shift*6)) ^ (in[7] << shift*7%64) ^ (in[8] << shift*8%64);
-
-	vpunpcklqdq	$D1, $D0, $T0		# T0 = [B2 A2 B0 A0]
-	vpunpcklqdq	$D3, $D2, $T1		# T1 = [D2 C2 D0 C0]
-	vpunpckhqdq	$D1, $D0, $T2		# T2 = [B3 A3 B1 A1]
-	vpunpckhqdq	$D3, $D2, $T3		# T3 = [D3 C3 D1 C1]
-
-	vperm2i128	\$0x20, $T1, $T0, $D0	# X0 = [D0 C0 B0 A0]
-	vperm2i128	\$0x20, $T3, $T2, $D1	# X1 = [D1 C1 B1 A1]
-	vperm2i128	\$0x31, $T1, $T0, $D2	# X2 = [D2 C2 B2 A2]
-	vperm2i128	\$0x31, $T3, $T2, $D3	# X3 = [D3 C3 B3 A3]
-
-	vmovdqa		$D0, 32*0(%rdi)
-	vmovdqa		$D1, 32*3(%rdi)
-	vmovdqa		$D2, 32*6(%rdi)
-	vmovdqa		$D3, 32*9(%rdi)
-
-	lea		32*9(%rsi), %rsi
-	lea		32*1(%rdi), %rdi
-
-	dec	%ecx
-	jnz	.Lconv_loop
-
-	vzeroupper
-___
-$code.=<<___	if ($win64);
-	movaps	16*0(%rsp), %xmm6
-	movaps	16*1(%rsp), %xmm7
-	movaps	16*2(%rsp), %xmm8
-	movaps	16*3(%rsp), %xmm9
-	movaps	16*4(%rsp), %xmm10
-	movaps	16*5(%rsp), %xmm11
-	movaps	16*6(%rsp), %xmm12
-	movaps	16*7(%rsp), %xmm13
-	movaps	16*8(%rsp), %xmm14
-	movaps	16*9(%rsp), %xmm15
-	lea	8+16*10(%rsp), %rsp
-___
-$code.=<<___;
-	ret
-.size	ecp_nistz256_avx2_convert_transpose_back,.-ecp_nistz256_avx2_convert_transpose_back
-___
-}
-{
-my ($r_ptr,$a_ptr,$b_ptr,$itr)=("%rdi","%rsi","%rdx","%ecx");
-my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4,$ACC5,$ACC6,$ACC7,$ACC8)=map("%ymm$_",(0..8));
-my ($B,$Y,$T0,$AND_MASK,$OVERFLOW)=map("%ymm$_",(9..13));
-
-sub NORMALIZE {
-my $ret=<<___;
-	vpsrlq		$digit_size, $ACC0, $T0
-	vpand		$AND_MASK, $ACC0, $ACC0
-	vpaddq		$T0, $ACC1, $ACC1
-
-	vpsrlq		$digit_size, $ACC1, $T0
-	vpand		$AND_MASK, $ACC1, $ACC1
-	vpaddq		$T0, $ACC2, $ACC2
-
-	vpsrlq		$digit_size, $ACC2, $T0
-	vpand		$AND_MASK, $ACC2, $ACC2
-	vpaddq		$T0, $ACC3, $ACC3
-
-	vpsrlq		$digit_size, $ACC3, $T0
-	vpand		$AND_MASK, $ACC3, $ACC3
-	vpaddq		$T0, $ACC4, $ACC4
-
-	vpsrlq		$digit_size, $ACC4, $T0
-	vpand		$AND_MASK, $ACC4, $ACC4
-	vpaddq		$T0, $ACC5, $ACC5
-
-	vpsrlq		$digit_size, $ACC5, $T0
-	vpand		$AND_MASK, $ACC5, $ACC5
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpsrlq		$digit_size, $ACC6, $T0
-	vpand		$AND_MASK, $ACC6, $ACC6
-	vpaddq		$T0, $ACC7, $ACC7
-
-	vpsrlq		$digit_size, $ACC7, $T0
-	vpand		$AND_MASK, $ACC7, $ACC7
-	vpaddq		$T0, $ACC8, $ACC8
-	#vpand		$AND_MASK, $ACC8, $ACC8
-___
-    $ret;
-}
-
-sub STORE {
-my $ret=<<___;
-	vmovdqa		$ACC0, 32*0(%rdi)
-	lea		160(%rdi), %rax		# size optimization
-	vmovdqa		$ACC1, 32*1(%rdi)
-	vmovdqa		$ACC2, 32*2(%rdi)
-	vmovdqa		$ACC3, 32*3(%rdi)
-	vmovdqa		$ACC4, 32*4-160(%rax)
-	vmovdqa		$ACC5, 32*5-160(%rax)
-	vmovdqa		$ACC6, 32*6-160(%rax)
-	vmovdqa		$ACC7, 32*7-160(%rax)
-	vmovdqa		$ACC8, 32*8-160(%rax)
-___
-    $ret;
-}
-
-$code.=<<___;
-.type	avx2_normalize,\@abi-omnipotent
-.align	32
-avx2_normalize:
-	vpsrlq		$digit_size, $ACC0, $T0
-	vpand		$AND_MASK, $ACC0, $ACC0
-	vpaddq		$T0, $ACC1, $ACC1
-
-	vpsrlq		$digit_size, $ACC1, $T0
-	vpand		$AND_MASK, $ACC1, $ACC1
-	vpaddq		$T0, $ACC2, $ACC2
-
-	vpsrlq		$digit_size, $ACC2, $T0
-	vpand		$AND_MASK, $ACC2, $ACC2
-	vpaddq		$T0, $ACC3, $ACC3
-
-	vpsrlq		$digit_size, $ACC3, $T0
-	vpand		$AND_MASK, $ACC3, $ACC3
-	vpaddq		$T0, $ACC4, $ACC4
-
-	vpsrlq		$digit_size, $ACC4, $T0
-	vpand		$AND_MASK, $ACC4, $ACC4
-	vpaddq		$T0, $ACC5, $ACC5
-
-	vpsrlq		$digit_size, $ACC5, $T0
-	vpand		$AND_MASK, $ACC5, $ACC5
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpsrlq		$digit_size, $ACC6, $T0
-	vpand		$AND_MASK, $ACC6, $ACC6
-	vpaddq		$T0, $ACC7, $ACC7
-
-	vpsrlq		$digit_size, $ACC7, $T0
-	vpand		$AND_MASK, $ACC7, $ACC7
-	vpaddq		$T0, $ACC8, $ACC8
-	#vpand		$AND_MASK, $ACC8, $ACC8
-
-	ret
-.size	avx2_normalize,.-avx2_normalize
-
-.type	avx2_normalize_n_store,\@abi-omnipotent
-.align	32
-avx2_normalize_n_store:
-	vpsrlq		$digit_size, $ACC0, $T0
-	vpand		$AND_MASK, $ACC0, $ACC0
-	vpaddq		$T0, $ACC1, $ACC1
-
-	vpsrlq		$digit_size, $ACC1, $T0
-	vpand		$AND_MASK, $ACC1, $ACC1
-	 vmovdqa	$ACC0, 32*0(%rdi)
-	 lea		160(%rdi), %rax		# size optimization
-	vpaddq		$T0, $ACC2, $ACC2
-
-	vpsrlq		$digit_size, $ACC2, $T0
-	vpand		$AND_MASK, $ACC2, $ACC2
-	 vmovdqa	$ACC1, 32*1(%rdi)
-	vpaddq		$T0, $ACC3, $ACC3
-
-	vpsrlq		$digit_size, $ACC3, $T0
-	vpand		$AND_MASK, $ACC3, $ACC3
-	 vmovdqa	$ACC2, 32*2(%rdi)
-	vpaddq		$T0, $ACC4, $ACC4
-
-	vpsrlq		$digit_size, $ACC4, $T0
-	vpand		$AND_MASK, $ACC4, $ACC4
-	 vmovdqa	$ACC3, 32*3(%rdi)
-	vpaddq		$T0, $ACC5, $ACC5
-
-	vpsrlq		$digit_size, $ACC5, $T0
-	vpand		$AND_MASK, $ACC5, $ACC5
-	 vmovdqa	$ACC4, 32*4-160(%rax)
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpsrlq		$digit_size, $ACC6, $T0
-	vpand		$AND_MASK, $ACC6, $ACC6
-	 vmovdqa	$ACC5, 32*5-160(%rax)
-	vpaddq		$T0, $ACC7, $ACC7
-
-	vpsrlq		$digit_size, $ACC7, $T0
-	vpand		$AND_MASK, $ACC7, $ACC7
-	 vmovdqa	$ACC6, 32*6-160(%rax)
-	vpaddq		$T0, $ACC8, $ACC8
-	#vpand		$AND_MASK, $ACC8, $ACC8
-	 vmovdqa	$ACC7, 32*7-160(%rax)
-	 vmovdqa	$ACC8, 32*8-160(%rax)
-
-	ret
-.size	avx2_normalize_n_store,.-avx2_normalize_n_store
-
-################################################################################
-# void avx2_mul_x4(void* RESULTx4, void *Ax4, void *Bx4);
-.type	avx2_mul_x4,\@abi-omnipotent
-.align	32
-avx2_mul_x4:
-	lea	.LAVX2_POLY(%rip), %rax
-
-	vpxor	$ACC0, $ACC0, $ACC0
-	vpxor	$ACC1, $ACC1, $ACC1
-	vpxor	$ACC2, $ACC2, $ACC2
-	vpxor	$ACC3, $ACC3, $ACC3
-	vpxor	$ACC4, $ACC4, $ACC4
-	vpxor	$ACC5, $ACC5, $ACC5
-	vpxor	$ACC6, $ACC6, $ACC6
-	vpxor	$ACC7, $ACC7, $ACC7
-
-	vmovdqa	32*7(%rax), %ymm14
-	vmovdqa	32*8(%rax), %ymm15
-
-	mov	$n_digits, $itr
-	lea	-512($a_ptr), $a_ptr	# strategic bias to control u-op density
-	jmp	.Lavx2_mul_x4_loop
-
-.align	32
-.Lavx2_mul_x4_loop:
-	vmovdqa		32*0($b_ptr), $B
-	lea		32*1($b_ptr), $b_ptr
-
-	vpmuludq	32*0+512($a_ptr), $B, $T0
-	vpmuludq	32*1+512($a_ptr), $B, $OVERFLOW	# borrow $OVERFLOW
-	vpaddq		$T0, $ACC0, $ACC0
-	vpmuludq	32*2+512($a_ptr), $B, $T0
-	vpaddq		$OVERFLOW, $ACC1, $ACC1
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpmuludq	32*3+512($a_ptr), $B, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC2
-	vpmuludq	32*4+512($a_ptr), $B, $T0
-	vpaddq		$OVERFLOW, $ACC3, $ACC3
-	vpmuludq	32*5+512($a_ptr), $B, $OVERFLOW
-	vpaddq		$T0, $ACC4, $ACC4
-	vpmuludq	32*6+512($a_ptr), $B, $T0
-	vpaddq		$OVERFLOW, $ACC5, $ACC5
-	vpmuludq	32*7+512($a_ptr), $B, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC6
-
-	# Skip some multiplications, optimizing for the constant poly
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*8+512($a_ptr), $B, $ACC8
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	.byte		0x67
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $OVERFLOW
-	.byte		0x67
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $T0
-	vpaddq		$OVERFLOW, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC7, $ACC6
-	vpaddq		$OVERFLOW, $ACC8, $ACC7
-
-	dec	$itr
-	jnz	.Lavx2_mul_x4_loop
-
-	vpxor	$ACC8, $ACC8, $ACC8
-
-	ret
-.size	avx2_mul_x4,.-avx2_mul_x4
-
-# Function optimized for the constant 1
-################################################################################
-# void avx2_mul_by1_x4(void* RESULTx4, void *Ax4);
-.type	avx2_mul_by1_x4,\@abi-omnipotent
-.align	32
-avx2_mul_by1_x4:
-	lea	.LAVX2_POLY(%rip), %rax
-
-	vpxor	$ACC0, $ACC0, $ACC0
-	vpxor	$ACC1, $ACC1, $ACC1
-	vpxor	$ACC2, $ACC2, $ACC2
-	vpxor	$ACC3, $ACC3, $ACC3
-	vpxor	$ACC4, $ACC4, $ACC4
-	vpxor	$ACC5, $ACC5, $ACC5
-	vpxor	$ACC6, $ACC6, $ACC6
-	vpxor	$ACC7, $ACC7, $ACC7
-	vpxor	$ACC8, $ACC8, $ACC8
-
-	vmovdqa	32*3+.LONE(%rip), %ymm14
-	vmovdqa	32*7+.LONE(%rip), %ymm15
-
-	mov	$n_digits, $itr
-	jmp	.Lavx2_mul_by1_x4_loop
-
-.align	32
-.Lavx2_mul_by1_x4_loop:
-	vmovdqa		32*0($a_ptr), $B
-	.byte		0x48,0x8d,0xb6,0x20,0,0,0	# lea	32*1($a_ptr), $a_ptr
-
-	vpsllq		\$5, $B, $OVERFLOW
-	vpmuludq	%ymm14, $B, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC3
-	.byte		0x67
-	vpmuludq	$AND_MASK, $B, $T0
-	vpand		$AND_MASK, $ACC0, $Y
-	vpaddq		$T0, $ACC4, $ACC4
-	vpaddq		$T0, $ACC5, $ACC5
-	vpaddq		$T0, $ACC6, $ACC6
-	vpsllq		\$23, $B, $T0
-
-	.byte		0x67,0x67
-	vpmuludq	%ymm15, $B, $OVERFLOW
-	vpsubq		$T0, $ACC6, $ACC6
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	vpaddq		$OVERFLOW, $ACC7, $ACC7
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	.byte		0x67,0x67
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $OVERFLOW
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	32*7(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC6, $ACC5
-	vpaddq		$T0, $ACC7, $ACC6
-	vpmuludq	32*8(%rax), $Y, $ACC7
-
-	dec	$itr
-	jnz	.Lavx2_mul_by1_x4_loop
-
-	ret
-.size	avx2_mul_by1_x4,.-avx2_mul_by1_x4
-
-################################################################################
-# void avx2_sqr_x4(void* RESULTx4, void *Ax4, void *Bx4);
-.type	avx2_sqr_x4,\@abi-omnipotent
-.align	32
-avx2_sqr_x4:
-	lea		.LAVX2_POLY(%rip), %rax
-
-	vmovdqa		32*7(%rax), %ymm14
-	vmovdqa		32*8(%rax), %ymm15
-
-	vmovdqa		32*0($a_ptr), $B
-	vmovdqa		32*1($a_ptr), $ACC1
-	vmovdqa		32*2($a_ptr), $ACC2
-	vmovdqa		32*3($a_ptr), $ACC3
-	vmovdqa		32*4($a_ptr), $ACC4
-	vmovdqa		32*5($a_ptr), $ACC5
-	vmovdqa		32*6($a_ptr), $ACC6
-	vmovdqa		32*7($a_ptr), $ACC7
-	vpaddq		$ACC1, $ACC1, $ACC1	# 2*$ACC0..7
-	vmovdqa		32*8($a_ptr), $ACC8
-	vpaddq		$ACC2, $ACC2, $ACC2
-	vmovdqa		$ACC1, 32*0(%rcx)
-	vpaddq		$ACC3, $ACC3, $ACC3
-	vmovdqa		$ACC2, 32*1(%rcx)
-	vpaddq		$ACC4, $ACC4, $ACC4
-	vmovdqa		$ACC3, 32*2(%rcx)
-	vpaddq		$ACC5, $ACC5, $ACC5
-	vmovdqa		$ACC4, 32*3(%rcx)
-	vpaddq		$ACC6, $ACC6, $ACC6
-	vmovdqa		$ACC5, 32*4(%rcx)
-	vpaddq		$ACC7, $ACC7, $ACC7
-	vmovdqa		$ACC6, 32*5(%rcx)
-	vpaddq		$ACC8, $ACC8, $ACC8
-	vmovdqa		$ACC7, 32*6(%rcx)
-	vmovdqa		$ACC8, 32*7(%rcx)
-
-	#itr		1
-	vpmuludq	$B, $B, $ACC0
-	vpmuludq	$B, $ACC1, $ACC1
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpmuludq	$B, $ACC2, $ACC2
-	vpmuludq	$B, $ACC3, $ACC3
-	vpmuludq	$B, $ACC4, $ACC4
-	vpmuludq	$B, $ACC5, $ACC5
-	vpmuludq	$B, $ACC6, $ACC6
-	 vpmuludq	$AND_MASK, $Y, $T0
-	vpmuludq	$B, $ACC7, $ACC7
-	vpmuludq	$B, $ACC8, $ACC8
-	 vmovdqa	32*1($a_ptr), $B
-
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		2
-	vpmuludq	$B, $B, $OVERFLOW
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpmuludq	32*1(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC1, $ACC1
-	vpmuludq	32*2(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC2
-	vpmuludq	32*3(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC3, $ACC3
-	vpmuludq	32*4(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC4, $ACC4
-	vpmuludq	32*5(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC5, $ACC5
-	vpmuludq	32*6(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*7(%rcx), $B, $ACC8
-	 vmovdqa	32*2($a_ptr), $B
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		3
-	vpmuludq	$B, $B, $T0
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpmuludq	32*2(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC2
-	vpmuludq	32*3(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC3, $ACC3
-	vpmuludq	32*4(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC4, $ACC4
-	vpmuludq	32*5(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC5, $ACC5
-	vpmuludq	32*6(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*7(%rcx), $B, $ACC8
-	 vmovdqa	32*3($a_ptr), $B
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		4
-	vpmuludq	$B, $B, $OVERFLOW
-	vpmuludq	32*3(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC3, $ACC3
-	vpmuludq	32*4(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC4, $ACC4
-	vpmuludq	32*5(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC5, $ACC5
-	vpmuludq	32*6(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*7(%rcx), $B, $ACC8
-	 vmovdqa	32*4($a_ptr), $B
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		5
-	vpmuludq	$B, $B, $T0
-	vpmuludq	32*4(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC4, $ACC4
-	vpmuludq	32*5(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC5, $ACC5
-	vpmuludq	32*6(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*7(%rcx), $B, $ACC8
-	 vmovdqa	32*5($a_ptr), $B
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3+.LAVX2_POLY(%rip), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		6
-	vpmuludq	$B, $B, $OVERFLOW
-	vpmuludq	32*5(%rcx), $B, $T0
-	vpaddq		$OVERFLOW, $ACC5, $ACC5
-	vpmuludq	32*6(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*7(%rcx), $B, $ACC8
-	 vmovdqa	32*6($a_ptr), $B
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		7
-	vpmuludq	$B, $B, $T0
-	vpmuludq	32*6(%rcx), $B, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC6
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*7(%rcx), $B, $ACC8
-	 vmovdqa	32*7($a_ptr), $B
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		8
-	vpmuludq	$B, $B, $OVERFLOW
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	 vpaddq		$OVERFLOW, $ACC7, $ACC7
-	 vpmuludq	32*7(%rcx), $B, $ACC8
-	 vmovdqa	32*8($a_ptr), $B
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	 vpand		$AND_MASK, $ACC0, $Y
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	#itr		9
-	vpmuludq	$B, $B, $ACC8
-
-	vpmuludq	$AND_MASK, $Y, $T0
-	vpaddq		$T0, $ACC0, $OVERFLOW
-	vpsrlq		$digit_size, $OVERFLOW, $OVERFLOW
-	vpaddq		$T0, $ACC1, $ACC0
-	vpaddq		$T0, $ACC2, $ACC1
-	vpmuludq	32*3(%rax), $Y, $T0
-	vpaddq		$OVERFLOW, $ACC0, $ACC0
-	vpaddq		$T0, $ACC3, $ACC2
-	vmovdqa		$ACC4, $ACC3
-	vpsllq		\$18, $Y, $T0
-	vmovdqa		$ACC5, $ACC4
-	vpmuludq	%ymm14, $Y, $OVERFLOW
-	vpaddq		$T0, $ACC6, $ACC5
-	vpmuludq	%ymm15, $Y, $T0
-	vpaddq		$OVERFLOW, $ACC7, $ACC6
-	vpaddq		$T0, $ACC8, $ACC7
-
-	vpxor		$ACC8, $ACC8, $ACC8
-
-	ret
-.size	avx2_sqr_x4,.-avx2_sqr_x4
-
-################################################################################
-# void avx2_sub_x4(void* RESULTx4, void *Ax4, void *Bx4);
-.type	avx2_sub_x4,\@abi-omnipotent
-.align	32
-avx2_sub_x4:
-	vmovdqa	32*0($a_ptr), $ACC0
-	lea	160($a_ptr), $a_ptr
-	lea	.LAVX2_POLY_x8+128(%rip), %rax
-	lea	128($b_ptr), $b_ptr
-	vmovdqa	32*1-160($a_ptr), $ACC1
-	vmovdqa	32*2-160($a_ptr), $ACC2
-	vmovdqa	32*3-160($a_ptr), $ACC3
-	vmovdqa	32*4-160($a_ptr), $ACC4
-	vmovdqa	32*5-160($a_ptr), $ACC5
-	vmovdqa	32*6-160($a_ptr), $ACC6
-	vmovdqa	32*7-160($a_ptr), $ACC7
-	vmovdqa	32*8-160($a_ptr), $ACC8
-
-	vpaddq	32*0-128(%rax), $ACC0, $ACC0
-	vpaddq	32*1-128(%rax), $ACC1, $ACC1
-	vpaddq	32*2-128(%rax), $ACC2, $ACC2
-	vpaddq	32*3-128(%rax), $ACC3, $ACC3
-	vpaddq	32*4-128(%rax), $ACC4, $ACC4
-	vpaddq	32*5-128(%rax), $ACC5, $ACC5
-	vpaddq	32*6-128(%rax), $ACC6, $ACC6
-	vpaddq	32*7-128(%rax), $ACC7, $ACC7
-	vpaddq	32*8-128(%rax), $ACC8, $ACC8
-
-	vpsubq	32*0-128($b_ptr), $ACC0, $ACC0
-	vpsubq	32*1-128($b_ptr), $ACC1, $ACC1
-	vpsubq	32*2-128($b_ptr), $ACC2, $ACC2
-	vpsubq	32*3-128($b_ptr), $ACC3, $ACC3
-	vpsubq	32*4-128($b_ptr), $ACC4, $ACC4
-	vpsubq	32*5-128($b_ptr), $ACC5, $ACC5
-	vpsubq	32*6-128($b_ptr), $ACC6, $ACC6
-	vpsubq	32*7-128($b_ptr), $ACC7, $ACC7
-	vpsubq	32*8-128($b_ptr), $ACC8, $ACC8
-
-	ret
-.size	avx2_sub_x4,.-avx2_sub_x4
-
-.type	avx2_select_n_store,\@abi-omnipotent
-.align	32
-avx2_select_n_store:
-	vmovdqa	`8+32*9*8`(%rsp), $Y
-	vpor	`8+32*9*8+32`(%rsp), $Y, $Y
-
-	vpandn	$ACC0, $Y, $ACC0
-	vpandn	$ACC1, $Y, $ACC1
-	vpandn	$ACC2, $Y, $ACC2
-	vpandn	$ACC3, $Y, $ACC3
-	vpandn	$ACC4, $Y, $ACC4
-	vpandn	$ACC5, $Y, $ACC5
-	vpandn	$ACC6, $Y, $ACC6
-	vmovdqa	`8+32*9*8+32`(%rsp), $B
-	vpandn	$ACC7, $Y, $ACC7
-	vpandn	`8+32*9*8`(%rsp), $B, $B
-	vpandn	$ACC8, $Y, $ACC8
-
-	vpand	32*0(%rsi), $B, $T0
-	lea	160(%rsi), %rax
-	vpand	32*1(%rsi), $B, $Y
-	vpxor	$T0, $ACC0, $ACC0
-	vpand	32*2(%rsi), $B, $T0
-	vpxor	$Y, $ACC1, $ACC1
-	vpand	32