poly1305/asm/poly1305-ppc.pl: add vector base 2^26 implementation.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8120)

2 files changed, 1452 insertions, 111 deletions

diff --git a/crypto/poly1305/asm/poly1305-ppc.pl b/crypto/poly1305/asm/poly1305-ppc.pl
index e9118ba18f..9f15c0d0be 100755
--- a/crypto/poly1305/asm/poly1305-ppc.pl
+++ b/crypto/poly1305/asm/poly1305-ppc.pl
@@ -8,10 +8,10 @@
 
 #
 # ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
+# Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
+# project. The module is dual licensed under OpenSSL and CRYPTOGAMS
+# licenses depending on where you obtain it. For further details see
+# https://github.com/dot-asm/cryptogams/.
 # ====================================================================
 #
 # This module implements Poly1305 hash for PowerPC.
@@ -44,6 +44,13 @@
 #
 # On side note, Power ISA 2.07 enables vector base 2^26 implementation,
 # and POWER8 might have capacity to break 1.0 cycle per byte barrier...
+#
+# January 2019
+#
+# ... Unfortunately not:-( Estimate was a projection of ARM result,
+# but ARM has vector multiply-n-add instruction, while PowerISA does
+# not, not one usable in the context. Improvement is ~40% over -m64
+# result above and is ~1.43 on little-endian systems.
 
 $flavour = shift;
 
@@ -99,6 +106,7 @@ $code.=<<___;
 	std	r0,0($ctx)		# zero hash value
 	std	r0,8($ctx)
 	std	r0,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
 
 	$UCMP	$inp,r0
 	beq-	Lno_key
@@ -140,6 +148,7 @@ Lno_key:
 .globl	.poly1305_blocks
 .align	4
 .poly1305_blocks:
+Lpoly1305_blocks:
 	srdi.	$len,$len,4
 	beq-	Labort
 
@@ -238,60 +247,120 @@ Labort:
 	.long	0
 	.byte	0,12,4,1,0x80,5,4,0
 .size	.poly1305_blocks,.-.poly1305_blocks
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));
 
+$code.=<<___;
 .globl	.poly1305_emit
-.align	4
+.align	5
 .poly1305_emit:
-	ld	$h0,0($ctx)		# load hash
-	ld	$h1,8($ctx)
-	ld	$h2,16($ctx)
-	ld	$padbit,0($nonce)	# load nonce
-	ld	$nonce,8($nonce)
-
-	addic	$d0,$h0,5		# compare to modulus
-	addze	$d1,$h1
-	addze	$d2,$h2
-
-	srdi	$mask,$d2,2		# did it carry/borrow?
-	neg	$mask,$mask
+	lwz	$h0,0($ctx)	# load hash value base 2^26
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+	lwz	r0,24($ctx)	# is_base2_26
+
+	sldi	$h1,$h1,26	# base 2^26 -> base 2^64
+	sldi	$t0,$h2,52
+	srdi	$h2,$h2,12
+	sldi	$h3,$h3,14
+	add	$h0,$h0,$h1
+	addc	$h0,$h0,$t0
+	sldi	$t0,$h4,40
+	srdi	$h4,$h4,24
+	adde	$h1,$h2,$h3
+	addc	$h1,$h1,$t0
+	addze	$h2,$h4
+
+	ld	$h3,0($ctx)	# load hash value base 2^64
+	ld	$h4,8($ctx)
+	ld	$t0,16($ctx)
+
+	neg	r0,r0
+	xor	$h0,$h0,$h3	# choose between radixes
+	xor	$h1,$h1,$h4
+	xor	$h2,$h2,$t0
+	and	$h0,$h0,r0
+	and	$h1,$h1,r0
+	and	$h2,$h2,r0
+	xor	$h0,$h0,$h3
+	xor	$h1,$h1,$h4
+	xor	$h2,$h2,$t0
+
+	addic	$h3,$h0,5	# compare to modulus
+	addze	$h4,$h1
+	addze	$t0,$h2
+
+	srdi	$t0,$t0,2	# see if it carried/borrowed
+	neg	$t0,$t0
+
+	andc	$h0,$h0,$t0
+	and	$h3,$h3,$t0
+	andc	$h1,$h1,$t0
+	and	$h4,$h4,$t0
+	or	$h0,$h0,$h3
+	or	$h1,$h1,$h4
+
+	lwz	$t0,4($nonce)
+	lwz	$h2,12($nonce)
+	lwz	$h3,0($nonce)
+	lwz	$h4,8($nonce)
+
+	insrdi	$h3,$t0,32,0
+	insrdi	$h4,$h2,32,0
+
+	addc	$h0,$h0,$h3	# accumulate nonce
+	adde	$h1,$h1,$h4
+
+	addi	$ctx,$mac,-1
+	addi	$mac,$mac,7
+
+	stbu	$h0,1($ctx)	# write [little-endian] result
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	stbu	$h1,1($mac)
 
-	andc	$h0,$h0,$mask
-	and	$d0,$d0,$mask
-	andc	$h1,$h1,$mask
-	and	$d1,$d1,$mask
-	or	$h0,$h0,$d0
-	or	$h1,$h1,$d1
-___
-$code.=<<___	if (!$LITTLE_ENDIAN);
-	rotldi	$padbit,$padbit,32	# flip nonce words
-	rotldi	$nonce,$nonce,32
-___
-$code.=<<___;
-	addc	$h0,$h0,$padbit		# accumulate nonce
-	adde	$h1,$h1,$nonce
-___
-$code.=<<___	if ($LITTLE_ENDIAN);
-	std	$h0,0($mac)		# write result
-	std	$h1,8($mac)
-___
-$code.=<<___	if (!$LITTLE_ENDIAN);
-	extrdi	r0,$h0,32,0
-	li	$d0,4
-	stwbrx	$h0,0,$mac		# write result
-	extrdi	$h0,$h1,32,0
-	li	$d1,8
-	stwbrx	r0,$d0,$mac
-	li	$d2,12
-	stwbrx	$h1,$d1,$mac
-	stwbrx	$h0,$d2,$mac
-___
-$code.=<<___;
 	blr
 	.long	0
 	.byte	0,12,0x14,0,0,0,3,0
 .size	.poly1305_emit,.-.poly1305_emit
 ___
-							} else {
+}							} else {
 ###############################################################################
 # base 2^32 implementation
 
@@ -309,6 +378,7 @@ $code.=<<___;
 	stw	r0,8($ctx)
 	stw	r0,12($ctx)
 	stw	r0,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
 
 	$UCMP	$inp,r0
 	beq-	Lno_key
@@ -353,6 +423,7 @@ Lno_key:
 .globl	.poly1305_blocks
 .align	4
 .poly1305_blocks:
+Lpoly1305_blocks:
 	srwi.	$len,$len,4
 	beq-	Labort
 
@@ -560,17 +631,389 @@ Labort:
 	.long	0
 	.byte	0,12,4,1,0x80,18,4,0
 .size	.poly1305_blocks,.-.poly1305_blocks
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));
 
+$code.=<<___;
 .globl	.poly1305_emit
-.align	4
+.align	5
 .poly1305_emit:
-	$STU	$sp,-$FRAME($sp)
+	lwz	r0,24($ctx)	# is_base2_26
+	lwz	$h0,0($ctx)	# load hash value
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+	cmplwi	r0,0
+	beq	Lemit_base2_32
+
+	slwi	$t0,$h1,26	# base 2^26 -> base 2^32
+	srwi	$h1,$h1,6
+	slwi	$t1,$h2,20
+	srwi	$h2,$h2,12
+	addc	$h0,$h0,$t0
+	slwi	$t0,$h3,14
+	srwi	$h3,$h3,18
+	adde	$h1,$h1,$t1
+	slwi	$t1,$h4,8
+	srwi	$h4,$h4,24
+	adde	$h2,$h2,$t0
+	adde	$h3,$h3,$t1
+	addze	$h4,$h4
+
+Lemit_base2_32:
+	addic	r0,$h0,5	# compare to modulus
+	addze	r0,$h1
+	addze	r0,$h2
+	addze	r0,$h3
+	addze	r0,$h4
+
+	srwi	r0,r0,2		# see if it carried/borrowed
+	neg	r0,r0
+	andi.	r0,r0,5
+
+	addc	$h0,$h0,r0
+	lwz	r0,0($nonce)
+	addze	$h1,$h1
+	lwz	$t0,4($nonce)
+	addze	$h2,$h2
+	lwz	$t1,8($nonce)
+	addze	$h3,$h3
+	lwz	$h4,12($nonce)
+
+	addc	$h0,$h0,r0	# accumulate nonce
+	adde	$h1,$h1,$t0
+	adde	$h2,$h2,$t1
+	adde	$h3,$h3,$h4
+
+	addi	$ctx,$mac,-1
+	addi	$mac,$mac,7
+
+	stbu	$h0,1($ctx)	# write [little-endian] result
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	stbu	$h2,1($mac)
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	stbu	$h3,1($mac)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,3,0
+.size	.poly1305_emit,.-.poly1305_emit
+___
+}							}
+{{{
+########################################################################
+# PowerISA 2.07/VSX section                                            #
+########################################################################
+
+my $LOCALS= 6*$SIZE_T;
+my $VSXFRAME = $LOCALS + 6*$SIZE_T;
+   $VSXFRAME += 128;	# local variables
+   $VSXFRAME += 13*16;	# v20-v31 offload
+
+my $BIG_ENDIAN = ($flavour !~ /le/) ? 4 : 0;
+
+########################################################################
+# Layout of opaque area is following:
+#
+#	unsigned __int32 h[5];		# current hash value base 2^26
+#	unsigned __int32 pad;
+#	unsigned __int32 is_base2_26, pad;
+#	unsigned __int64 r[2];		# key value base 2^64
+#	struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
+#
+# where r^n are base 2^26 digits of powers of multiplier key. There are
+# 5 digits, but last four are interleaved with multiples of 5, totalling
+# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
+# powers is as they appear in register, not memory.
+
+my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
+my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
+my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
+my      ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
+my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
+my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
+my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
+my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
+my ($ctx_,$_ctx,$const) = map("r$_",(10..12));
+
+							if ($flavour =~ /64/) {
+###############################################################################
+# setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
+# but the base 2^26 computational part is same...
+
+my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
+my $mask = "r0";
+
+$code.=<<___;
+.globl	.poly1305_blocks_vsx
+.align	5
+.poly1305_blocks_vsx:
+	lwz	r7,24($ctx)		# is_base2_26
+	cmpldi	$len,128
+	bge	__poly1305_blocks_vsx
+
+	neg	r0,r7			# is_base2_26 as mask
+	lwz	r7,0($ctx)		# load hash base 2^26
+	lwz	r8,4($ctx)
+	lwz	r9,8($ctx)
+	lwz	r10,12($ctx)
+	lwz	r11,16($ctx)
+
+	sldi	r8,r8,26		# base 2^26 -> base 2^64
+	sldi	r12,r9,52
+	add	r7,r7,r8
+	srdi	r9,r9,12
+	sldi	r10,r10,14
+	addc	r7,r7,r12
+	sldi	r8,r11,40
+	adde	r9,r9,r10
+	srdi	r11,r11,24
+	addc	r9,r9,r8
+	addze	r11,r11
+
+	ld	r8,0($ctx)		# load hash base 2^64
+	ld	r10,8($ctx)
+	ld	r12,16($ctx)
+
+	xor	r7,r7,r8		# select between radixes
+	xor	r9,r9,r10
+	xor	r11,r11,r12
+	and	r7,r7,r0
+	and	r9,r9,r0
+	and	r11,r11,r0
+	xor	r7,r7,r8
+	xor	r9,r9,r10
+	xor	r11,r11,r12
+
+	li	r0,0
+	std	r7,0($ctx)		# store hash base 2^64
+	std	r9,8($ctx)
+	std	r11,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
+
+	b	Lpoly1305_blocks
+	.long	0
+	.byte	0,12,0x14,0,0,0,4,0
+.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align	5
+__poly1305_mul:
+	mulld	$d0,$h0,$r0		# h0*r0
+	mulhdu	$d1,$h0,$r0
+
+	mulld	$t0,$h1,$s1		# h1*5*r1
+	mulhdu	$t1,$h1,$s1
+	addc	$d0,$d0,$t0
+	adde	$d1,$d1,$t1
+
+	mulld	$t0,$h0,$r1		# h0*r1
+	mulhdu	$d2,$h0,$r1
+	addc	$d1,$d1,$t0
+	addze	$d2,$d2
+
+	mulld	$t0,$h1,$r0		# h1*r0
+	mulhdu	$t1,$h1,$r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	mulld	$t0,$h2,$s1		# h2*5*r1
+	mulld	$t1,$h2,$r0		# h2*r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	andc	$t0,$d2,$mask		# final reduction step
+	and	$h2,$d2,$mask
+	srdi	$t1,$t0,2
+	add	$t0,$t0,$t1
+	addc	$h0,$d0,$t0
+	addze	$h1,$d1
+	addze	$h2,$h2
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_mul,.-__poly1305_mul
+
+.align	5
+__poly1305_splat:
+	extrdi	$d0,$h0,26,38
+	extrdi	$d1,$h0,26,12
+	stw	$d0,0x00($t1)
+
+	extrdi	$d2,$h0,12,0
+	slwi	$d0,$d1,2
+	stw	$d1,0x10($t1)
+	add	$d0,$d0,$d1		# * 5
+	stw	$d0,0x20($t1)
+
+	insrdi	$d2,$h1,14,38
+	slwi	$d0,$d2,2
+	stw	$d2,0x30($t1)
+	add	$d0,$d0,$d2		# * 5
+	stw	$d0,0x40($t1)
+
+	extrdi	$d1,$h1,26,24
+	extrdi	$d2,$h1,24,0
+	slwi	$d0,$d1,2
+	stw	$d1,0x50($t1)
+	add	$d0,$d0,$d1		# * 5
+	stw	$d0,0x60($t1)
+
+	insrdi	$d2,$h2,3,37
+	slwi	$d0,$d2,2
+	stw	$d2,0x70($t1)
+	add	$d0,$d0,$d2		# * 5
+	stw	$d0,0x80($t1)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_splat,.-__poly1305_splat
+
+.align	5
+__poly1305_blocks_vsx:
+	$STU	$sp,-$VSXFRAME($sp)
 	mflr	r0
-	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
-	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
-	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
-	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
-	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mfspr	r12,256
+	stvx	v20,r10,$sp
+	addi	r10,r10,32
+	stvx	v21,r11,$sp
+	addi	r11,r11,32
+	stvx	v22,r10,$sp
+	addi	r10,r10,32
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+	li	r12,-1
+	mtspr	256,r12			# preserve all AltiVec registers
+	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)
+
+	bl	LPICmeup
+
+	li	$x10,0x10
+	li	$x20,0x20
+	li	$x30,0x30
+	li	$x40,0x40
+	li	$x50,0x50
+	lvx_u	$mask26,$x00,$const
+	lvx_u	$_26,$x10,$const
+	lvx_u	$_40,$x20,$const
+	lvx_u	$I2perm,$x30,$const
+	lvx_u	$padbits,$x40,$const
+
+	cmplwi	r7,0			# is_base2_26?
+	bne	Lskip_init_vsx
+
+	ld	$r0,32($ctx)		# load key base 2^64
+	ld	$r1,40($ctx)
+	srdi	$s1,$r1,2
+	li	$mask,3
+	add	$s1,$s1,$r1		# s1 = r1 + r1>>2
+
+	mr	$h0,$r0			# "calculate" r^1
+	mr	$h1,$r1
+	li	$h2,0
+	addi	$t1,$ctx,`48+(12^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^2
+	addi	$t1,$ctx,`48+(4^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^3
+	addi	$t1,$ctx,`48+(8^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^4
+	addi	$t1,$ctx,`48+(0^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	ld	$h0,0($ctx)		# load hash
+	ld	$h1,8($ctx)
+	ld	$h2,16($ctx)
+
+	extrdi	$d0,$h0,26,38		# base 2^64 -> base 2^26
+	extrdi	$d1,$h0,26,12
+	extrdi	$d2,$h0,12,0
+	mtvrwz	$H0,$d0
+	insrdi	$d2,$h1,14,38
+	mtvrwz	$H1,$d1
+	extrdi	$d1,$h1,26,24
+	mtvrwz	$H2,$d2
+	extrdi	$d2,$h1,24,0
+	mtvrwz	$H3,$d1
+	insrdi	$d2,$h2,3,37
+	mtvrwz	$H4,$d2
+___
+							} else {
+###############################################################################
+# 32-bit initialization
+
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
+my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);
+
+$code.=<<___;
+.globl	.poly1305_blocks_vsx
+.align	5
+.poly1305_blocks_vsx:
+	lwz	r7,24($ctx)		# is_base2_26
+	cmplwi	$len,128
+	bge	__poly1305_blocks_vsx
+	cmplwi	r7,0
+	beq	Lpoly1305_blocks
 
 	lwz	$h0,0($ctx)		# load hash
 	lwz	$h1,4($ctx)
@@ -578,68 +1021,957 @@ Labort:
 	lwz	$h3,12($ctx)
 	lwz	$h4,16($ctx)
 
-	addic	$d0,$h0,5		# compare to modulus
-	addze	$d1,$h1
-	addze	$d2,$h2
-	addze	$d3,$h3
-	addze	$mask,$h4
+	slwi	$t0,$h1,26		# base 2^26 -> base 2^32
+	srwi	$h1,$h1,6
+	slwi	$t1,$h2,20
+	srwi	$h2,$h2,12
+	addc	$h0,$h0,$t0
+	slwi	$t0,$h3,14
+	srwi	$h3,$h3,18
+	adde	$h1,$h1,$t1
+	slwi	$t1,$h4,8
+	srwi	$h4,$h4,24
+	adde	$h2,$h2,$t0
+	li	$t0,0
+	adde	$h3,$h3,$t1
+	addze	$h4,$h4
 
-	srwi	$mask,$mask,2		# did it carry/borrow?
-	neg	$mask,$mask
+	stw	$h0,0($ctx)		# store hash base 2^32
+	stw	$h1,4($ctx)
+	stw	$h2,8($ctx)
+	stw	$h3,12($ctx)
+	stw	$h4,16($ctx)
+	stw	$t0,24($ctx)		# clear is_base2_26
 
-	andc	$h0,$h0,$mask
-	and	$d0,$d0,$mask
-	andc	$h1,$h1,$mask
-	and	$d1,$d1,$mask
-	or	$h0,$h0,$d0
-	lwz	$d0,0($nonce)		# load nonce
-	andc	$h2,$h2,$mask
-	and	$d2,$d2,$mask
-	or	$h1,$h1,$d1
-	lwz	$d1,4($nonce)
-	andc	$h3,$h3,$mask
-	and	$d3,$d3,$mask
-	or	$h2,$h2,$d2
-	lwz	$d2,8($nonce)
-	or	$h3,$h3,$d3
-	lwz	$d3,12($nonce)
-
-	addc	$h0,$h0,$d0		# accumulate nonce
-	adde	$h1,$h1,$d1
-	adde	$h2,$h2,$d2
-	adde	$h3,$h3,$d3
-___
-$code.=<<___	if ($LITTLE_ENDIAN);
-	stw	$h0,0($mac)		# write result
-	stw	$h1,4($mac)
-	stw	$h2,8($mac)
-	stw	$h3,12($mac)
-___
-$code.=<<___	if (!$LITTLE_ENDIAN);
-	li	$d1,4
-	stwbrx	$h0,0,$mac		# write result
-	li	$d2,8
-	stwbrx	$h1,$d1,$mac
-	li	$d3,12
-	stwbrx	$h2,$d2,$mac
-	stwbrx	$h3,$d3,$mac
+	b	Lpoly1305_blocks
+	.long	0
+	.byte	0,12,0x14,0,0,0,4,0
+.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align	5
+__poly1305_mul:
+	vmulouw		$ACC0,$H0,$R0
+	vmulouw		$ACC1,$H1,$R0
+	vmulouw		$ACC2,$H2,$R0
+	vmulouw		$ACC3,$H3,$R0
+	vmulouw		$ACC4,$H4,$R0
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	################################################################
+	# lazy reduction
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_mul,.-__poly1305_mul
+
+.align	5
+__poly1305_blocks_vsx:
+	$STU	$sp,-$VSXFRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mfspr	r12,256
+	stvx	v20,r10,$sp
+	addi	r10,r10,32
+	stvx	v21,r11,$sp
+	addi	r11,r11,32
+	stvx	v22,r10,$sp
+	addi	r10,r10,32
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+	li	r12,-1
+	mtspr	256,r12			# preserve all AltiVec registers
+	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)
+
+	bl	LPICmeup
+
+	li	$x10,0x10
+	li	$x20,0x20
+	li	$x30,0x30
+	li	$x40,0x40
+	li	$x50,0x50
+	lvx_u	$mask26,$x00,$const
+	lvx_u	$_26,$x10,$const
+	lvx_u	$_40,$x20,$const
+	lvx_u	$I2perm,$x30,$const
+	lvx_u	$padbits,$x40,$const
+
+	cmplwi	r7,0			# is_base2_26?
+	bne	Lskip_init_vsx
+
+	lwz	$h1,32($ctx)		# load key base 2^32
+	lwz	$h2,36($ctx)
+	lwz	$h3,40($ctx)
+	lwz	$h4,44($ctx)
+
+	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
+	extrwi	$h1,$h1,6,0
+	insrwi	$h1,$h2,20,6
+	extrwi	$h2,$h2,12,0
+	insrwi	$h2,$h3,14,6
+	extrwi	$h3,$h3,18,0
+	insrwi	$h3,$h4,8,6
+	extrwi	$h4,$h4,24,0
+
+	mtvrwz	$R0,$h0
+	slwi	$h0,$h1,2
+	mtvrwz	$R1,$h1
+	add	$h1,$h1,$h0
+	mtvrwz	$S1,$h1
+	slwi	$h1,$h2,2
+	mtvrwz	$R2,$h2
+	add	$h2,$h2,$h1
+	mtvrwz	$S2,$h2
+	slwi	$h2,$h3,2
+	mtvrwz	$R3,$h3
+	add	$h3,$h3,$h2
+	mtvrwz	$S3,$h3
+	slwi	$h3,$h4,2
+	mtvrwz	$R4,$h4
+	add	$h4,$h4,$h3
+	mtvrwz	$S4,$h4
+
+	vmr	$H0,$R0
+	vmr	$H1,$R1
+	vmr	$H2,$R2
+	vmr	$H3,$R3
+	vmr	$H4,$R4
+
+	bl	__poly1305_mul		# r^1:- * r^1:-
+
+	vpermdi	$R0,$H0,$R0,0b00
+	vpermdi	$R1,$H1,$R1,0b00
+	vpermdi	$R2,$H2,$R2,0b00
+	vpermdi	$R3,$H3,$R3,0b00
+	vpermdi	$R4,$H4,$R4,0b00
+	vpermdi	$H0,$H0,$H0,0b00
+	vpermdi	$H1,$H1,$H1,0b00
+	vpermdi	$H2,$H2,$H2,0b00
+	vpermdi	$H3,$H3,$H3,0b00
+	vpermdi	$H4,$H4,$H4,0b00
+	vsld	$S1,$R1,$T0		# <<2
+	vsld	$S2,$R2,$T0
+	vsld	$S3,$R3,$T0
+	vsld	$S4,$R4,$T0
+	vaddudm	$S1,$S1,$R1
+	vaddudm	$S2,$S2,$R2
+	vaddudm	$S3,$S3,$R3
+	vaddudm	$S4,$S4,$R4
+
+	bl	__poly1305_mul		# r^2:r^2 * r^2:r^1
+
+	addi	$h0,$ctx,0x60
+	lwz	$h1,0($ctx)		# load hash
+	lwz	$h2,4($ctx)
+	lwz	$h3,8($ctx)
+	lwz	$h4,12($ctx)
+	lwz	$t0,16($ctx)
+
+	vmrgow	$R0,$R0,$H0		# r^2:r^4:r^1:r^3
+	vmrgow	$R1,$R1,$H1
+	vmrgow	$R2,$R2,$H2
+	vmrgow	$R3,$R3,$H3
+	vmrgow	$R4,$R4,$H4
+	vslw	$S1,$R1,$T0		# <<2
+	vslw	$S2,$R2,$T0
+	vslw	$S3,$R3,$T0
+	vslw	$S4,$R4,$T0
+	vadduwm	$S1,$S1,$R1
+	vadduwm	$S2,$S2,$R2
+	vadduwm	$S3,$S3,$R3
+	vadduwm	$S4,$S4,$R4
+
+	stvx_u	$R0,$x30,$ctx
+	stvx_u	$R1,$x40,$ctx
+	stvx_u	$S1,$x50,$ctx
+	stvx_u	$R2,$x00,$h0
+	stvx_u	$S2,$x10,$h0
+	stvx_u	$R3,$x20,$h0
+	stvx_u	$S3,$x30,$h0
+	stvx_u	$R4,$x40,$h0
+	stvx_u	$S4,$x50,$h0
+
+	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
+	extrwi	$h1,$h1,6,0
+	mtvrwz	$H0,$h0
+	insrwi	$h1,$h2,20,6
+	extrwi	$h2,$h2,12,0
+	mtvrwz	$H1,$h1
+	insrwi	$h2,$h3,14,6
+	extrwi	$h3,$h3,18,0
+	mtvrwz	$H2,$h2
+	insrwi	$h3,$h4,8,6
+	extrwi	$h4,$h4,24,0
+	mtvrwz	$H3,$h3
+	insrwi	$h4,$t0,3,5
+	mtvrwz	$H4,$h4
 ___
+							}
 $code.=<<___;
-	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
-	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
-	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
-	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
-	addi	$sp,$sp,$FRAME
+	li	r0,1
+	stw	r0,24($ctx)		# set is_base2_26
+	b	Loaded_vsx
+
+.align	4
+Lskip_init_vsx:
+	li		$x10,4
+	li		$x20,8
+	li		$x30,12
+	li		$x40,16
+	lvwzx_u		$H0,$x00,$ctx
+	lvwzx_u		$H1,$x10,$ctx
+	lvwzx_u		$H2,$x20,$ctx
+	lvwzx_u		$H3,$x30,$ctx
+	lvwzx_u		$H4,$x40,$ctx
+
+Loaded_vsx:
+	li		$x10,0x10
+	li		$x20,0x20
+	li		$x30,0x30
+	li		$x40,0x40
+	li		$x50,0x50
+	li		$x60,0x60
+	li		$x70,0x70
+	addi		$ctx_,$ctx,64		# &ctx->r[1]
+	addi		$_ctx,$sp,`$LOCALS+15`	# &ctx->r[1], r^2:r^4 shadow
+
+	vxor		$T0,$T0,$T0		# ensure second half is zero
+	vpermdi		$H0,$H0,$T0,0b00
+	vpermdi		$H1,$H1,$T0,0b00
+	vpermdi		$H2,$H2,$T0,0b00
+	vpermdi		$H3,$H3,$T0,0b00
+	vpermdi		$H4,$H4,$T0,0b00
+
+	be?lvx_u	$_4,$x50,$const		# byte swap mask
+	lvx_u		$T1,$x00,$inp		# load first input block
+	lvx_u		$T2,$x10,$inp
+	lvx_u		$T3,$x20,$inp
+	lvx_u		$T4,$x30,$inp
+	be?vperm	$T1,$T1,$T1,$_4
+	be?vperm	$T2,$T2,$T2,$_4
+	be?vperm	$T3,$T3,$T3,$_4
+	be?vperm	$T4,$T4,$T4,$_4
+
+	vpermdi		$I0,$T1,$T2,0b00	# smash input to base 2^26
+	vspltisb	$_4,4
+	vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	vspltisb	$_14,14
+	vpermdi		$I3,$T1,$T2,0b11
+
+	vsrd		$I1,$I0,$_26
+	vsrd		$I2,$I2,$_4
+	vsrd		$I4,$I3,$_40
+	vsrd		$I3,$I3,$_14
+	vand		$I0,$I0,$mask26
+	vand		$I1,$I1,$mask26
+	vand		$I2,$I2,$mask26
+	vand		$I3,$I3,$mask26
+
+	vpermdi		$T1,$T3,$T4,0b00
+	vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	vpermdi		$T3,$T3,$T4,0b11
+
+	vsrd		$T0,$T1,$_26
+	vsrd		$T2,$T2,$_4
+	vsrd		$T4,$T3,$_40
+	vsrd		$T3,$T3,$_14
+	vand		$T1,$T1,$mask26
+	vand		$T0,$T0,$mask26
+	vand		$T2,$T2,$mask26
+	vand		$T3,$T3,$mask26
+
+	# inp[2]:inp[0]:inp[3]:inp[1]
+	vmrgow		$I4,$T4,$I4
+	vmrgow		$I0,$T1,$I0
+	vmrgow		$I1,$T0,$I1
+	vmrgow		$I2,$T2,$I2
+	vmrgow		$I3,$T3,$I3
+	vor		$I4,$I4,$padbits
+
+	lvx_splt	$R0,$x30,$ctx		# taking lvx_vsplt out of loop
+	lvx_splt	$R1,$x00,$ctx_		# gives ~8% improvement
+	lvx_splt	$S1,$x10,$ctx_
+	lvx_splt	$R2,$x20,$ctx_
+	lvx_splt	$S2,$x30,$ctx_
+	lvx_splt	$T1,$x40,$ctx_
+	lvx_splt	$T2,$x50,$ctx_
+	lvx_splt	$T3,$x60,$ctx_
+	lvx_splt	$T4,$x70,$ctx_
+	stvx		$R1,$x00,$_ctx
+	stvx		$S1,$x10,$_ctx
+	stvx		$R2,$x20,$_ctx
+	stvx		$S2,$x30,$_ctx
+	stvx		$T1,$x40,$_ctx
+	stvx		$T2,$x50,$_ctx
+	stvx		$T3,$x60,$_ctx
+	stvx		$T4,$x70,$_ctx
+
+	addi		$inp,$inp,0x40
+	addi		$const,$const,0x50
+	addi		r0,$len,-64
+	srdi		r0,r0,6
+	mtctr		r0
+	b		Loop_vsx
+
+.align	4
+Loop_vsx:
+	################################################################
+	## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+	## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+	##   \___________________/
+	##
+	## Note that we start with inp[2:3]*r^2. This is because it
+	## doesn't depend on reduction in previous iteration.
+	################################################################
+	## d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	## d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	## d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	## d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+
+	vmuleuw		$ACC0,$I0,$R0
+	vmuleuw		$ACC1,$I0,$R1
+	vmuleuw		$ACC2,$I0,$R2
+	vmuleuw		$ACC3,$I1,$R2
+
+	vmuleuw		$T0,$I1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	 vmuleuw	$ACC4,$I2,$R2
+	vmuleuw		$T0,$I4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S3,$x50,$_ctx
+	vmuleuw		$T0,$I3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R3,$x40,$_ctx
+
+	 vaddudm	$H2,$H2,$I2
+	 vaddudm	$H0,$H0,$I0
+	 vaddudm	$H3,$H3,$I3
+	 vaddudm	$H1,$H1,$I1
+	 vaddudm	$H4,$H4,$I4
+
+	vmuleuw		$T0,$I3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S4,$x70,$_ctx
+	vmuleuw		$T0,$I4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R4,$x60,$_ctx
+
+	vmuleuw		$T0,$I2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 be?lvx_u	$_4,$x00,$const		# byte swap mask
+	 lvx_u		$T1,$x00,$inp		# load next input block
+	 lvx_u		$T2,$x10,$inp
+	 lvx_u		$T3,$x20,$inp
+	 lvx_u		$T4,$x30,$inp
+	 be?vperm	$T1,$T1,$T1,$_4
+	 be?vperm	$T2,$T2,$T2,$_4
+	 be?vperm	$T3,$T3,$T3,$_4
+	 be?vperm	$T4,$T4,$T4,$_4
+
+	vmuleuw		$T0,$I1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vpermdi	$I0,$T1,$T2,0b00	# smash input to base 2^26
+	 vspltisb	$_4,4
+	 vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	 vpermdi	$I3,$T1,$T2,0b11
+
+	# (hash + inp[0:1]) * r^4
+	vmulouw		$T0,$H0,$R0
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vpermdi	$T1,$T3,$T4,0b00
+	 vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	 vpermdi	$T3,$T3,$T4,0b11
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S1,$x10,$_ctx
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R1,$x00,$_ctx
+
+	 vsrd		$I1,$I0,$_26
+	 vsrd		$I2,$I2,$_4
+	 vsrd		$I4,$I3,$_40
+	 vsrd		$I3,$I3,$_14
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S2,$x30,$_ctx
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R2,$x20,$_ctx
+
+	 vand		$I0,$I0,$mask26
+	 vand		$I1,$I1,$mask26
+	 vand		$I2,$I2,$mask26
+	 vand		$I3,$I3,$mask26
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vsrd		$T2,$T2,$_4
+	 vsrd		$_4,$T1,$_26
+	 vsrd		$T4,$T3,$_40
+	 vsrd		$T3,$T3,$_14
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vand		$T1,$T1,$mask26
+	 vand		$_4,$_4,$mask26
+	 vand		$T2,$T2,$mask26
+	 vand		$T3,$T3,$mask26
+
+	################################################################
+	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+	# and P. Schwabe
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	 vmrgow		$I4,$T4,$I4
+	 vmrgow		$I0,$T1,$I0
+	 vmrgow		$I1,$_4,$I1
+	 vmrgow		$I2,$T2,$I2
+	 vmrgow		$I3,$T3,$I3
+	 vor		$I4,$I4,$padbits
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	addi		$inp,$inp,0x40
+	bdnz		Loop_vsx
+
+	neg		$len,$len
+	andi.		$len,$len,0x30
+	sub		$inp,$inp,$len
+
+	lvx_u		$R0,$x30,$ctx		# load all powers
+	lvx_u		$R1,$x00,$ctx_
+	lvx_u		$S1,$x10,