chacha20 performance optimizations for ppc64le with 8x lanes,

Performance increase around 50%.

Co-authored-by: Madhusudhanan Duraisamy <madurais@in.ibm.com>

Co-authored-by: Nilamjyoti Goswami <nilamgoswami@in.ibm.com>

Co-authored-by: Siva Sundar Anbareeswaran <srisivasundar@in.ibm.com>

Reviewed-by: Danny Tsen <dtsen@us.ibm.com>
Tested-by: Danny Tsen <dtsen@us.ibm.com>
Signed-off-by: Danny <dtsen@us.ibm.com>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16637)

3 files changed, 1366 insertions, 6 deletions

diff --git a/crypto/chacha/asm/chachap10-ppc.pl b/crypto/chacha/asm/chachap10-ppc.pl
new file mode 100755
index 0000000000..752fc731e2
--- /dev/null
+++ b/crypto/chacha/asm/chachap10-ppc.pl
@@ -0,0 +1,1354 @@
+#! /usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# October 2015
+#
+# ChaCha20 for PowerPC/AltiVec.
+#
+# June 2018
+#
+# Add VSX 2.07 code path. Original 3xAltiVec+1xIALU is well-suited for
+# processors that can't issue more than one vector instruction per
+# cycle. But POWER8 (and POWER9) can issue a pair, and vector-only 4x
+# interleave would perform better. Incidentally PowerISA 2.07 (first
+# implemented by POWER8) defined new usable instructions, hence 4xVSX
+# code path...
+#
+# Performance in cycles per byte out of large buffer.
+#
+#			IALU/gcc-4.x    3xAltiVec+1xIALU	4xVSX
+#
+# Freescale e300	13.6/+115%	-			-
+# PPC74x0/G4e		6.81/+310%	3.81			-
+# PPC970/G5		9.29/+160%	?			-
+# POWER7		8.62/+61%	3.35			-
+# POWER8		8.70/+51%	2.91			2.09
+# POWER9		8.80/+29%	4.44(*)			2.45(**)
+#
+# (*)	this is trade-off result, it's possible to improve it, but
+#	then it would negatively affect all others;
+# (**)	POWER9 seems to be "allergic" to mixing vector and integer
+#	instructions, which is why switch to vector-only code pays
+#	off that much;
+
+# $output is the last argument if it looks like a file (it has an extension)
+# $flavour is the first argument if it doesn't look like a file
+$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
+$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+	$UCMP	="cmpld";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+	$UCMP	="cmplw";
+} else { die "nonsense $flavour"; }
+
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour \"$output\""
+    or die "can't call $xlate: $!";
+
+$LOCALS=6*$SIZE_T;
+$FRAME=$LOCALS+64+18*$SIZE_T;	# 64 is for local variables
+
+sub AUTOLOAD()		# thunk [simplified] x86-style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
+    $code .= "\t$opcode\t".join(',',@_)."\n";
+}
+
+my $sp = "r1";
+
+my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
+
+
+{{{
+my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+    $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = map("v$_",(0..15));
+my @K = map("v$_",(16..19));
+my $CTR = "v26";
+my ($xt0,$xt1,$xt2,$xt3) = map("v$_",(27..30));
+my ($sixteen,$twelve,$eight,$seven) = ($xt0,$xt1,$xt2,$xt3);
+my $beperm = "v31";
+
+my ($x00,$x10,$x20,$x30) = (0, map("r$_",(8..10)));
+
+my $FRAME=$LOCALS+64+7*16;	# 7*16 is for v26-v31 offload
+
+sub VSX_lane_ROUND_1x {
+my $a=@_[0];
+my $b=@_[1];
+my $c=@_[2];
+my $d=@_[3];
+my $odd=@_[4];
+	vadduwm		($a,$a,$b);
+	vxor		($d,$d,$a);
+	vrlw		($d,$d,$sixteen);
+	vadduwm		($c,$c,$d);
+	vxor		($b,$b,$c);
+	vrlw		($b,$b,$twelve);
+	vadduwm		($a,$a,$b);
+	vxor		($d,$d,$a);
+	vrlw		($d,$d,$eight);
+	vadduwm		($c,$c,$d);
+	vxor		($b,$b,$c);
+	vrlw		($b,$b,$seven);
+	xxsldwi		($c,$c,$c,2);
+	xxsldwi		($b,$b,$b,$odd?3:1);
+	xxsldwi		($d,$d,$d,$odd?1:3);
+}
+
+
+sub VSX_lane_ROUND_4x {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my @x=map("\"v$_\"",(0..15));
+
+	(
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$sixteen')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$sixteen')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$sixteen')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$sixteen')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$twelve')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$twelve')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$twelve')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$twelve')",
+
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$eight')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$eight')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$eight')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$eight')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$seven')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$seven')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$seven')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$seven')"
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vsx_p10
+.align	5
+.ChaCha20_ctr32_vsx_p10:
+	${UCMP}i $len,256
+	bgt 	ChaCha20_ctr32_vsx_8x
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$FRAME-4`($sp)		# save vrsave
+	li	r12,-4096+63
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+	lvx_4w	@K[0],0,r12			# load sigma
+	addi	r12,r12,0x70
+	li	$x10,16
+	li	$x20,32
+	li	$x30,48
+	li	r11,64
+
+	lvx_4w	@K[1],0,$key			# load key
+	lvx_4w	@K[2],$x10,$key
+	lvx_4w	@K[3],0,$ctr			# load counter
+
+	vxor	$xt0,$xt0,$xt0
+	lvx_4w	$xt1,r11,r12
+	vspltw	$CTR,@K[3],0
+	vsldoi	@K[3],@K[3],$xt0,4
+	vsldoi	@K[3],$xt0,@K[3],12		# clear @K[3].word[0]
+	vadduwm	$CTR,$CTR,$xt1
+
+	be?lvsl	$beperm,0,$x10			# 0x00..0f
+	be?vspltisb $xt0,3			# 0x03..03
+	be?vxor	$beperm,$beperm,$xt0		# swap bytes within words
+
+	li	r0,10				# inner loop counter
+	mtctr	r0
+	b	Loop_outer_vsx
+
+.align	5
+Loop_outer_vsx:
+	lvx	$xa0,$x00,r12			# load [smashed] sigma
+	lvx	$xa1,$x10,r12
+	lvx	$xa2,$x20,r12
+	lvx	$xa3,$x30,r12
+
+	vspltw	$xb0,@K[1],0			# smash the key
+	vspltw	$xb1,@K[1],1
+	vspltw	$xb2,@K[1],2
+	vspltw	$xb3,@K[1],3
+
+	vspltw	$xc0,@K[2],0
+	vspltw	$xc1,@K[2],1
+	vspltw	$xc2,@K[2],2
+	vspltw	$xc3,@K[2],3
+
+	vmr	$xd0,$CTR			# smash the counter
+	vspltw	$xd1,@K[3],1
+	vspltw	$xd2,@K[3],2
+	vspltw	$xd3,@K[3],3
+
+	vspltisw $sixteen,-16			# synthesize constants
+	vspltisw $twelve,12
+	vspltisw $eight,8
+	vspltisw $seven,7
+
+	${UCMP}i $len,64
+	bgt 	Loop_vsx_4x
+
+	vmr	$xa0,@K[0]
+	vmr	$xb0,@K[1]
+	vmr	$xc0,@K[2]
+	vmr 	$xd0,@K[3]
+
+Loop_vsx_1x:
+___
+	VSX_lane_ROUND_1x($xa0, $xb0, $xc0,$xd0,0);
+	VSX_lane_ROUND_1x($xa0, $xb0, $xc0,$xd0,1);
+
+$code.=<<___;
+
+	bdnz	Loop_vsx_1x
+
+	vadduwm $xa0, $xa0, @K[0]
+	vadduwm $xb0, $xb0, @K[1]
+	vadduwm $xc0, $xc0, @K[2]
+	vadduwm $xd0, $xd0, @K[3]
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w  $xt0,$x00, $inp
+	lvx_4w  $xt1,$x10, $inp
+	lvx_4w  $xt2,$x20, $inp
+	lvx_4w  $xt3,$x30, $inp
+
+	vxor	$xa0,$xa0,$xt0
+	vxor	$xb0,$xb0,$xt1
+	vxor	$xc0,$xc0,$xt2
+	vxor	$xd0,$xd0,$xt3
+
+	stvx_4w	$xa0,$x00,$out
+	stvx_4w	$xb0,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xc0,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xd0,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+Loop_vsx_4x:
+___
+	foreach (&VSX_lane_ROUND_4x(0, 4, 8,12)) { eval; }
+	foreach (&VSX_lane_ROUND_4x(0, 5,10,15)) { eval; }
+$code.=<<___;
+
+	bdnz	Loop_vsx_4x
+
+	vadduwm	$xd0,$xd0,$CTR
+
+	vmrgew	$xt0,$xa0,$xa1			# transpose data
+	vmrgew	$xt1,$xa2,$xa3
+	vmrgow	$xa0,$xa0,$xa1
+	vmrgow	$xa2,$xa2,$xa3
+	vmrgew	$xt2,$xb0,$xb1
+	vmrgew	$xt3,$xb2,$xb3
+	vpermdi	$xa1,$xa0,$xa2,0b00
+	vpermdi	$xa3,$xa0,$xa2,0b11
+	vpermdi	$xa0,$xt0,$xt1,0b00
+	vpermdi	$xa2,$xt0,$xt1,0b11
+
+	vmrgow	$xb0,$xb0,$xb1
+	vmrgow	$xb2,$xb2,$xb3
+	vmrgew	$xt0,$xc0,$xc1
+	vmrgew	$xt1,$xc2,$xc3
+	vpermdi	$xb1,$xb0,$xb2,0b00
+	vpermdi	$xb3,$xb0,$xb2,0b11
+	vpermdi	$xb0,$xt2,$xt3,0b00
+	vpermdi	$xb2,$xt2,$xt3,0b11
+
+	vmrgow	$xc0,$xc0,$xc1
+	vmrgow	$xc2,$xc2,$xc3
+	vmrgew	$xt2,$xd0,$xd1
+	vmrgew	$xt3,$xd2,$xd3
+	vpermdi	$xc1,$xc0,$xc2,0b00
+	vpermdi	$xc3,$xc0,$xc2,0b11
+	vpermdi	$xc0,$xt0,$xt1,0b00
+	vpermdi	$xc2,$xt0,$xt1,0b11
+
+	vmrgow	$xd0,$xd0,$xd1
+	vmrgow	$xd2,$xd2,$xd3
+	vspltisw $xt0,4
+	vadduwm  $CTR,$CTR,$xt0		# next counter value
+	vpermdi	$xd1,$xd0,$xd2,0b00
+	vpermdi	$xd3,$xd0,$xd2,0b11
+	vpermdi	$xd0,$xt2,$xt3,0b00
+	vpermdi	$xd2,$xt2,$xt3,0b11
+
+	vadduwm	$xa0,$xa0,@K[0]
+	vadduwm	$xb0,$xb0,@K[1]
+	vadduwm	$xc0,$xc0,@K[2]
+	vadduwm	$xd0,$xd0,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa1,@K[0]
+	vadduwm	$xb0,$xb1,@K[1]
+	vadduwm	$xc0,$xc1,@K[2]
+	vadduwm	$xd0,$xd1,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa2,@K[0]
+	vadduwm	$xb0,$xb2,@K[1]
+	vadduwm	$xc0,$xc2,@K[2]
+	vadduwm	$xd0,$xd2,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa3,@K[0]
+	vadduwm	$xb0,$xb3,@K[1]
+	vadduwm	$xc0,$xc3,@K[2]
+	vadduwm	$xd0,$xd3,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	mtctr	r0
+	bne	Loop_outer_vsx
+
+Ldone_vsx:
+	lwz	r12,`$FRAME-4`($sp)		# pull vrsave
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	$POP	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# restore vrsave
+	lvx	v26,r10,$sp
+	addi	r10,r10,32
+	lvx	v27,r11,$sp
+	addi	r11,r11,32
+	lvx	v28,r10,$sp
+	addi	r10,r10,32
+	lvx	v29,r11,$sp
+	addi	r11,r11,32
+	lvx	v30,r10,$sp
+	lvx	v31,r11,$sp
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+
+.align	4
+Ltail_vsx:
+	addi	r11,$sp,$LOCALS
+	mtctr	$len
+	stvx_4w	$xa0,$x00,r11			# offload block to stack
+	stvx_4w	$xb0,$x10,r11
+	stvx_4w	$xc0,$x20,r11
+	stvx_4w	$xd0,$x30,r11
+	subi	r12,r11,1			# prepare for *++ptr
+	subi	$inp,$inp,1
+	subi	$out,$out,1
+
+Loop_tail_vsx:
+	lbzu	r6,1(r12)
+	lbzu	r7,1($inp)
+	xor	r6,r6,r7
+	stbu	r6,1($out)
+	bdnz	Loop_tail_vsx
+
+	stvx_4w	$K[0],$x00,r11			# wipe copy of the block
+	stvx_4w	$K[0],$x10,r11
+	stvx_4w	$K[0],$x20,r11
+	stvx_4w	$K[0],$x30,r11
+
+	b	Ldone_vsx
+	.long	0
+	.byte	0,12,0x04,1,0x80,0,5,0
+	.long	0
+.size	.ChaCha20_ctr32_vsx_p10,.-.ChaCha20_ctr32_vsx_p10
+___
+}}}
+
+##This is 8 block in parallel implementation. The heart of chacha round uses vector instruction that has access to 
+# vsr[32+X]. To perform the 8 parallel block we tend to use all 32 register to hold the 8 block info.
+# WE need to store few register value on side, so we can use VSR{32+X} for few vector instructions used in round op and hold intermediate value.
+# WE use the VSR[0]-VSR[31] for holding intermediate value and perform 8 block in parallel.
+#
+{{{
+#### ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
+my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+    $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3,
+    $xa4,$xa5,$xa6,$xa7, $xb4,$xb5,$xb6,$xb7,
+    $xc4,$xc5,$xc6,$xc7, $xd4,$xd5,$xd6,$xd7) = map("v$_",(0..31));
+my ($xcn4,$xcn5,$xcn6,$xcn7, $xdn4,$xdn5,$xdn6,$xdn7) = map("v$_",(8..15));
+my ($xan0,$xbn0,$xcn0,$xdn0) = map("v$_",(0..3));
+my @K = map("v$_",27,(24..26));
+my ($xt0,$xt1,$xt2,$xt3,$xt4) = map("v$_",23,(28..31));
+my $xr0 = "v4";
+my $CTR0 = "v22";
+my $CTR1 = "v5";
+my $beperm = "v31";
+my ($x00,$x10,$x20,$x30) = (0, map("r$_",(8..10)));
+my ($xv0,$xv1,$xv2,$xv3,$xv4,$xv5,$xv6,$xv7) = map("v$_",(0..7));
+my ($xv8,$xv9,$xv10,$xv11,$xv12,$xv13,$xv14,$xv15,$xv16,$xv17) = map("v$_",(8..17));
+my ($xv18,$xv19,$xv20,$xv21) = map("v$_",(18..21));
+my ($xv22,$xv23,$xv24,$xv25,$xv26) = map("v$_",(22..26));
+
+my $FRAME=$LOCALS+64+9*16;	# 8*16 is for v24-v31 offload
+
+sub VSX_lane_ROUND_8x {
+my ($a0,$b0,$c0,$d0,$a4,$b4,$c4,$d4)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my ($a5,$b5,$c5,$d5)=map(($_&~3)+(($_+1)&3),($a4,$b4,$c4,$d4));
+my ($a6,$b6,$c6,$d6)=map(($_&~3)+(($_+1)&3),($a5,$b5,$c5,$d5));
+my ($a7,$b7,$c7,$d7)=map(($_&~3)+(($_+1)&3),($a6,$b6,$c6,$d6));
+my ($xv8,$xv9,$xv10,$xv11,$xv12,$xv13,$xv14,$xv15,$xv16,$xv17) = map("\"v$_\"",(8..17));
+my @x=map("\"v$_\"",(0..31));
+
+	(
+	"&vxxlor        ($xv15 ,@x[$c7],@x[$c7])",      #copy v30 to v13
+	"&vxxlorc       (@x[$c7], $xv9,$xv9)",
+
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	"&vadduwm	(@x[$a4],@x[$a4],@x[$b4])",	# Q1
+	 "&vadduwm	(@x[$a5],@x[$a5],@x[$b5])",	# Q2
+	  "&vadduwm	(@x[$a6],@x[$a6],@x[$b6])",	# Q3
+	   "&vadduwm	(@x[$a7],@x[$a7],@x[$b7])",	# Q4
+
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vxor		(@x[$d4],@x[$d4],@x[$a4])",
+	 "&vxor		(@x[$d5],@x[$d5],@x[$a5])",
+	  "&vxor	(@x[$d6],@x[$d6],@x[$a6])",
+	   "&vxor	(@x[$d7],@x[$d7],@x[$a7])",
+
+	"&vrlw		(@x[$d0],@x[$d0],@x[$c7])",
+	 "&vrlw		(@x[$d1],@x[$d1],@x[$c7])",
+	  "&vrlw	(@x[$d2],@x[$d2],@x[$c7])",
+	   "&vrlw	(@x[$d3],@x[$d3],@x[$c7])",
+	"&vrlw		(@x[$d4],@x[$d4],@x[$c7])",
+	 "&vrlw		(@x[$d5],@x[$d5],@x[$c7])",
+	  "&vrlw	(@x[$d6],@x[$d6],@x[$c7])",
+	   "&vrlw	(@x[$d7],@x[$d7],@x[$c7])",
+
+	"&vxxlor        ($xv13 ,@x[$a7],@x[$a7])",
+	"&vxxlorc       (@x[$c7], $xv15,$xv15)",
+	"&vxxlorc       (@x[$a7], $xv10,$xv10)",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vadduwm	(@x[$c4],@x[$c4],@x[$d4])",
+	 "&vadduwm	(@x[$c5],@x[$c5],@x[$d5])",
+	  "&vadduwm	(@x[$c6],@x[$c6],@x[$d6])",
+	   "&vadduwm	(@x[$c7],@x[$c7],@x[$d7])",
+
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vxor		(@x[$b4],@x[$b4],@x[$c4])",
+	 "&vxor		(@x[$b5],@x[$b5],@x[$c5])",
+	  "&vxor	(@x[$b6],@x[$b6],@x[$c6])",
+	   "&vxor	(@x[$b7],@x[$b7],@x[$c7])",
+
+	"&vrlw		(@x[$b0],@x[$b0],@x[$a7])",
+	 "&vrlw		(@x[$b1],@x[$b1],@x[$a7])",
+	  "&vrlw	(@x[$b2],@x[$b2],@x[$a7])",
+	   "&vrlw	(@x[$b3],@x[$b3],@x[$a7])",
+	"&vrlw		(@x[$b4],@x[$b4],@x[$a7])",
+	 "&vrlw		(@x[$b5],@x[$b5],@x[$a7])",
+	  "&vrlw	(@x[$b6],@x[$b6],@x[$a7])",
+	   "&vrlw	(@x[$b7],@x[$b7],@x[$a7])",
+
+	"&vxxlorc       (@x[$a7], $xv13,$xv13)",
+	"&vxxlor	($xv15 ,@x[$c7],@x[$c7])",                 
+	"&vxxlorc       (@x[$c7], $xv11,$xv11)",
+
+
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",
+	"&vadduwm	(@x[$a4],@x[$a4],@x[$b4])",
+	 "&vadduwm	(@x[$a5],@x[$a5],@x[$b5])",
+	  "&vadduwm	(@x[$a6],@x[$a6],@x[$b6])",
+	   "&vadduwm	(@x[$a7],@x[$a7],@x[$b7])",
+
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vxor		(@x[$d4],@x[$d4],@x[$a4])",
+	 "&vxor		(@x[$d5],@x[$d5],@x[$a5])",
+	  "&vxor	(@x[$d6],@x[$d6],@x[$a6])",
+	   "&vxor	(@x[$d7],@x[$d7],@x[$a7])",
+
+	"&vrlw		(@x[$d0],@x[$d0],@x[$c7])",
+	 "&vrlw		(@x[$d1],@x[$d1],@x[$c7])",
+	  "&vrlw	(@x[$d2],@x[$d2],@x[$c7])",
+	   "&vrlw	(@x[$d3],@x[$d3],@x[$c7])",
+	"&vrlw		(@x[$d4],@x[$d4],@x[$c7])",
+	 "&vrlw		(@x[$d5],@x[$d5],@x[$c7])",
+	  "&vrlw	(@x[$d6],@x[$d6],@x[$c7])",
+	   "&vrlw	(@x[$d7],@x[$d7],@x[$c7])",
+
+	"&vxxlorc       (@x[$c7], $xv15,$xv15)",
+	"&vxxlor        ($xv13 ,@x[$a7],@x[$a7])",               
+	"&vxxlorc       (@x[$a7], $xv12,$xv12)",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vadduwm	(@x[$c4],@x[$c4],@x[$d4])",
+	 "&vadduwm	(@x[$c5],@x[$c5],@x[$d5])",
+	  "&vadduwm	(@x[$c6],@x[$c6],@x[$d6])",
+	   "&vadduwm	(@x[$c7],@x[$c7],@x[$d7])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vxor		(@x[$b4],@x[$b4],@x[$c4])",
+	 "&vxor		(@x[$b5],@x[$b5],@x[$c5])",
+	  "&vxor	(@x[$b6],@x[$b6],@x[$c6])",
+	   "&vxor	(@x[$b7],@x[$b7],@x[$c7])",
+	"&vrlw		(@x[$b0],@x[$b0],@x[$a7])",
+	 "&vrlw		(@x[$b1],@x[$b1],@x[$a7])",
+	  "&vrlw	(@x[$b2],@x[$b2],@x[$a7])",
+	   "&vrlw	(@x[$b3],@x[$b3],@x[$a7])",
+	"&vrlw		(@x[$b4],@x[$b4],@x[$a7])",
+	 "&vrlw		(@x[$b5],@x[$b5],@x[$a7])",
+	  "&vrlw	(@x[$b6],@x[$b6],@x[$a7])",
+	   "&vrlw	(@x[$b7],@x[$b7],@x[$a7])",
+
+	"&vxxlorc       (@x[$a7], $xv13,$xv13)",
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vsx_8x
+.align	5
+.ChaCha20_ctr32_vsx_8x:
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v24,r10,$sp
+	addi	r10,r10,32
+	stvx	v25,r11,$sp
+	addi	r11,r11,32
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$FRAME-4`($sp)		# save vrsave
+	li	r12,-4096+63
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+
+	lvx_4w	@K[0],0,r12			# load sigma
+	addi	r12,r12,0x70
+	li	$x10,16
+	li	$x20,32
+	li	$x30,48
+	li	r11,64
+
+	vspltisw $xa4,-16			# synthesize constants
+	vspltisw $xb4,12			# synthesize constants
+	vspltisw $xc4,8			# synthesize constants
+	vspltisw $xd4,7			# synthesize constants
+
+	lvx	$xa0,$x00,r12			# load [smashed] sigma
+	lvx	$xa1,$x10,r12
+	lvx	$xa2,$x20,r12
+	lvx	$xa3,$x30,r12
+
+	vxxlor	$xv9   ,$xa4,$xa4               #save shift val in vr9-12
+	vxxlor	$xv10  ,$xb4,$xb4
+	vxxlor	$xv11  ,$xc4,$xc4
+	vxxlor	$xv12  ,$xd4,$xd4
+	vxxlor	$xv22  ,$xa0,$xa0               #save sigma in vr22-25
+	vxxlor	$xv23  ,$xa1,$xa1
+	vxxlor	$xv24  ,$xa2,$xa2
+	vxxlor	$xv25  ,$xa3,$xa3
+
+	lvx_4w	@K[1],0,$key			# load key
+	lvx_4w	@K[2],$x10,$key
+	lvx_4w	@K[3],0,$ctr			# load counter
+	vspltisw $xt3,4
+
+
+	vxor	$xt2,$xt2,$xt2
+	lvx_4w	$xt1,r11,r12
+	vspltw	$xa2,@K[3],0			#save the original count after spltw
+	vsldoi	@K[3],@K[3],$xt2,4
+	vsldoi	@K[3],$xt2,@K[3],12		# clear @K[3].word[0]
+	vadduwm	$xt1,$xa2,$xt1
+	vadduwm $xt3,$xt1,$xt3     		# next counter value
+	vspltw	$xa0,@K[2],2                    # save the K[2] spltw 2 and save v8.
+
+	be?lvsl	  $beperm,0,$x10			# 0x00..0f
+	be?vspltisb $xt0,3			# 0x03..03
+	be?vxor   $beperm,$beperm,$xt0		# swap bytes within words
+	be?vxxlor $xv26 ,$beperm,$beperm
+
+	vxxlor	$xv0 ,@K[0],@K[0]               # K0,k1,k2 to vr0,1,2
+	vxxlor	$xv1 ,@K[1],@K[1]
+	vxxlor	$xv2 ,@K[2],@K[2]
+	vxxlor	$xv3 ,@K[3],@K[3]
+	vxxlor	$xv4 ,$xt1,$xt1                #CTR ->4, CTR+4-> 5
+	vxxlor	$xv5 ,$xt3,$xt3
+	vxxlor	$xv8 ,$xa0,$xa0
+
+	li	r0,10				# inner loop counter
+	mtctr	r0
+	b	Loop_outer_vsx_8x
+
+.align	5
+Loop_outer_vsx_8x:
+	vxxlorc	$xa0,$xv22,$xv22	        # load [smashed] sigma
+	vxxlorc	$xa1,$xv23,$xv23
+	vxxlorc	$xa2,$xv24,$xv24
+	vxxlorc	$xa3,$xv25,$xv25
+	vxxlorc	$xa4,$xv22,$xv22
+	vxxlorc	$xa5,$xv23,$xv23
+	vxxlorc	$xa6,$xv24,$xv24
+	vxxlorc	$xa7,$xv25,$xv25
+
+	vspltw	$xb0,@K[1],0			# smash the key
+	vspltw	$xb1,@K[1],1
+	vspltw	$xb2,@K[1],2
+	vspltw	$xb3,@K[1],3
+	vspltw	$xb4,@K[1],0			# smash the key
+	vspltw	$xb5,@K[1],1
+	vspltw	$xb6,@K[1],2
+	vspltw	$xb7,@K[1],3
+
+	vspltw	$xc0,@K[2],0
+	vspltw	$xc1,@K[2],1
+	vspltw	$xc2,@K[2],2
+	vspltw	$xc3,@K[2],3
+	vspltw	$xc4,@K[2],0
+	vspltw	$xc7,@K[2],3
+	vspltw	$xc5,@K[2],1
+
+	vxxlorc	$xd0,$xv4,$xv4			# smash the counter
+	vspltw	$xd1,@K[3],1
+	vspltw	$xd2,@K[3],2
+	vspltw	$xd3,@K[3],3
+	vxxlorc	$xd4,$xv5,$xv5			# smash the counter
+	vspltw	$xd5,@K[3],1
+	vspltw	$xd6,@K[3],2
+	vspltw	$xd7,@K[3],3
+	vxxlorc	$xc6,$xv8,$xv8                  #copy of vlspt k[2],2 is in v8.v26 ->k[3] so need to wait until k3 is done
+
+Loop_vsx_8x:
+___
+	foreach (&VSX_lane_ROUND_8x(0,4, 8,12,16,20,24,28)) { eval; }
+	foreach (&VSX_lane_ROUND_8x(0,5,10,15,16,21,26,31)) { eval; }
+$code.=<<___;
+
+	bdnz	        Loop_vsx_8x
+	vxxlor	        $xv13 ,$xd4,$xd4                # save the register vr24-31
+	vxxlor	        $xv14 ,$xd5,$xd5                #
+	vxxlor	        $xv15 ,$xd6,$xd6                #
+	vxxlor	        $xv16 ,$xd7,$xd7                #
+
+	vxxlor	        $xv18 ,$xc4,$xc4                #
+	vxxlor	        $xv19 ,$xc5,$xc5                #
+	vxxlor	        $xv20 ,$xc6,$xc6                #
+	vxxlor	        $xv21 ,$xc7,$xc7                #
+
+	vxxlor	        $xv6  ,$xb6,$xb6                # save vr23, so we get 8 regs
+	vxxlor	        $xv7  ,$xb7,$xb7                # save vr23, so we get 8 regs
+	be?vxxlorc      $beperm,$xv26,$xv26             # copy back the the beperm.
+
+	vxxlorc	   @K[0],$xv0,$xv0                #27
+	vxxlorc	   @K[1],$xv1,$xv1 		  #24
+	vxxlorc	   @K[2],$xv2,$xv2		  #25
+	vxxlorc	   @K[3],$xv3,$xv3		  #26
+	vxxlorc	   $CTR0,$xv4,$xv4
+###changing to vertical
+
+	vmrgew	$xt0,$xa0,$xa1			# transpose data
+	vmrgew	$xt1,$xa2,$xa3
+	vmrgow	$xa0,$xa0,$xa1
+	vmrgow	$xa2,$xa2,$xa3
+
+	vmrgew	$xt2,$xb0,$xb1
+	vmrgew	$xt3,$xb2,$xb3
+	vmrgow	$xb0,$xb0,$xb1
+	vmrgow	$xb2,$xb2,$xb3
+
+	vadduwm	$xd0,$xd0,$CTR0
+
+	vpermdi	$xa1,$xa0,$xa2,0b00
+	vpermdi	$xa3,$xa0,$xa2,0b11
+	vpermdi	$xa0,$xt0,$xt1,0b00
+	vpermdi	$xa2,$xt0,$xt1,0b11
+	vpermdi	$xb1,$xb0,$xb2,0b00
+	vpermdi	$xb3,$xb0,$xb2,0b11
+	vpermdi	$xb0,$xt2,$xt3,0b00
+	vpermdi	$xb2,$xt2,$xt3,0b11
+
+	vmrgew	$xt0,$xc0,$xc1
+	vmrgew	$xt1,$xc2,$xc3
+	vmrgow	$xc0,$xc0,$xc1
+	vmrgow	$xc2,$xc2,$xc3
+	vmrgew	$xt2,$xd0,$xd1
+	vmrgew	$xt3,$xd2,$xd3
+	vmrgow	$xd0,$xd0,$xd1
+	vmrgow	$xd2,$xd2,$xd3
+
+	vpermdi	$xc1,$xc0,$xc2,0b00
+	vpermdi	$xc3,$xc0,$xc2,0b11
+	vpermdi	$xc0,$xt0,$xt1,0b00
+	vpermdi	$xc2,$xt0,$xt1,0b11
+	vpermdi	$xd1,$xd0,$xd2,0b00
+	vpermdi	$xd3,$xd0,$xd2,0b11
+	vpermdi	$xd0,$xt2,$xt3,0b00
+	vpermdi	$xd2,$xt2,$xt3,0b11
+
+	vspltisw $xt0,8
+	vadduwm  $CTR0,$CTR0,$xt0		# next counter value
+	vxxlor	 $xv4 ,$CTR0,$CTR0	        #CTR+4-> 5
+
+	vadduwm	$xa0,$xa0,@K[0]
+	vadduwm	$xb0,$xb0,@K[1]
+	vadduwm	$xc0,$xc0,@K[2]
+	vadduwm	$xd0,$xd0,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx_8x
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx_8x
+
+	vadduwm	$xa0,$xa1,@K[0]
+	vadduwm	$xb0,$xb1,@K[1]
+	vadduwm	$xc0,$xc1,@K[2]
+	vadduwm	$xd0,$xd1,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx_8x
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx_8x
+
+	vadduwm	$xa0,$xa2,@K[0]
+	vadduwm	$xb0,$xb2,@K[1]
+	vadduwm	$xc0,$xc2,@K[2]
+	vadduwm	$xd0,$xd2,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx_8x
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx_8x
+
+	vadduwm	$xa0,$xa3,@K[0]
+	vadduwm	$xb0,$xb3,@K[1]
+	vadduwm	$xc0,$xc3,@K[2]
+	vadduwm	$xd0,$xd3,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx_8x
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx_8x
+
+#blk4-7: 24:31 remain the same as we can use the same logic above . Reg a4-b7 remain same.Load c4,d7--> position 8-15.we can reuse vr24-31.
+#VR0-3 : are used to load temp value, vr4 --> as xr0 instead of xt0.
+
+	vxxlorc	   $CTR1 ,$xv5,$xv5
+
+	vxxlorc	   $xcn4 ,$xv18,$xv18
+	vxxlorc	   $xcn5 ,$xv19,$xv19
+	vxxlorc	   $xcn6 ,$xv20,$xv20
+	vxxlorc	   $xcn7 ,$xv21,$xv21
+
+	vxxlorc	   $xdn4 ,$xv13,$xv13
+	vxxlorc	   $xdn5 ,$xv14,$xv14
+	vxxlorc	   $xdn6 ,$xv15,$xv15
+	vxxlorc	   $xdn7 ,$xv16,$xv16
+	vadduwm	   $xdn4,$xdn4,$CTR1
+
+	vxxlorc	   $xb6 ,$xv6,$xv6
+	vxxlorc	   $xb7 ,$xv7,$xv7
+#use xa1->xr0, as xt0...in the block 4-7
+
+	vmrgew	$xr0,$xa4,$xa5			# transpose data
+	vmrgew	$xt1,$xa6,$xa7
+	vmrgow	$xa4,$xa4,$xa5
+	vmrgow	$xa6,$xa6,$xa7
+	vmrgew	$xt2,$xb4,$xb5
+	vmrgew	$xt3,$xb6,$xb7
+	vmrgow	$xb4,$xb4,$xb5
+	vmrgow	$xb6,$xb6,$xb7
+
+	vpermdi	$xa5,$xa4,$xa6,0b00
+	vpermdi	$xa7,$xa4,$xa6,0b11
+	vpermdi	$xa4,$xr0,$xt1,0b00
+	vpermdi	$xa6,$xr0,$xt1,0b11
+	vpermdi	$xb5,$xb4,$xb6,0b00
+	vpermdi	$xb7,$xb4,$xb6,0b11
+	vpermdi	$xb4,$xt2,$xt3,0b00
+	vpermdi	$xb6,$xt2,$xt3,0b11
+
+	vmrgew	$xr0,$xcn4,$xcn5
+	vmrgew	$xt1,$xcn6,$xcn7
+	vmrgow	$xcn4,$xcn4,$xcn5
+	vmrgow	$xcn6,$xcn6,$xcn7
+	vmrgew	$xt2,$xdn4,$xdn5
+	vmrgew	$xt3,$xdn6,$xdn7
+	vmrgow	$xdn4,$xdn4,$xdn5
+	vmrgow	$xdn6,$xdn6,$xdn7
+
+	vpermdi	$xcn5,$xcn4,$xcn6,0b00
+	vpermdi	$xcn7,$xcn4,$xcn6,0b11
+	vpermdi	$xcn4,$xr0,$xt1,0b00
+	vpermdi	$xcn6,$xr0,$xt1,0b11
+	vpermdi	$xdn5,$xdn4,$xdn6,0b00
+	vpermdi	$xdn7,$xdn4,$xdn6,0b11
+	vpermdi	$xdn4,$xt2,$xt3,0b00
+	vpermdi	$xdn6,$xt2,$xt3,0b11
+
+	vspltisw $xr0,8
+	vadduwm  $CTR1,$CTR1,$xr0		# next counter value
+	vxxlor	 $xv5 ,$CTR1,$CTR1	        #CTR+4-> 5
+
+	vadduwm	$xan0,$xa4,@K[0]
+	vadduwm	$xbn0,$xb4,@K[1]
+	vadduwm	$xcn0,$xcn4,@K[2]
+	vadduwm	$xdn0,$xdn4,@K[3]
+
+	be?vperm $xan0,$xa4,$xa4,$beperm
+	be?vperm $xbn0,$xb4,$xb4,$beperm
+	be?vperm $xcn0,$xcn4,$xcn4,$beperm
+	be?vperm $xdn0,$xdn4,$xdn4,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx_8x_1
+
+	lvx_4w	$xr0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xr0,$xr0,$xan0
+	vxor	$xt1,$xt1,$xbn0
+	vxor	$xt2,$xt2,$xcn0
+	vxor	$xt3,$xt3,$xdn0
+
+	stvx_4w	$xr0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x3