aesp8-ppc.pl: optimize CBC decrypt even further.

10-19% improvement depending on key length and endianness.

1 files changed, 231 insertions, 334 deletions

diff --git a/crypto/aes/asm/aesp8-ppc.pl b/crypto/aes/asm/aesp8-ppc.pl
index b19fd6a1be..b1da5965fb 100755
--- a/crypto/aes/asm/aesp8-ppc.pl
+++ b/crypto/aes/asm/aesp8-ppc.pl
@@ -10,11 +10,16 @@
 # This module implements support for AES instructions as per PowerISA
 # specification version 2.07, first implemented by POWER8 processor.
 # The module is endian-agnostic in sense that it supports both big-
-# and little-endian cases. As well as alignment-agnostic, and it is
-# guaranteed not to cause alignment exceptions. [One of options was
-# to use VSX loads and stores, which tolerate unaligned references,
-# but even then specification doesn't prohibit exceptions on page
-# boundaries.]
+# and little-endian cases. Data alignment in parallelizable modes is
+# handled with VSX loads and stores, which implies MSR.VSX flag being
+# set. It should also be noted that ISA specification doesn't prohibit
+# alignment exceptions for these instructions on page boundaries.
+# Initially alignment was handled in pure AltiVec/VMX way [when data
+# is aligned programmatically, which in turn guarantees exception-
+# free execution], but it turned to hamper performance when vcipher
+# instructions are interleaved. It's reckoned that eventual
+# misalignment penalties at page boundaries are in average lower
+# than additional overhead in pure AltiVec approach.
 
 $flavour = shift;
 
@@ -49,7 +54,8 @@ $prefix="AES";
 $sp="r1";
 $vrsave="r12";
 
-{{{
+#########################################################################
+{{{	Key setup procedures						#
 my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
 my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
 my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
@@ -96,13 +102,9 @@ Lset_encrypt_key:
 	li		r8,0x20
 	cmpwi		$bits,192
 	lvx		$in1,0,$inp
-___
-$code.=<<___		if ($LITTLE_ENDIAN);
-	vspltisb	$mask,0x0f		# borrow $mask
-	vxor		$key,$key,$mask		# adjust for byte swap
-___
-$code.=<<___;
+	le?vspltisb	$mask,0x0f		# borrow $mask
 	lvx		$rcon,0,$ptr
+	le?vxor		$key,$key,$mask		# adjust for byte swap
 	lvx		$mask,r8,$ptr
 	addi		$ptr,$ptr,0x10
 	vperm		$in0,$in0,$in1,$key	# align [and byte swap in LE]
@@ -324,6 +326,7 @@ Ldone:
 	blr
 	.long		0
 	.byte		0,12,0x14,1,0,0,3,0
+	.long		0
 .size	.${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
 
 .globl	.${prefix}_set_decrypt_key
@@ -367,10 +370,12 @@ Ldeckey:
 	blr
 	.long		0
 	.byte		0,12,4,1,0x80,0,3,0
+	.long		0
 .size	.${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
 ___
 }}}
-{{{
+#########################################################################
+{{{	Single block en- and decrypt procedures				#
 sub gen_block () {
 my $dir = shift;
 my $n   = $dir eq "de" ? "n" : "";
@@ -390,9 +395,9 @@ $code.=<<___;
 	neg		r11,$out
 	lvx		v1,$idx,$inp
 	lvsl		v2,0,$inp		# inpperm
-	`"vspltisb	v4,0x0f"		if ($LITTLE_ENDIAN)`
+	le?vspltisb	v4,0x0f
 	?lvsl		v3,0,r11		# outperm
-	`"vxor		v2,v2,v4"		if ($LITTLE_ENDIAN)`
+	le?vxor		v2,v2,v4
 	li		$idx,16
 	vperm		v0,v0,v1,v2		# align [and byte swap in LE]
 	lvx		v1,0,$key
@@ -429,7 +434,7 @@ Loop_${dir}c:
 	vxor		v1,v1,v1
 	li		$idx,15			# 15 is not typo
 	?vperm		v2,v1,v2,v3		# outmask
-	`"vxor		v3,v3,v4"		if ($LITTLE_ENDIAN)`
+	le?vxor		v3,v3,v4
 	lvx		v1,0,$out		# outhead
 	vperm		v0,v0,v0,v3		# rotate [and byte swap in LE]
 	vsel		v1,v1,v0,v2
@@ -442,17 +447,19 @@ Loop_${dir}c:
 	blr
 	.long		0
 	.byte		0,12,0x14,0,0,0,3,0
+	.long		0
 .size	.${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
 ___
 }
 &gen_block("en");
 &gen_block("de");
 }}}
-{{{
+#########################################################################
+{{{	CBC en- and decrypt procedures					#
 my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
-my ($rndkey0,$rndkey1,$inout,$tmp)=map("v$_",(0..3));
-my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=map("v$_",(4..10));
-
+my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
+my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
+						map("v$_",(4..10));
 $code.=<<___;
 .globl	.${prefix}_cbc_encrypt
 .align	5
@@ -467,12 +474,12 @@ $code.=<<___;
 
 	li		$idx,15
 	vxor		$rndkey0,$rndkey0,$rndkey0
-	`"vspltisb	$tmp,0x0f"		if ($LITTLE_ENDIAN)`
+	le?vspltisb	$tmp,0x0f
 
 	lvx		$ivec,0,$ivp		# load [unaligned] iv
 	lvsl		$inpperm,0,$ivp
 	lvx		$inptail,$idx,$ivp
-	`"vxor		$inpperm,$inpperm,$tmp"	if ($LITTLE_ENDIAN)`
+	le?vxor		$inpperm,$inpperm,$tmp
 	vperm		$ivec,$ivec,$inptail,$inpperm
 
 	neg		r11,$inp
@@ -482,13 +489,13 @@ $code.=<<___;
 	lvsr		$inpperm,0,r11		# prepare for unaligned load
 	lvx		$inptail,0,$inp
 	addi		$inp,$inp,15		# 15 is not typo
-	`"vxor		$inpperm,$inpperm,$tmp"	if ($LITTLE_ENDIAN)`
+	le?vxor		$inpperm,$inpperm,$tmp
 
 	?lvsr		$outperm,0,$out		# prepare for unaligned store
 	vspltisb	$outmask,-1
 	lvx		$outhead,0,$out
 	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	`"vxor		$outperm,$outperm,$tmp"	if ($LITTLE_ENDIAN)`
+	le?vxor		$outperm,$outperm,$tmp
 
 	srwi		$rounds,$rounds,1
 	li		$idx,16
@@ -597,10 +604,10 @@ Lcbc_done:
 	li		$idx,15			# 15 is not typo
 	vxor		$rndkey0,$rndkey0,$rndkey0
 	vspltisb	$outmask,-1
-	`"vspltisb	$tmp,0x0f"		if ($LITTLE_ENDIAN)`
+	le?vspltisb	$tmp,0x0f
 	?lvsl		$outperm,0,$enc
 	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	`"vxor		$outperm,$outperm,$tmp"	if ($LITTLE_ENDIAN)`
+	le?vxor		$outperm,$outperm,$tmp
 	lvx		$outhead,0,$ivp
 	vperm		$ivec,$ivec,$ivec,$outperm
 	vsel		$inout,$outhead,$ivec,$outmask
@@ -613,9 +620,12 @@ Lcbc_done:
 	blr
 	.long		0
 	.byte		0,12,0x14,0,0,0,6,0
+	.long		0
 ___
-{{
+#########################################################################
+{{	Optimized CBC decrypt procedure					#
 my $key_="r11";
+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
 my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
@@ -625,7 +635,7 @@ my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
 $code.=<<___;
 .align	5
 _aesp8_cbc_decrypt8x:
-	$STU		$sp,-`($FRAME+21*16)`($sp)
+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
 	li		r10,`$FRAME+8*16+15`
 	li		r11,`$FRAME+8*16+31`
 	stvx		v20,r10,$sp		# ABI says so
@@ -650,104 +660,103 @@ _aesp8_cbc_decrypt8x:
 	addi		r11,r11,32
 	stvx		v30,r10,$sp
 	stvx		v31,r11,$sp
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
 	li		r0,-1
+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
+	li		$x10,0x10
+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	li		$x20,0x20
+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	li		$x30,0x30
+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	li		$x40,0x40
+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	li		$x50,0x50
+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	li		$x60,0x60
+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	li		$x70,0x70
 	mtspr		256,r0
 
 	subi		$rounds,$rounds,3	# -4 in total
 	subi		$len,$len,128		# bias
 
-	addi		$idx,$key,16		# load key schedule
-	lvx		$rndkey0,0,$key
-	addi		$key,$key,32
-	lvx		v30,0,$idx
-	addi		$idx,$idx,32
-	lvx		v31,0,$key
-	addi		$key,$key,32
+	lvx		$rndkey0,$x00,$key	# load key schedule
+	lvx		v30,$x10,$key
+	addi		$key,$key,0x20
+	lvx		v31,$x00,$key
 	?vperm		$rndkey0,$rndkey0,v30,$keyperm
 	addi		$key_,$sp,$FRAME+15
 	mtctr		$rounds
 
 Load_cbc_dec_key:
 	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,0,$idx
-	addi		$idx,$idx,32
-	stvx		v24,0,$key_		# off-load round[1]
-	addi		$key_,$key_,16
+	lvx		v30,$x10,$key
+	addi		$key,$key,0x20
+	stvx		v24,$x00,$key_		# off-load round[1]
 	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,0,$key
-	addi		$key,$key,32
-	stvx		v25,0,$key_		# off-load round[2]
-	addi		$key_,$key_,16
+	lvx		v31,$x00,$key
+	stvx		v25,$x10,$key_		# off-load round[2]
+	addi		$key_,$key_,0x20
 	bdnz		Load_cbc_dec_key
 
-	lvx		v26,0,$idx
-	addi		$idx,$idx,32
+	lvx		v26,$x10,$key
 	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,0,$key
-	addi		$key,$key,32
-	stvx		v24,0,$key_		# off-load round[3]
-	addi		$key_,$key_,16
+	lvx		v27,$x20,$key
+	stvx		v24,$x00,$key_		# off-load round[3]
 	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,0,$idx
-	addi		$idx,$idx,32
-	stvx		v25,0,$key_		# off-load round[4]
+	lvx		v28,$x30,$key
+	stvx		v25,$x10,$key_		# off-load round[4]
 	addi		$key_,$sp,$FRAME+15	# rewind $key_
 	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,0,$key
-	addi		$key,$key,32
+	lvx		v29,$x40,$key
 	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,0,$idx
-	addi		$idx,$idx,32
+	lvx		v30,$x50,$key
 	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,0,$key
+	lvx		v31,$x60,$key
 	?vperm		v29,v29,v30,$keyperm
-	lvx		$out0,0,$idx		# borrow $out0
+	lvx		$out0,$x70,$key		# borrow $out0
 	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,0,$key_		# pre-load round[1]
-	addi		$key_,$key_,16
+	lvx		v24,$x00,$key_		# pre-load round[1]
 	?vperm		v31,v31,$out0,$keyperm
-	lvx		v25,0,$key_		# pre-load round[2]
-	addi		$key_,$key_,16
-
+	lvx		v25,$x10,$key_		# pre-load round[2]
 
 	#lvx		$inptail,0,$inp		# "caller" already did this
 	#addi		$inp,$inp,15		# 15 is not typo
-
-	lvx		$in1,0,$inp		# load first 8 "words"
-	addi		$inp,$inp,16
-	lvx		$in2,0,$inp
-	addi		$inp,$inp,16
-	lvx		$in3,0,$inp
-	addi		$inp,$inp,16
-	vperm		$in0,$inptail,$in1,$inpperm
-	lvx		$in4,0,$inp
-	addi		$inp,$inp,16
-	vperm		$in1,$in1,$in2,$inpperm
-	lvx		$in5,0,$inp
-	addi		$inp,$inp,16
-	vperm		$in2,$in2,$in3,$inpperm
+	subi		$inp,$inp,15		# undo "caller"
+
+	 le?li		$idx,8
+	lvx_u		$in0,$x00,$inp		# load first 8 "words"
+	 le?lvsl	$inpperm,0,$idx
+	 le?vspltisb	$tmp,0x0f
+	lvx_u		$in1,$x10,$inp
+	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
+	lvx_u		$in2,$x20,$inp
+	 le?vperm	$in0,$in0,$in0,$inpperm
+	lvx_u		$in3,$x30,$inp
+	 le?vperm	$in1,$in1,$in1,$inpperm
+	lvx_u		$in4,$x40,$inp
+	 le?vperm	$in2,$in2,$in2,$inpperm
 	vxor		$out0,$in0,$rndkey0
-	lvx		$in6,0,$inp
-	addi		$inp,$inp,16
-	vperm		$in3,$in3,$in4,$inpperm
+	lvx_u		$in5,$x50,$inp
+	 le?vperm	$in3,$in3,$in3,$inpperm
 	vxor		$out1,$in1,$rndkey0
-	lvx		$in7,0,$inp
-	addi		$inp,$inp,16
-	vperm		$in4,$in4,$in5,$inpperm
+	lvx_u		$in6,$x60,$inp
+	 le?vperm	$in4,$in4,$in4,$inpperm
 	vxor		$out2,$in2,$rndkey0
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	vperm		$in5,$in5,$in6,$inpperm
+	lvx_u		$in7,$x70,$inp
+	addi		$inp,$inp,0x80
+	 le?vperm	$in5,$in5,$in5,$inpperm
 	vxor		$out3,$in3,$rndkey0
-	vperm		$in6,$in6,$in7,$inpperm
+	 le?vperm	$in6,$in6,$in6,$inpperm
 	vxor		$out4,$in4,$rndkey0
-	vperm		$in7,$in7,$inptail,$inpperm
+	 le?vperm	$in7,$in7,$in7,$inpperm
 	vxor		$out5,$in5,$rndkey0
 	vxor		$out6,$in6,$rndkey0
 	vxor		$out7,$in7,$rndkey0
 
 	mtctr		$rounds
+	b		Loop_cbc_dec8x
+.align	5
 Loop_cbc_dec8x:
 	vncipher	$out0,$out0,v24
 	vncipher	$out1,$out1,v24
@@ -757,8 +766,8 @@ Loop_cbc_dec8x:
 	vncipher	$out5,$out5,v24
 	vncipher	$out6,$out6,v24
 	vncipher	$out7,$out7,v24
-	lvx		v24,0,$key_		# round[3]
-	addi		$key_,$key_,16
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
 
 	vncipher	$out0,$out0,v25
 	vncipher	$out1,$out1,v25
@@ -768,8 +777,7 @@ Loop_cbc_dec8x:
 	vncipher	$out5,$out5,v25
 	vncipher	$out6,$out6,v25
 	vncipher	$out7,$out7,v25
-	lvx		v25,0,$key_		# round[4]
-	addi		$key_,$key_,16
+	lvx		v25,$x10,$key_		# round[4]
 	bdnz		Loop_cbc_dec8x
 
 	subic		$len,$len,128		# $len-=128
@@ -824,8 +832,7 @@ Loop_cbc_dec8x:
 	vncipher	$out5,$out5,v28
 	vncipher	$out6,$out6,v28
 	vncipher	$out7,$out7,v28
-	lvx		v24,0,$key_		# re-pre-load round[1]
-	addi		$key_,$key_,16
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
 
 	vncipher	$out0,$out0,v29
 	vncipher	$out1,$out1,v29
@@ -835,8 +842,7 @@ Loop_cbc_dec8x:
 	vncipher	$out5,$out5,v29
 	vncipher	$out6,$out6,v29
 	vncipher	$out7,$out7,v29
-	lvx		v25,0,$key_		# re-pre-load round[2]
-	addi		$key_,$key_,16
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
 
 	vncipher	$out0,$out0,v30
 	 vxor		$ivec,$ivec,v31		# xor with last round key
@@ -857,86 +863,55 @@ Loop_cbc_dec8x:
 
 	vncipherlast	$out0,$out0,$ivec
 	vncipherlast	$out1,$out1,$in0
+	 lvx_u		$in0,$x00,$inp		# load next input block
 	vncipherlast	$out2,$out2,$in1
-	 lvx		$in1,0,$inp		# load next input block
-	 addi		$inp,$inp,16
+	 lvx_u		$in1,$x10,$inp
 	vncipherlast	$out3,$out3,$in2
-	 lvx		$in2,0,$inp
-	 addi		$inp,$inp,16
+	 le?vperm	$in0,$in0,$in0,$inpperm
+	 lvx_u		$in2,$x20,$inp
 	vncipherlast	$out4,$out4,$in3
-	 lvx		$in3,0,$inp
-	 addi		$inp,$inp,16
-	 vperm		$in0,$inptail,$in1,$inpperm
+	 le?vperm	$in1,$in1,$in1,$inpperm
+	 lvx_u		$in3,$x30,$inp
 	vncipherlast	$out5,$out5,$in4
-	 lvx		$in4,0,$inp
-	 addi		$inp,$inp,16
-	 vperm		$in1,$in1,$in2,$inpperm
+	 le?vperm	$in2,$in2,$in2,$inpperm
+	 lvx_u		$in4,$x40,$inp
 	vncipherlast	$out6,$out6,$in5
-	 lvx		$in5,0,$inp
-	 addi		$inp,$inp,16
-	 vperm		$in2,$in2,$in3,$inpperm
+	 le?vperm	$in3,$in3,$in3,$inpperm
+	 lvx_u		$in5,$x50,$inp
 	vncipherlast	$out7,$out7,$in6
-	 lvx		$in6,0,$inp
-	 addi		$inp,$inp,16
-	 vperm		$in3,$in3,$in4,$inpperm
+	 le?vperm	$in4,$in4,$in4,$inpperm
+	 lvx_u		$in6,$x60,$inp
 	vmr		$ivec,$in7
-
-
-	vperm		$out0,$out0,$out0,$outperm
-	 lvx		$in7,0,$inp
-	 addi		$inp,$inp,16
-	vperm		$out1,$out1,$out1,$outperm
-	vsel		$outhead,$outhead,$out0,$outmask
-	 vperm		$in4,$in4,$in5,$inpperm
-	 lvx		$inptail,0,$inp
-	 addi		$inp,$inp,16
-	vsel		$out0,$out0,$out1,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-
-	vperm		$out2,$out2,$out2,$outperm
-	 vperm		$in5,$in5,$in6,$inpperm
-	vsel		$out1,$out1,$out2,$outmask
-	stvx		$out0,0,$out
-	addi		$out,$out,16
-
-	vperm		$out3,$out3,$out3,$outperm
+	 le?vperm	$in5,$in5,$in5,$inpperm
+	 lvx_u		$in7,$x70,$inp
+	 addi		$inp,$inp,0x80
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	 le?vperm	$in6,$in6,$in6,$inpperm
 	 vxor		$out0,$in0,$rndkey0
-	 vperm		$in6,$in6,$in7,$inpperm
-	vsel		$out2,$out2,$out3,$outmask
-	stvx		$out1,0,$out
-	addi		$out,$out,16
-
-	vperm		$out4,$out4,$out4,$outperm
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	 le?vperm	$in7,$in7,$in7,$inpperm
 	 vxor		$out1,$in1,$rndkey0
-	 vperm		$in7,$in7,$inptail,$inpperm
-	vsel		$out3,$out3,$out4,$outmask
-	stvx		$out2,0,$out
-	addi		$out,$out,16
-
-	vperm		$out5,$out5,$out5,$outperm
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x20,$out
 	 vxor		$out2,$in2,$rndkey0
-	vsel		$out4,$out4,$out5,$outmask
-	stvx		$out3,0,$out
-	addi		$out,$out,16
-
-	vperm		$out6,$out6,$out6,$outperm
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x30,$out
 	 vxor		$out3,$in3,$rndkey0
-	vsel		$out5,$out5,$out6,$outmask
-	stvx		$out4,0,$out
-	addi		$out,$out,16
-
-	vperm		$outhead,$out7,$out7,$outperm
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x40,$out
 	 vxor		$out4,$in4,$rndkey0
-	vsel		$out6,$out6,$outhead,$outmask
-	stvx		$out5,0,$out
-	addi		$out,$out,16
-
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x50,$out
 	 vxor		$out5,$in5,$rndkey0
-	stvx		$out6,0,$out
-	addi		$out,$out,16
-
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x60,$out
 	 vxor		$out6,$in6,$rndkey0
+	stvx_u		$out7,$x70,$out
+	addi		$out,$out,0x80
 	 vxor		$out7,$in7,$rndkey0
 
 	mtctr		$rounds
@@ -945,6 +920,7 @@ Loop_cbc_dec8x:
 	addic.		$len,$len,128
 	beq		Lcbc_dec8x_done
 	nop
+	nop
 
 Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
 	vncipher	$out1,$out1,v24
@@ -954,8 +930,8 @@ Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
 	vncipher	$out5,$out5,v24
 	vncipher	$out6,$out6,v24
 	vncipher	$out7,$out7,v24
-	lvx		v24,0,$key_		# round[3]
-	addi		$key_,$key_,16
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
 
 	vncipher	$out1,$out1,v25
 	vncipher	$out2,$out2,v25
@@ -964,8 +940,7 @@ Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
 	vncipher	$out5,$out5,v25
 	vncipher	$out6,$out6,v25
 	vncipher	$out7,$out7,v25
-	lvx		v25,0,$key_		# round[4]
-	addi		$key_,$key_,16
+	lvx		v25,$x10,$key_		# round[4]
 	bdnz		Loop_cbc_dec8x_tail
 
 	vncipher	$out1,$out1,v24
@@ -1044,6 +1019,7 @@ Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
 	nop
 	beq		Lcbc_dec8x_six
 
+Lcbc_dec8x_seven:
 	vncipherlast	$out1,$out1,$ivec
 	vncipherlast	$out2,$out2,$in1
 	vncipherlast	$out3,$out3,$in2
@@ -1053,40 +1029,21 @@ Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
 	vncipherlast	$out7,$out7,$in6
 	vmr		$ivec,$in7
 
-	vperm		$out1,$out1,$out1,$outperm
-	vsel		$outhead,$outhead,$out1,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-
-	vperm		$out2,$out2,$out2,$outperm
-	vsel		$out1,$out1,$out2,$outmask
-	stvx		$out1,0,$out
-	addi		$out,$out,16
-
-	vperm		$out3,$out3,$out3,$outperm
-	vsel		$out2,$out2,$out3,$outmask
-	stvx		$out2,0,$out
-	addi		$out,$out,16
-
-	vperm		$out4,$out4,$out4,$outperm
-	vsel		$out3,$out3,$out4,$outmask
-	stvx		$out3,0,$out
-	addi		$out,$out,16
-
-	vperm		$out5,$out5,$out5,$outperm
-	vsel		$out4,$out4,$out5,$outmask
-	stvx		$out4,0,$out
-	addi		$out,$out,16
-
-	vperm		$out6,$out6,$out6,$outperm
-	vsel		$out5,$out5,$out6,$outmask
-	stvx		$out5,0,$out
-	addi		$out,$out,16
-
-	vperm		$outhead,$out7,$out7,$outperm
-	vsel		$out6,$out6,$outhead,$outmask
-	stvx		$out6,0,$out
-	addi		$out,$out,16
+	le?vperm	$out1,$out1,$out1,$inpperm
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x00,$out
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x10,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x20,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x30,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x40,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x50,$out
+	stvx_u		$out7,$x60,$out
+	addi		$out,$out,0x70
 	b		Lcbc_dec8x_done
 
 .align	5
@@ -1099,35 +1056,19 @@ Lcbc_dec8x_six:
 	vncipherlast	$out7,$out7,$in6
 	vmr		$ivec,$in7
 
-	vperm		$out2,$out2,$out2,$outperm
-	vsel		$outhead,$outhead,$out2,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-
-	vperm		$out3,$out3,$out3,$outperm
-	vsel		$out2,$out2,$out3,$outmask
-	stvx		$out2,0,$out
-	addi		$out,$out,16
-
-	vperm		$out4,$out4,$out4,$outperm
-	vsel		$out3,$out3,$out4,$outmask
-	stvx		$out3,0,$out
-	addi		$out,$out,16
-
-	vperm		$out5,$out5,$out5,$outperm
-	vsel		$out4,$out4,$out5,$outmask
-	stvx		$out4,0,$out
-	addi		$out,$out,16
-
-	vperm		$out6,$out6,$out6,$outperm
-	vsel		$out5,$out5,$out6,$outmask
-	stvx		$out5,0,$out
-	addi		$out,$out,16
-
-	vperm		$outhead,$out7,$out7,$outperm
-	vsel		$out6,$out6,$outhead,$outmask
-	stvx		$out6,0,$out
-	addi		$out,$out,16
+	le?vperm	$out2,$out2,$out2,$inpperm
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x00,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x10,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x20,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x30,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x40,$out
+	stvx_u		$out7,$x50,$out
+	addi		$out,$out,0x60
 	b		Lcbc_dec8x_done
 
 .align	5
@@ -1139,30 +1080,17 @@ Lcbc_dec8x_five:
 	vncipherlast	$out7,$out7,$in6
 	vmr		$ivec,$in7
 
-	vperm		$out3,$out3,$out3,$outperm
-	vsel		$outhead,$outhead,$out3,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-
-	vperm		$out4,$out4,$out4,$outperm
-	vsel		$out3,$out3,$out4,$outmask
-	stvx		$out3,0,$out
-	addi		$out,$out,16
-
-	vperm		$out5,$out5,$out5,$outperm
-	vsel		$out4,$out4,$out5,$outmask
-	stvx		$out4,0,$out
-	addi		$out,$out,16
-
-	vperm		$out6,$out6,$out6,$outperm
-	vsel		$out5,$out5,$out6,$outmask
-	stvx		$out5,0,$out
-	addi		$out,$out,16
-
-	vperm		$outhead,$out7,$out7,$outperm
-	vsel		$out6,$out6,$outhead,$outmask
-	stvx		$out6,0,$out
-	addi		$out,$out,16
+	le?vperm	$out3,$out3,$out3,$inpperm
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x00,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x10,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x20,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x30,$out
+	stvx_u		$out7,$x40,$out
+	addi		$out,$out,0x50
 	b		Lcbc_dec8x_done
 
 .align	5
@@ -1173,25 +1101,15 @@ Lcbc_dec8x_four:
 	vncipherlast	$out7,$out7,$in6
 	vmr		$ivec,$in7
 
-	vperm		$out4,$out4,$out4,$outperm
-	vsel		$outhead,$outhead,$out4,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-
-	vperm		$out5,$out5,$out5,$outperm
-	vsel		$out4,$out4,$out5,$outmask
-	stvx		$out4,0,$out
-	addi		$out,$out,16
-
-	vperm		$out6,$out6,$out6,$outperm
-	vsel		$out5,$out5,$out6,$outmask
-	stvx		$out5,0,$out
-	addi		$out,$out,16
-
-	vperm		$outhead,$out7,$out7,$outperm
-	vsel		$out6,$out6,$outhead,$outmask
-	stvx		$out6,0,$out
-	addi		$out,$out,16
+	le?vperm	$out4,$out4,$out4,$inpperm
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x00,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x10,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x20,$out
+	stvx_u		$out7,$x30,$out
+	addi		$out,$out,0x40
 	b		Lcbc_dec8x_done
 
 .align	5
@@ -1201,20 +1119,13 @@ Lcbc_dec8x_three:
 	vncipherlast	$out7,$out7,$in6
 	vmr		$ivec,$in7
 
-	vperm		$out5,$out5,$out5,$outperm
-	vsel		$outhead,$outhead,$out5,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-
-	vperm		$out6,$out6,$out6,$outperm
-	vsel		$out5,$out5,$out6,$outmask
-	stvx		$out5,0,$out
-	addi		$out,$out,16
-
-	vperm		$outhead,$out7,$out7,$outperm
-	vsel		$out6,$out6,$outhead,$outmask
-	stvx		$out6,0,$out
-	addi		$out,$out,16
+	le?vperm	$out5,$out5,$out5,$inpperm
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x00,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x10,$out
+	stvx_u		$out7,$x20,$out
+	addi		$out,$out,0x30
 	b		Lcbc_dec8x_done
 
 .align	5
@@ -1223,15 +1134,11 @@ Lcbc_dec8x_two:
 	vncipherlast	$out7,$out7,$in6
 	vmr		$ivec,$in7
 
-	vperm		$out6,$out6,$out6,$outperm
-	vsel		$outhead,$outhead,$out6,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-
-	vperm		$outhead,$out7,$out7,$outperm
-	vsel		$out6,$out6,$outhead,$outmask
-	stvx		$out6,0,$out
-	addi		$out,$out,16
+	le?vperm	$out6,$out6,$out6,$inpperm
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x00,$out
+	stvx_u		$out7,$x10,$out
+	addi		$out,$out,0x20
 	b		Lcbc_dec8x_done
 
 .align	5
@@ -1239,52 +1146,31 @@ Lcbc_dec8x_one:
 	vncipherlast	$out7,$out7,$ivec
 	vmr		$ivec,$in7
 
-	vperm		$out7,$out7,$out7,$outperm
-	vsel		$outhead,$outhead,$out7,$outmask
-	stvx		$outhead,0,$out
-	addi		$out,$out,16
-	vmr		$outhead,$out7
-	nop
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out7,0,$out
+	addi		$out,$out,0x10
 
 Lcbc_dec8x_done:
-	addi		$out,$out,-1
-	lvx		$out7,0,$out		# redundant in aligned case
-	vsel		$out7,$outhead,$out7,$outmask
-	stvx		$out7,0,$out
-
-	neg		$enc,$ivp		# write [unaligned] iv
-	li		$idx,15			# 15 is not typo
-	vxor		$rndkey0,$rndkey0,$rndkey0
-	vspltisb	$outmask,-1
-	`"vspltisb	$tmp,0x0f"		if ($LITTLE_ENDIAN)`
-	?lvsl		$outperm,0,$enc
-	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	`"vxor		$outperm,$outperm,$tmp"	if ($LITTLE_ENDIAN)`
-	lvx		$outhead,0,$ivp
-	vperm		$ivec,$ivec,$ivec,$outperm
-	vsel		$in0,$outhead,$ivec,$outmask
-	lvx		$inptail,$idx,$ivp
-	stvx		$in0,0,$ivp
-	vsel		$in0,$ivec,$inptail,$outmask
-	stvx		$in0,$idx,$ivp
+	le?vperm	$ivec,$ivec,$ivec,$inpperm
+	stvx_u		$ivec,0,$ivp		# write [unaligned] iv
 
 	li		r10,`$FRAME+15`
 	li		r11,`$FRAME+31`
-	stvx		$outmask,r10,$sp	# wipe copies of rounds keys
+	stvx		$inpperm,r10,$sp	# wipe copies of round keys
 	addi		r10,r10,32
-	stvx		$outmask,r11,$sp
+	stvx		$inpperm,r11,$sp
 	addi		r11,r11,32
-	stvx		$outmask,r10,$sp
+	stvx		$inpperm,r10,$sp
 	addi		r10,r10,32
-	stvx		$outmask,r11,$sp
+	stvx		$inpperm,r11,$sp
 	addi		r11,r11,32
-	stvx		$outmask,r10,$sp
+	stvx		$inpperm,r10,$sp
 	addi		r10,r10,32
-	stvx		$outmask,r11,$sp
+	stvx		$inpperm,r11,$sp
 	addi		r11,r11,32
-	stvx		$outmask,r10,$sp
+	stvx		$inpperm,r10,$sp
 	addi		r10,r10,32
-	stvx		$outmask,r11,$sp
+	stvx		$inpperm,r11,$sp
 	addi		r11,r11,32
 
 	mtspr		256,$vrsave
@@ -1310,10 +1196,17 @@ Lcbc_dec8x_done:
 	addi		r11,r11,32
 	lvx		v30,r10,$sp
 	lvx		v31,r11,$sp
-	addi		$sp,$sp,`$FRAME+21*16`
+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
 	blr
 	.long		0
-	.byte		0,12,0x14,0,0x80,0,6,0
+	.byte		0,12,0x14,0,0x80,6,6,0
+	.long		0
 .size	.${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
 ___
 }}	}}}
@@ -1354,12 +1247,16 @@ foreach(split("\n",$code)) {
 	# instructions prefixed with '?' are endian-specific and need
 	# to be adjusted accordingly...
 	if ($flavour =~ /le$/o) {	# little-endian
-	    s/\?lvsr/lvsl/o or
-	    s/\?lvsl/lvsr/o or
+	    s/le\?//o		or
+	    s/be\?/#be#/o	or
+	    s/\?lvsr/lvsl/o	or
+	    s/\?lvsl/lvsr/o	or
 	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
 	    s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
 	    s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
 	} else {			# big-endian
+	    s/le\?/#le#/o	or
+	    s/be\?//o		or
 	    s/\?([a-z]+)/$1/o;
 	}