SM4 AESE optimization for ARMv8

Signed-off-by: Xu Yizhou <xuyizhou1@huawei.com>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19914)

3 files changed, 2005 insertions, 1 deletions

diff --git a/crypto/sm4/asm/vpsm4-armv8.pl b/crypto/sm4/asm/vpsm4-armv8.pl
index ac979c074e..cb1be122e4 100755
--- a/crypto/sm4/asm/vpsm4-armv8.pl
+++ b/crypto/sm4/asm/vpsm4-armv8.pl
@@ -28,6 +28,7 @@ open OUT,"| \"$^X\" $xlate $flavour \"$output\""
 
 $prefix="vpsm4";
 my @vtmp=map("v$_",(0..3));
+my @qtmp=map("q$_",(0..3));
 my @data=map("v$_",(4..7));
 my @datax=map("v$_",(8..11));
 my ($rk0,$rk1)=("v12","v13");
@@ -36,6 +37,7 @@ my @vtmpx=map("v$_",(12..15));
 my @sbox=map("v$_",(16..31));
 my ($inp,$outp,$blocks,$rks)=("x0","x1","w2","x3");
 my ($tmpw,$tmp,$wtmp0,$wtmp1,$wtmp2)=("w6","x6","w7","w8","w9");
+my ($xtmp1,$xtmp2)=("x8","x9");
 my ($ptr,$counter)=("x10","w11");
 my ($word0,$word1,$word2,$word3)=("w12","w13","w14","w15");
 
@@ -60,6 +62,51 @@ ___
 	}
 }
 
+sub rev32_armeb() {
+	my $dst = shift;
+	my $src = shift;
+
+	if ($src and ("$src" ne "$dst")) {
+$code.=<<___;
+#ifdef __AARCH64EB__
+	rev32	$dst.16b,$src.16b
+#else
+	mov	$dst.16b,$src.16b
+#endif
+___
+	} else {
+$code.=<<___;
+#ifdef __AARCH64EB__
+	rev32	$dst.16b,$dst.16b
+#endif
+___
+	}
+}
+
+sub rbit() {
+	my $dst = shift;
+	my $src = shift;
+	my $std = shift;
+
+	if ($src and ("$src" ne "$dst")) {
+		if ($std eq "_gb") {
+$code.=<<___;
+			rbit $dst.16b,$src.16b
+___
+		} else {
+$code.=<<___;
+			mov $dst.16b,$src.16b
+___
+		}
+	} else {
+		if ($std eq "_gb") {
+$code.=<<___;
+			rbit $dst.16b,$src.16b
+___
+		}
+	}
+}
+
 sub transpose() {
 	my ($dat0,$dat1,$dat2,$dat3,$vt0,$vt1,$vt2,$vt3) = @_;
 
@@ -435,6 +482,58 @@ $code.=<<___;
 ___
 }
 
+
+sub mov_reg_to_vec() {
+	my $src0 = shift;
+	my $src1 = shift;
+	my $desv = shift;
+$code.=<<___;
+	mov $desv.d[0],$src0
+	mov $desv.d[1],$src1
+___
+	&rev32_armeb($desv,$desv);
+}
+
+sub mov_vec_to_reg() {
+	my $srcv = shift;
+	my $des0 = shift;
+	my $des1 = shift;
+$code.=<<___;
+	mov $des0,$srcv.d[0]
+	mov $des1,$srcv.d[1]
+___
+}
+
+sub compute_tweak() {
+	my $src0 = shift;
+	my $src1 = shift;
+	my $des0 = shift;
+	my $des1 = shift;
+$code.=<<___;
+	mov $wtmp0,0x87
+	extr	$xtmp2,$src1,$src1,#32
+	extr	$des1,$src1,$src0,#63
+	and	$wtmp1,$wtmp0,$wtmp2,asr#31
+	eor	$des0,$xtmp1,$src0,lsl#1
+___
+}
+
+sub compute_tweak_vec() {
+	my $src = shift;
+	my $des = shift;
+	my $std = shift;
+	&rbit(@vtmp[2],$src,$std);
+$code.=<<___;
+	ldr  @qtmp[0], =0x01010101010101010101010101010187
+	shl  $des.16b, @vtmp[2].16b, #1
+	ext  @vtmp[1].16b, @vtmp[2].16b, @vtmp[2].16b,#15
+	ushr @vtmp[1].16b, @vtmp[1].16b, #7
+	mul  @vtmp[1].16b, @vtmp[1].16b, @vtmp[0].16b
+	eor  $des.16b, $des.16b, @vtmp[1].16b
+___
+	&rbit($des,$des,$std);
+}
+
 $code=<<___;
 #include "arm_arch.h"
 .arch	armv8-a
@@ -1101,6 +1200,365 @@ $code.=<<___;
 .size	${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks
 ___
 }}}
+
+{{{
+my ($blocks,$len)=("x2","x2");
+my $ivp=("x5");
+my @twx=map("x$_",(12..27));
+my ($rks1,$rks2)=("x26","x27");
+my $lastBlk=("x26");
+my $enc=("w28");
+my $remain=("x29");
+
+my @tweak=@datax;
+
+sub gen_xts_cipher() {
+	my $std = shift;
+$code.=<<___;
+.globl	${prefix}_xts_encrypt${std}
+.type	${prefix}_xts_encrypt${std},%function
+.align	5
+${prefix}_xts_encrypt${std}:
+	AARCH64_SIGN_LINK_REGISTER
+	stp	x15, x16, [sp, #-0x10]!
+	stp	x17, x18, [sp, #-0x10]!
+	stp	x19, x20, [sp, #-0x10]!
+	stp	x21, x22, [sp, #-0x10]!
+	stp	x23, x24, [sp, #-0x10]!
+	stp	x25, x26, [sp, #-0x10]!
+	stp	x27, x28, [sp, #-0x10]!
+	stp	x29, x30, [sp, #-0x10]!
+	stp	d8, d9, [sp, #-0x10]!
+	stp	d10, d11, [sp, #-0x10]!
+	stp	d12, d13, [sp, #-0x10]!
+	stp	d14, d15, [sp, #-0x10]!
+	mov	$rks1,x3
+	mov	$rks2,x4
+	mov	$enc,w6
+	ld1	{@tweak[0].4s}, [$ivp]
+	mov	$rks,$rks2
+___
+	&load_sbox();
+	&rev32(@tweak[0],@tweak[0]);
+	&encrypt_1blk(@tweak[0]);
+$code.=<<___;
+	mov	$rks,$rks1
+	and	$remain,$len,#0x0F
+	// convert length into blocks
+	lsr	$blocks,$len,4
+	cmp	$blocks,#1
+	b.lt .return${std}
+
+	cmp $remain,0
+	// If the encryption/decryption Length is N times of 16,
+	// the all blocks are encrypted/decrypted in .xts_encrypt_blocks${std}
+	b.eq .xts_encrypt_blocks${std}
+
+	// If the encryption/decryption length is not N times of 16,
+	// the last two blocks are encrypted/decrypted in .last_2blks_tweak${std} or .only_2blks_tweak${std}
+	// the other blocks are encrypted/decrypted in .xts_encrypt_blocks${std}
+	subs $blocks,$blocks,#1
+	b.eq .only_2blks_tweak${std}
+.xts_encrypt_blocks${std}:
+___
+	&rbit(@tweak[0],@tweak[0],$std);
+	&rev32_armeb(@tweak[0],@tweak[0]);
+	&mov_vec_to_reg(@tweak[0],@twx[0],@twx[1]);
+	&compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]);
+	&compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]);
+	&compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]);
+	&compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]);
+	&compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]);
+	&compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]);
+	&compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]);
+$code.=<<___;
+.Lxts_8_blocks_process${std}:
+	cmp	$blocks,#8
+	b.lt	.Lxts_4_blocks_process${std}
+___
+	&mov_reg_to_vec(@twx[0],@twx[1],@vtmp[0]);
+	&mov_reg_to_vec(@twx[2],@twx[3],@vtmp[1]);
+	&mov_reg_to_vec(@twx[4],@twx[5],@vtmp[2]);
+	&mov_reg_to_vec(@twx[6],@twx[7],@vtmp[3]);
+	&mov_reg_to_vec(@twx[8],@twx[9],@vtmpx[0]);
+	&mov_reg_to_vec(@twx[10],@twx[11],@vtmpx[1]);
+	&mov_reg_to_vec(@twx[12],@twx[13],@vtmpx[2]);
+	&mov_reg_to_vec(@twx[14],@twx[15],@vtmpx[3]);
+$code.=<<___;
+	ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
+___
+	&rbit(@vtmp[0],@vtmp[0],$std);
+	&rbit(@vtmp[1],@vtmp[1],$std);
+	&rbit(@vtmp[2],@vtmp[2],$std);
+	&rbit(@vtmp[3],@vtmp[3],$std);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @vtmp[0].16b
+	eor @data[1].16b, @data[1].16b, @vtmp[1].16b
+	eor @data[2].16b, @data[2].16b, @vtmp[2].16b
+	eor @data[3].16b, @data[3].16b, @vtmp[3].16b
+	ld1	{@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
+___
+	&rbit(@vtmpx[0],@vtmpx[0],$std);
+	&rbit(@vtmpx[1],@vtmpx[1],$std);
+	&rbit(@vtmpx[2],@vtmpx[2],$std);
+	&rbit(@vtmpx[3],@vtmpx[3],$std);
+$code.=<<___;
+	eor @datax[0].16b, @datax[0].16b, @vtmpx[0].16b
+	eor @datax[1].16b, @datax[1].16b, @vtmpx[1].16b
+	eor @datax[2].16b, @datax[2].16b, @vtmpx[2].16b
+	eor @datax[3].16b, @datax[3].16b, @vtmpx[3].16b
+___
+	&rev32(@data[0],@data[0]);
+	&rev32(@data[1],@data[1]);
+	&rev32(@data[2],@data[2]);
+	&rev32(@data[3],@data[3]);
+	&rev32(@datax[0],@datax[0]);
+	&rev32(@datax[1],@datax[1]);
+	&rev32(@datax[2],@datax[2]);
+	&rev32(@datax[3],@datax[3]);
+	&transpose(@data,@vtmp);
+	&transpose(@datax,@vtmp);
+$code.=<<___;
+	bl	_${prefix}_enc_8blks
+___
+	&transpose(@vtmp,@datax);
+	&transpose(@data,@datax);
+
+	&mov_reg_to_vec(@twx[0],@twx[1],@vtmpx[0]);
+	&compute_tweak(@twx[14],@twx[15],@twx[0],@twx[1]);
+	&mov_reg_to_vec(@twx[2],@twx[3],@vtmpx[1]);
+	&compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]);
+	&mov_reg_to_vec(@twx[4],@twx[5],@vtmpx[2]);
+	&compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]);
+	&mov_reg_to_vec(@twx[6],@twx[7],@vtmpx[3]);
+	&compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]);
+	&mov_reg_to_vec(@twx[8],@twx[9],@tweak[0]);
+	&compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]);
+	&mov_reg_to_vec(@twx[10],@twx[11],@tweak[1]);
+	&compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]);
+	&mov_reg_to_vec(@twx[12],@twx[13],@tweak[2]);
+	&compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]);
+	&mov_reg_to_vec(@twx[14],@twx[15],@tweak[3]);
+	&compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]);
+$code.=<<___;
+	eor @vtmp[0].16b, @vtmp[0].16b, @vtmpx[0].16b
+	eor @vtmp[1].16b, @vtmp[1].16b, @vtmpx[1].16b
+	eor @vtmp[2].16b, @vtmp[2].16b, @vtmpx[2].16b
+	eor @vtmp[3].16b, @vtmp[3].16b, @vtmpx[3].16b
+	eor @data[0].16b, @data[0].16b, @tweak[0].16b
+	eor @data[1].16b, @data[1].16b, @tweak[1].16b
+	eor @data[2].16b, @data[2].16b, @tweak[2].16b
+	eor @data[3].16b, @data[3].16b, @tweak[3].16b
+
+	// save the last tweak
+	st1	{@tweak[3].4s},[$ivp]
+	st1	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
+	st1	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
+	subs	$blocks,$blocks,#8
+	b.gt	.Lxts_8_blocks_process${std}
+	b	100f
+.Lxts_4_blocks_process${std}:
+___
+	&mov_reg_to_vec(@twx[0],@twx[1],@tweak[0]);
+	&mov_reg_to_vec(@twx[2],@twx[3],@tweak[1]);
+	&mov_reg_to_vec(@twx[4],@twx[5],@tweak[2]);
+	&mov_reg_to_vec(@twx[6],@twx[7],@tweak[3]);
+$code.=<<___;
+	cmp	$blocks,#4
+	b.lt	1f
+	ld1	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
+___
+	&rbit(@tweak[0],@tweak[0],$std);
+	&rbit(@tweak[1],@tweak[1],$std);
+	&rbit(@tweak[2],@tweak[2],$std);
+	&rbit(@tweak[3],@tweak[3],$std);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @tweak[0].16b
+	eor @data[1].16b, @data[1].16b, @tweak[1].16b
+	eor @data[2].16b, @data[2].16b, @tweak[2].16b
+	eor @data[3].16b, @data[3].16b, @tweak[3].16b
+___
+	&rev32(@data[0],@data[0]);
+	&rev32(@data[1],@data[1]);
+	&rev32(@data[2],@data[2]);
+	&rev32(@data[3],@data[3]);
+	&transpose(@data,@vtmp);
+$code.=<<___;
+	bl	_${prefix}_enc_4blks
+___
+	&transpose(@vtmp,@data);
+$code.=<<___;
+	eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b
+	eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b
+	eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b
+	eor @vtmp[3].16b, @vtmp[3].16b, @tweak[3].16b
+	st1	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
+	sub	$blocks,$blocks,#4
+___
+	&mov_reg_to_vec(@twx[8],@twx[9],@tweak[0]);
+	&mov_reg_to_vec(@twx[10],@twx[11],@tweak[1]);
+	&mov_reg_to_vec(@twx[12],@twx[13],@tweak[2]);
+$code.=<<___;
+	// save the last tweak
+	st1	{@tweak[3].4s},[$ivp]
+1:
+	// process last block
+	cmp	$blocks,#1
+	b.lt	100f
+	b.gt	1f
+	ld1	{@data[0].4s},[$inp],#16
+___
+	&rbit(@tweak[0],@tweak[0],$std);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @tweak[0].16b
+___
+	&rev32(@data[0],@data[0]);
+	&encrypt_1blk(@data[0]);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @tweak[0].16b
+	st1	{@data[0].4s},[$outp],#16
+	// save the last tweak
+	st1	{@tweak[0].4s},[$ivp]
+	b	100f
+1:  // process last 2 blocks
+	cmp	$blocks,#2
+	b.gt	1f
+	ld1	{@data[0].4s,@data[1].4s},[$inp],#32
+___
+	&rbit(@tweak[0],@tweak[0],$std);
+	&rbit(@tweak[1],@tweak[1],$std);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @tweak[0].16b
+	eor @data[1].16b, @data[1].16b, @tweak[1].16b
+___
+	&rev32(@data[0],@data[0]);
+	&rev32(@data[1],@data[1]);
+	&transpose(@data,@vtmp);
+$code.=<<___;
+	bl	_${prefix}_enc_4blks
+___
+	&transpose(@vtmp,@data);
+$code.=<<___;
+	eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b
+	eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b
+	st1	{@vtmp[0].4s,@vtmp[1].4s},[$outp],#32
+	// save the last tweak
+	st1	{@tweak[1].4s},[$ivp]
+	b	100f
+1:  // process last 3 blocks
+	ld1	{@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48
+___
+	&rbit(@tweak[0],@tweak[0],$std);
+	&rbit(@tweak[1],@tweak[1],$std);
+	&rbit(@tweak[2],@tweak[2],$std);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @tweak[0].16b
+	eor @data[1].16b, @data[1].16b, @tweak[1].16b
+	eor @data[2].16b, @data[2].16b, @tweak[2].16b
+___
+	&rev32(@data[0],@data[0]);
+	&rev32(@data[1],@data[1]);
+	&rev32(@data[2],@data[2]);
+	&transpose(@data,@vtmp);
+$code.=<<___;
+	bl	_${prefix}_enc_4blks
+___
+	&transpose(@vtmp,@data);
+$code.=<<___;
+	eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b
+	eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b
+	eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b
+	st1	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48
+	// save the last tweak
+	st1	{@tweak[2].4s},[$ivp]
+100:
+	cmp $remain,0
+	b.eq .return${std}
+
+// This brance calculates the last two tweaks, 
+// while the encryption/decryption length is larger than 32
+.last_2blks_tweak${std}:
+	ld1	{@tweak[0].4s},[$ivp]
+___
+	&rev32_armeb(@tweak[0],@tweak[0]);
+	&compute_tweak_vec(@tweak[0],@tweak[1],$std);
+	&compute_tweak_vec(@tweak[1],@tweak[2],$std);
+$code.=<<___;
+	b .check_dec${std}
+
+
+// This brance calculates the last two tweaks, 
+// while the encryption/decryption length is equal to 32, who only need two tweaks
+.only_2blks_tweak${std}:
+	mov @tweak[1].16b,@tweak[0].16b
+___
+	&rev32_armeb(@tweak[1],@tweak[1]);
+	&compute_tweak_vec(@tweak[1],@tweak[2]);
+$code.=<<___;
+	b .check_dec${std}
+
+
+// Determine whether encryption or decryption is required.
+// The last two tweaks need to be swapped for decryption.
+.check_dec${std}:
+	// encryption:1 decryption:0
+	cmp $enc,1
+	b.eq .prcess_last_2blks${std}
+	mov @vtmp[0].16B,@tweak[1].16b
+	mov @tweak[1].16B,@tweak[2].16b
+	mov @tweak[2].16B,@vtmp[0].16b
+
+.prcess_last_2blks${std}:
+___
+	&rev32_armeb(@tweak[1],@tweak[1]);
+	&rev32_armeb(@tweak[2],@tweak[2]);
+$code.=<<___;
+	ld1	{@data[0].4s},[$inp],#16
+	eor @data[0].16b, @data[0].16b, @tweak[1].16b
+___
+	&rev32(@data[0],@data[0]);
+	&encrypt_1blk(@data[0]);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @tweak[1].16b
+	st1	{@data[0].4s},[$outp],#16
+
+	sub $lastBlk,$outp,16
+	.loop${std}:
+		subs $remain,$remain,1
+		ldrb	$wtmp0,[$lastBlk,$remain]
+		ldrb	$wtmp1,[$inp,$remain]
+		strb	$wtmp1,[$lastBlk,$remain]
+		strb	$wtmp0,[$outp,$remain]
+	b.gt .loop${std}
+	ld1		{@data[0].4s}, [$lastBlk]	
+	eor @data[0].16b, @data[0].16b, @tweak[2].16b
+___
+	&rev32(@data[0],@data[0]);
+	&encrypt_1blk(@data[0]);
+$code.=<<___;
+	eor @data[0].16b, @data[0].16b, @tweak[2].16b
+	st1		{@data[0].4s}, [$lastBlk]
+.return${std}:
+	ldp		d14, d15, [sp], #0x10
+	ldp		d12, d13, [sp], #0x10
+	ldp		d10, d11, [sp], #0x10
+	ldp		d8, d9, [sp], #0x10
+	ldp		x29, x30, [sp], #0x10
+	ldp		x27, x28, [sp], #0x10
+	ldp		x25, x26, [sp], #0x10
+	ldp		x23, x24, [sp], #0x10
+	ldp		x21, x22, [sp], #0x10
+	ldp		x19, x20, [sp], #0x10
+	ldp		x17, x18, [sp], #0x10
+	ldp		x15, x16, [sp], #0x10
+	AARCH64_VALIDATE_LINK_REGISTER
+	ret
+.size	${prefix}_xts_encrypt${std},.-${prefix}_xts_encrypt${std}
+___
+} # end of gen_xts_cipher
+&gen_xts_cipher("_gb");
+&gen_xts_cipher("");
+}}}
 ########################################
 open SELF,$0;
 while(<SELF>) {
diff --git a/crypto/sm4/asm/vpsm4_ex-armv8.pl b/crypto/sm4/asm/vpsm4_ex-armv8.pl
new file mode 100644
index 0000000000..3d094aa535
--- /dev/null
+++ b/crypto/sm4/asm/vpsm4_ex-armv8.pl
@@ -0,0 +1,1544 @@
+#! /usr/bin/env perl
+# Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# This module implements SM4 with ASIMD and AESE on AARCH64
+#
+# Dec 2022
+#
+
+# $output is the last argument if it looks like a file (it has an extension)
+# $flavour is the first argument if it doesn't look like a file
+$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
+$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+die "can't locate arm-xlate.pl";
+
+open OUT,"| \"$^X\" $xlate $flavour \"$output\""
+	or die "can't call $xlate: $!";
+*STDOUT=*OUT;
+
+$prefix="vpsm4_ex";
+my @vtmp=map("v$_",(0..3));
+my @qtmp=map("q$_",(0..3));
+my @data=map("v$_",(4..7));
+my @datax=map("v$_",(8..11));
+my ($rk0,$rk1)=("v12","v13");
+my ($rka,$rkb)=("v14","v15");
+my @vtmpx=map("v$_",(12..15));
+my ($vtmp4,$vtmp5)=("v24","v25");
+my ($MaskV,$TAHMatV,$TALMatV,$ATAHMatV,$ATALMatV,$ANDMaskV)=("v26","v27","v28","v29","v30","v31");
+my ($MaskQ,$TAHMatQ,$TALMatQ,$ATAHMatQ,$ATALMatQ,$ANDMaskQ)=("q26","q27","q28","q29","q30","q31");
+
+my ($inp,$outp,$blocks,$rks)=("x0","x1","w2","x3");
+my ($tmpw,$tmp,$wtmp0,$wtmp1,$wtmp2)=("w6","x6","w7","w8","w9");
+my ($xtmp1,$xtmp2)=("x8","x9");
+my ($ptr,$counter)=("x10","w11");
+my ($word0,$word1,$word2,$word3)=("w12","w13","w14","w15");
+
+sub rev32() {
+	my $dst = shift;
+	my $src = shift;
+
+	if ($src and ("$src" ne "$dst")) {
+$code.=<<___;
+#ifndef __AARCH64EB__
+	rev32	$dst.16b,$src.16b
+#else
+	mov	$dst.16b,$src.16b
+#endif
+___
+	} else {
+$code.=<<___;
+#ifndef __AARCH64EB__
+	rev32	$dst.16b,$dst.16b
+#endif
+___
+	}
+}
+
+sub rev32_armeb() {
+	my $dst = shift;
+	my $src = shift;
+
+	if ($src and ("$src" ne "$dst")) {
+$code.=<<___;
+#ifdef __AARCH64EB__
+	rev32	$dst.16b,$src.16b
+#else
+	mov	$dst.16b,$src.16b
+#endif
+___
+	} else {
+$code.=<<___;
+#ifdef __AARCH64EB__
+	rev32	$dst.16b,$dst.16b
+#endif
+___
+	}
+}
+
+sub rbit() {
+	my $dst = shift;
+	my $src = shift;
+	my $std = shift;
+
+	if ($src and ("$src" ne "$dst")) {
+		if ($std eq "_gb") {
+$code.=<<___;
+			rbit $dst.16b,$src.16b
+___
+		} else {
+$code.=<<___;
+			mov $dst.16b,$src.16b
+___
+		}
+	} else {
+		if ($std eq "_gb") {
+$code.=<<___;
+			rbit $dst.16b,$src.16b
+___
+		}
+	}
+}
+
+sub transpose() {
+	my ($dat0,$dat1,$dat2,$dat3,$vt0,$vt1,$vt2,$vt3) = @_;
+
+$code.=<<___;
+	zip1	$vt0.4s,$dat0.4s,$dat1.4s
+	zip2	$vt1.4s,$dat0.4s,$dat1.4s
+	zip1	$vt2.4s,$dat2.4s,$dat3.4s
+	zip2	$vt3.4s,$dat2.4s,$dat3.4s
+	zip1	$dat0.2d,$vt0.2d,$vt2.2d
+	zip2	$dat1.2d,$vt0.2d,$vt2.2d
+	zip1	$dat2.2d,$vt1.2d,$vt3.2d
+	zip2	$dat3.2d,$vt1.2d,$vt3.2d
+___
+}
+
+# matrix multiplication Mat*x = (lowerMat*x) ^ (higherMat*x)
+sub mul_matrix() {
+	my $x = shift;
+	my $higherMat = shift;
+	my $lowerMat = shift;
+	my $tmp = shift;
+$code.=<<___;
+	ushr	$tmp.16b, $x.16b, 4
+	and		$x.16b, $x.16b, $ANDMaskV.16b
+	tbl		$x.16b, {$lowerMat.16b}, $x.16b
+	tbl		$tmp.16b, {$higherMat.16b}, $tmp.16b
+	eor		$x.16b, $x.16b, $tmp.16b
+___
+}
+
+# sbox operations for 4-lane of words
+# sbox operation for 4-lane of words
+sub sbox() {
+	my $dat = shift;
+
+$code.=<<___;
+	// optimize sbox using AESE instruction
+	tbl	@vtmp[0].16b, {$dat.16b}, $MaskV.16b
+___
+	&mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, $vtmp4);
+$code.=<<___;
+	eor @vtmp[1].16b, @vtmp[1].16b, @vtmp[1].16b
+	aese @vtmp[0].16b,@vtmp[1].16b
+___
+	&mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV, $vtmp4);
+$code.=<<___;
+	mov	$dat.16b,@vtmp[0].16b
+
+	// linear transformation
+	ushr	@vtmp[0].4s,$dat.4s,32-2
+	ushr	@vtmp[1].4s,$dat.4s,32-10
+	ushr	@vtmp[2].4s,$dat.4s,32-18
+	ushr	@vtmp[3].4s,$dat.4s,32-24
+	sli	@vtmp[0].4s,$dat.4s,2
+	sli	@vtmp[1].4s,$dat.4s,10
+	sli	@vtmp[2].4s,$dat.4s,18
+	sli	@vtmp[3].4s,$dat.4s,24
+	eor	$vtmp4.16b,@vtmp[0].16b,$dat.16b
+	eor	$vtmp4.16b,$vtmp4.16b,$vtmp[1].16b
+	eor	$dat.16b,@vtmp[2].16b,@vtmp[3].16b
+	eor	$dat.16b,$dat.16b,$vtmp4.16b
+___
+}
+
+# sbox operation for 8-lane of words
+sub sbox_double() {
+	my $dat = shift;
+	my $datx = shift;
+
+$code.=<<___;
+	// optimize sbox using AESE instruction
+	tbl	@vtmp[0].16b, {$dat.16b}, $MaskV.16b
+	tbl	@vtmp[1].16b, {$datx.16b}, $MaskV.16b
+___
+	&mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, $vtmp4);
+	&mul_matrix(@vtmp[1], $TAHMatV, $TALMatV, $vtmp4);
+$code.=<<___;
+	eor $vtmp5.16b, $vtmp5.16b, $vtmp5.16b
+	aese @vtmp[0].16b,$vtmp5.16b
+	aese @vtmp[1].16b,$vtmp5.16b
+___
+	&mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV,$vtmp4);
+	&mul_matrix(@vtmp[1], $ATAHMatV, $ATALMatV,$vtmp4);
+$code.=<<___;
+	mov	$dat.16b,@vtmp[0].16b
+	mov	$datx.16b,@vtmp[1].16b
+
+	// linear transformation
+	ushr	@vtmp[0].4s,$dat.4s,32-2
+	ushr	$vtmp5.4s,$datx.4s,32-2
+	ushr	@vtmp[1].4s,$dat.4s,32-10
+	ushr	@vtmp[2].4s,$dat.4s,32-18
+	ushr	@vtmp[3].4s,$dat.4s,32-24
+	sli	@vtmp[0].4s,$dat.4s,2
+	sli	$vtmp5.4s,$datx.4s,2
+	sli	@vtmp[1].4s,$dat.4s,10
+	sli	@vtmp[2].4s,$dat.4s,18
+	sli	@vtmp[3].4s,$dat.4s,24
+	eor	$vtmp4.16b,@vtmp[0].16b,$dat.16b
+	eor	$vtmp4.16b,$vtmp4.16b,@vtmp[1].16b
+	eor	$dat.16b,@vtmp[2].16b,@vtmp[3].16b
+	eor	$dat.16b,$dat.16b,$vtmp4.16b
+	ushr	@vtmp[1].4s,$datx.4s,32-10
+	ushr	@vtmp[2].4s,$datx.4s,32-18
+	ushr	@vtmp[3].4s,$datx.4s,32-24
+	sli	@vtmp[1].4s,$datx.4s,10
+	sli	@vtmp[2].4s,$datx.4s,18
+	sli	@vtmp[3].4s,$datx.4s,24
+	eor	$vtmp4.16b,$vtmp5.16b,$datx.16b
+	eor	$vtmp4.16b,$vtmp4.16b,@vtmp[1].16b
+	eor	$datx.16b,@vtmp[2].16b,@vtmp[3].16b
+	eor	$datx.16b,$datx.16b,$vtmp4.16b
+___
+}
+
+# sbox operation for one single word
+sub sbox_1word () {
+	my $word = shift;
+
+$code.=<<___;
+	mov	@vtmp[3].s[0],$word
+	// optimize sbox using AESE instruction
+	tbl	@vtmp[0].16b, {@vtmp[3].16b}, $MaskV.16b
+___
+	&mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, @vtmp[2]);
+$code.=<<___;
+	eor @vtmp[1].16b, @vtmp[1].16b, @vtmp[1].16b
+	aese @vtmp[0].16b,@vtmp[1].16b
+___
+	&mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV, @vtmp[2]);
+$code.=<<___;
+
+	mov	$wtmp0,@vtmp[0].s[0]
+	eor	$word,$wtmp0,$wtmp0,ror #32-2
+	eor	$word,$word,$wtmp0,ror #32-10
+	eor	$word,$word,$wtmp0,ror #32-18
+	eor	$word,$word,$wtmp0,ror #32-24
+___
+}
+
+# sm4 for one block of data, in scalar registers word0/word1/word2/word3
+sub sm4_1blk () {
+	my $kptr = shift;
+
+$code.=<<___;
+	ldp	$wtmp0,$wtmp1,[$kptr],8
+	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
+	eor	$tmpw,$word2,$word3
+	eor	$wtmp2,$wtmp0,$word1
+	eor	$tmpw,$tmpw,$wtmp2
+___
+	&sbox_1word($tmpw);
+$code.=<<___;
+	eor	$word0,$word0,$tmpw
+	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
+	eor	$tmpw,$word2,$word3
+	eor	$wtmp2,$word0,$wtmp1
+	eor	$tmpw,$tmpw,$wtmp2
+___
+	&sbox_1word($tmpw);
+$code.=<<___;
+	ldp	$wtmp0,$wtmp1,[$kptr],8
+	eor	$word1,$word1,$tmpw
+	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
+	eor	$tmpw,$word0,$word1
+	eor	$wtmp2,$wtmp0,$word3
+	eor	$tmpw,$tmpw,$wtmp2
+___
+	&sbox_1word($tmpw);
+$code.=<<___;
+	eor	$word2,$word2,$tmpw
+	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
+	eor	$tmpw,$word0,$word1
+	eor	$wtmp2,$word2,$wtmp1
+	eor	$tmpw,$tmpw,$wtmp2
+___
+	&sbox_1word($tmpw);
+$code.=<<___;
+	eor	$word3,$word3,$tmpw
+___
+}
+
+# sm4 for 4-lanes of data, in neon registers data0/data1/data2/data3
+sub sm4_4blks () {
+	my $kptr = shift;
+
+$code.=<<___;
+	ldp	$wtmp0,$wtmp1,[$kptr],8
+	dup	$rk0.4s,$wtmp0
+	dup	$rk1.4s,$wtmp1
+
+	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
+	eor	$rka.16b,@data[2].16b,@data[3].16b
+	eor	$rk0.16b,@data[1].16b,$rk0.16b
+	eor	$rk0.16b,$rka.16b,$rk0.16b
+___
+	&sbox($rk0);
+$code.=<<___;
+	eor	@data[0].16b,@data[0].16b,$rk0.16b
+
+	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
+	eor	$rka.16b,$rka.16b,@data[0].16b
+	eor	$rk1.16b,$rka.16b,$rk1.16b
+___
+	&sbox($rk1);
+$code.=<<___;
+	ldp	$wtmp0,$wtmp1,[$kptr],8
+	eor	@data[1].16b,@data[1].16b,$rk1.16b
+
+	dup	$rk0.4s,$wtmp0
+	dup	$rk1.4s,$wtmp1
+
+	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
+	eor	$rka.16b,@data[0].16b,@data[1].16b
+	eor	$rk0.16b,@data[3].16b,$rk0.16b
+	eor	$rk0.16b,$rka.16b,$rk0.16b
+___
+	&sbox($rk0);
+$code.=<<___;
+	eor	@data[2].16b,@data[2].16b,$rk0.16b
+
+	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
+	eor	$rka.16b,$rka.16b,@data[2].16b
+	eor	$rk1.16b,$rka.16b,$rk1.16b
+___
+	&sbox($rk1);
+$code.=<<___;
+	eor	@data[3].16b,@data[3].16b,$rk1.16b
+___
+}
+
+# sm4 for 8 lanes of data, in neon registers
+# data0/data1/data2/data3 datax0/datax1/datax2/datax3
+sub sm4_8blks () {
+	my $kptr = shift;
+
+$code.=<<___;
+	ldp	$wtmp0,$wtmp1,[$kptr],8
+	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
+	dup	$rk0.4s,$wtmp0
+	eor	$rka.16b,@data[2].16b,@data[3].16b
+	eor	$rkb.16b,@datax[2].16b,@datax[3].16b
+	eor	@vtmp[0].16b,@data[1].16b,$rk0.16b
+	eor	@vtmp[1].16b,@datax[1].16b,$rk0.16b
+	eor	$rk0.16b,$rka.16b,@vtmp[0].16b
+	eor	$rk1.16b,$rkb.16b,@vtmp[1].16b
+___
+	&sbox_double($rk0,$rk1);
+$code.=<<___;
+	eor	@data[0].16b,@data[0].16b,$rk0.16b
+	eor	@datax[0].16b,@datax[0].16b,$rk1.16b
+
+	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
+	dup	$rk1.4s,$wtmp1
+	eor	$rka.16b,$rka.16b,@data[0].16b
+	eor	$rkb.16b,$rkb.16b,@datax[0].16b
+	eor	$rk0.16b,$rka.16b,$rk1.16b
+	eor	$rk1.16b,$rkb.16b,$rk1.16b
+___
+	&sbox_double($rk0,$rk1);
+$code.=<<___;
+	ldp	$wtmp0,$wtmp1,[$kptr],8
+	eor	@data[1].16b,@data[1].16b,$rk0.16b
+	eor	@datax[1].16b,@datax[1].16b,$rk1.16b
+
+	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
+	dup	$rk0.4s,$wtmp0
+	eor	$rka.16b,@data[0].16b,@data[1].16b
+	eor	$rkb.16b,@datax[0].16b,@datax[1].16b
+	eor	@vtmp[0].16b,@data[3].16b,$rk0.16b
+	eor	@vtmp[1].16b,@datax[3].16b,$rk0.16b
+	eor	$rk0.16b,$rka.16b,@vtmp[0].16b
+	eor	$rk1.16b,$rkb.16b,@vtmp[1].16b
+___
+	&sbox_double($rk0,$rk1);
+$code.=<<___;
+	eor	@data[2].16b,@data[2].16b,$rk0.16b
+	eor	@datax[2].16b,@datax[2].16b,$rk1.16b
+
+	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
+	dup	$rk1.4s,$wtmp1
+	eor	$rka.16b,$rka.16b,@data[2].16b
+	eor	$rkb.16b,$rkb.16b,@datax[2].16b
+	eor	$rk0.16b,$rka.16b,$rk1.16b
+	eor	$rk1.16b,$rkb.16b,$rk1.16b
+___
+	&sbox_double($rk0,$rk1);
+$code.=<<___;
+	eor	@data[3].16b,@data[3].16b,$rk0.16b
+	eor	@datax[3].16b,@datax[3].16b,$rk1.16b
+___
+}
+
+sub encrypt_1blk_norev() {
+	my $dat = shift;
+
+$code.=<<___;
+	mov	$ptr,$rks
+	mov	$counter,#8
+	mov	$word0,$dat.s[0]
+	mov	$word1,$dat.s[1]
+	mov	$word2,$dat.s[2]
+	mov	$word3,$dat.s[3]
+10:
+___
+	&sm4_1blk($ptr);
+$code.=<<___;
+	subs	$counter,$counter,#1
+	b.ne	10b
+	mov	$dat.s[0],$word3
+	mov	$dat.s[1],$word2
+	mov	$dat.s[2],$word1
+	mov	$dat.s[3],$word0
+___
+}
+
+sub encrypt_1blk() {
+	my $dat = shift;
+
+	&encrypt_1blk_norev($dat);
+	&rev32($dat,$dat);
+}
+
+sub encrypt_4blks() {
+$code.=<<___;
+	mov	$ptr,$rks
+	mov	$counter,#8
+10:
+___
+	&sm4_4blks($ptr);
+$code.=<<___;
+	subs	$counter,$counter,#1
+	b.ne	10b
+___
+	&rev32(@vtmp[3],@data[0]);
+	&rev32(@vtmp[2],@data[1]);
+	&rev32(@vtmp[1],@data[2]);
+	&rev32(@vtmp[0],@data[3]);
+}
+
+sub encrypt_8blks() {
+$code.=<<___;
+	mov	$ptr,$rks
+	mov	$counter,#8
+10:
+___
+	&sm4_8blks($ptr);
+$code.=<<___;
+	subs	$counter,$counter,#1
+	b.ne	10b
+___
+	&rev32(@vtmp[3],@data[0]);
+	&rev32(@vtmp[2],@data[1]);
+	&rev32(@vtmp[1],@data[2]);
+	&rev32(@vtmp[0],@data[3]);
+	&rev32(@data[3],@datax[0]);
+	&rev32(@data[2],@datax[1]);
+	&rev32(@data[1],@datax[2]);
+	&rev32(@data[0],@datax[3]);
+}
+
+sub load_sbox () {
+	my $data = shift;
+
+$code.=<<___;
+	ldr $MaskQ,	   =0x0306090c0f0205080b0e0104070a0d00
+	ldr $TAHMatQ,	=0x22581a6002783a4062185a2042387a00
+	ldr $TALMatQ,	=0xc10bb67c4a803df715df62a89e54e923
+	ldr $ATAHMatQ,   =0x1407c6d56c7fbeadb9aa6b78c1d21300
+	ldr $ATALMatQ,   =0xe383c1a1fe9edcbc6404462679195b3b
+	ldr $ANDMaskQ,	=0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+___
+}
+
+sub mov_reg_to_vec() {
+	my $src0 = shift;
+	my $src1 = shift;
+	my $desv = shift;
+$code.=<<___;
+	mov $desv.d[0],$src0
+	mov $desv.d[1],$src1
+___
+	&rev32_armeb($desv,$desv);
+}
+
+sub mov_vec_to_reg() {
+	my $srcv = shift;
+	my $des0 = shift;
+	my $des1 = shift;
+$code.=<<___;
+	mov $des0,$srcv.d[0]
+	mov $des1,$srcv.d[1]
+___
+}
+
+sub compute_tweak() {
+	my $src0 = shift;
+	my $src1 = shift;
+	my $des0 = shift;
+	my $des1 = shift;
+$code.=<<___;
+	mov $wtmp0,0x87
+	extr	$xtmp2,$src1,$src1,#32
+	extr	$des1,$src1,$src0,#63
+	and	$wtmp1,$wtmp0,$wtmp2,asr#31
+	eor	$des0,$xtmp1,$src0,lsl#1
+___
+}
+
+sub compute_tweak_vec() {
+	my $src = shift;
+	my $des = shift;
+	my $std = shift;
+	&rbit(@vtmp[2],$src,$std);
+$code.=<<___;
+	ldr  @qtmp[0], =0x01010101010101010101010101010187
+	shl  $des.16b, @vtmp[2].16b, #1
+	ext  @vtmp[1].16b, @vtmp[2].16b, @vtmp[2].16b,#15
+	ushr @vtmp[1].16b, @vtmp[1].16b, #7
+	mul  @vtmp[1].16b, @vtmp[1].16b, @vtmp[0].16b
+	eor  $des.16b, $des.16b, @vtmp[1].16b
+___
+	&rbit($des,$des,$std);
+}
+
+$code=<<___;
+#include "arm_arch.h"
+.arch	armv8-a+crypto
+.text
+
+.type	_${prefix}_consts,%object
+.align	7
+_${prefix}_consts:
+.Lck:
+	.long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269
+	.long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9
+	.long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249
+	.long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9
+	.long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229
+	.long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299
+	.long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209
+	.long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279
+.Lfk:
+	.dword 0x56aa3350a3b1bac6,0xb27022dc677d9197
+.Lshuffles:
+	.dword 0x0B0A090807060504,0x030201000F0E0D0C
+
+.size	_${prefix}_consts,.-_${prefix}_consts
+___
+
+{{{
+my ($key,$keys,$enc)=("x0","x1","w2");
+my ($pointer,$schedules,$wtmp,$roundkey)=("x5","x6","w7","w8");
+my ($vkey,$vfk,$vmap)=("v5","v6","v7");
+$code.=<<___;
+.type	_${prefix}_set_key,%function
+.align	4
+_${prefix}_set_key:
+	AARCH64_VALID_CALL_TARGET
+	ld1	{$vkey.4s},[$key]
+___
+	&load_sbox();
+	&rev32($vkey,$vkey);
+$code.=<<___;
+	adr	$pointer,.Lshuffles
+	ld1	{$vmap.2d},[$pointer]
+	adr	$pointer,.Lfk
+	ld1	{$vfk.2d},[$pointer]
+	eor	$vkey.16b,$vkey.16b,$vfk.16b
+	mov	$schedules,#32
+	adr	$pointer,.Lck
+	movi	@vtmp[0].16b,#64
+	cbnz	$enc,1f
+	add	$keys,$keys,124
+1:
+	mov	$wtmp,$vkey.s[1]
+	ldr	$roundkey,[$pointer],#4
+	eor	$roundkey,$roundkey,$wtmp
+	mov	$wtmp,$vkey.s[2]
+	eor	$roundkey,$roundkey,$wtmp
+	mov	$wtmp,$vkey.s[3]
+	eor	$roundkey,$roundkey,$wtmp
+	// optimize sbox using AESE instruction
+	mov	@data[0].s[0],$roundkey
+	tbl	@vtmp[0].16b, {@data[0].16b}, $MaskV.16b
+___
+	&mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, @vtmp[2]);
+$code.=<<___;
+	eor @vtmp[1].16b, @vtmp[1].16b, @vtmp[1].16b
+	aese @vtmp[0].16b,@vtmp[1].16b
+___
+	&mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV, @vtmp[2]);
+$code.=<<___;
+	mov	$wtmp,@vtmp[0].s[0]
+	eor	$roundkey,$wtmp,$wtmp,ror #19
+	eor	$roundkey,$roundkey,$wtmp,ror #9
+	mov	$wtmp,$vkey.s[0]
+	eor	$roundkey,$roundkey,$wtmp
+	mov	$vkey.s[0],$roundkey
+	cbz	$enc,2f
+	str	$roundkey,[$keys],#4
+	b	3f
+2:
+	str	$roundkey,[$keys],#-4
+3:
+	tbl	$vkey.16b,{$vkey.16b},$vmap.16b
+	subs	$schedules,$schedules,#1
+	b.ne	1b
+	ret
+.size	_${prefix}_set_key,.-_${prefix}_set_key
+___
+}}}
+
+
+{{{
+$code.=<<___;
+.type	_${prefix}_enc_4blks,%function
+.align	4
+_${prefix}_enc_4blks:
+	AARCH64_VALID_CALL_TARGET
+___
+	&encrypt_4blks();
+$code.=<<___;
+	ret
+.size	_${prefix}_enc_4blks,.-_${prefix}_enc_4blks
+___
+}}}
+
+{{{
+$code.=<<___;
+.type	_${prefix}_enc_8blks,%function
+.align	4
+_${prefix}_enc_8blks:
+	AARCH64_VALID_CALL_TARGET
+___
+	&encrypt_8blks();
+$code.=<<___;
+	ret
+.size	_${prefix}_enc_8blks,.-_${prefix}_enc_8blks
+___
+}}}
+
+
+{{{
+my ($key,$keys)=("x0","x1");
+$code.=<<___;
+.globl	${prefix}_set_encrypt_key