From 31b59078c8245a4ee7f7fa4e6ea98bba7f9a29a5 Mon Sep 17 00:00:00 2001 From: "Fangming.Fang" Date: Fri, 31 May 2019 10:15:10 +0000 Subject: Optimize AES-GCM implementation on aarch64 Comparing to current implementation, this change can get more performance improved by tunning the loop-unrolling factor in interleave implementation as well as by enabling high level parallelism. Performance(A72) new type 16 bytes 64 bytes 256 bytes 1024 bytes 8192 bytes 16384 bytes aes-128-gcm 113065.51k 375743.00k 848359.51k 1517865.98k 1964040.19k 1986663.77k aes-192-gcm 110679.32k 364470.63k 799322.88k 1428084.05k 1826917.03k 1848967.17k aes-256-gcm 104919.86k 352939.29k 759477.76k 1330683.56k 1663175.34k 1670430.72k old type 16 bytes 64 bytes 256 bytes 1024 bytes 8192 bytes 16384 bytes aes-128-gcm 115595.32k 382348.65k 855891.29k 1236452.35k 1425670.14k 1429793.45k aes-192-gcm 112227.02k 369543.47k 810046.55k 1147948.37k 1286288.73k 1296941.06k aes-256-gcm 111543.90k 361902.36k 769543.59k 1070693.03k 1208576.68k 1207511.72k Change-Id: I28a2dca85c001a63a2a942e80c7c64f7a4fdfcf7 Reviewed-by: Bernd Edlinger Reviewed-by: Paul Dale (Merged from https://github.com/openssl/openssl/pull/9818) --- crypto/modes/asm/aes-gcm-armv8_64.pl | 5722 ++++++++++++++++++++++++++++++++++ 1 file changed, 5722 insertions(+) create mode 100755 crypto/modes/asm/aes-gcm-armv8_64.pl (limited to 'crypto/modes/asm/aes-gcm-armv8_64.pl') diff --git a/crypto/modes/asm/aes-gcm-armv8_64.pl b/crypto/modes/asm/aes-gcm-armv8_64.pl new file mode 100755 index 0000000000..ee88906d85 --- /dev/null +++ b/crypto/modes/asm/aes-gcm-armv8_64.pl @@ -0,0 +1,5722 @@ +#! /usr/bin/env perl +# Copyright 2019 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# +#======================================================================== +# Written by Fangming Fang for the OpenSSL project, +# derived from https://github.com/ARM-software/AArch64cryptolib, original +# author Samuel Lee . The module is, however, dual +# licensed under OpenSSL and CRYPTOGAMS licenses depending on where you +# obtain it. For further details see http://www.openssl.org/~appro/cryptogams/. +#======================================================================== +# +# Approach - assume we don't want to reload constants, so reserve ~half of vector register file for constants +# +# main loop to act on 4 16B blocks per iteration, and then do modulo of the accumulated intermediate hashes from the 4 blocks +# +# ____________________________________________________ +# | | +# | PRE | +# |____________________________________________________| +# | | | | +# | CTR block 4k+8 | AES block 4k+4 | GHASH block 4k+0 | +# |________________|________________|__________________| +# | | | | +# | CTR block 4k+9 | AES block 4k+5 | GHASH block 4k+1 | +# |________________|________________|__________________| +# | | | | +# | CTR block 4k+10| AES block 4k+6 | GHASH block 4k+2 | +# |________________|________________|__________________| +# | | | | +# | CTR block 4k+11| AES block 4k+7 | GHASH block 4k+3 | +# |________________|____(mostly)____|__________________| +# | | +# | MODULO | +# |____________________________________________________| +# +# PRE: +# Ensure previous generated intermediate hash is aligned and merged with result for GHASH 4k+0 +# EXT low_acc, low_acc, low_acc, #8 +# EOR res_curr (4k+0), res_curr (4k+0), low_acc +# +# CTR block: +# Increment and byte reverse counter in scalar registers and transfer to SIMD registers +# REV ctr32, rev_ctr32 +# ORR ctr64, constctr96_top32, ctr32, LSL #32 +# INS ctr_next.d[0], constctr96_bottom64 // Keeping this in scalar registers to free up space in SIMD RF +# INS ctr_next.d[1], ctr64X +# ADD rev_ctr32, #1 +# +# AES block: +# Do AES encryption/decryption on CTR block X and EOR it with input block X. Take 256 bytes key below for example. +# Doing small trick here of loading input in scalar registers, EORing with last key and then transferring +# Given we are very constrained in our ASIMD registers this is quite important +# +# Encrypt: +# LDR input_low, [ input_ptr ], #8 +# LDR input_high, [ input_ptr ], #8 +# EOR input_low, k14_low +# EOR input_high, k14_high +# INS res_curr.d[0], input_low +# INS res_curr.d[1], input_high +# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k13 +# EOR res_curr, res_curr, ctr_curr +# ST1 { res_curr.16b }, [ output_ptr ], #16 +# +# Decrypt: +# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k13 +# LDR res_curr, [ input_ptr ], #16 +# EOR res_curr, res_curr, ctr_curr +# MOV output_low, res_curr.d[0] +# MOV output_high, res_curr.d[1] +# EOR output_low, k14_low +# EOR output_high, k14_high +# STP output_low, output_high, [ output_ptr ], #16 +# +# GHASH block X: +# do 128b karatsuba polynomial multiplication on block +# We only have 64b->128b polynomial multipliers, naively that means we need to do 4 64b multiplies to generate a 128b +# +# multiplication: +# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah,Bl) ^ Pmull(Al,Bh))<<64 +# +# The idea behind Karatsuba multiplication is that we can do just 3 64b multiplies: +# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah^Al,Bh^Bl) ^ Pmull(Ah,Bh) ^ Pmull(Al,Bl))<<64 +# +# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are +# multiplying with "twisted" powers of H +# +# Note: We can PMULL directly into the acc_x in first GHASH of the loop +# Note: For scheduling big cores we want to split the processing to happen over two loop iterations - otherwise the critical +# path latency dominates the performance +# +# This has a knock on effect on register pressure, so we have to be a bit more clever with our temporary registers +# than indicated here +# REV64 res_curr, res_curr +# INS t_m.d[0], res_curr.d[1] +# EOR t_m.8B, t_m.8B, res_curr.8B +# PMULL2 t_h, res_curr, HX +# PMULL t_l, res_curr, HX +# PMULL t_m, t_m, HX_k +# EOR acc_h, acc_h, t_h +# EOR acc_l, acc_l, t_l +# EOR acc_m, acc_m, t_m +# +# MODULO: take the partial accumulators (~representing sum of 256b multiplication results), from GHASH and do modulo reduction on them +# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are doing modulo +# with a reversed constant +# EOR acc_m, acc_m, acc_h +# EOR acc_m, acc_m, acc_l // Finish off karatsuba processing +# PMULL t_mod, acc_h, mod_constant +# EXT acc_h, acc_h, acc_h, #8 +# EOR acc_m, acc_m, acc_h +# EOR acc_m, acc_m, t_mod +# PMULL acc_h, acc_m, mod_constant +# EXT acc_m, acc_m, acc_m, #8 +# EOR acc_l, acc_l, acc_h +# EOR acc_l, acc_l, acc_m + +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate ) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +$input_ptr="x0"; #argument block +$bit_length="x1"; +$output_ptr="x2"; +$current_tag="x3"; +$counter="x16"; +$cc="x8"; + +{ +my ($end_input_ptr,$main_end_input_ptr,$input_l0,$input_h0)=map("x$_",(4..7)); +my ($input_l1,$input_h1,$input_l2,$input_h2,$input_l3,$input_h3)=map("x$_",(19..24)); +my ($output_l1,$output_h1,$output_l2,$output_h2,$output_l3,$output_h3)=map("x$_",(19..24)); +my ($output_l0,$output_h0)=map("x$_",(6..7)); + +my $ctr32w="w9"; +my ($ctr32x,$ctr96_b64x,$ctr96_t32x,$rctr32x,$rk10_l,$rk10_h,$len)=map("x$_",(9..15)); +my ($ctr96_t32w,$rctr32w)=map("w$_",(11..12)); + +my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$res0b,$res1b,$res2b,$res3b)=map("v$_.16b",(0..7)); +my ($ctr0,$ctr1,$ctr2,$ctr3,$res0,$res1,$res2,$res3)=map("v$_",(0..7)); +my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$res0d,$res1d,$res2d,$res3d)=map("d$_",(0..7)); +my ($res0q,$res1q,$res2q,$res3q)=map("q$_",(4..7)); + +my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(9..11)); +my ($acc_h,$acc_m,$acc_l)=map("v$_",(9..11)); +my ($acc_hd,$acc_md,$acc_ld)=map("d$_",(9..11)); + +my ($h1,$h2,$h3,$h4,$h12k,$h34k)=map("v$_",(12..17)); +my ($h1q,$h2q,$h3q,$h4q)=map("q$_",(12..15)); +my ($h1b,$h2b,$h3b,$h4b)=map("v$_.16b",(12..15)); + +my $t0="v8"; +my $t0d="d8"; + +my ($t1,$t2,$t3)=map("v$_",(28..30)); +my ($t1d,$t2d,$t3d)=map("d$_",(28..30)); + +my $t4="v8"; +my $t4d="d8"; +my $t5="v28"; +my $t5d="d28"; +my $t6="v31"; +my $t6d="d31"; + +my $t7="v4"; +my $t7d="d4"; +my $t8="v29"; +my $t8d="d29"; +my $t9="v30"; +my $t9d="d30"; + +my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3)=map("v$_",(4..7)); +my ($ctr_t0d,$ctr_t1d,$ctr_t2d,$ctr_t3d)=map("d$_",(4..7)); +my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b)=map("v$_.16b",(4..7)); + +my $mod_constantd="d8"; +my $mod_constant="v8"; +my $mod_t="v31"; + +my ($rk0,$rk1,$rk2,$rk3,$rk4,$rk5,$rk6,$rk7,$rk8,$rk9)=map("v$_.16b",(18..27)); +my ($rk0q,$rk1q,$rk2q,$rk3q,$rk4q,$rk5q,$rk6q,$rk7q,$rk8q,$rk9q)=map("q$_",(18..27)); +my $rk2q1="v20.1q"; +my $rk3q1="v21.1q"; +my $rk4v="v22"; +my $rk4d="d22"; + +$code=<<___; +#include "arm_arch.h" + +#if __ARM_MAX_ARCH__>=8 +___ +$code.=".arch armv8-a+crypto\n.text\n" if ($flavour =~ /64/); +$code.=<<___ if ($flavour !~ /64/); +.fpu neon +#ifdef __thumb2__ +.syntax unified +.thumb +# define INST(a,b,c,d) $_byte c,0xef,a,b +#else +.code 32 +# define INST(a,b,c,d) $_byte a,b,c,0xf2 +#endif + +.text +___ + +######################################################################################### +# size_t aes_gcm_enc_128_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# const void *key, +# unsigned char ivec[16], +# u64 *Xi); +# +$code.=<<___; +.global aes_gcm_enc_128_kernel +.type aes_gcm_enc_128_kernel,%function +.align 4 +aes_gcm_enc_128_kernel: + cbz x1, .L128_enc_ret + stp x19, x20, [sp, #-112]! + mov x16, x4 + mov x8, x5 + stp x21, x22, [sp, #16] + stp x23, x24, [sp, #32] + stp d8, d9, [sp, #48] + stp d10, d11, [sp, #64] + stp d12, d13, [sp, #80] + stp d14, d15, [sp, #96] + + ldp $ctr96_b64x, $ctr96_t32x, [$counter] @ ctr96_b64, ctr96_t32 + ldp $rk10_l, $rk10_h, [$cc, #160] @ load rk10 + + ld1 {$acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + mov $len, $main_end_input_ptr + + ldr $rk9q, [$cc, #144] @ load rk9 + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + lsr $rctr32x, $ctr96_t32x, #32 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4b, $h4b, $h4b, #8 + + fmov $ctr1d, $ctr96_b64x @ CTR block 1 + rev $rctr32w, $rctr32w @ rev_ctr32 + + add $rctr32w, $rctr32w, #1 @ increment rev_ctr32 + orr $ctr96_t32w, $ctr96_t32w, $ctr96_t32w + ldr $rk0q, [$cc, #0] @ load rk0 + + rev $ctr32w, $rctr32w @ CTR block 1 + add $rctr32w, $rctr32w, #1 @ CTR block 1 + fmov $ctr3d, $ctr96_b64x @ CTR block 3 + + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1 + ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible + + fmov $ctr1.d[1], $ctr32x @ CTR block 1 + rev $ctr32w, $rctr32w @ CTR block 2 + + fmov $ctr2d, $ctr96_b64x @ CTR block 2 + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2 + add $rctr32w, $rctr32w, #1 @ CTR block 2 + + fmov $ctr2.d[1], $ctr32x @ CTR block 2 + rev $ctr32w, $rctr32w @ CTR block 3 + + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3 + ldr $rk1q, [$cc, #16] @ load rk1 + + add $rctr32w, $rctr32w, #1 @ CTR block 3 + fmov $ctr3.d[1], $ctr32x @ CTR block 3 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3b, $h3b, $h3b, #8 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + ldr $rk2q, [$cc, #32] @ load rk2 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1b, $h1b, $h1b, #8 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + ldr $rk8q, [$cc, #128] @ load rk8 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + ldr $rk3q, [$cc, #48] @ load rk3 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + trn2 $h34k.2d, $h3.2d, $h4.2d @ h4l | h3l + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + ldr $rk6q, [$cc, #96] @ load rk6 + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + ldr $rk7q, [$cc, #112] @ load rk7 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + trn1 $acc_h.2d, $h3.2d, $h4.2d @ h4h | h3h + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + ldr $rk5q, [$cc, #80] @ load rk5 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2b, $h2b, $h2b, #8 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + eor $h34k.16b, $h34k.16b, $acc_h.16b @ h4k | h3k + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + ldr $rk4q, [$cc, #64] @ load rk4 + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + trn2 $h12k.2d, $h1.2d, $h2.2d @ h2l | h1l + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 4 blocks + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + trn1 $t0.2d, $h1.2d, $h2.2d @ h2h | h1h + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + + aese $ctr2b, $rk9 @ AES block 2 - round 9 + + aese $ctr0b, $rk9 @ AES block 0 - round 9 + + eor $h12k.16b, $h12k.16b, $t0.16b @ h2k | h1k + + aese $ctr1b, $rk9 @ AES block 1 - round 9 + + aese $ctr3b, $rk9 @ AES block 3 - round 9 + b.ge .L128_enc_tail @ handle tail + + ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 0 - load plaintext + + ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 2 - load plaintext + + ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 1 - load plaintext + + ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 3 - load plaintext + + eor $input_l0, $input_l0, $rk10_l @ AES block 0 - round 10 low + eor $input_h0, $input_h0, $rk10_h @ AES block 0 - round 10 high + + eor $input_l2, $input_l2, $rk10_l @ AES block 2 - round 10 low + fmov $ctr_t0d, $input_l0 @ AES block 0 - mov low + + eor $input_l1, $input_l1, $rk10_l @ AES block 1 - round 10 low + eor $input_h2, $input_h2, $rk10_h @ AES block 2 - round 10 high + fmov $ctr_t0.d[1], $input_h0 @ AES block 0 - mov high + + fmov $ctr_t1d, $input_l1 @ AES block 1 - mov low + eor $input_h1, $input_h1, $rk10_h @ AES block 1 - round 10 high + + eor $input_l3, $input_l3, $rk10_l @ AES block 3 - round 10 low + fmov $ctr_t1.d[1], $input_h1 @ AES block 1 - mov high + + fmov $ctr_t2d, $input_l2 @ AES block 2 - mov low + eor $input_h3, $input_h3, $rk10_h @ AES block 3 - round 10 high + rev $ctr32w, $rctr32w @ CTR block 4 + + fmov $ctr_t2.d[1], $input_h2 @ AES block 2 - mov high + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4 + + eor $res0b, $ctr_t0b, $ctr0b @ AES block 0 - result + fmov $ctr0d, $ctr96_b64x @ CTR block 4 + add $rctr32w, $rctr32w, #1 @ CTR block 4 + + fmov $ctr0.d[1], $ctr32x @ CTR block 4 + rev $ctr32w, $rctr32w @ CTR block 5 + + eor $res1b, $ctr_t1b, $ctr1b @ AES block 1 - result + fmov $ctr1d, $ctr96_b64x @ CTR block 5 + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5 + + add $rctr32w, $rctr32w, #1 @ CTR block 5 + add $input_ptr, $input_ptr, #64 @ AES input_ptr update + fmov $ctr1.d[1], $ctr32x @ CTR block 5 + + fmov $ctr_t3d, $input_l3 @ AES block 3 - mov low + rev $ctr32w, $rctr32w @ CTR block 6 + st1 { $res0b}, [$output_ptr], #16 @ AES block 0 - store result + + fmov $ctr_t3.d[1], $input_h3 @ AES block 3 - mov high + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6 + + add $rctr32w, $rctr32w, #1 @ CTR block 6 + eor $res2b, $ctr_t2b, $ctr2b @ AES block 2 - result + st1 { $res1b}, [$output_ptr], #16 @ AES block 1 - store result + + fmov $ctr2d, $ctr96_b64x @ CTR block 6 + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + fmov $ctr2.d[1], $ctr32x @ CTR block 6 + rev $ctr32w, $rctr32w @ CTR block 7 + st1 { $res2b}, [$output_ptr], #16 @ AES block 2 - store result + + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 7 + + eor $res3b, $ctr_t3b, $ctr3b @ AES block 3 - result + st1 { $res3b}, [$output_ptr], #16 @ AES block 3 - store result + b.ge .L128_enc_prepretail @ do prepretail + + .L128_enc_main_loop: @ main loop start + ldp $input_l3, $input_h3, [$input_ptr, #48] @ AES block 4k+3 - load plaintext + rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free) + rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free) + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0 + fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3 + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free) + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0 + add $rctr32w, $rctr32w, #1 @ CTR block 4k+3 + fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0 + mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1 + mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1 + eor $res0b, $res0b, $acc_lb @ PRE 1 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0 + eor $input_h3, $input_h3, $rk10_h @ AES block 4k+3 - round 10 high + + pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high + eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid + ldp $input_l0, $input_h0, [$input_ptr, #0] @ AES block 4k+4 - load plaintext + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1 + rev $ctr32w, $rctr32w @ CTR block 4k+8 + + eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid + mov $t0d, $res0.d[1] @ GHASH block 4k - mid + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+8 + + pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high + add $rctr32w, $rctr32w, #1 @ CTR block 4k+8 + mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2 + + pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low + eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3 + eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high + + pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low + + pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid + rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free) + + pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid + + pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low + ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid + + pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high + eor $input_h0, $input_h0, $rk10_h @ AES block 4k+4 - round 10 high + + eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid + mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1 + eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2 + eor $input_l0, $input_l0, $rk10_l @ AES block 4k+4 - round 10 low + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3 + eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid + + pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3 + eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high + + pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid + + pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low + movi $mod_constant.8b, #0xc2 + + pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid + eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2 + shl $mod_constantd, $mod_constantd, #56 @ mod_constant + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4 + eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5 + ldp $input_l1, $input_h1, [$input_ptr, #16] @ AES block 4k+5 - load plaintext + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3 + eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5 + ldp $input_l2, $input_h2, [$input_ptr, #32] @ AES block 4k+6 - load plaintext + + pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4 + eor $input_l1, $input_l1, $rk10_l @ AES block 4k+5 - round 10 low + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4 + eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6 + eor $input_l3, $input_l3, $rk10_l @ AES block 4k+3 - round 10 low + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5 + eor $t9.16b, $acc_lb, $acc_hb @ MODULO - karatsuba tidy up + + fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6 + fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high + + add $input_ptr, $input_ptr, #64 @ AES input_ptr update + fmov $ctr_t3d, $input_l3 @ AES block 4k+3 - mov low + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5 + fmov $ctr_t1d, $input_l1 @ AES block 4k+5 - mov low + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7 + eor $acc_mb, $acc_mb, $t9.16b @ MODULO - karatsuba tidy up + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6 + eor $input_h1, $input_h1, $rk10_h @ AES block 4k+5 - round 10 high + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7 + fmov $ctr_t1.d[1], $input_h1 @ AES block 4k+5 - mov high + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8 + fmov $ctr_t3.d[1], $input_h3 @ AES block 4k+3 - mov high + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6 + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8 + eor $acc_mb, $acc_mb, $mod_t.16b @ MODULO - fold into mid + + aese $ctr0b, $rk9 @ AES block 4k+4 - round 9 + eor $input_l2, $input_l2, $rk10_l @ AES block 4k+6 - round 10 low + eor $input_h2, $input_h2, $rk10_h @ AES block 4k+6 - round 10 high + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7 + fmov $ctr_t2d, $input_l2 @ AES block 4k+6 - mov low + + aese $ctr1b, $rk9 @ AES block 4k+5 - round 9 + fmov $ctr_t2.d[1], $input_h2 @ AES block 4k+6 - mov high + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7 + eor $res0b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result + + fmov $ctr0d, $ctr96_b64x @ CTR block 4k+8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8 + + fmov $ctr0.d[1], $ctr32x @ CTR block 4k+8 + rev $ctr32w, $rctr32w @ CTR block 4k+9 + eor $acc_mb, $acc_mb, $acc_hb @ MODULO - fold into mid + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8 + eor $res1b, $ctr_t1b, $ctr1b @ AES block 4k+5 - result + + add $rctr32w, $rctr32w, #1 @ CTR block 4k+9 + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+9 + fmov $ctr1d, $ctr96_b64x @ CTR block 4k+9 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + fmov $ctr1.d[1], $ctr32x @ CTR block 4k+9 + rev $ctr32w, $rctr32w @ CTR block 4k+10 + + aese $ctr2b, $rk9 @ AES block 4k+6 - round 9 + st1 { $res0b}, [$output_ptr], #16 @ AES block 4k+4 - store result + eor $res2b, $ctr_t2b, $ctr2b @ AES block 4k+6 - result + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+10 + + aese $ctr3b, $rk9 @ AES block 4k+7 - round 9 + add $rctr32w, $rctr32w, #1 @ CTR block 4k+10 + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + fmov $ctr2d, $ctr96_b64x @ CTR block 4k+10 + + eor $acc_lb, $acc_lb, $acc_hb @ MODULO - fold into low + st1 { $res1b}, [$output_ptr], #16 @ AES block 4k+5 - store result + + fmov $ctr2.d[1], $ctr32x @ CTR block 4k+10 + st1 { $res2b}, [$output_ptr], #16 @ AES block 4k+6 - store result + rev $ctr32w, $rctr32w @ CTR block 4k+11 + + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4k+11 + eor $res3b, $ctr_t3b, $ctr3b @ AES block 4k+3 - result + + eor $acc_lb, $acc_lb, $acc_mb @ MODULO - fold into low + st1 { $res3b}, [$output_ptr], #16 @ AES block 4k+3 - store result + b.lt .L128_enc_main_loop + + .L128_enc_prepretail: @ PREPRETAIL + rev64 $res0b, $res0b @ GHASH block 4k (only t0 is free) + fmov $ctr3d, $ctr96_b64x @ CTR block 4k+3 + rev64 $res1b, $res1b @ GHASH block 4k+1 (t0 and t1 free) + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + add $rctr32w, $rctr32w, #1 @ CTR block 4k+3 + fmov $ctr3.d[1], $ctr32x @ CTR block 4k+3 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 0 + rev64 $res2b, $res2b @ GHASH block 4k+2 (t0, t1, and t2 free) + + pmull $t2.1q, $res1.1d, $h3.1d @ GHASH block 4k+1 - low + + rev64 $res3b, $res3b @ GHASH block 4k+3 (t0, t1, t2 and t3 free) + eor $res0b, $res0b, $acc_lb @ PRE 1 + + pmull2 $t1.1q, $res1.2d, $h3.2d @ GHASH block 4k+1 - high + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 0 + mov $t3d, $res1.d[1] @ GHASH block 4k+1 - mid + + pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH block 4k - low + mov $t0d, $res0.d[1] @ GHASH block 4k - mid + + mov $t6d, $res2.d[1] @ GHASH block 4k+2 - mid + mov $acc_md, $h34k.d[1] @ GHASH block 4k - mid + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 1 + eor $t3.8b, $t3.8b, $res1.8b @ GHASH block 4k+1 - mid + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH block 4k - mid + + pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH block 4k - high + eor $t6.8b, $t6.8b, $res2.8b @ GHASH block 4k+2 - mid + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 1 + + pmull $t3.1q, $t3.1d, $h34k.1d @ GHASH block 4k+1 - mid + eor $acc_lb, $acc_lb, $t2.16b @ GHASH block 4k+1 - low + + pmull $acc_m.1q, $t0.1d, $acc_m.1d @ GHASH block 4k - mid + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 0 + ins $t6.d[1], $t6.d[0] @ GHASH block 4k+2 - mid + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 0 + + eor $acc_mb, $acc_mb, $t3.16b @ GHASH block 4k+1 - mid + mov $t9d, $res3.d[1] @ GHASH block 4k+3 - mid + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 1 + eor $acc_hb, $acc_hb, $t1.16b @ GHASH block 4k+1 - high + + pmull2 $t6.1q, $t6.2d, $h12k.2d @ GHASH block 4k+2 - mid + + pmull2 $t4.1q, $res2.2d, $h2.2d @ GHASH block 4k+2 - high + eor $t9.8b, $t9.8b, $res3.8b @ GHASH block 4k+3 - mid + + pmull2 $t7.1q, $res3.2d, $h1.2d @ GHASH block 4k+3 - high + + pmull $t5.1q, $res2.1d, $h2.1d @ GHASH block 4k+2 - low + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 1 + eor $acc_hb, $acc_hb, $t4.16b @ GHASH block 4k+2 - high + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 2 + + pmull $t8.1q, $res3.1d, $h1.1d @ GHASH block 4k+3 - low + movi $mod_constant.8b, #0xc2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 2 + eor $acc_lb, $acc_lb, $t5.16b @ GHASH block 4k+2 - low + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 2 + + pmull $t9.1q, $t9.1d, $h12k.1d @ GHASH block 4k+3 - mid + eor $acc_mb, $acc_mb, $t6.16b @ GHASH block 4k+2 - mid + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 3 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 2 + eor $acc_hb, $acc_hb, $t7.16b @ GHASH block 4k+3 - high + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 3 + + eor $acc_mb, $acc_mb, $t9.16b @ GHASH block 4k+3 - mid + shl $mod_constantd, $mod_constantd, #56 @ mod_constant + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 3 + eor $acc_lb, $acc_lb, $t8.16b @ GHASH block 4k+3 - low + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 4 + + pmull $t1.1q, $acc_h.1d, $mod_constant.1d + eor $acc_mb, $acc_mb, $acc_hb @ karatsuba tidy up + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 5 + ext $acc_hb, $acc_hb, $acc_hb, #8 + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 3 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 4 + eor $acc_mb, $acc_mb, $acc_lb + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 6 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 4 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 5 + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 5 + eor $acc_mb, $acc_mb, $t1.16b + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 5 + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 6 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 6 + eor $acc_mb, $acc_mb, $acc_hb + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 7 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 7 + + pmull $t1.1q, $acc_m.1d, $mod_constant.1d + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 7 + ext $acc_mb, $acc_mb, $acc_mb, #8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 4k+7 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 4k+4 - round 8 + eor $acc_lb, $acc_lb, $t1.16b + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 4k+5 - round 8 + + aese $ctr3b, $rk9 @ AES block 4k+7 - round 9 + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 4k+6 - round 8 + + aese $ctr0b, $rk9 @ AES block 4k+4 - round 9 + + aese $ctr1b, $rk9 @ AES block 4k+5 - round 9 + eor $acc_lb, $acc_lb, $acc_mb + + aese $ctr2b, $rk9 @ AES block 4k+6 - round 9 + .L128_enc_tail: @ TAIL + + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES block 4k+4 - load plaintext + + cmp $main_end_input_ptr, #48 + + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + eor $input_l0, $input_l0, $rk10_l @ AES block 4k+4 - round 10 low + eor $input_h0, $input_h0, $rk10_h @ AES block 4k+4 - round 10 high + + fmov $ctr_t0d, $input_l0 @ AES block 4k+4 - mov low + + fmov $ctr_t0.d[1], $input_h0 @ AES block 4k+4 - mov high + + eor $res1b, $ctr_t0b, $ctr0b @ AES block 4k+4 - result + + b.gt .L128_enc_blocks_more_than_3 + + sub $rctr32w, $rctr32w, #1 + movi $acc_l.8b, #0 + mov $ctr3b, $ctr2b + + cmp $main_end_input_ptr, #32 + mov $ctr2b, $ctr1b + movi $acc_h.8b, #0 + + movi $acc_m.8b, #0 + b.gt .L128_enc_blocks_more_than_2 + + mov $ctr3b, $ctr1b + cmp $main_end_input_ptr, #16 + + sub $rctr32w, $rctr32w, #1 + b.gt .L128_enc_blocks_more_than_1 + + sub $rctr32w, $rctr32w, #1 + b .L128_enc_blocks_less_than_1 + .L128_enc_blocks_more_than_3: @ blocks left > 3 + st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result + + ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-2 block - load input low & high + + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + eor $input_h0, $input_h0, $rk10_h @ AES final-2 block - round 10 high + eor $input_l0, $input_l0, $rk10_l @ AES final-2 block - round 10 low + + fmov $res1d, $input_l0 @ AES final-2 block - mov low + + movi $t0.8b, #0 @ suppress further partial tag feed in + fmov $res1.d[1], $input_h0 @ AES final-2 block - mov high + + pmull $acc_l.1q, $res0.1d, $h4.1d @ GHASH final-3 block - low + mov $rk4d, $res0.d[1] @ GHASH final-3 block - mid + + pmull2 $acc_h.1q, $res0.2d, $h4.2d @ GHASH final-3 block - high + + mov $acc_md, $h34k.d[1] @ GHASH final-3 block - mid + + eor $res1b, $res1b, $ctr1b @ AES final-2 block - result + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-3 block - mid + .L128_enc_blocks_more_than_2: @ blocks left > 2 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result + + rev64 $res0b, $res1b @ GHASH final-2 block + ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final-1 block - load input low & high + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + eor $input_l0, $input_l0, $rk10_l @ AES final-1 block - round 10 low + + fmov $res1d, $input_l0 @ AES final-1 block - mov low + eor $input_h0, $input_h0, $rk10_h @ AES final-1 block - round 10 high + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + fmov $res1.d[1], $input_h0 @ AES final-1 block - mov high + + mov $rk4d, $res0.d[1] @ GHASH final-2 block - mid + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + eor $res1b, $res1b, $ctr2b @ AES final-1 block - result + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + .L128_enc_blocks_more_than_1: @ blocks left > 1 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result + + rev64 $res0b, $res1b @ GHASH final-1 block + ldp $input_l0, $input_h0, [$input_ptr], #16 @ AES final block - load input low & high + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + eor $input_h0, $input_h0, $rk10_h @ AES final block - round 10 high + eor $input_l0, $input_l0, $rk10_l @ AES final block - round 10 low + + fmov $res1d, $input_l0 @ AES final block - mov low + + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + fmov $res1.d[1], $input_h0 @ AES final block - mov high + + mov $rk4d, $res0.d[1] @ GHASH final-1 block - mid + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + eor $res1b, $res1b, $ctr3b @ AES final block - result + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + .L128_enc_blocks_less_than_1: @ blocks left <= 1 + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + mvn $rk10_l, xzr @ rk10_l = 0xffffffffffffffff + + mvn $rk10_h, xzr @ rk10_h = 0xffffffffffffffff + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $rk10_h, $rk10_h, $bit_length @ rk10_h is mask for top 64b of last block + cmp $bit_length, #64 + + csel $input_l0, $rk10_l, $rk10_h, lt + csel $input_h0, $rk10_h, xzr, lt + + fmov $ctr0d, $input_l0 @ ctr0b is mask for last block + + fmov $ctr0.d[1], $input_h0 + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + mov $t0d, $res0.d[1] @ GHASH final block - mid + + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + rev $ctr32w, $rctr32w + + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + movi $mod_constant.8b, #0xc2 + + eor $t9.16b, $acc_lb, $acc_hb @ MODULO - karatsuba tidy up + + shl $mod_constantd, $mod_constantd, #56 @ mod_constant + + eor $acc_mb, $acc_mb, $t9.16b @ MODULO - karatsuba tidy up + + pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + eor $acc_mb, $acc_mb, $mod_t.16b @ MODULO - fold into mid + + eor $acc_mb, $acc_mb, $acc_hb @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + eor $acc_lb, $acc_lb, $acc_hb @ MODULO - fold into low + st1 { $res1b}, [$output_ptr] @ store all 16B + + str $ctr32w, [$counter, #12] @ store the updated counter + + eor $acc_lb, $acc_lb, $acc_mb @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + mov x0, $len + st1 { $acc_l.16b }, [$current_tag] + ldp x21, x22, [sp, #16] + ldp x23, x24, [sp, #32] + ldp d8, d9, [sp, #48] + ldp d10, d11, [sp, #64] + ldp d12, d13, [sp, #80] + ldp d14, d15, [sp, #96] + ldp x19, x20, [sp], #112 + ret + +.L128_enc_ret: + mov w0, #0x0 + ret +.size aes_gcm_enc_128_kernel,.-aes_gcm_enc_128_kernel +___ + +######################################################################################### +# size_t aes_gcm_dec_128_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# const void *key, +# unsigned char ivec[16], +# u64 *Xi); +# +$code.=<<___; +.global aes_gcm_dec_128_kernel +.type aes_gcm_dec_128_kernel,%function +.align 4 +aes_gcm_dec_128_kernel: + cbz x1, .L128_dec_ret + stp x19, x20, [sp, #-112]! + mov x16, x4 + mov x8, x5 + stp x21, x22, [sp, #16] + stp x23, x24, [sp, #32] + stp d8, d9, [sp, #48] + stp d10, d11, [sp, #64] + stp d12, d13, [sp, #80] + stp d14, d15, [sp, #96] + + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + mov $len, $main_end_input_ptr + ldp $ctr96_b64x, $ctr96_t32x, [$counter] @ ctr96_b64, ctr96_t32 + + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + ldr $rk0q, [$cc, #0] @ load rk0 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + ld1 { $ctr0b}, [$counter] @ special case vector load initial counter so we can start first AES block as quickly as possible + + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2b, $h2b, $h2b, #8 + + lsr $rctr32x, $ctr96_t32x, #32 + fmov $ctr2d, $ctr96_b64x @ CTR block 2 + + ldr $rk1q, [$cc, #16] @ load rk1 + orr $ctr96_t32w, $ctr96_t32w, $ctr96_t32w + rev $rctr32w, $rctr32w @ rev_ctr32 + + fmov $ctr1d, $ctr96_b64x @ CTR block 1 + add $rctr32w, $rctr32w, #1 @ increment rev_ctr32 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + rev $ctr32w, $rctr32w @ CTR block 1 + + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 1 + ldr $rk2q, [$cc, #32] @ load rk2 + add $rctr32w, $rctr32w, #1 @ CTR block 1 + + fmov $ctr1.d[1], $ctr32x @ CTR block 1 + rev $ctr32w, $rctr32w @ CTR block 2 + add $rctr32w, $rctr32w, #1 @ CTR block 2 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 2 + + fmov $ctr2.d[1], $ctr32x @ CTR block 2 + rev $ctr32w, $rctr32w @ CTR block 3 + + fmov $ctr3d, $ctr96_b64x @ CTR block 3 + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 3 + add $rctr32w, $rctr32w, #1 @ CTR block 3 + + fmov $ctr3.d[1], $ctr32x @ CTR block 3 + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + ldr $rk3q, [$cc, #48] @ load rk3 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + ldr $rk6q, [$cc, #96] @ load rk6 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + ldr $rk7q, [$cc, #112] @ load rk7 + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + ldr $rk4q, [$cc, #64] @ load rk4 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + ldp $rk10_l, $rk10_h, [$cc, #160] @ load rk10 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + ldr $rk5q, [$cc, #80] @ load rk5 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + ldr $rk9q, [$cc, #144] @ load rk9 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3b, $h3b, $h3b, #8 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + ldr $rk8q, [$cc, #128] @ load rk8 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1b, $h1b, $h1b, #8 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + trn1 $t0.2d, $h1.2d, $h2.2d @ h2h | h1h + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4b, $h4b, $h4b, #8 + trn2 $h12k.2d, $h1.2d, $h2.2d @ h2l | h1l + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + eor $h12k.16b, $h12k.16b, $t0.16b @ h2k | h1k + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + trn2 $h34k.2d, $h3.2d, $h4.2d @ h4l | h3l + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + trn1 $acc_h.2d, $h3.2d, $h4.2d @ h4h | h3h + + aese $ctr2b, $rk9 @ AES block 2 - round 9 + + aese $ctr3b, $rk9 @ AES block 3 - round 9 + + aese $ctr0b, $rk9 @ AES block 0 - round 9 + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 4 blocks + + aese $ctr1b, $rk9 @ AES block 1 - round 9 + eor $h34k.16b, $h34k.16b, $acc_h.16b @ h4k | h3k + b.ge .L128_dec_tail @ handle tail + + ldr $res1q, [$input_ptr, #16] @ AES block 1 - load ciphertext + + ldr $res0q, [$input_ptr, #0] @ AES block 0 - load ciphertext + + eor $ctr1b, $res1b, $ctr1b @ AES block 1 - result + ldr $res2q, [$input_ptr, #32] @ AES block 2 - load ciphertext + + eor $ctr0b, $res0b, $ctr0b @ AES block 0 - result + rev64 $res0b, $res0b @ GHASH block 0 + rev $ctr32w, $rctr32w @ CTR block 4 + + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 4 + add $rctr32w, $rctr32w, #1 @ CTR block 4 + ldr $res3q, [$input_ptr, #48] @ AES block 3 - load ciphertext + + rev64 $res1b, $res1b @ GHASH block 1 + add $input_ptr, $input_ptr, #64 @ AES input_ptr update + mov $output_l1, $ctr1.d[0] @ AES block 1 - mov low + + mov $output_h1, $ctr1.d[1] @ AES block 1 - mov high + + mov $output_l0, $ctr0.d[0] @ AES block 0 - mov low + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + mov $output_h0, $ctr0.d[1] @ AES block 0 - mov high + + fmov $ctr0d, $ctr96_b64x @ CTR block 4 + + fmov $ctr0.d[1], $ctr32x @ CTR block 4 + rev $ctr32w, $rctr32w @ CTR block 5 + eor $output_l1, $output_l1, $rk10_l @ AES block 1 - round 10 low + + fmov $ctr1d, $ctr96_b64x @ CTR block 5 + add $rctr32w, $rctr32w, #1 @ CTR block 5 + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 5 + + fmov $ctr1.d[1], $ctr32x @ CTR block 5 + rev $ctr32w, $rctr32w @ CTR block 6 + add $rctr32w, $rctr32w, #1 @ CTR block 6 + + orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 @ CTR block 6 + + eor $output_h1, $output_h1, $rk10_h @ AES block 1 - round 10 high + eor $output_l0, $output_l0, $rk10_l @ AES block 0 - round 10 low + eor $ctr2b, $res2b, $ctr2b @ AES block 2 - result + + eor $output_h0, $output_h0, $rk10_h @ AES block 0 - round 10 high + stp $output_l0, $output_h0, [$output_ptr], #16 @ AES block 0 - store result + + stp $output_l1, $output_h1, [$output_ptr], #16 @ AES block 1 - store result + b.ge .L128_dec_prepretail