e_padlock: add CTR mode.

2 files changed, 75 insertions, 13 deletions

diff --git a/engines/asm/e_padlock-x86.pl b/engines/asm/e_padlock-x86.pl
index df8f56b521..61e91d889f 100644
--- a/engines/asm/e_padlock-x86.pl
+++ b/engines/asm/e_padlock-x86.pl
@@ -183,7 +183,7 @@ my ($mode,$opcode) = @_;
 &set_label("${mode}_pic_point");
 	&lea	($ctx,&DWP(16,$ctx));	# control word
 	&xor	("eax","eax");
-					if ($mode eq "ctr16") {
+					if ($mode eq "ctr32") {
 	&movq	("mm0",&QWP(-16,$ctx));	# load [upper part of] counter
 					} else {
 	&xor	("ebx","ebx");
@@ -216,7 +216,7 @@ my ($mode,$opcode) = @_;
 	&mov	(&DWP(8,"ebp"),$len);
 	&mov	($len,$chunk);
 	&mov	(&DWP(12,"ebp"),$chunk);	# chunk
-						if ($mode eq "ctr16") {
+						if ($mode eq "ctr32") {
 	&mov	("ecx",&DWP(-4,$ctx));
 	&xor	($out,$out);
 	&mov	("eax",&DWP(-8,$ctx));		# borrow $len
@@ -257,7 +257,7 @@ my ($mode,$opcode) = @_;
 						}
 	&mov	($out,&DWP(0,"ebp"));		# restore parameters
 	&mov	($chunk,&DWP(12,"ebp"));
-						if ($mode eq "ctr16") {
+						if ($mode eq "ctr32") {
 	&mov	($inp,&DWP(4,"ebp"));
 	&xor	($len,$len);
 &set_label("${mode}_xor");
@@ -284,7 +284,7 @@ my ($mode,$opcode) = @_;
 	&sub	($len,$chunk);
 	&mov	($chunk,$PADLOCK_CHUNK);
 	&jnz	(&label("${mode}_loop"));
-						if ($mode ne "ctr16") {
+						if ($mode ne "ctr32") {
 	&test	($out,0x0f);			# out_misaligned
 	&jz	(&label("${mode}_done"));
 						}
@@ -296,7 +296,7 @@ my ($mode,$opcode) = @_;
 	&data_byte(0xf3,0xab);			# rep stosl
 &set_label("${mode}_done");
 	&lea	("esp",&DWP(24,"ebp"));
-						if ($mode ne "ctr16") {
+						if ($mode ne "ctr32") {
 	&jmp	(&label("${mode}_exit"));
 
 &set_label("${mode}_aligned",16);
@@ -311,7 +311,7 @@ my ($mode,$opcode) = @_;
 &set_label("${mode}_exit");			}
 	&mov	("eax",1);
 	&lea	("esp",&DWP(4,"esp"));		# popf
-	&emms	()				if ($mode eq "ctr16");
+	&emms	()				if ($mode eq "ctr32");
 &set_label("${mode}_abort");
 &function_end("padlock_${mode}_encrypt");
 }
@@ -320,10 +320,11 @@ my ($mode,$opcode) = @_;
 &generate_mode("cbc",0xd0);
 &generate_mode("cfb",0xe0);
 &generate_mode("ofb",0xe8);
-&generate_mode("ctr16",0xc8);	# yes, it implements own ctr with ecb opcode,
-				# because hardware ctr was introduced later
-				# and even has errata on certain CPU stepping.
-				# own implementation *always* works...
+&generate_mode("ctr32",0xc8);	# yes, it implements own CTR with ECB opcode,
+				# because hardware CTR was introduced later
+				# and even has errata on certain C7 stepping.
+				# own implementation *always* works, though
+				# ~15% slower than dedicated hardware...
 
 &function_begin_B("padlock_xstore");
 	&push	("edi");
diff --git a/engines/asm/e_padlock-x86_64.pl b/engines/asm/e_padlock-x86_64.pl
index d5cc5fe00e..13c371be67 100644
--- a/engines/asm/e_padlock-x86_64.pl
+++ b/engines/asm/e_padlock-x86_64.pl
@@ -9,7 +9,8 @@
 
 # September 2011
 #
-# Assembler helpers for Padlock engine.
+# Assembler helpers for Padlock engine. See even e_padlock-x86.pl for
+# details.
 
 $flavour = shift;
 $output  = shift;
@@ -26,7 +27,7 @@ open STDOUT,"| $^X $xlate $flavour $output";
 
 $code=".text\n";
 
-$PADLOCK_CHUNK=512;	# Must be a power of 2 larger than 16
+$PADLOCK_CHUNK=512;	# Must be a power of 2 between 32 and 2^20
 
 $ctx="%rdx";
 $out="%rdi";
@@ -234,9 +235,23 @@ padlock_${mode}_encrypt:
 	neg	%rax
 	and	\$$PADLOCK_CHUNK-1,$chunk	# chunk%=PADLOCK_CHUNK
 	lea	(%rax,%rbp),%rsp
+___
+$code.=<<___				if ($mode eq "ctr32");
+	mov	-4($ctx),%eax		# pull 32-bit counter
+	bswap	%eax
+	neg	%eax
+	and	\$`$PADLOCK_CHUNK/16-1`,%eax
+	jz	.L${mode}_loop
+	shl	\$4,%eax
+	cmp	%rax,$len
+	cmova	%rax,$chunk		# don't let counter cross PADLOCK_CHUNK
+___
+$code.=<<___;
 	jmp	.L${mode}_loop
 .align	16
 .L${mode}_loop:
+	cmp	$len,$chunk		# ctr32 artefact
+	cmova	$len,$chunk		# ctr32 artefact
 	mov	$out,%r8		# save parameters
 	mov	$inp,%r9
 	mov	$len,%r10
@@ -261,6 +276,16 @@ $code.=<<___				if ($mode !~ /ecb|ctr/);
 	movdqa	(%rax),%xmm0
 	movdqa	%xmm0,-16($ctx)		# copy [or refresh] iv
 ___
+$code.=<<___				if ($mode eq "ctr32");
+	mov	-4($ctx),%eax		# pull 32-bit counter
+	test	\$0xffff0000,%eax
+	jnz	.L${mode}_no_corr
+	bswap	%eax
+	add	\$0x10000,%eax
+	bswap	%eax
+	mov	%eax,-4($ctx)
+.L${mode}_no_corr:
+___
 $code.=<<___;
 	mov	%r8,$out		# restore paramters
 	mov	%r11,$chunk
@@ -295,6 +320,29 @@ $code.=<<___;
 
 .align	16
 .L${mode}_aligned:
+___
+$code.=<<___				if ($mode eq "ctr32");
+	mov	-4($ctx),%eax		# pull 32-bit counter
+	mov	\$`16*0x10000`,$chunk
+	bswap	%eax
+	cmp	$len,$chunk
+	cmova	$len,$chunk
+	neg	%eax
+	and	\$0xffff,%eax
+	jz	.L${mode}_aligned_loop
+	shl	\$4,%eax
+	cmp	%rax,$len
+	cmova	%rax,$chunk		# don't let counter cross 2^16
+	jmp	.L${mode}_aligned_loop
+.align	16
+.L${mode}_aligned_loop:
+	cmp	$len,$chunk
+	cmova	$len,$chunk
+	mov	$len,%r10		# save parameters
+	mov	$chunk,$len
+	mov	$chunk,%r11
+___
+$code.=<<___;
 	lea	-16($ctx),%rax		# ivp
 	lea	16($ctx),%rbx		# key
 	shr	\$4,$len		# len/=AES_BLOCK_SIZE
@@ -304,6 +352,19 @@ $code.=<<___				if ($mode !~ /ecb|ctr/);
 	movdqa	(%rax),%xmm0
 	movdqa	%xmm0,-16($ctx)		# copy [or refresh] iv
 ___
+$code.=<<___				if ($mode eq "ctr32");
+	mov	-4($ctx),%eax		# pull 32-bit counter
+	bswap	%eax
+	add	\$0x10000,%eax
+	bswap	%eax
+	mov	%eax,-4($ctx)
+
+	mov	%r11,$chunk		# restore paramters
+	mov	%r10,$len
+	sub	$chunk,$len
+	mov	\$`16*0x10000`,$chunk
+	jnz	.L${mode}_aligned_loop
+___
 $code.=<<___;
 .L${mode}_exit:
 	mov	\$1,%eax
@@ -320,7 +381,7 @@ ___
 &generate_mode("cbc",0xd0);
 &generate_mode("cfb",0xe0);
 &generate_mode("ofb",0xe8);
-&generate_mode("ctr16",0xd8);
+&generate_mode("ctr32",0xd8);	# all 64-bit CPUs have working CTR...
 
 $code.=<<___;
 .asciz	"VIA Padlock x86_64 module, CRYPTOGAMS by <appro\@openssl.org>"