1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
|
#! /usr/bin/env perl
# Copyright 2011-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
# September 2011
#
# Assembler helpers for Padlock engine. See even e_padlock-x86.pl for
# details.
$flavour = shift;
$output = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../crypto/perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;
$code=".text\n";
%PADLOCK_PREFETCH=(ecb=>128, cbc=>64, ctr32=>32); # prefetch errata
$PADLOCK_CHUNK=512; # Must be a power of 2 between 32 and 2^20
$ctx="%rdx";
$out="%rdi";
$inp="%rsi";
$len="%rcx";
$chunk="%rbx";
($arg1,$arg2,$arg3,$arg4)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order
("%rdi","%rsi","%rdx","%rcx"); # Unix order
$code.=<<___;
.globl padlock_capability
.type padlock_capability,\@abi-omnipotent
.align 16
padlock_capability:
mov %rbx,%r8
xor %eax,%eax
cpuid
xor %eax,%eax
cmp \$`"0x".unpack("H*",'tneC')`,%ebx
jne .Lnoluck
cmp \$`"0x".unpack("H*",'Hrua')`,%edx
jne .Lnoluck
cmp \$`"0x".unpack("H*",'slua')`,%ecx
jne .Lnoluck
mov \$0xC0000000,%eax
cpuid
mov %eax,%edx
xor %eax,%eax
cmp \$0xC0000001,%edx
jb .Lnoluck
mov \$0xC0000001,%eax
cpuid
mov %edx,%eax
and \$0xffffffef,%eax
or \$0x10,%eax # set Nano bit#4
.Lnoluck:
mov %r8,%rbx
ret
.size padlock_capability,.-padlock_capability
.globl padlock_key_bswap
.type padlock_key_bswap,\@abi-omnipotent,0
.align 16
padlock_key_bswap:
mov 240($arg1),%edx
.Lbswap_loop:
mov ($arg1),%eax
bswap %eax
mov %eax,($arg1)
lea 4($arg1),$arg1
sub \$1,%edx
jnz .Lbswap_loop
ret
.size padlock_key_bswap,.-padlock_key_bswap
.globl padlock_verify_context
.type padlock_verify_context,\@abi-omnipotent
.align 16
padlock_verify_context:
mov $arg1,$ctx
pushf
lea .Lpadlock_saved_context(%rip),%rax
call _padlock_verify_ctx
lea 8(%rsp),%rsp
ret
.size padlock_verify_context,.-padlock_verify_context
.type _padlock_verify_ctx,\@abi-omnipotent
.align 16
_padlock_verify_ctx:
mov 8(%rsp),%r8
bt \$30,%r8
jnc .Lverified
cmp (%rax),$ctx
je .Lverified
pushf
popf
.Lverified:
mov $ctx,(%rax)
ret
.size _padlock_verify_ctx,.-_padlock_verify_ctx
.globl padlock_reload_key
.type padlock_reload_key,\@abi-omnipotent
.align 16
padlock_reload_key:
pushf
popf
ret
.size padlock_reload_key,.-padlock_reload_key
.globl padlock_aes_block
.type padlock_aes_block,\@function,3
.align 16
padlock_aes_block:
mov %rbx,%r8
mov \$1,$len
lea 32($ctx),%rbx # key
lea 16($ctx),$ctx # control word
.byte 0xf3,0x0f,0xa7,0xc8 # rep xcryptecb
mov %r8,%rbx
ret
.size padlock_aes_block,.-padlock_aes_block
.globl padlock_xstore
.type padlock_xstore,\@function,2
.align 16
padlock_xstore:
mov %esi,%edx
.byte 0x0f,0xa7,0xc0 # xstore
ret
.size padlock_xstore,.-padlock_xstore
.globl padlock_sha1_oneshot
.type padlock_sha1_oneshot,\@function,3
.align 16
padlock_sha1_oneshot:
mov %rdx,%rcx
mov %rdi,%rdx # put aside %rdi
movups (%rdi),%xmm0 # copy-in context
sub \$128+8,%rsp
mov 16(%rdi),%eax
movaps %xmm0,(%rsp)
mov %rsp,%rdi
mov %eax,16(%rsp)
xor %rax,%rax
.byte 0xf3,0x0f,0xa6,0xc8 # rep xsha1
movaps (%rsp),%xmm0
mov 16(%rsp),%eax
add \$128+8,%rsp
movups %xmm0,(%rdx) # copy-out context
mov %eax,16(%rdx)
ret
.size padlock_sha1_oneshot,.-padlock_sha1_oneshot
.globl padlock_sha1_blocks
.type padlock_sha1_blocks,\@function,3
.align 16
padlock_sha1_blocks:
mov %rdx,%rcx
mov %rdi,%rdx # put aside %rdi
movups (%rdi),%xmm0 # copy-in context
sub \$128+8,%rsp
mov 16(%rdi),%eax
movaps %xmm0,(%rsp)
mov %rsp,%rdi
mov %eax,16(%rsp)
mov \$-1,%rax
.byte 0xf3,0x0f,0xa6,0xc8 # rep xsha1
movaps (%rsp),%xmm0
mov 16(%rsp),%eax
add \$128+8,%rsp
movups %xmm0,(%rdx) # copy-out context
mov %eax,16(%rdx)
ret
.size padlock_sha1_blocks,.-padlock_sha1_blocks
.globl padlock_sha256_oneshot
.type padlock_sha256_oneshot,\@function,3
.align 16
padlock_sha256_oneshot:
mov %rdx,%rcx
mov %rdi,%rdx # put aside %rdi
movups (%rdi),%xmm0 # copy-in context
sub \$128+8,%rsp
movups 16(%rdi),%xmm1
movaps %xmm0,(%rsp)
mov %rsp,%rdi
movaps %xmm1,16(%rsp)
xor %rax,%rax
.byte 0xf3,0x0f,0xa6,0xd0 # rep xsha256
movaps (%rsp),%xmm0
movaps 16(%rsp),%xmm1
add \$128+8,%rsp
movups %xmm0,(%rdx) # copy-out context
movups %xmm1,16(%rdx)
ret
.size padlock_sha256_oneshot,.-padlock_sha256_oneshot
.globl padlock_sha256_blocks
.type padlock_sha256_blocks,\@function,3
.align 16
padlock_sha256_blocks:
mov %rdx,%rcx
mov %rdi,%rdx # put aside %rdi
movups (%rdi),%xmm0 # copy-in context
sub \$128+8,%rsp
movups 16(%rdi),%xmm1
movaps %xmm0,(%rsp)
mov %rsp,%rdi
movaps %xmm1,16(%rsp)
mov \$-1,%rax
.byte 0xf3,0x0f,0xa6,0xd0 # rep xsha256
movaps (%rsp),%xmm0
movaps 16(%rsp),%xmm1
add \$128+8,%rsp
movups %xmm0,(%rdx) # copy-out context
movups %xmm1,16(%rdx)
ret
.size padlock_sha256_blocks,.-padlock_sha256_blocks
.globl padlock_sha512_blocks
.type padlock_sha512_blocks,\@function,3
.align 16
padlock_sha512_blocks:
mov %rdx,%rcx
mov %rdi,%rdx # put aside %rdi
movups (%rdi),%xmm0 # copy-in context
sub \$128+8,%rsp
movups 16(%rdi),%xmm1
movups 32(%rdi),%xmm2
movups 48(%rdi),%xmm3
movaps %xmm0,(%rsp)
mov %rsp,%rdi
movaps %xmm1,16(%rsp)
movaps %xmm2,32(%rsp)
movaps %xmm3,48(%rsp)
.byte 0xf3,0x0f,0xa6,0xe0 # rep xha512
movaps (%rsp),%xmm0
movaps 16(%rsp),%xmm1
movaps 32(%rsp),%xmm2
movaps 48(%rsp),%xmm3
add \$128+8,%rsp
movups %xmm0,(%rdx) # copy-out context
movups %xmm1,16(%rdx)
movups %xmm2,32(%rdx)
movups %xmm3,48(%rdx)
ret
.size padlock_sha512_blocks,.-padlock_sha512_blocks
___
sub generate_mode {
my ($mode,$opcode) = @_;
# int padlock_$mode_encrypt(void *out, const void *inp,
# struct padlock_cipher_data *ctx, size_t len);
$code.=<<___;
.globl padlock_${mode}_encrypt
.type padlock_${mode}_encrypt,\@function,4
.align 16
padlock_${mode}_encrypt:
push %rbp
push %rbx
xor %eax,%eax
test \$15,$ctx
jnz .L${mode}_abort
test \$15,$len
jnz .L${mode}_abort
lea .Lpadlock_saved_context(%rip),%rax
pushf
cld
call _padlock_verify_ctx
lea 16($ctx),$ctx # control word
xor %eax,%eax
xor %ebx,%ebx
testl \$`1<<5`,($ctx) # align bit in control word
jnz .L${mode}_aligned
test \$0x0f,$out
setz %al # !out_misaligned
test \$0x0f,$inp
setz %bl # !inp_misaligned
test %ebx,%eax
jnz .L${mode}_aligned
neg %rax
mov \$$PADLOCK_CHUNK,$chunk
not %rax # out_misaligned?-1:0
lea (%rsp),%rbp
cmp $chunk,$len
cmovc $len,$chunk # chunk=len>PADLOCK_CHUNK?PADLOCK_CHUNK:len
and $chunk,%rax # out_misaligned?chunk:0
mov $len,$chunk
neg %rax
and \$$PADLOCK_CHUNK-1,$chunk # chunk%=PADLOCK_CHUNK
lea (%rax,%rbp),%rsp
mov \$$PADLOCK_CHUNK,%rax
cmovz %rax,$chunk # chunk=chunk?:PADLOCK_CHUNK
___
$code.=<<___ if ($mode eq "ctr32");
.L${mode}_reenter:
mov -4($ctx),%eax # pull 32-bit counter
bswap %eax
neg %eax
and \$`$PADLOCK_CHUNK/16-1`,%eax
mov \$$PADLOCK_CHUNK,$chunk
shl \$4,%eax
cmovz $chunk,%rax
cmp %rax,$len
cmova %rax,$chunk # don't let counter cross PADLOCK_CHUNK
cmovbe $len,$chunk
___
$code.=<<___ if ($PADLOCK_PREFETCH{$mode});
cmp $chunk,$len
ja .L${mode}_loop
mov $inp,%rax # check if prefetch crosses page
cmp %rsp,%rbp
cmove $out,%rax
add $len,%rax
neg %rax
and \$0xfff,%rax # distance to page boundary
cmp \$$PADLOCK_PREFETCH{$mode},%rax
mov \$-$PADLOCK_PREFETCH{$mode},%rax
cmovae $chunk,%rax # mask=distance<prefetch?-prefetch:-1
and %rax,$chunk
jz .L${mode}_unaligned_tail
___
$code.=<<___;
jmp .L${mode}_loop
.align 16
.L${mode}_loop:
cmp $len,$chunk # ctr32 artefact
cmova $len,$chunk # ctr32 artefact
mov $out,%r8 # save parameters
mov $inp,%r9
mov $len,%r10
mov $chunk,$len
mov $chunk,%r11
test \$0x0f,$out # out_misaligned
cmovnz %rsp,$out
test \$0x0f,$inp # inp_misaligned
jz .L${mode}_inp_aligned
shr \$3,$len
.byte 0xf3,0x48,0xa5 # rep movsq
sub $chunk,$out
mov $chunk,$len
mov $out,$inp
.L${mode}_inp_aligned:
lea -16($ctx),%rax # ivp
lea 16($ctx),%rbx # key
shr \$4,$len
.byte 0xf3,0x0f,0xa7,$opcode # rep xcrypt*
___
$code.=<<___ if ($mode !~ /ecb|ctr/);
movdqa (%rax),%xmm0
movdqa %xmm0,-16($ctx) # copy [or refresh] iv
___
$code.=<<___ if ($mode eq "ctr32");
mov -4($ctx),%eax # pull 32-bit counter
test \$0xffff0000,%eax
jnz .L${mode}_no_carry
bswap %eax
add \$0x10000,%eax
bswap %eax
mov %eax,-4($ctx)
.L${mode}_no_carry:
___
$code.=<<___;
mov %r8,$out # restore parameters
mov %r11,$chunk
test \$0x0f,$out
jz .L${mode}_out_aligned
mov $chunk,$len
lea (%rsp),$inp
shr \$3,$len
.byte 0xf3,0x48,0xa5 # rep movsq
sub $chunk,$out
.L${mode}_out_aligned:
mov %r9,$inp
mov %r10,$len
add $chunk,$out
add $chunk,$inp
sub $chunk,$len
mov \$$PADLOCK_CHUNK,$chunk
___
if (!$PADLOCK_PREFETCH{$mode}) {
$code.=<<___;
jnz .L${mode}_loop
___
} else {
$code.=<<___;
jz .L${mode}_break
cmp $chunk,$len
jae .L${mode}_loop
___
$code.=<<___ if ($mode eq "ctr32");
mov $len,$chunk
mov $inp,%rax # check if prefetch crosses page
cmp %rsp,%rbp
cmove $out,%rax
add $len,%rax
neg %rax
and \$0xfff,%rax # distance to page boundary
cmp \$$PADLOCK_PREFETCH{$mode},%rax
mov \$-$PADLOCK_PREFETCH{$mode},%rax
cmovae $chunk,%rax
and %rax,$chunk
jnz .L${mode}_loop
___
$code.=<<___;
.L${mode}_unaligned_tail:
xor %eax,%eax
cmp %rsp,%rbp
cmove $len,%rax
mov $out,%r8 # save parameters
mov $len,$chunk
sub %rax,%rsp # alloca
shr \$3,$len
lea (%rsp),$out
.byte 0xf3,0x48,0xa5 # rep movsq
mov %rsp,$inp
mov %r8, $out # restore parameters
mov $chunk,$len
jmp .L${mode}_loop
.align 16
.L${mode}_break:
___
}
$code.=<<___;
cmp %rbp,%rsp
je .L${mode}_done
pxor %xmm0,%xmm0
lea (%rsp),%rax
.L${mode}_bzero:
movaps %xmm0,(%rax)
lea 16(%rax),%rax
cmp %rax,%rbp
ja .L${mode}_bzero
.L${mode}_done:
lea (%rbp),%rsp
jmp .L${mode}_exit
.align 16
.L${mode}_aligned:
___
$code.=<<___ if ($mode eq "ctr32");
mov -4($ctx),%eax # pull 32-bit counter
bswap %eax
neg %eax
and \$0xffff,%eax
mov \$`16*0x10000`,$chunk
shl \$4,%eax
cmovz $chunk,%rax
cmp %rax,$len
cmova %rax,$chunk # don't let counter cross 2^16
cmovbe $len,$chunk
jbe .L${mode}_aligned_skip
.L${mode}_aligned_loop:
mov $len,%r10 # save parameters
mov $chunk,$len
mov $chunk,%r11
lea -16($ctx),%rax # ivp
lea 16($ctx),%rbx # key
shr \$4,$len # len/=AES_BLOCK_SIZE
.byte 0xf3,0x0f,0xa7,$opcode # rep xcrypt*
mov -4($ctx),%eax # pull 32-bit counter
bswap %eax
add \$0x10000,%eax
bswap %eax
mov %eax,-4($ctx)
mov %r10,$len # restore parameters
sub %r11,$len
mov \$`16*0x10000`,$chunk
jz .L${mode}_exit
cmp $chunk,$len
jae .L${mode}_aligned_loop
.L${mode}_aligned_skip:
___
$code.=<<___ if ($PADLOCK_PREFETCH{$mode});
lea ($inp,$len),%rbp
neg %rbp
and \$0xfff,%rbp # distance to page boundary
xor %eax,%eax
cmp \$$PADLOCK_PREFETCH{$mode},%rbp
mov \$$PADLOCK_PREFETCH{$mode}-1,%rbp
cmovae %rax,%rbp
and $len,%rbp # remainder
sub %rbp,$len
jz .L${mode}_aligned_tail
___
$code.=<<___;
lea -16($ctx),%rax # ivp
lea 16($ctx),%rbx # key
shr \$4,$len # len/=AES_BLOCK_SIZE
.byte 0xf3,0x0f,0xa7,$opcode # rep xcrypt*
___
$code.=<<___ if ($mode !~ /ecb|ctr/);
movdqa (%rax),%xmm0
movdqa %xmm0,-16($ctx) # copy [or refresh] iv
___
$code.=<<___ if ($PADLOCK_PREFETCH{$mode});
test %rbp,%rbp # check remainder
jz .L${mode}_exit
.L${mode}_aligned_tail:
mov $out,%r8
mov %rbp,$chunk
mov %rbp,$len
lea (%rsp),%rbp
sub $len,%rsp
shr \$3,$len
lea (%rsp),$out
.byte 0xf3,0x48,0xa5 # rep movsq
lea (%r8),$out
lea (%rsp),$inp
mov $chunk,$len
jmp .L${mode}_loop
___
$code
|