path: root/arch/x86/crypto/sha1_ssse3_asm.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * This is a SIMD SHA-1 implementation. It requires the Intel(R) Supplemental
 * SSE3 instruction set extensions introduced in Intel Core Microarchitecture
 * processors. CPUs supporting Intel(R) AVX extensions will get an additional
 * boost.
 *
 * This work was inspired by the vectorized implementation of Dean Gaudet.
 * Additional information on it can be found at:
 *    http://www.arctic.org/~dean/crypto/sha1.html
 *
 * It was improved upon with more efficient vectorization of the message
 * scheduling. This implementation has also been optimized for all current and
 * several future generations of Intel CPUs.
 *
 * See this article for more information about the implementation details:
 *   http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/
 *
 * Copyright (C) 2010, Intel Corp.
 *   Authors: Maxim Locktyukhin <maxim.locktyukhin@intel.com>
 *            Ronen Zohar <ronen.zohar@intel.com>
 *
 * Converted to AT&T syntax and adapted for inclusion in the Linux kernel:
 *   Author: Mathias Krause <minipli@googlemail.com>
 */

#include <linux/linkage.h>

#define CTX	%rdi	// arg1
#define BUF	%rsi	// arg2
#define CNT	%rdx	// arg3

#define REG_A	%ecx
#define REG_B	%esi
#define REG_C	%edi
#define REG_D	%r12d
#define REG_E	%edx

#define REG_T1	%eax
#define REG_T2	%ebx

#define K_BASE		%r8
#define HASH_PTR	%r9
#define BUFFER_PTR	%r10
#define BUFFER_END	%r11

#define W_TMP1	%xmm0
#define W_TMP2	%xmm9

#define W0	%xmm1
#define W4	%xmm2
#define W8	%xmm3
#define W12	%xmm4
#define W16	%xmm5
#define W20	%xmm6
#define W24	%xmm7
#define W28	%xmm8

#define XMM_SHUFB_BSWAP	%xmm10

/* we keep window of 64 w[i]+K pre-calculated values in a circular buffer */
#define WK(t)	(((t) & 15) * 4)(%rsp)
#define W_PRECALC_AHEAD	16

/*
 * This macro implements the SHA-1 function's body for single 64-byte block
 * param: function's name
 */
.macro SHA1_VECTOR_ASM  name
	SYM_FUNC_START(\name)

	push	%rbx
	push	%r12
	push	%rbp
	mov	%rsp, %rbp

	sub	$64, %rsp		# allocate workspace
	and	$~15, %rsp		# align stack

	mov	CTX, HASH_PTR
	mov	BUF, BUFFER_PTR

	shl	$6, CNT			# multiply by 64
	add	BUF, CNT
	mov	CNT, BUFFER_END

	lea	K_XMM_AR(%rip), K_BASE
	xmm_mov	BSWAP_SHUFB_CTL(%rip), XMM_SHUFB_BSWAP

	SHA1_PIPELINED_MAIN_BODY

	# cleanup workspace
	mov	$8, %ecx
	mov	%rsp, %rdi
	xor	%eax, %eax
	rep stosq

	mov	%rbp, %rsp		# deallocate workspace
	pop	%rbp
	pop	%r12
	pop	%rbx
	ret

	SYM_FUNC_END(\name)
.endm

/*
 * This macro implements 80 rounds of SHA-1 for one 64-byte block
 */
.macro SHA1_PIPELINED_MAIN_BODY
	INIT_REGALLOC

	mov	  (HASH_PTR), A
	mov	 4(HASH_PTR), B
	mov	 8(HASH_PTR), C
	mov	12(HASH_PTR), D
	mov	16(HASH_PTR), E

  .set i, 0
  .rept W_PRECALC_AHEAD
	W_PRECALC i
    .set i, (i+1)
  .endr

.align 4
1:
	RR F1,A,B,C,D,E,0
	RR F1,D,E,A,B,C,2
	RR F1,B,C,D,E,A,4
	RR F1,E,A,B,C,D,6
	RR F1,C,D,E,A,B,8

	RR F1,A,B,C,D,E,10
	RR F1,D,E,A,B,C,12
	RR F1,B,C,D,E,A,14
	RR F1,E,A,B,C,D,16
	RR F1,C,D,E,A,B,18

	RR F2,A,B,C,D,E,20
	RR F2,D,E,A,B,C,22
	RR F2,B,C,D,E,A,24
	RR F2,E,A,B,C,D,26
	RR F2,C,D,E,A,B,28

	RR F2,A,B,C,D,E,30
	RR F2,D,E,A,B,C,32
	RR F2,B,C,D,E,A,34
	RR F2,E,A,B,C,D,36
	RR F2,C,D,E,A,B,38

	RR F3,A,B,C,D,E,40
	RR F3,D,E,A,B,C,42
	RR F3,B,C,D,E,A,44
	RR F3,E,A,B,C,D,46
	RR F3,C,D,E,A,B,48

	RR F3,A,B,C,D,E,50
	RR F3,D,E,A,B,C,52
	RR F3,B,C,D,E,A,54
	RR F3,E,A,B,C,D,56
	RR F3,C,D,E,A,B,58

	add	$64, BUFFER_PTR		# move to the next 64-byte block
	cmp	BUFFER_END, BUFFER_PTR	# if the current is the last one use
	cmovae	K_BASE, BUFFER_PTR	# dummy source to avoid buffer overrun

	RR F4,A,B,C,D,E,60
	RR F4,D,E,A,B,C,62
	RR F4,B,C,D,E,A,64
	RR F4,E,A,B,C,D,66
	RR F4,C,D,E,A,B,68

	RR F4,A,B,C,D,E,70
	RR F4,D,E,A,B,C,72
	RR F4,B,C,D,E,A,74
	RR F4,E,A,B,C,D,76
	RR F4,C,D,E,A,B,78

	UPDATE_HASH   (HASH_PTR), A
	UPDATE_HASH  4(HASH_PTR), B
	UPDATE_HASH  8(HASH_PTR), C
	UPDATE_HASH 12(HASH_PTR), D
	UPDATE_HASH 16(HASH_PTR), E

	RESTORE_RENAMED_REGS
	cmp	K_BASE, BUFFER_PTR	# K_BASE means, we reached the end
	jne	1b
.endm

.macro INIT_REGALLOC
  .set A, REG_A
  .set B, REG_B
  .set C, REG_C
  .set D, REG_D
  .set E, REG_E
  .set T1, REG_T1
  .set T2, REG_T2
.endm

.macro RESTORE_RENAMED_REGS
	# order is important (REG_C is where it should be)
	mov	B, REG_B
	mov	D, REG_D
	mov	A, REG_A
	mov	E, REG_E
.endm

.macro SWAP_REG_NAMES  a, b
  .set _T, \a
  .set \a, \b
  .set \b, _T
.endm

.macro F1  b, c, d
	mov	\c, T1
	SWAP_REG_NAMES \c, T1
	xor	\d, T1
	and	\b, T1
	xor	\d, T1
.endm

.macro F2  b, c, d
	mov	\d, T1
	SWAP_REG_NAMES \d, T1
	xor	\c, T1
	xor	\b, T1
.endm

.macro F3  b, c ,d
	mov	\c, T1
	SWAP_REG_NAMES \c, T1
	mov	\b, T2
	or	\b, T1
	and	\c, T2
	and	\d, T1
	or	T2, T1
.endm

.macro F4  b, c, d
	F2 \b, \c, \d
.endm

.macro UPDATE_HASH  hash, val
	add	\hash, \val
	mov	\val, \hash
.endm

/*
 * RR does two rounds of SHA-1 back to back with W[] pre-calc
 *   t1 = F(b, c, d);   e += w(i)
 *   e += t1;           b <<= 30;   d  += w(i+1);
 *   t1 = F(a, b, c);
 *   d += t1;           a <<= 5;
 *   e += a;
 *   t1 = e;            a >>= 7;
 *   t1 <<= 5;
 *   d += t1;
 */
.macro RR  F, a,