crypto: amcc - Add crypt4xx driver

This patch adds support for AMCC ppc4xx security device driver. This is the
initial release that includes the driver framework with AES and SHA1 algorithms
support.

The remaining algorithms will be released in the near future.

Signed-off-by: James Hsiao <jhsiao@amcc.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

7 files changed, 2417 insertions, 0 deletions

diff --git a/drivers/crypto/amcc/Makefile b/drivers/crypto/amcc/Makefile
new file mode 100644
index 000000000000..aa376e8d5ed5
--- /dev/null
+++ b/drivers/crypto/amcc/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += crypto4xx.o
+crypto4xx-objs :=  crypto4xx_core.o crypto4xx_alg.o crypto4xx_sa.o
diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c
new file mode 100644
index 000000000000..61b6e1bec8c6
--- /dev/null
+++ b/drivers/crypto/amcc/crypto4xx_alg.c
@@ -0,0 +1,293 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * This file implements the Linux crypto algorithms.
+ */
+
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock_types.h>
+#include <linux/scatterlist.h>
+#include <linux/crypto.h>
+#include <linux/hash.h>
+#include <crypto/internal/hash.h>
+#include <linux/dma-mapping.h>
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/sha.h>
+#include "crypto4xx_reg_def.h"
+#include "crypto4xx_sa.h"
+#include "crypto4xx_core.h"
+
+void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
+			      u32 save_iv, u32 ld_h, u32 ld_iv, u32 hdr_proc,
+			      u32 h, u32 c, u32 pad_type, u32 op_grp, u32 op,
+			      u32 dir)
+{
+	sa->sa_command_0.w = 0;
+	sa->sa_command_0.bf.save_hash_state = save_h;
+	sa->sa_command_0.bf.save_iv = save_iv;
+	sa->sa_command_0.bf.load_hash_state = ld_h;
+	sa->sa_command_0.bf.load_iv = ld_iv;
+	sa->sa_command_0.bf.hdr_proc = hdr_proc;
+	sa->sa_command_0.bf.hash_alg = h;
+	sa->sa_command_0.bf.cipher_alg = c;
+	sa->sa_command_0.bf.pad_type = pad_type & 3;
+	sa->sa_command_0.bf.extend_pad = pad_type >> 2;
+	sa->sa_command_0.bf.op_group = op_grp;
+	sa->sa_command_0.bf.opcode = op;
+	sa->sa_command_0.bf.dir = dir;
+}
+
+void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, u32 hmac_mc,
+			      u32 cfb, u32 esn, u32 sn_mask, u32 mute,
+			      u32 cp_pad, u32 cp_pay, u32 cp_hdr)
+{
+	sa->sa_command_1.w = 0;
+	sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
+	sa->sa_command_1.bf.crypto_mode9_8 = cm & 3;
+	sa->sa_command_1.bf.feedback_mode = cfb,
+	sa->sa_command_1.bf.sa_rev = 1;
+	sa->sa_command_1.bf.extended_seq_num = esn;
+	sa->sa_command_1.bf.seq_num_mask = sn_mask;
+	sa->sa_command_1.bf.mutable_bit_proc = mute;
+	sa->sa_command_1.bf.copy_pad = cp_pad;
+	sa->sa_command_1.bf.copy_payload = cp_pay;
+	sa->sa_command_1.bf.copy_hdr = cp_hdr;
+}
+
+int crypto4xx_encrypt(struct ablkcipher_request *req)
+{
+	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+	ctx->direction = DIR_OUTBOUND;
+	ctx->hash_final = 0;
+	ctx->is_hash = 0;
+	ctx->pd_ctl = 0x1;
+
+	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
+				  req->nbytes, req->info,
+				  get_dynamic_sa_iv_size(ctx));
+}
+
+int crypto4xx_decrypt(struct ablkcipher_request *req)
+{
+	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+	ctx->direction = DIR_INBOUND;
+	ctx->hash_final = 0;
+	ctx->is_hash = 0;
+	ctx->pd_ctl = 1;
+
+	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
+				  req->nbytes, req->info,
+				  get_dynamic_sa_iv_size(ctx));
+}
+
+/**
+ * AES Functions
+ */
+static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher,
+				const u8 *key,
+				unsigned int keylen,
+				unsigned char cm,
+				u8 fb)
+{
+	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct dynamic_sa_ctl *sa;
+	int    rc;
+
+	if (keylen != AES_KEYSIZE_256 &&
+		keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_128) {
+		crypto_ablkcipher_set_flags(cipher,
+				CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	/* Create SA */
+	if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr)
+		crypto4xx_free_sa(ctx);
+
+	rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4);
+	if (rc)
+		return rc;
+
+	if (ctx->state_record_dma_addr == 0) {
+		rc = crypto4xx_alloc_state_record(ctx);
+		if (rc) {
+			crypto4xx_free_sa(ctx);
+			return rc;
+		}
+	}
+	/* Setup SA */
+	sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+	ctx->hash_final = 0;
+
+	set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV,
+				 SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
+				 SA_NO_HEADER_PROC, SA_HASH_ALG_NULL,
+				 SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
+				 SA_OP_GROUP_BASIC, SA_OPCODE_DECRYPT,
+				 DIR_INBOUND);
+
+	set_dynamic_sa_command_1(sa, cm, SA_HASH_MODE_HASH,
+				 fb, SA_EXTENDED_SN_OFF,
+				 SA_SEQ_MASK_OFF, SA_MC_ENABLE,
+				 SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
+				 SA_NOT_COPY_HDR);
+	crypto4xx_memcpy_le(ctx->sa_in + get_dynamic_sa_offset_key_field(ctx),
+			    key, keylen);
+	sa->sa_contents = SA_AES_CONTENTS | (keylen << 2);
+	sa->sa_command_1.bf.key_len = keylen >> 3;
+	ctx->is_hash = 0;
+	ctx->direction = DIR_INBOUND;
+	memcpy(ctx->sa_in + get_dynamic_sa_offset_state_ptr_field(ctx),
+			(void *)&ctx->state_record_dma_addr, 4);
+	ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx);
+
+	memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
+	sa = (struct dynamic_sa_ctl *) ctx->sa_out;
+	sa->sa_command_0.bf.dir = DIR_OUTBOUND;
+
+	return 0;
+}
+
+int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher,
+			     const u8 *key, unsigned int keylen)
+{
+	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CBC,
+				    CRYPTO_FEEDBACK_MODE_NO_FB);
+}
+
+/**
+ * HASH SHA1 Functions
+ */
+static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm,
+				   unsigned int sa_len,
+				   unsigned char ha,
+				   unsigned char hm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct crypto4xx_alg *my_alg = crypto_alg_to_crypto4xx_alg(alg);
+	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct dynamic_sa_ctl *sa;
+	struct dynamic_sa_hash160 *sa_in;
+	int rc;
+
+	ctx->dev   = my_alg->dev;
+	ctx->is_hash = 1;
+	ctx->hash_final = 0;
+
+	/* Create SA */
+	if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr)
+		crypto4xx_free_sa(ctx);
+
+	rc = crypto4xx_alloc_sa(ctx, sa_len);
+	if (rc)
+		return rc;
+
+	if (ctx->state_record_dma_addr == 0) {
+		crypto4xx_alloc_state_record(ctx);
+		if (!ctx->state_record_dma_addr) {
+			crypto4xx_free_sa(ctx);
+			return -ENOMEM;
+		}
+	}
+
+	tfm->crt_ahash.reqsize = sizeof(struct crypto4xx_ctx);
+	sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+	set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
+				 SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA,
+				 SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL,
+				 SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
+				 SA_OPCODE_HASH, DIR_INBOUND);
+	set_dynamic_sa_command_1(sa, 0, SA_HASH_MODE_HASH,
+				 CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
+				 SA_SEQ_MASK_OFF, SA_MC_ENABLE,
+				 SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
+				 SA_NOT_COPY_HDR);
+	ctx->direction = DIR_INBOUND;
+	sa->sa_contents = SA_HASH160_CONTENTS;
+	sa_in = (struct dynamic_sa_hash160 *) ctx->sa_in;
+	/* Need to zero hash digest in SA */
+	memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest));
+	memset(sa_in->outer_digest, 0, sizeof(sa_in->outer_digest));
+	sa_in->state_ptr = ctx->state_record_dma_addr;
+	ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx);
+
+	return 0;
+}
+
+int crypto4xx_hash_init(struct ahash_request *req)
+{
+	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	int ds;
+	struct dynamic_sa_ctl *sa;
+
+	sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+	ds = crypto_ahash_digestsize(
+			__crypto_ahash_cast(req->base.tfm));
+	sa->sa_command_0.bf.digest_len = ds >> 2;
+	sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA;
+	ctx->is_hash = 1;
+	ctx->direction = DIR_INBOUND;
+
+	return 0;
+}
+
+int crypto4xx_hash_update(struct ahash_request *req)
+{
+	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+	ctx->is_hash = 1;
+	ctx->hash_final = 0;
+	ctx->pd_ctl = 0x11;
+	ctx->direction = DIR_INBOUND;
+
+	return crypto4xx_build_pd(&req->base, ctx, req->src,
+				  (struct scatterlist *) req->result,
+				  req->nbytes, NULL, 0);
+}
+
+int crypto4xx_hash_final(struct ahash_request *req)
+{
+	return 0;
+}
+
+int crypto4xx_hash_digest(struct ahash_request *req)
+{
+	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+	ctx->hash_final = 1;
+	ctx->pd_ctl = 0x11;
+	ctx->direction = DIR_INBOUND;
+
+	return crypto4xx_build_pd(&req->base, ctx, req->src,
+				  (struct scatterlist *) req->result,
+				  req->nbytes, NULL, 0);
+}
+
+/**
+ * SHA1 Algorithm
+ */
+int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm)
+{
+	return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1,
+				       SA_HASH_MODE_HASH);
+}
+
+
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
new file mode 100644
index 000000000000..4c0dfb2b872e
--- /dev/null
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -0,0 +1,1310 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * This file implements AMCC crypto offload Linux device driver for use with
+ * Linux CryptoAPI.
+ */
+
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock_types.h>
+#include <linux/random.h>
+#include <linux/scatterlist.h>
+#include <linux/crypto.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/init.h>
+#include <linux/of_platform.h>
+#include <asm/dcr.h>
+#include <asm/dcr-regs.h>
+#include <asm/cacheflush.h>
+#include <crypto/internal/hash.h>
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/sha.h>
+#include "crypto4xx_reg_def.h"
+#include "crypto4xx_core.h"
+#include "crypto4xx_sa.h"
+
+#define PPC4XX_SEC_VERSION_STR			"0.5"
+
+/**
+ * PPC4xx Crypto Engine Initialization Routine
+ */
+static void crypto4xx_hw_init(struct crypto4xx_device *dev)
+{
+	union ce_ring_size ring_size;
+	union ce_ring_contol ring_ctrl;
+	union ce_part_ring_size part_ring_size;
+	union ce_io_threshold io_threshold;
+	u32 rand_num;
+	union ce_pe_dma_cfg pe_dma_cfg;
+
+	writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG);
+	/* setup pe dma, include reset sg, pdr and pe, then release reset */
+	pe_dma_cfg.w = 0;
+	pe_dma_cfg.bf.bo_sgpd_en = 1;
+	pe_dma_cfg.bf.bo_data_en = 0;
+	pe_dma_cfg.bf.bo_sa_en = 1;
+	pe_dma_cfg.bf.bo_pd_en = 1;
+	pe_dma_cfg.bf.dynamic_sa_en = 1;
+	pe_dma_cfg.bf.reset_sg = 1;
+	pe_dma_cfg.bf.reset_pdr = 1;
+	pe_dma_cfg.bf.reset_pe = 1;
+	writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG);
+	/* un reset pe,sg and pdr */
+	pe_dma_cfg.bf.pe_mode = 0;
+	pe_dma_cfg.bf.reset_sg = 0;
+	pe_dma_cfg.bf.reset_pdr = 0;
+	pe_dma_cfg.bf.reset_pe = 0;
+	pe_dma_cfg.bf.bo_td_en = 0;
+	writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG);
+	writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_PDR_BASE);
+	writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_RDR_BASE);
+	writel(PPC4XX_PRNG_CTRL_AUTO_EN, dev->ce_base + CRYPTO4XX_PRNG_CTRL);
+	get_random_bytes(&rand_num, sizeof(rand_num));
+	writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_L);
+	get_random_bytes(&rand_num, sizeof(rand_num));
+	writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_H);
+	ring_size.w = 0;
+	ring_size.bf.ring_offset = PPC4XX_PD_SIZE;
+	ring_size.bf.ring_size   = PPC4XX_NUM_PD;
+	writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE);
+	ring_ctrl.w = 0;
+	writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL);
+	writel(PPC4XX_DC_3DES_EN, dev->ce_base + CRYPTO4XX_DEVICE_CTRL);
+	writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE);
+	writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE);
+	part_ring_size.w = 0;
+	part_ring_size.bf.sdr_size = PPC4XX_SDR_SIZE;
+	part_ring_size.bf.gdr_size = PPC4XX_GDR_SIZE;
+	writel(part_ring_size.w, dev->ce_base + CRYPTO4XX_PART_RING_SIZE);
+	writel(PPC4XX_SD_BUFFER_SIZE, dev->ce_base + CRYPTO4XX_PART_RING_CFG);
+	io_threshold.w = 0;
+	io_threshold.bf.output_threshold = PPC4XX_OUTPUT_THRESHOLD;
+	io_threshold.bf.input_threshold  = PPC4XX_INPUT_THRESHOLD;
+	writel(io_threshold.w, dev->ce_base + CRYPTO4XX_IO_THRESHOLD);
+	writel(0, dev->ce_base + CRYPTO4XX_PDR_BASE_UADDR);
+	writel(0, dev->ce_base + CRYPTO4XX_RDR_BASE_UADDR);
+	writel(0, dev->ce_base + CRYPTO4XX_PKT_SRC_UADDR);
+	writel(0, dev->ce_base + CRYPTO4XX_PKT_DEST_UADDR);
+	writel(0, dev->ce_base + CRYPTO4XX_SA_UADDR);
+	writel(0, dev->ce_base + CRYPTO4XX_GATH_RING_BASE_UADDR);
+	writel(0, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE_UADDR);
+	/* un reset pe,sg and pdr */
+	pe_dma_cfg.bf.pe_mode = 1;
+	pe_dma_cfg.bf.reset_sg = 0;
+	pe_dma_cfg.bf.reset_pdr = 0;
+	pe_dma_cfg.bf.reset_pe = 0;
+	pe_dma_cfg.bf.bo_td_en = 0;
+	writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG);
+	/*clear all pending interrupt*/
+	writel(PPC4XX_INTERRUPT_CLR, dev->ce_base + CRYPTO4XX_INT_CLR);
+	writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT);
+	writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT);
+	writel(PPC4XX_INT_CFG, dev->ce_base + CRYPTO4XX_INT_CFG);
+	writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN);
+}
+
+int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size)
+{
+	ctx->sa_in = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4,
+					&ctx->sa_in_dma_addr, GFP_ATOMIC);
+	if (ctx->sa_in == NULL)
+		return -ENOMEM;
+
+	ctx->sa_out = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4,
+					 &ctx->sa_out_dma_addr, GFP_ATOMIC);
+	if (ctx->sa_out == NULL) {
+		dma_free_coherent(ctx->dev->core_dev->device,
+				  ctx->sa_len * 4,
+				  ctx->sa_in, ctx->sa_in_dma_addr);
+		return -ENOMEM;
+	}
+
+	memset(ctx->sa_in, 0, size * 4);
+	memset(ctx->sa_out, 0, size * 4);
+	ctx->sa_len = size;
+
+	return 0;
+}
+
+void crypto4xx_free_sa(struct crypto4xx_ctx *ctx)
+{
+	if (ctx->sa_in != NULL)
+		dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4,
+				  ctx->sa_in, ctx->sa_in_dma_addr);
+	if (ctx->sa_out != NULL)
+		dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4,
+				  ctx->sa_out, ctx->sa_out_dma_addr);
+
+	ctx->sa_in_dma_addr = 0;
+	ctx->sa_out_dma_addr = 0;
+	ctx->sa_len = 0;
+}
+
+u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx)
+{
+	ctx->state_record = dma_alloc_coherent(ctx->dev->core_dev->device,
+				sizeof(struct sa_state_record),
+				&ctx->state_record_dma_addr, GFP_ATOMIC);
+	if (!ctx->state_record_dma_addr)
+		return -ENOMEM;
+	memset(ctx->state_record, 0, sizeof(struct sa_state_record));
+
+	return 0;
+}
+
+void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx)
+{
+	if (ctx->state_record != NULL)
+		dma_free_coherent(ctx->dev->core_dev->device,
+				  sizeof(struct sa_state_record),
+				  ctx->state_record,
+				  ctx->state_record_dma_addr);
+	ctx->state_record_dma_addr = 0;
+}
+
+/**
+ * alloc memory for the gather ring
+ * no need to alloc buf for the ring
+ * gdr_tail, gdr_head and gdr_count are initialized by this function
+ */
+static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev)
+{
+	int i;
+	struct pd_uinfo *pd_uinfo;
+	dev->pdr = dma_alloc_coherent(dev->core_dev->device,
+				      sizeof(struct ce_pd) * PPC4XX_NUM_PD,
+				      &dev->pdr_pa, GFP_ATOMIC);
+	if (!dev->pdr)
+		return -ENOMEM;
+
+	dev->pdr_uinfo = kzalloc(sizeof(struct pd_uinfo) * PPC4XX_NUM_PD,
+				GFP_KERNEL);
+	if (!dev->pdr_uinfo) {
+		dma_free_coherent(dev->core_dev->device,
+				  sizeof(struct ce_pd) * PPC4XX_NUM_PD,
+				  dev->pdr,
+				  dev->pdr_pa);
+		return -ENOMEM;
+	}
+	memset(dev->pdr, 0,  sizeof(struct ce_pd) * PPC4XX_NUM_PD);
+	dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device,
+				   256 * PPC4XX_NUM_PD,
+				   &dev->shadow_sa_pool_pa,
+				   GFP_ATOMIC);
+	if (!dev->shadow_sa_pool)
+		return -ENOMEM;
+
+	dev->shadow_sr_pool = dma_alloc_coherent(dev->core_dev->device,
+			 sizeof(struct sa_state_record) * PPC4XX_NUM_PD,
+			 &dev->shadow_sr_pool_pa, GFP_ATOMIC);
+	if (!dev->shadow_sr_pool)
+		return -ENOMEM;
+	for (i = 0; i < PPC4XX_NUM_PD; i++) {
+		pd_uinfo = (struct pd_uinfo *) (dev->pdr_uinfo +
+						sizeof(struct pd_uinfo) * i);
+
+		/* alloc 256 bytes which is enough for any kind of dynamic sa */
+		pd_uinfo->sa_va = dev->shadow_sa_pool + 256 * i;
+		pd_uinfo->sa_pa = dev->shadow_sa_pool_pa + 256 * i;
+
+		/* alloc state record */
+		pd_uinfo->sr_va = dev->shadow_sr_pool +
+		    sizeof(struct sa_state_record) * i;
+		pd_uinfo->sr_pa = dev->shadow_sr_pool_pa +
+		    sizeof(struct sa_state_record) * i;
+	}
+
+	return 0;
+}
+
+static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev)
+{
+	if (dev->pdr != NULL)
+		dma_free_coherent(dev->core_dev->device,
+				  sizeof(struct ce_pd) * PPC4XX_NUM_PD,
+				  dev->pdr, dev->pdr_pa);
+	if (dev->shadow_sa_pool)
+		dma_free_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD,
+				  dev->shadow_sa_pool, dev->shadow_sa_pool_pa);
+	if (dev->shadow_sr_pool)
+		dma_free_coherent(dev->core_dev->device,
+			sizeof(struct sa_state_record) * PPC4XX_NUM_PD,
+			dev->shadow_sr_pool, dev->shadow_sr_pool_pa);
+
+	kfree(dev->pdr_uinfo);
+}
+
+static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev)
+{
+	u32 retval;
+	u32 tmp;
+
+	retval = dev->pdr_head;
+	tmp = (dev->pdr_head + 1) % PPC4XX_NUM_PD;
+
+	if (tmp == dev->pdr_tail)
+		return ERING_WAS_FULL;
+
+	dev->pdr_head = tmp;
+
+	return retval;
+}
+
+static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx)
+{
+	struct pd_uinfo *pd_uinfo;
+	unsigned long flags;
+
+	pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo +
+				       sizeof(struct pd_uinfo) * idx);
+	spin_lock_irqsave(&dev->core_dev->lock, flags);
+	if (dev->pdr_tail != PPC4XX_LAST_PD)
+		dev->pdr_tail++;
+	else
+		dev->pdr_tail = 0;
+	pd_uinfo->state = PD_ENTRY_FREE;
+	spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+
+	return 0;
+}
+
+static struct ce_pd *crypto4xx_get_pdp(struct crypto4xx_device *dev,
+				       dma_addr_t *pd_dma, u32 idx)
+{
+	*pd_dma = dev->pdr_pa + sizeof(struct ce_pd) * idx;
+
+	return dev->pdr + sizeof(struct ce_pd) * idx;
+}
+
+/**
+ * alloc memory for the gather ring
+ * no need to alloc buf for the ring
+ * gdr_tail, gdr_head and gdr_count are initialized by this function
+ */
+static u32 crypto4xx_build_gdr(struct crypto4xx_device *dev)
+{
+	dev->gdr = dma_alloc_coherent(dev->core_dev->device,
+				      sizeof(struct ce_gd) * PPC4XX_NUM_GD,
+				      &dev->gdr_pa, GFP_ATOMIC);
+	if (!dev->gdr)
+		return -ENOMEM;
+
+	memset(dev->gdr, 0, sizeof(struct ce_gd) * PPC4XX_NUM_GD);
+
+	return 0;
+}
+
+static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev)
+{
+	dma_free_coherent(dev->core_dev->device,
+			  sizeof(struct ce_gd) * PPC4XX_NUM_GD,
+			  dev->gdr, dev->gdr_pa);
+}
+
+/*
+ * when this function is called.
+ * preemption or interrupt must be disabled
+ */
+u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n)
+{
+	u32 retval;
+	u32 tmp;
+	if (n >= PPC4XX_NUM_GD)
+		return ERING_WAS_FULL;
+
+	retval = dev->gdr_head;
+	tmp = (dev->gdr_head + n) % PPC4XX_NUM_GD;
+	if (dev->gdr_head > dev->gdr_tail) {
+		if (tmp < dev->gdr_head && tmp >= dev->gdr_tail)
+			return ERING_WAS_FULL;
+	} else if (dev->gdr_head < dev->gdr_tail) {
+		if (tmp < dev->gdr_head || tmp >= dev->gdr_tail)
+			return ERING_WAS_FULL;
+	}
+	dev->gdr_head = tmp;
+
+	return retval;
+}
+
+static u32 crypto4xx_put_gd_to_gdr(struct crypto4xx_device *dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->core_dev->lock, flags);
+	if (dev->gdr_tail == dev->gdr_head) {
+		spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+		return 0;
+	}
+
+	if (dev->gdr_tail != PPC4XX_LAST_GD)
+		dev->gdr_tail++;
+	else
+		dev->gdr_tail = 0;
+
+	spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+
+	return 0;
+}
+
+static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev,
+					      dma_addr_t *gd_dma, u32 idx)
+{
+	*gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx;
+
+	return (struct ce_gd *) (dev->gdr + sizeof(struct ce_gd) * idx);
+}
+
+/**
+ * alloc memory for the scatter ring
+ * need to alloc buf for the ring
+ * sdr_tail, sdr_head and sdr_count are initialized by this function
+ */
+static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev)
+{
+	int i;
+	struct ce_sd *sd_array;
+
+	/* alloc memory for scatter descriptor ring */
+	dev->sdr = dma_alloc_coherent(dev->core_dev->device,
+				      sizeof(struct ce_sd) * PPC4XX_NUM_SD,
+				      &dev->sdr_pa, GFP_ATOMIC);
+	if (!dev->sdr)
+		return -ENOMEM;
+
+	dev->scatter_buffer_size = PPC4XX_SD_BUFFER_SIZE;
+	dev->scatter_buffer_va =
+		dma_alloc_coherent(dev->core_dev->device,
+			dev->scatter_buffer_size * PPC4XX_NUM_SD,
+			&dev->scatter_buffer_pa, GFP_ATOMIC);
+	if (!dev->scatter_buffer_va) {
+		dma_free_coherent(dev->core_dev->device,
+				  sizeof(struct ce_sd) * PPC4XX_NUM_SD,
+				  dev->sdr, dev->sdr_pa);
+		return -ENOMEM;
+	}
+
+	sd_array = dev->sdr;
+
+	for (i = 0; i < PPC4XX_NUM_SD; i++) {
+		sd_array[i].ptr = dev->scatter_buffer_pa +
+				  dev->scatter_buffer_size * i;
+	}
+
+	return 0;
+}
+
+static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev)
+{
+	if (dev->sdr != NULL)
+		dma_free_coherent(dev->core_dev->device,
+				  sizeof(struct ce_sd) * PPC4XX_NUM_SD,
+				  dev->sdr, dev->sdr_pa);
+
+	if (dev->scatter_buffer_va != NULL)
+		dma_free_coherent(dev->core_dev->device,
+				  dev->scatter_buffer_size * PPC4XX_NUM_SD,
+				  dev->scatter_buffer_va,
+				  dev->scatter_buffer_pa);
+}
+
+/*
+ * when this function is called.
+ * preemption or interrupt must be disabled
+ */
+static u32 crypto4xx_get_n_sd(struct crypto4xx_device *dev, int n)
+{
+	u32 retval;
+	u32 tmp;
+
+	if (n >= PPC4XX_NUM_SD)
+		return ERING_WAS_FULL;
+
+	retval = dev->sdr_head;
+	tmp = (dev->sdr_head + n) % PPC4XX_NUM_SD;
+	if (dev->sdr_head > dev->gdr_tail) {
+		if (tmp < dev->sdr_head && tmp >= dev->sdr_tail)
+			return ERING_WAS_FULL;
+	} else if (dev->sdr_head < dev->sdr_tail) {
+		if (tmp < dev->sdr_head || tmp >= dev->sdr_tail)
+			return ERING_WAS_FULL;
+	} /* the head = tail, or empty case is already take cared */
+	dev->sdr_head = tmp;
+
+	return retval;
+}
+
+static u32 crypto4xx_put_sd_to_sdr(struct crypto4xx_device *dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->core_dev->lock, flags);
+	if (dev->sdr_tail == dev->sdr_head) {
+		spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+		return 0;
+	}
+	if (dev->sdr_tail != PPC4XX_LAST_SD)
+		dev->sdr_tail++;
+	else
+		dev->sdr_tail = 0;
+	spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+
+	return 0;
+}
+
+static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev,
+					      dma_addr_t *sd_dma, u32 idx)
+{
+	*sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx;
+
+	return  (struct ce_sd *)(dev->sdr + sizeof(struct ce_sd) * idx);
+}
+
+static u32 crypto4xx_fill_one_page(struct crypto4xx_device *dev,
+				   dma_addr_t *addr, u32 *length,
+				   u32 *idx, u32 *offset, u32 *nbytes)
+{
+	u32 len;
+
+	if (*length > dev->scatter_buffer_size) {
+		memcpy(phys_to_virt(*addr),
+			dev->scatter_buffer_va +
+			*idx * dev->scatter_buffer_size + *offset,
+			dev->scatter_buffer_size);
+		*offset = 0;
+		*length -= dev->scatter_buffer_size;
+		*nbytes -= dev->scatter_buffer_size;
+		if (*idx == PPC4XX_LAST_SD)
+			*idx = 0;
+		else
+			(*idx)++;
+		*addr = *addr +  dev->scatter_buffer_size;
+		return 1;
+	} else if (*length < dev->scatter_buffer_size) {
+		memcpy(phys_to_virt(*addr),
+			dev->scatter_buffer_va +
+			*idx * dev->scatter_buffer_size + *offset, *length);
+		if ((*offset + *length) == dev->scatter_buffer_size) {
+			if (*idx == PPC4XX_LAST_SD)
+				*idx = 0;
+			else
+				(*idx)++;
+			*nbytes -= *length;
+			*offset = 0;
+		} else {
+			*nbytes -= *length;
+			*offset += *length;
+		}
+
+		return 0;
+	} else {
+		len = (*nbytes <= dev->scatter_buffer_size) ?
+				(*nbytes) : dev->scatter_buffer_size;
+		memcpy(phys_to_virt(*addr),
+			dev->scatter_buffer_va +
+			*idx * dev->scatter_buffer_size + *offset,
+			len);
+		*offset = 0;
+		*nbytes -= len;
+
+		if (*idx == PPC4XX_LAST_SD)
+			*idx = 0;
+		else
+			(*idx)++;
+
+		return 0;
+    }
+}
+
+static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev,
+				      struct ce_pd *pd,
+				      struct pd_uinfo *pd_uinfo,
+				      u32 nbytes,
+				      struct scatterlist *dst)
+{
+	dma_addr_t addr;
+	u32 this_sd;
+	u32 offset;
+	u32 len;
+	u32 i;
+	u32 sg_len;
+	struct scatterlist *sg;
+
+	this_sd = pd_uinfo->first_sd;
+	offset = 0;
+	i = 0;
+
+	while (nbytes) {
+		sg = &dst[i];
+		sg_len = sg->length;
+		addr = dma_map_page(dev->core_dev->device, sg_page(sg),
+				sg->offset, sg->length, DMA_TO_DEVICE);
+
+		if (offset == 0) {
+			len = (nbytes <= sg->length) ? nbytes : sg->length;
+			while (crypto4xx_fill_one_page(dev, &addr, &len,
+				&this_sd, &offset, &nbytes))
+				;
+			if (!nbytes)
+				return;
+			i++;
+		} else {
+			len = (nbytes <= (dev->scatter_buffer_size - offset)) ?
+				nbytes : (dev->scatter_buffer_size - offset);
+			len = (sg->length < len) ? sg->length : len;
+			while (crypto4xx_fill_one_page(dev, &addr, &len,
+					       &this_sd, &offset, &nbytes))
+				;
+			if (!nbytes)
+				return;
+			sg_len -= len;
+			if (sg_len) {
+				addr += len;
+				while (crypto4xx_fill_one_page(dev, &addr,
+					&sg_len, &this_sd, &offset, &nbytes))
+					;
+			}
+			i++;
+		}
+	}
+}
+
+static u32 crypto4xx_copy_digest_to_dst(struct pd_uinfo *pd_uinfo,
+					struct crypto4xx_ctx *ctx)
+{
+	struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+	struct sa_state_record *state_record =
+				(struct sa_state_record *) pd_uinfo->sr_va;
+
+	if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) {
+		memcpy((void *) pd_uinfo->dest_va, state_record->save_digest,
+		       SA_HASH_ALG_SHA1_DIGEST_SIZE);
+	}
+
+	return 0;
+}
+
+static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev,
+				  struct pd_uinfo *pd_uinfo)
+{
+	int i;
+	if (pd_uinfo->num_gd) {
+		for (i = 0; i < pd_uinfo->num_gd; i++)
+			crypto4xx_put_gd_to_gdr(dev);
+		pd_uinfo->first_gd = 0xffffffff;
+		pd_uinfo->num_gd = 0;
+	}
+	if (pd_uinfo->num_sd) {
+		for (i = 0; i < pd_uinfo->num_sd; i++)
+			crypto4xx_put_sd_to_sdr(dev);
+
+		pd_uinfo->first_sd = 0xffffffff;
+		pd_uinfo->num_sd = 0;
+	}
+}
+
+static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev,
+				     struct pd_uinfo *pd_uinfo,
+				     struct ce_pd *pd)
+{
+	struct crypto4xx_ctx *ctx;
+	struct ablkcipher_request *ablk_req;
+	struct scatterlist *dst;
+	dma_addr_t addr;
+
+	ablk_req = ablkcipher_request_cast(pd_uinfo->async_req);
+	ctx  = crypto_tfm_ctx(ablk_req->base.tfm);
+
+	if (pd_uinfo->using_sd) {
+		crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, ablk_req->nbytes,
+					  ablk_req->dst);
+	} else {
+		dst = pd_uinfo->dest_va;
+		addr = dma_map_page(dev->core_dev->device, sg_page(dst),
+				    dst->offset, dst->length, DMA_FROM_DEVICE);
+	}
+	crypto4xx_ret_sg_desc(dev, pd_uinfo);
+	if (ablk_req->base.complete != NULL)
+		ablk_req->base.complete(&ablk_req->base, 0);
+
+	return 0;
+}
+
+static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev,
+				struct pd_uinfo *pd_uinfo)
+{
+	struct crypto4xx_ctx *ctx;
+	struct ahash_request *ahash_req;
+
+	ahash_req = ahash_request_cast(pd_uinfo->async_req);
+	ctx  = crypto_tfm_ctx(ahash_req->base.tfm);
+
+	crypto4xx_copy_digest_to_dst(pd_uinfo,
+				     crypto_tfm_ctx(ahash_req->base.tfm));
+	crypto4xx_ret_sg_desc(dev, pd_uinfo);
+	/* call user provided callback function x */
+	if (ahash_req->base.complete != NULL)
+		ahash_req->base.complete(&ahash_req->base, 0);
+
+	return 0;
+}
+
+static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx)
+{
+	struct ce_pd *pd;
+	struct pd_uinfo *pd_uinfo;
+
+	pd =  dev->pdr + sizeof(struct ce_pd)*idx;
+	pd_uinfo = dev->pdr_uinfo + sizeof(struct pd_uinfo)*idx;
+	if (crypto_tfm_alg_type(pd_uinfo->async_req->tfm) ==
+			CRYPTO_ALG_TYPE_ABLKCIPHER)
+		return crypto4xx_ablkcipher_done(dev, pd_uinfo, pd);
+	else
+		return crypto4xx_ahash_done(dev, pd_uinfo);
+}
+
+/**
+ * Note: Only use this function to copy items that is word aligned.
+ */
+void crypto4xx_memcpy_le(unsigned int *dst,
+			 const unsigned char *buf,
+			 int len)
+{
+	u8 *tmp;
+	for (; len >= 4; buf += 4, len -= 4)
+		*dst++ = cpu_to_le32(*(unsigned int *) buf);
+
+	tmp = (u8 *)dst;
+	switch (len) {
+	case 3:
+		*tmp++ = 0;
+		*tmp++ = *(buf+2);
+		*tmp++ = *(buf+1);
+		*tmp++ = *buf;
+		break;
+	case 2:
+		*tmp++ = 0;
+		*tmp++ = 0;
+		*tmp++ = *(buf+1);
+		*tmp++ = *buf;
+		break;
+	case 1:
+		*tmp++ = 0;
+		*tmp++ = 0;
+		*tmp++ = 0;
+		*tmp++ = *buf;
+		break;
+	default:
+		break;
+	}
+}
+
+static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev)
+{
+	crypto4xx_destroy_pdr(core_dev->dev);
+	crypto4xx_destroy_gdr(core_dev->dev);
+	crypto4xx_destroy_sdr(core_dev->dev);
+	dev_set_drvdata(core_dev->device, NULL);
+	iounmap(core_dev->dev->ce_base);
+	kfree(core_dev->dev);
+	kfree(core_dev);
+}
+
+void crypto4xx_return_pd(struct crypto4xx_device *dev,
+			 u32 pd_entry, struct ce_pd *pd,
+			 struct pd_uinfo *pd_uinfo)
+{
+	/* irq should be already disabled */
+	dev->pdr_head = pd_entry;
+	pd->pd_ctl.w = 0;
+	pd->pd_ctl_len.w = 0;
+	pd_uinfo->state = PD_ENTRY_FREE;
+}
+
+/*
+ * derive number of elements in scatterlist
+ * Shamlessly copy from talitos.c
+ */
+static int get_sg_count(struct scatterlist *sg_list, int nbytes)
+{
+	struct scatterlist *sg = sg_list;
+	int sg_nents = 0;
+
+	while (nbytes) {
+		sg_nents++;
+		if (sg->length > nbytes)
+			break;
+		nbytes -= sg->length;
+		sg = sg_next(sg);
+	}
+
+	return sg_nents;
+}
+
+static u32 get_next_gd(u32 current)
+