diff options
author | Mahipal Challa <mahipalreddy2006@gmail.com> | 2017-02-15 10:45:08 +0530 |
---|---|---|
committer | Herbert Xu <herbert@gondor.apana.org.au> | 2017-03-09 18:34:25 +0800 |
commit | 640035a2dc5534b49cd64580e41874b71f131a1c (patch) | |
tree | 81762417b92fd4c4f2fb4f14413ba0d3edcbe721 /drivers/crypto/cavium | |
parent | 27c539aeffe2851bf9aeeeba8a58038187a05019 (diff) |
crypto: zip - Add ThunderX ZIP driver core
Add a driver for the ZIP engine found on Cavium ThunderX SOCs.
The ZIP engine supports hardware accelerated compression and
decompression. It includes 2 independent ZIP cores and supports:
- DEFLATE compression and decompression (RFC 1951)
- LZS compression and decompression (RFC 2395 and ANSI X3.241-1994)
- ADLER32 and CRC32 checksums for ZLIB (RFC 1950) and GZIP (RFC 1952)
The ZIP engine is presented as a PCI device. It supports DMA and
scatter-gather.
Signed-off-by: Mahipal Challa <Mahipal.Challa@cavium.com>
Signed-off-by: Jan Glauber <jglauber@cavium.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Diffstat (limited to 'drivers/crypto/cavium')
-rw-r--r-- | drivers/crypto/cavium/Makefile | 4 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/Makefile | 8 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/common.h | 202 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_crypto.h | 77 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_device.c | 201 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_device.h | 108 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_main.c | 552 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_main.h | 106 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_mem.c | 114 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_mem.h | 78 | ||||
-rw-r--r-- | drivers/crypto/cavium/zip/zip_regs.h | 1347 |
11 files changed, 2797 insertions, 0 deletions
diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile new file mode 100644 index 000000000000..641268b784be --- /dev/null +++ b/drivers/crypto/cavium/Makefile @@ -0,0 +1,4 @@ +# +# Makefile for Cavium crypto device drivers +# +obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += zip/ diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile new file mode 100644 index 000000000000..2c0750885165 --- /dev/null +++ b/drivers/crypto/cavium/zip/Makefile @@ -0,0 +1,8 @@ +# +# Makefile for Cavium's ZIP Driver. +# + +obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += thunderx_zip.o +thunderx_zip-y := zip_main.o \ + zip_device.o \ + zip_mem.o diff --git a/drivers/crypto/cavium/zip/common.h b/drivers/crypto/cavium/zip/common.h new file mode 100644 index 000000000000..dc451e0a43c5 --- /dev/null +++ b/drivers/crypto/cavium/zip/common.h @@ -0,0 +1,202 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __COMMON_H__ +#define __COMMON_H__ + +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/seq_file.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/version.h> + +/* Device specific zlib function definitions */ +#include "zip_device.h" + +/* ZIP device definitions */ +#include "zip_main.h" + +/* ZIP memory allocation/deallocation related definitions */ +#include "zip_mem.h" + +/* Device specific structure definitions */ +#include "zip_regs.h" + +#define ZIP_ERROR -1 + +#define ZIP_FLUSH_FINISH 4 + +#define RAW_FORMAT 0 /* for rawpipe */ +#define ZLIB_FORMAT 1 /* for zpipe */ +#define GZIP_FORMAT 2 /* for gzpipe */ +#define LZS_FORMAT 3 /* for lzspipe */ + +/* Max number of ZIP devices supported */ +#define MAX_ZIP_DEVICES 2 + +/* Configures the number of zip queues to be used */ +#define ZIP_NUM_QUEUES 2 + +#define DYNAMIC_STOP_EXCESS 1024 + +/* Maximum buffer sizes in direct mode */ +#define MAX_INPUT_BUFFER_SIZE (64 * 1024) +#define MAX_OUTPUT_BUFFER_SIZE (64 * 1024) + +/** + * struct zip_operation - common data structure for comp and decomp operations + * @input: Next input byte is read from here + * @output: Next output byte written here + * @ctx_addr: Inflate context buffer address + * @history: Pointer to the history buffer + * @input_len: Number of bytes available at next_in + * @input_total_len: Total number of input bytes read + * @output_len: Remaining free space at next_out + * @output_total_len: Total number of bytes output so far + * @csum: Checksum value of the uncompressed data + * @flush: Flush flag + * @format: Format (depends on stream's wrap) + * @speed: Speed depends on stream's level + * @ccode: Compression code ( stream's strategy) + * @lzs_flag: Flag for LZS support + * @begin_file: Beginning of file indication for inflate + * @history_len: Size of the history data + * @end_file: Ending of the file indication for inflate + * @compcode: Completion status of the ZIP invocation + * @bytes_read: Input bytes read in current instruction + * @bits_processed: Total bits processed for entire file + * @sizeofptr: To distinguish between ILP32 and LP64 + * @sizeofzops: Optional just for padding + * + * This structure is used to maintain the required meta data for the + * comp and decomp operations. + */ +struct zip_operation { + u8 *input; + u8 *output; + u64 ctx_addr; + u64 history; + + u32 input_len; + u32 input_total_len; + + u32 output_len; + u32 output_total_len; + + u32 csum; + u32 flush; + + u32 format; + u32 speed; + u32 ccode; + u32 lzs_flag; + + u32 begin_file; + u32 history_len; + + u32 end_file; + u32 compcode; + u32 bytes_read; + u32 bits_processed; + + u32 sizeofptr; + u32 sizeofzops; +}; + +/* error messages */ +#define zip_err(fmt, args...) pr_err("ZIP ERR:%s():%d: " \ + fmt "\n", __func__, __LINE__, ## args) + +#ifdef MSG_ENABLE +/* Enable all messages */ +#define zip_msg(fmt, args...) pr_info("ZIP_MSG:" fmt "\n", ## args) +#else +#define zip_msg(fmt, args...) +#endif + +#if defined(ZIP_DEBUG_ENABLE) && defined(MSG_ENABLE) + +#ifdef DEBUG_LEVEL + +#define FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : \ + strrchr(__FILE__, '\\') ? strrchr(__FILE__, '\\') + 1 : __FILE__) + +#if DEBUG_LEVEL >= 4 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ + fmt "\n", FILE_NAME, __func__, __LINE__, ## args) + +#elif DEBUG_LEVEL >= 3 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ + fmt "\n", FILE_NAME, __func__, __LINE__, ## args) + +#elif DEBUG_LEVEL >= 2 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s() : %d: " \ + fmt "\n", __func__, __LINE__, ## args) + +#else + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) + +#endif /* DEBUG LEVEL >=4 */ + +#else + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) + +#endif /* DEBUG_LEVEL */ +#else + +#define zip_dbg(fmt, args...) + +#endif /* ZIP_DEBUG_ENABLE && MSG_ENABLE*/ + +#endif diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h new file mode 100644 index 000000000000..2d91b05a94b8 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_crypto.h @@ -0,0 +1,77 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_CRYPTO_H__ +#define __ZIP_CRYPTO_H__ + +#include <linux/crypto.h> +#include <crypto/internal/scompress.h> +#include "common.h" + +struct zip_kernel_ctx { + struct zip_operation zip_comp; + struct zip_operation zip_decomp; +}; + +int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm); +int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm); +void zip_free_comp_ctx(struct crypto_tfm *tfm); +int zip_comp_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); +int zip_comp_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); + +void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm); +void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm); +void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx); +int zip_scomp_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); +int zip_scomp_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); +#endif diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c new file mode 100644 index 000000000000..e980f0894546 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_device.c @@ -0,0 +1,201 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "common.h" + +/** + * zip_cmd_queue_consumed - Calculates the space consumed in the command queue. + * + * @zip_dev: Pointer to zip device structure + * @queue: Queue number + * + * Return: Bytes consumed in the command queue buffer. + */ +static inline u32 zip_cmd_queue_consumed(struct zip_device *zip_dev, int queue) +{ + return ((zip_dev->iq[queue].sw_head - zip_dev->iq[queue].sw_tail) * + sizeof(u64 *)); +} + +/** + * zip_load_instr - Submits the instruction into the ZIP command queue + * @instr: Pointer to the instruction to be submitted + * @zip_dev: Pointer to ZIP device structure to which the instruction is to + * be submitted + * + * This function copies the ZIP instruction to the command queue and rings the + * doorbell to notify the engine of the instruction submission. The command + * queue is maintained in a circular fashion. When there is space for exactly + * one instruction in the queue, next chunk pointer of the queue is made to + * point to the head of the queue, thus maintaining a circular queue. + * + * Return: Queue number to which the instruction was submitted + */ +u32 zip_load_instr(union zip_inst_s *instr, + struct zip_device *zip_dev) +{ + union zip_quex_doorbell dbell; + u32 queue = 0; + u32 consumed = 0; + u64 *ncb_ptr = NULL; + union zip_nptr_s ncp; + + /* + * Distribute the instructions between the enabled queues based on + * the CPU id. + */ + if (smp_processor_id() % 2 == 0) + queue = 0; + else + queue = 1; + + zip_dbg("CPU Core: %d Queue number:%d", smp_processor_id(), queue); + + /* Take cmd buffer lock */ + spin_lock(&zip_dev->iq[queue].lock); + + /* + * Command Queue implementation + * 1. If there is place for new instructions, push the cmd at sw_head. + * 2. If there is place for exactly one instruction, push the new cmd + * at the sw_head. Make sw_head point to the sw_tail to make it + * circular. Write sw_head's physical address to the "Next-Chunk + * Buffer Ptr" to make it cmd_hw_tail. + * 3. Ring the door bell. + */ + zip_dbg("sw_head : %lx", zip_dev->iq[queue].sw_head); + zip_dbg("sw_tail : %lx", zip_dev->iq[queue].sw_tail); + + consumed = zip_cmd_queue_consumed(zip_dev, queue); + /* Check if there is space to push just one cmd */ + if ((consumed + 128) == (ZIP_CMD_QBUF_SIZE - 8)) { + zip_dbg("Cmd queue space available for single command"); + /* Space for one cmd, pust it and make it circular queue */ + memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, + sizeof(union zip_inst_s)); + zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ + + /* Now, point the "Next-Chunk Buffer Ptr" to sw_head */ + ncb_ptr = zip_dev->iq[queue].sw_head; + + zip_dbg("ncb addr :0x%lx sw_head addr :0x%lx", + ncb_ptr, zip_dev->iq[queue].sw_head - 16); + + /* Using Circular command queue */ + zip_dev->iq[queue].sw_head = zip_dev->iq[queue].sw_tail; + /* Mark this buffer for free */ + zip_dev->iq[queue].free_flag = 1; + + /* Write new chunk buffer address at "Next-Chunk Buffer Ptr" */ + ncp.u_reg64 = 0ull; + ncp.s.addr = __pa(zip_dev->iq[queue].sw_head); + *ncb_ptr = ncp.u_reg64; + zip_dbg("*ncb_ptr :0x%lx sw_head[phys] :0x%lx", + *ncb_ptr, __pa(zip_dev->iq[queue].sw_head)); + + zip_dev->iq[queue].pend_cnt++; + + } else { + zip_dbg("Enough space is available for commands"); + /* Push this cmd to cmd queue buffer */ + memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, + sizeof(union zip_inst_s)); + zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ + + zip_dev->iq[queue].pend_cnt++; + } + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, + zip_dev->iq[queue].hw_tail); + + zip_dbg(" Pushed the new cmd : pend_cnt : %d", + zip_dev->iq[queue].pend_cnt); + + /* Ring the doorbell */ + dbell.u_reg64 = 0ull; + dbell.s.dbell_cnt = 1; + zip_reg_write(dbell.u_reg64, + (zip_dev->reg_base + ZIP_QUEX_DOORBELL(queue))); + + /* Unlock cmd buffer lock */ + spin_unlock(&zip_dev->iq[queue].lock); + + return queue; +} + +/** + * zip_update_cmd_bufs - Updates the queue statistics after posting the + * instruction + * @zip_dev: Pointer to zip device structure + * @queue: Queue number + */ +void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue) +{ + /* Take cmd buffer lock */ + spin_lock(&zip_dev->iq[queue].lock); + + /* Check if the previous buffer can be freed */ + if (zip_dev->iq[queue].free_flag == 1) { + zip_dbg("Free flag. Free cmd buffer, adjust sw head and tail"); + /* Reset the free flag */ + zip_dev->iq[queue].free_flag = 0; + + /* Point the hw_tail to start of the new chunk buffer */ + zip_dev->iq[queue].hw_tail = zip_dev->iq[queue].sw_head; + } else { + zip_dbg("Free flag not set. increment hw tail"); + zip_dev->iq[queue].hw_tail += 16; /* 16 64_bit words = 128B */ + } + + zip_dev->iq[queue].done_cnt++; + zip_dev->iq[queue].pend_cnt--; + + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, + zip_dev->iq[queue].hw_tail); + zip_dbg(" Got CC : pend_cnt : %d\n", zip_dev->iq[queue].pend_cnt); + + spin_unlock(&zip_dev->iq[queue].lock); +} diff --git a/drivers/crypto/cavium/zip/zip_device.h b/drivers/crypto/cavium/zip/zip_device.h new file mode 100644 index 000000000000..9e18b3b93d38 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_device.h @@ -0,0 +1,108 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_DEVICE_H__ +#define __ZIP_DEVICE_H__ + +#include <linux/types.h> +#include "zip_main.h" + +struct sg_info { + /* + * Pointer to the input data when scatter_gather == 0 and + * pointer to the input gather list buffer when scatter_gather == 1 + */ + union zip_zptr_s *gather; + + /* + * Pointer to the output data when scatter_gather == 0 and + * pointer to the output scatter list buffer when scatter_gather == 1 + */ + union zip_zptr_s *scatter; + + /* + * Holds size of the output buffer pointed by scatter list + * when scatter_gather == 1 + */ + u64 scatter_buf_size; + + /* for gather data */ + u64 gather_enable; + + /* for scatter data */ + u64 scatter_enable; + + /* Number of gather list pointers for gather data */ + u32 gbuf_cnt; + + /* Number of scatter list pointers for scatter data */ + u32 sbuf_cnt; + + /* Buffers allocation state */ + u8 alloc_state; +}; + +/** + * struct zip_state - Structure representing the required information related + * to a command + * @zip_cmd: Pointer to zip instruction structure + * @result: Pointer to zip result structure + * @ctx: Context pointer for inflate + * @history: Decompression history pointer + * @sginfo: Scatter-gather info structure + */ +struct zip_state { + union zip_inst_s zip_cmd; + union zip_zres_s result; + union zip_zptr_s *ctx; + union zip_zptr_s *history; + struct sg_info sginfo; +}; + +#define ZIP_CONTEXT_SIZE 2048 +#define ZIP_INFLATE_HISTORY_SIZE 32768 +#define ZIP_DEFLATE_HISTORY_SIZE 32768 + +#endif diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c new file mode 100644 index 000000000000..89ea360d410d --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -0,0 +1,552 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "common.h" +#include "zip_crypto.h" + +#define DRV_NAME "ThunderX-ZIP" + +static struct zip_device *zip_dev[MAX_ZIP_DEVICES]; + +static const struct pci_device_id zip_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) }, + { 0, } +}; + +void zip_reg_write(u64 val, u64 __iomem *addr) +{ + writeq(val, addr); +} + +u64 zip_reg_read(u64 __iomem *addr) +{ + return readq(addr); +} + +/* + * Allocates new ZIP device structure + * Returns zip_device pointer or NULL if cannot allocate memory for zip_device + */ +static struct zip_device *zip_alloc_device(struct pci_dev *pdev) +{ + struct zip_device *zip = NULL; + int idx; + + for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) { + if (!zip_dev[idx]) + break; + } + + /* To ensure that the index is within the limit */ + if (idx < MAX_ZIP_DEVICES) + zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL); + + if (!zip) + return NULL; + + zip_dev[idx] = zip; + zip->index = idx; + return zip; +} + +/** + * zip_get_device - Get ZIP device based on node id of cpu + * + * @node: Node id of the current cpu + * Return: Pointer to Zip device structure + */ +struct zip_device *zip_get_device(int node) +{ + if ((node < MAX_ZIP_DEVICES) && (node >= 0)) + return zip_dev[node]; + + zip_err("ZIP device not found for node id %d\n", node); + return NULL; +} + +/** + * zip_get_node_id - Get the node id of the current cpu + * + * Return: Node id of the current cpu + */ +int zip_get_node_id(void) +{ + return cpu_to_node(smp_processor_id()); +} + +/* Initializes the ZIP h/w sub-system */ +static int zip_init_hw(struct zip_device *zip) +{ + union zip_cmd_ctl cmd_ctl; + union zip_constants constants; + union zip_que_ena que_ena; + union zip_quex_map que_map; + union zip_que_pri que_pri; + + union zip_quex_sbuf_addr que_sbuf_addr; + union zip_quex_sbuf_ctl que_sbuf_ctl; + + int q = 0; + + /* Enable the ZIP Engine(Core) Clock */ + cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL); + cmd_ctl.s.forceclk = 1; + zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL)); + + zip_msg("ZIP_CMD_CTL : 0x%016llx", + zip_reg_read(zip->reg_base + ZIP_CMD_CTL)); + + constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS); + zip->depth = constants.s.depth; + zip->onfsize = constants.s.onfsize; + zip->ctxsize = constants.s.ctxsize; + + zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx", + zip->depth, zip->onfsize, zip->ctxsize); + + /* + * Program ZIP_QUE(0..7)_SBUF_ADDR and ZIP_QUE(0..7)_SBUF_CTL to + * have the correct buffer pointer and size configured for each + * instruction queue. + */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + que_sbuf_ctl.u_reg64 = 0ull; + que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64)); + que_sbuf_ctl.s.inst_be = 0; + que_sbuf_ctl.s.stream_id = 0; + zip_reg_write(que_sbuf_ctl.u_reg64, + (zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); + + zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); + } + + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + memset(&zip->iq[q], 0x0, sizeof(struct zip_iq)); + + spin_lock_init(&zip->iq[q].lock); + + if (zip_cmd_qbuf_alloc(zip, q)) { + while (q != 0) { + q--; + zip_cmd_qbuf_free(zip, q); + } + return -ENOMEM; + } + + /* Initialize tail ptr to head */ + zip->iq[q].sw_tail = zip->iq[q].sw_head; + zip->iq[q].hw_tail = zip->iq[q].sw_head; + + /* Write the physical addr to register */ + que_sbuf_addr.u_reg64 = 0ull; + que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >> + ZIP_128B_ALIGN); + + zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q, + (u64)que_sbuf_addr.s.ptr); + + zip_reg_write(que_sbuf_addr.u_reg64, + (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); + + zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); + + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip->iq[q].sw_head, zip->iq[q].sw_tail, + zip->iq[q].hw_tail); + zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr); + } + + /* + * Queue-to-ZIP core mapping + * If a queue is not mapped to a particular core, it is equivalent to + * the ZIP core being disabled. + */ + que_ena.u_reg64 = 0x0ull; + /* Enabling queues based on ZIP_NUM_QUEUES */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) + que_ena.s.ena |= (0x1 << q); + zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA)); + + zip_msg("QUE_ENA : 0x%016llx", + zip_reg_read(zip->reg_base + ZIP_QUE_ENA)); + + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + que_map.u_reg64 = 0ull; + /* Mapping each queue to two ZIP cores */ + que_map.s.zce = 0x3; + zip_reg_write(que_map.u_reg64, + (zip->reg_base + ZIP_QUEX_MAP(q))); + + zip_msg("QUE_MAP(%d) : 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q))); + } + + que_pri.u_reg64 = 0ull; + for (q = 0; q < ZIP_NUM_QUEUES; q++) + que_pri.s.pri |= (0x1 << q); /* Higher Priority RR */ + zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI)); + + zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI)); + + return 0; +} + +static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct zip_device *zip = NULL; + int err; + + zip = zip_alloc_device(pdev); + if (!zip) + return -ENOMEM; + + dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index, + pdev->vendor, pdev->device, dev_to_node(dev)); + + pci_set_drvdata(pdev, zip); + zip->pdev = pdev; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device"); + goto err_free_device; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x", err); + goto err_disable_device; + } + + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto err_release_regions; + } + + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get 48-bit DMA for allocations\n"); + goto err_release_regions; + } + + /* MAP configuration registers */ + zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0); + if (!zip->reg_base) { + dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting"); + err = -ENOMEM; + goto err_release_regions; + } + + /* Initialize ZIP Hardware */ + err = zip_init_hw(zip); + if (err) + goto err_release_regions; + + return 0; + +err_release_regions: + if (zip->reg_base) + iounmap(zip->reg_base); + pci_release_regions(pdev); + +err_disable_device: + pci_disable_device(pdev); + +err_free_device: + pci_set_drvdata(pdev, NULL); + + /* Remove zip_dev from zip_device |