From 14fa93cdcd9bbd50018196c00ca16da636f965c2 Mon Sep 17 00:00:00 2001 From: Srikanth Jampala Date: Tue, 30 May 2017 17:28:01 +0530 Subject: crypto: cavium - Add support for CNN55XX adapters. Add Physical Function driver support for CNN55XX crypto adapters. CNN55XX adapters belongs to Cavium NITROX family series, which accelerate both Symmetric and Asymmetric crypto workloads. These adapters have crypto engines that need firmware to become operational. Signed-off-by: Srikanth Jampala Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 + drivers/crypto/Makefile | 1 + drivers/crypto/cavium/nitrox/Kconfig | 21 + drivers/crypto/cavium/nitrox/Makefile | 7 + drivers/crypto/cavium/nitrox/nitrox_common.h | 35 + drivers/crypto/cavium/nitrox/nitrox_csr.h | 1080 ++++++++++++++++++++++++++ drivers/crypto/cavium/nitrox/nitrox_dev.h | 175 +++++ drivers/crypto/cavium/nitrox/nitrox_hal.c | 401 ++++++++++ drivers/crypto/cavium/nitrox/nitrox_isr.c | 467 +++++++++++ drivers/crypto/cavium/nitrox/nitrox_lib.c | 172 ++++ drivers/crypto/cavium/nitrox/nitrox_main.c | 465 +++++++++++ drivers/crypto/cavium/nitrox/nitrox_req.h | 445 +++++++++++ drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 732 +++++++++++++++++ 13 files changed, 4002 insertions(+) create mode 100644 drivers/crypto/cavium/nitrox/Kconfig create mode 100644 drivers/crypto/cavium/nitrox/Makefile create mode 100644 drivers/crypto/cavium/nitrox/nitrox_common.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_csr.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_dev.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_hal.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_isr.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_lib.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_main.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_req.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_reqmgr.c diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 0a652baf3cb3..0528a62a39a6 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -541,6 +541,7 @@ config CRYPTO_DEV_MXS_DCP source "drivers/crypto/qat/Kconfig" source "drivers/crypto/cavium/cpt/Kconfig" +source "drivers/crypto/cavium/nitrox/Kconfig" config CRYPTO_DEV_CAVIUM_ZIP tristate "Cavium ZIP driver" diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 28786321b118..2c555a3393b2 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/ +obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/ obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig new file mode 100644 index 000000000000..731e6a57218a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Kconfig @@ -0,0 +1,21 @@ +# +# Cavium NITROX Crypto Device configuration +# +config CRYPTO_DEV_NITROX + tristate + select CRYPTO_BLKCIPHER + select CRYPTO_AES + select CRYPTO_DES + select FW_LOADER + +config CRYPTO_DEV_NITROX_CNN55XX + tristate "Support for Cavium CNN55XX driver" + depends on PCI_MSI && 64BIT + select CRYPTO_DEV_NITROX + default m + help + Support for Cavium NITROX family CNN55XX driver + for accelerating crypto workloads. + + To compile this as a module, choose M here: the module + will be called n5pf. diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile new file mode 100644 index 000000000000..ef457f609d88 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -0,0 +1,7 @@ +obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o + +n5pf-objs := nitrox_main.o \ + nitrox_isr.o \ + nitrox_lib.o \ + nitrox_hal.o \ + nitrox_reqmgr.o diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h new file mode 100644 index 000000000000..3284ffe4837a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -0,0 +1,35 @@ +#ifndef __NITROX_COMMON_H +#define __NITROX_COMMON_H + +#include "nitrox_dev.h" +#include "nitrox_req.h" + +void nitrox_pf_cleanup_isr(struct nitrox_device *ndev); +int nitrox_pf_init_isr(struct nitrox_device *ndev); + +int nitrox_common_sw_init(struct nitrox_device *ndev); +void nitrox_common_sw_cleanup(struct nitrox_device *ndev); + +void pkt_slc_resp_handler(unsigned long data); +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t cb, + struct skcipher_request *skreq); +void backlog_qflush_work(struct work_struct *work); + +void nitrox_config_emu_unit(struct nitrox_device *ndev); +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev); +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev); +void nitrox_config_vfmode(struct nitrox_device *ndev, int mode); +void nitrox_config_nps_unit(struct nitrox_device *ndev); +void nitrox_config_pom_unit(struct nitrox_device *ndev); +void nitrox_config_rand_unit(struct nitrox_device *ndev); +void nitrox_config_efl_unit(struct nitrox_device *ndev); +void nitrox_config_bmi_unit(struct nitrox_device *ndev); +void nitrox_config_bmo_unit(struct nitrox_device *ndev); +void nitrox_config_lbc_unit(struct nitrox_device *ndev); +void invalidate_lbc(struct nitrox_device *ndev); +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring); +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port); + +#endif /* __NITROX_COMMON_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h new file mode 100644 index 000000000000..583465190882 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -0,0 +1,1080 @@ +#ifndef __NITROX_CSR_H +#define __NITROX_CSR_H + +#include +#include + +/* EMU clusters */ +#define NR_CLUSTERS 4 +#define AE_CORES_PER_CLUSTER 20 +#define SE_CORES_PER_CLUSTER 16 + +/* BIST registers */ +#define EMU_BIST_STATUSX(_i) (0x1402700 + ((_i) * 0x40000)) +#define UCD_BIST_STATUS 0x12C0070 +#define NPS_CORE_BIST_REG 0x10000E8 +#define NPS_CORE_NPC_BIST_REG 0x1000128 +#define NPS_PKT_SLC_BIST_REG 0x1040088 +#define NPS_PKT_IN_BIST_REG 0x1040100 +#define POM_BIST_REG 0x11C0100 +#define BMI_BIST_REG 0x1140080 +#define EFL_CORE_BIST_REGX(_i) (0x1240100 + ((_i) * 0x400)) +#define EFL_TOP_BIST_STAT 0x1241090 +#define BMO_BIST_REG 0x1180080 +#define LBC_BIST_STATUS 0x1200020 +#define PEM_BIST_STATUSX(_i) (0x1080468 | ((_i) << 18)) + +/* EMU registers */ +#define EMU_SE_ENABLEX(_i) (0x1400000 + ((_i) * 0x40000)) +#define EMU_AE_ENABLEX(_i) (0x1400008 + ((_i) * 0x40000)) +#define EMU_WD_INT_ENA_W1SX(_i) (0x1402318 + ((_i) * 0x40000)) +#define EMU_GE_INT_ENA_W1SX(_i) (0x1402518 + ((_i) * 0x40000)) +#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000)) + +/* UCD registers */ +#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010 +#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20)) +#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000)) + +/* NPS core registers */ +#define NPS_CORE_GBL_VFCFG 0x1000000 +#define NPS_CORE_CONTROL 0x1000008 +#define NPS_CORE_INT_ACTIVE 0x1000080 +#define NPS_CORE_INT 0x10000A0 +#define NPS_CORE_INT_ENA_W1S 0x10000B8 + +/* NPS packet registers */ +#define NPS_PKT_INT 0x1040018 +#define NPS_PKT_IN_RERR_HI 0x1040108 +#define NPS_PKT_IN_RERR_HI_ENA_W1S 0x1040120 +#define NPS_PKT_IN_RERR_LO 0x1040128 +#define NPS_PKT_IN_RERR_LO_ENA_W1S 0x1040140 +#define NPS_PKT_IN_ERR_TYPE 0x1040148 +#define NPS_PKT_IN_ERR_TYPE_ENA_W1S 0x1040160 +#define NPS_PKT_IN_INSTR_CTLX(_i) (0x10060 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BADDRX(_i) (0x10068 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_RSIZEX(_i) (0x10070 + ((_i) * 0x40000)) +#define NPS_PKT_IN_DONE_CNTSX(_i) (0x10080 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BAOFF_DBELLX(_i) (0x10078 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INT_LEVELSX(_i) (0x10088 + ((_i) * 0x40000)) + +#define NPS_PKT_SLC_RERR_HI 0x1040208 +#define NPS_PKT_SLC_RERR_HI_ENA_W1S 0x1040220 +#define NPS_PKT_SLC_RERR_LO 0x1040228 +#define NPS_PKT_SLC_RERR_LO_ENA_W1S 0x1040240 +#define NPS_PKT_SLC_ERR_TYPE 0x1040248 +#define NPS_PKT_SLC_ERR_TYPE_ENA_W1S 0x1040260 +#define NPS_PKT_SLC_CTLX(_i) (0x10000 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_CNTSX(_i) (0x10008 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_INT_LEVELSX(_i) (0x10010 + ((_i) * 0x40000)) + +/* POM registers */ +#define POM_INT_ENA_W1S 0x11C0018 +#define POM_GRP_EXECMASKX(_i) (0x11C1100 | ((_i) * 8)) +#define POM_INT 0x11C0000 +#define POM_PERF_CTL 0x11CC400 + +/* BMI registers */ +#define BMI_INT 0x1140000 +#define BMI_CTL 0x1140020 +#define BMI_INT_ENA_W1S 0x1140018 + +/* EFL registers */ +#define EFL_CORE_INT_ENA_W1SX(_i) (0x1240018 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0X(_i) (0x1240050 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0_ENA_W1SX(_i) (0x1240068 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1X(_i) (0x1240070 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1_ENA_W1SX(_i) (0x1240088 + ((_i) * 0x400)) +#define EFL_CORE_SE_ERR_INTX(_i) (0x12400A0 + ((_i) * 0x400)) +#define EFL_RNM_CTL_STATUS 0x1241800 +#define EFL_CORE_INTX(_i) (0x1240000 + ((_i) * 0x400)) + +/* BMO registers */ +#define BMO_CTL2 0x1180028 + +/* LBC registers */ +#define LBC_INT 0x1200000 +#define LBC_INVAL_CTL 0x1201010 +#define LBC_PLM_VF1_64_INT 0x1202008 +#define LBC_INVAL_STATUS 0x1202010 +#define LBC_INT_ENA_W1S 0x1203000 +#define LBC_PLM_VF1_64_INT_ENA_W1S 0x1205008 +#define LBC_PLM_VF65_128_INT 0x1206008 +#define LBC_ELM_VF1_64_INT 0x1208000 +#define LBC_PLM_VF65_128_INT_ENA_W1S 0x1209008 +#define LBC_ELM_VF1_64_INT_ENA_W1S 0x120B000 +#define LBC_ELM_VF65_128_INT 0x120C000 +#define LBC_ELM_VF65_128_INT_ENA_W1S 0x120F000 + +/* PEM registers */ +#define PEM0_INT 0x1080428 + +/** + * struct emu_fuse_map - EMU Fuse Map Registers + * @ae_fuse: Fuse settings for AE 19..0 + * @se_fuse: Fuse settings for SE 15..0 + * + * A set bit indicates the unit is fuse disabled. + */ +union emu_fuse_map { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 valid : 1; + u64 raz_52_62 : 11; + u64 ae_fuse : 20; + u64 raz_16_31 : 16; + u64 se_fuse : 16; +#else + u64 se_fuse : 16; + u64 raz_16_31 : 16; + u64 ae_fuse : 20; + u64 raz_52_62 : 11; + u64 valid : 1; +#endif + } s; +}; + +/** + * struct emu_se_enable - Symmetric Engine Enable Registers + * @enable: Individual enables for each of the clusters + * 16 symmetric engines. + */ +union emu_se_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 48; + u64 enable : 16; +#else + u64 enable : 16; + u64 raz : 48; +#endif + } s; +}; + +/** + * struct emu_ae_enable - EMU Asymmetric engines. + * @enable: Individual enables for each of the cluster's + * 20 Asymmetric Engines. + */ +union emu_ae_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 44; + u64 enable : 20; +#else + u64 enable : 20; + u64 raz : 44; +#endif + } s; +}; + +/** + * struct emu_wd_int_ena_w1s - EMU Interrupt Enable Registers + * @ae_wd: Reads or sets enable for EMU(0..3)_WD_INT[AE_WD] + * @se_wd: Reads or sets enable for EMU(0..3)_WD_INT[SE_WD] + */ +union emu_wd_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 12; + u64 ae_wd : 20; + u64 raz1 : 16; + u64 se_wd : 16; +#else + u64 se_wd : 16; + u64 raz1 : 16; + u64 ae_wd : 20; + u64 raz2 : 12; +#endif + } s; +}; + +/** + * struct emu_ge_int_ena_w1s - EMU Interrupt Enable set registers + * @ae_ge: Reads or sets enable for EMU(0..3)_GE_INT[AE_GE] + * @se_ge: Reads or sets enable for EMU(0..3)_GE_INT[SE_GE] + */ +union emu_ge_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_52_63 : 12; + u64 ae_ge : 20; + u64 raz_16_31: 16; + u64 se_ge : 16; +#else + u64 se_ge : 16; + u64 raz_16_31: 16; + u64 ae_ge : 20; + u64 raz_52_63 : 12; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_ctl - Solicited Packet Out Control Registers + * @rh: Indicates whether to remove or include the response header + * 1 = Include, 0 = Remove + * @z: If set, 8 trailing 0x00 bytes will be added to the end of the + * outgoing packet. + * @enb: Enable for this port. + */ +union nps_pkt_slc_ctl { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 61; + u64 rh : 1; + u64 z : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 z : 1; + u64 rh : 1; + u64 raz : 61; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_cnts - Solicited Packet Out Count Registers + * @slc_int: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET]. + * To clear the bit, the CNTS register must be written to clear. + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]. + * To clear the bit, the DONE_CNTS register must be written to clear. + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. To clear the bit, + * write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] with 1. + * @timer: Timer, incremented every 2048 coprocessor clock cycles + * when [CNT] is not zero. The hardware clears both [TIMER] and + * [INT] when [CNT] goes to 0. + * @cnt: Packet counter. Hardware adds to [CNT] as it sends packets out. + * On a write to this CSR, hardware subtracts the amount written to the + * [CNT] field from [CNT]. + */ +union nps_pkt_slc_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 5; + u64 timer : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timer : 22; + u64 raz : 5; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_int_levels - Solicited Packet Out Interrupt Levels + * Registers. + * @bmode: Determines whether NPS_PKT_SLC_CNTS[CNT] is a byte or + * packet counter. + * @timet: Output port counter time interrupt threshold. + * @cnt: Output port counter interrupt threshold. + */ +union nps_pkt_slc_int_levels { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 bmode : 1; + u64 raz : 9; + u64 timet : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timet : 22; + u64 raz : 9; + u64 bmode : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_inst - NPS Packet Interrupt Register + * @in_err: Set when any NPS_PKT_IN_RERR_HI/LO bit and + * corresponding NPS_PKT_IN_RERR_*_ENA_* bit are bot set. + * @uns_err: Set when any NSP_PKT_UNS_RERR_HI/LO bit and + * corresponding NPS_PKT_UNS_RERR_*_ENA_* bit are both set. + * @slc_er: Set when any NSP_PKT_SLC_RERR_HI/LO bit and + * corresponding NPS_PKT_SLC_RERR_*_ENA_* bit are both set. + */ +union nps_pkt_int { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 54; + u64 uns_wto : 1; + u64 in_err : 1; + u64 uns_err : 1; + u64 slc_err : 1; + u64 in_dbe : 1; + u64 in_sbe : 1; + u64 uns_dbe : 1; + u64 uns_sbe : 1; + u64 slc_dbe : 1; + u64 slc_sbe : 1; +#else + u64 slc_sbe : 1; + u64 slc_dbe : 1; + u64 uns_sbe : 1; + u64 uns_dbe : 1; + u64 in_sbe : 1; + u64 in_dbe : 1; + u64 slc_err : 1; + u64 uns_err : 1; + u64 in_err : 1; + u64 uns_wto : 1; + u64 raz : 54; +#endif + } s; +}; + +/** + * struct nps_pkt_in_done_cnts - Input instruction ring counts registers + * @slc_cnt: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SCL(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @uns_int: Return a 1 when: + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT], or + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * To clear the bit, the DONE_CNTS register + * must be written to clear the underlying condition + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. + * To clear the bit, write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] + * with 1. + * @resend: A write of 1 will resend an MSI-X interrupt message if any + * of the following conditions are true for this ring "i". + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT] + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET] + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT] + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set + * @cnt: Packet counter. Hardware adds to [CNT] as it reads + * packets. On a write to this CSR, hardware substracts the + * amount written to the [CNT] field from [CNT], which will + * clear PKT_IN(i)_INT_STATUS[INTR] if [CNT] becomes <= + * NPS_PKT_IN(i)_INT_LEVELS[CNT]. This register should be + * cleared before enabling a ring by reading the current + * value and writing it back. + */ +union nps_pkt_in_done_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 27; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 raz : 27; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_ctl - Input Instruction Ring Control Registers. + * @is64b: If 1, the ring uses 64-byte instructions. If 0, the + * ring uses 32-byte instructions. + * @enb: Enable for the input ring. + */ +union nps_pkt_in_instr_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 62; + u64 is64b : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 is64b : 1; + u64 raz : 62; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_rsize - Input instruction ring size registers + * @rsize: Ring size (number of instructions) + */ +union nps_pkt_in_instr_rsize { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 32; + u64 rsize : 32; +#else + u64 rsize : 32; + u64 raz : 32; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_baoff_dbell - Input instruction ring + * base address offset and doorbell registers + * @aoff: Address offset. The offset from the NPS_PKT_IN_INSTR_BADDR + * where the next pointer is read. + * @dbell: Pointer list doorbell count. Write operations to this field + * increments the present value here. Read operations return the + * present value. + */ +union nps_pkt_in_instr_baoff_dbell { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 aoff : 32; + u64 dbell : 32; +#else + u64 dbell : 32; + u64 aoff : 32; +#endif + } s; +}; + +/** + * struct nps_core_int_ena_w1s - NPS core interrupt enable set register + * @host_nps_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_NPS_WR_ERR]. + * @npco_dma_malform: Reads or sets enable for + * NPS_CORE_INT[NPCO_DMA_MALFORM]. + * @exec_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[EXEC_WR_TIMEOUT]. + * @host_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_TIMEOUT]. + * @host_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_ERR] + */ +union nps_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz4 : 55; + u64 host_nps_wr_err : 1; + u64 npco_dma_malform : 1; + u64 exec_wr_timeout : 1; + u64 host_wr_timeout : 1; + u64 host_wr_err : 1; + u64 raz3 : 1; + u64 raz2 : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 raz2 : 1; + u64 raz3 : 1; + u64 host_wr_err : 1; + u64 host_wr_timeout : 1; + u64 exec_wr_timeout : 1; + u64 npco_dma_malform : 1; + u64 host_nps_wr_err : 1; + u64 raz4 : 55; +#endif + } s; +}; + +/** + * struct nps_core_gbl_vfcfg - Global VF Configuration Register. + * @ilk_disable: When set, this bit indicates that the ILK interface has + * been disabled. + * @obaf: BMO allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @ibaf: BMI allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @zaf: ZIP allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @aeaf: AE allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @seaf: SE allocation control + * 0 = allocation per queue + * 1 = allocate per VF + * @cfg: VF/PF mode. + */ +union nps_core_gbl_vfcfg { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz :55; + u64 ilk_disable :1; + u64 obaf :1; + u64 ibaf :1; + u64 zaf :1; + u64 aeaf :1; + u64 seaf :1; + u64 cfg :3; +#else + u64 cfg :3; + u64 seaf :1; + u64 aeaf :1; + u64 zaf :1; + u64 ibaf :1; + u64 obaf :1; + u64 ilk_disable :1; + u64 raz :55; +#endif + } s; +}; + +/** + * struct nps_core_int_active - NPS Core Interrupt Active Register + * @resend: Resend MSI-X interrupt if needs to handle interrupts + * Sofware can set this bit and then exit the ISR. + * @ocla: Set when any OCLA(0)_INT and corresponding OCLA(0_INT_ENA_W1C + * bit are set + * @mbox: Set when any NPS_PKT_MBOX_INT_LO/HI and corresponding + * NPS_PKT_MBOX_INT_LO_ENA_W1C/HI_ENA_W1C bits are set + * @emu: bit i is set in [EMU] when any EMU(i)_INT bit is set + * @bmo: Set when any BMO_INT bit is set + * @bmi: Set when any BMI_INT bit is set or when any non-RO + * BMI_INT and corresponding BMI_INT_ENA_W1C bits are both set + * @aqm: Set when any AQM_INT bit is set + * @zqm: Set when any ZQM_INT bit is set + * @efl: Set when any EFL_INT RO bit is set or when any non-RO EFL_INT + * and corresponding EFL_INT_ENA_W1C bits are both set + * @ilk: Set when any ILK_INT bit is set + * @lbc: Set when any LBC_INT RO bit is set or when any non-RO LBC_INT + * and corresponding LBC_INT_ENA_W1C bits are bot set + * @pem: Set when any PEM(0)_INT RO bit is set or when any non-RO + * PEM(0)_INT and corresponding PEM(0)_INT_ENA_W1C bit are both set + * @ucd: Set when any UCD_INT bit is set + * @zctl: Set when any ZIP_INT RO bit is set or when any non-RO ZIP_INT + * and corresponding ZIP_INT_ENA_W1C bits are both set + * @lbm: Set when any LBM_INT bit is set + * @nps_pkt: Set when any NPS_PKT_INT bit is set + * @nps_core: Set when any NPS_CORE_INT RO bit is set or when non-RO + * NPS_CORE_INT and corresponding NSP_CORE_INT_ENA_W1C bits are both set + */ +union nps_core_int_active { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 resend : 1; + u64 raz : 43; + u64 ocla : 1; + u64 mbox : 1; + u64 emu : 4; + u64 bmo : 1; + u64 bmi : 1; + u64 aqm : 1; + u64 zqm : 1; + u64 efl : 1; + u64 ilk : 1; + u64 lbc : 1; + u64 pem : 1; + u64 pom : 1; + u64 ucd : 1; + u64 zctl : 1; + u64 lbm : 1; + u64 nps_pkt : 1; + u64 nps_core : 1; +#else + u64 nps_core : 1; + u64 nps_pkt : 1; + u64 lbm : 1; + u64 zctl: 1; + u64 ucd : 1; + u64 pom : 1; + u64 pem : 1; + u64 lbc : 1; + u64 ilk : 1; + u64 efl : 1; + u64 zqm : 1; + u64 aqm : 1; + u64 bmi : 1; + u64 bmo : 1; + u64 emu : 4; + u64 mbox : 1; + u64 ocla : 1; + u64 raz : 43; + u64 resend : 1; +#endif + } s; +}; + +/** + * struct efl_core_int - EFL Interrupt Registers + * @epci_decode_err: EPCI decoded a transacation that was unknown + * This error should only occurred when there is a micrcode/SE error + * and should be considered fatal + * @ae_err: An AE uncorrectable error occurred. + * See EFL_CORE(0..3)_AE_ERR_INT + * @se_err: An SE uncorrectable error occurred. + * See EFL_CORE(0..3)_SE_ERR_INT + * @dbe: Double-bit error occurred in EFL + * @sbe: Single-bit error occurred in EFL + * @d_left: Asserted when new POM-Header-BMI-data is + * being sent to an Exec, and that Exec has Not read all BMI + * data associated with the previous POM header + * @len_ovr: Asserted when an Exec-Read is issued that is more than + * 14 greater in length that the BMI data left to be read + */ +union efl_core_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 57; + u64 epci_decode_err : 1; + u64 ae_err : 1; + u64 se_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 se_err : 1; + u64 ae_err : 1; + u64 epci_decode_err : 1; + u64 raz : 57; +#endif + } s; +}; + +/** + * struct efl_core_int_ena_w1s - EFL core interrupt enable set register + * @epci_decode_err: Reads or sets enable for + * EFL_CORE(0..3)_INT[EPCI_DECODE_ERR]. + * @d_left: Reads or sets enable for + * EFL_CORE(0..3)_INT[D_LEFT]. + * @len_ovr: Reads or sets enable for + * EFL_CORE(0..3)_INT[LEN_OVR]. + */ +union efl_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_7_63 : 57; + u64 epci_decode_err : 1; + u64 raz_2_5 : 4; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 raz_2_5 : 4; + u64 epci_decode_err : 1; + u64 raz_7_63 : 57; +#endif + } s; +}; + +/** + * struct efl_rnm_ctl_status - RNM Control and Status Register + * @ent_sel: Select input to RNM FIFO + * @exp_ent: Exported entropy enable for random number generator + * @rng_rst: Reset to RNG. Setting this bit to 1 cancels the generation + * of the current random number. + * @rnm_rst: Reset the RNM. Setting this bit to 1 clears all sorted numbers + * in the random number memory. + * @rng_en: Enabled the output of the RNG. + * @ent_en: Entropy enable for random number generator. + */ +union efl_rnm_ctl_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_9_63 : 55; + u64 ent_sel : 4; + u64 exp_ent : 1; + u64 rng_rst : 1; + u64 rnm_rst : 1; + u64 rng_en : 1; + u64 ent_en : 1; +#else + u64 ent_en : 1; + u64 rng_en : 1; + u64 rnm_rst : 1; + u64 rng_rst : 1; + u64 exp_ent : 1; + u64 ent_sel : 4; + u64 raz_9_63 : 55; +#endif + } s; +}; + +/** + * struct bmi_ctl - BMI control register + * @ilk_hdrq_thrsh: Maximum number of header queue locations + * that ILK packets may consume. When the threshold is + * exceeded ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_hdrq_thrsh: Maximum number of header queue locations + * that NPS packets may consume. When the threshold is + * exceeded NPS_XOFF is sent to the BMI_X2P_ARB. + * @totl_hdrq_thrsh: Maximum number of header queue locations + * that the sum of ILK and NPS packets may consume. + * @ilk_free_thrsh: Maximum number of buffers that ILK packet + * flows may consume before ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_free_thrsh: Maximum number of buffers that NPS packet + * flows may consume before NPS XOFF is sent to the BMI_X2p_ARB. + * @totl_free_thrsh: Maximum number of buffers that bot ILK and NPS + * packet flows may consume before both NPS_XOFF and ILK_XOFF + * are asserted to the BMI_X2P_ARB. + * @max_pkt_len: Maximum packet length, integral number of 256B + * buffers. + */ +union bmi_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_56_63 : 8; + u64 ilk_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 totl_free_thrsh : 8; + u64 max_pkt_len : 8; +#else + u64 max_pkt_len : 8; + u64 totl_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 ilk_hdrq_thrsh : 8; + u64 raz_56_63 : 8; +#endif + } s; +}; + +/** + * struct bmi_int_ena_w1s - BMI interrupt enable set register + * @ilk_req_oflw: Reads or sets enable for + * BMI_INT[ILK_REQ_OFLW]. + * @nps_req_oflw: Reads or sets enable for + * BMI_INT[NPS_REQ_OFLW]. + * @fpf_undrrn: Reads or sets enable for + * BMI_INT[FPF_UNDRRN]. + * @eop_err_ilk: Reads or sets enable for + * BMI_INT[EOP_ERR_ILK]. + * @eop_err_nps: Reads or sets enable for + * BMI_INT[EOP_ERR_NPS]. + * @sop_err_ilk: Reads or sets enable for + * BMI_INT[SOP_ERR_ILK]. + * @sop_err_nps: Reads or sets enable for + * BMI_INT[SOP_ERR_NPS]. + * @pkt_rcv_err_ilk: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_ILK]. + * @pkt_rcv_err_nps: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_NPS]. + * @max_len_err_ilk: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_ILK]. + * @max_len_err_nps: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_NPS]. + */ +union bmi_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_13_63 : 51; + u64 ilk_req_oflw : 1; + u64 nps_req_oflw : 1; + u64 raz_10 : 1; + u64 raz_9 : 1; + u64 fpf_undrrn : 1; + u64 eop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 sop_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 max_len_err_nps : 1; +#else + u64 max_len_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 sop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 eop_err_ilk : 1; + u64 fpf_undrrn : 1; + u64 raz_9 : 1; + u64 raz_10 : 1; + u64 nps_req_oflw : 1; + u64 ilk_req_oflw : 1; + u64 raz_13_63 : 51; +#endif + } s; +}; + +/** + * struct bmo_ctl2 - BMO Control2 Register + * @arb_sel: Determines P2X Arbitration + * @ilk_buf_thrsh: Maximum number of buffers that the + * ILK packet flows may consume before ILK XOFF is + * asserted to the POM. + * @nps_slc_buf_thrsh: Maximum number of buffers that the + * NPS_SLC packet flow may consume before NPS_SLC XOFF is + * asserted to the POM. + * @nps_uns_buf_thrsh: Maximum number of buffers that the + * NPS_UNS packet flow may consume before NPS_UNS XOFF is + * asserted to the POM. + * @totl_buf_thrsh: Maximum number of buffers that ILK, NPS_UNS and + * NPS_SLC packet flows may consume before NPS_UNS XOFF, NSP_SLC and + * ILK_XOFF are all asserted POM. + */ +union bmo_ctl2 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 arb_sel : 1; + u64 raz_32_62 : 31; + u64 ilk_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 totl_buf_thrsh : 8; +#else + u64 totl_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 ilk_buf_thrsh : 8; + u64 raz_32_62 : 31; + u64 arb_sel : 1; +#endif + } s; +}; + +/** + * struct pom_int_ena_w1s - POM interrupt enable set register + * @illegal_intf: Reads or sets enable for POM_INT[ILLEGAL_INTF]. + * @illegal_dport: Reads or sets enable for POM_INT[ILLEGAL_DPORT]. + */ +union pom_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 60; + u64 illegal_intf : 1; + u64 illegal_dport : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 illegal_dport : 1; + u64 illegal_intf : 1; + u64 raz2 : 60; +#endif + } s; +}; + +/** + * struct lbc_inval_ctl - LBC invalidation control register + * @wait_timer: Wait timer for wait state. [WAIT_TIMER] must + * always be written with its reset value. + * @cam_inval_start: Software should write [CAM_INVAL_START]=1 + * to initiate an LBC cache invalidation. After this, software + * should read LBC_INVAL_STATUS until LBC_INVAL_STATUS[DONE] is set. + * LBC hardware clears [CAVM_INVAL_START] before software can + * observed LBC_INVAL_STATUS[DONE] to be set + */ +union lbc_inval_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 48; + u64 wait_timer : 8; + u64 raz1 : 6; + u64 cam_inval_start : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 cam_inval_start : 1; + u64 raz1 : 6; + u64 wait_timer : 8; + u64 raz2 : 48; +#endif + } s; +}; + +/** + * struct lbc_int_ena_w1s - LBC interrupt enable set register + * @cam_hard_err: Reads or sets enable for LBC_INT[CAM_HARD_ERR]. + * @cam_inval_abort: Reads or sets enable for LBC_INT[CAM_INVAL_ABORT]. + * @over_fetch_err: Reads or sets enable for LBC_INT[OVER_FETCH_ERR]. + * @cache_line_to_err: Reads or sets enable for + * LBC_INT[CACHE_LINE_TO_ERR]. + * @cam_soft_err: Reads or sets enable for + * LBC_INT[CAM_SOFT_ERR]. + * @dma_rd_err: Reads or sets enable for + * LBC_INT[DMA_RD_ERR]. + */ +union lbc_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 raz_2_5 : 4; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 raz_2_5 : 4; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_int - LBC interrupt summary register + * @cam_hard_err: indicates a fatal hardware error. + * It requires system reset. + * When [CAM_HARD_ERR] is set, LBC stops logging any new information in + * LBC_POM_MISS_INFO_LOG, + * LBC_POM_MISS_ADDR_LOG, + * LBC_EFL_MISS_INFO_LOG, and + * LBC_EFL_MISS_ADDR_LOG. + * Software should sample them. + * @cam_inval_abort: indicates a fatal hardware error. + * System reset is required. + * @over_fetch_err: indicates a fatal hardware error + * System reset is required + * @cache_line_to_err: is a debug feature. + * This timeout interrupt bit tells the software that + * a cacheline in LBC has non-zero usage and the context + * has not been used for greater than the + * LBC_TO_CNT[TO_CNT] time interval. + * @sbe: Memory SBE error. This is recoverable via ECC. + * See LBC_ECC_INT for more details. + * @dbe: Memory DBE error. This is a fatal and requires a + * system reset. + * @pref_dat_len_mismatch_err: Summary bit for context length + * mismatch errors. + * @rd_dat_len_mismatch_err: Summary bit for SE read data length + * greater than data prefect length errors. + * @cam_soft_err: is recoverable. Software must complete a + * LBC_INVAL_CTL[CAM_INVAL_START] invalidation sequence and + * then clear [CAM_SOFT_ERR]. + * @dma_rd_err: A context prefect read of host memory returned with + * a read error. + */ +union lbc_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_inval_status: LBC Invalidation status register + * @cam_clean_entry_complete_cnt: The number of entries that are + * cleaned up successfully. + * @cam_clean_entry_cnt: The number of entries that have the CAM + * inval command issued. + * @cam_inval_state: cam invalidation FSM state + * @cam_inval_abort: cam invalidation abort + * @cam_rst_rdy: lbc_cam reset ready + * @done: LBC clears [DONE] when + * LBC_INVAL_CTL[CAM_INVAL_START] is written with a one, + * and sets [DONE] when it completes the invalidation + * sequence. + */ +union lbc_inval_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz3 : 23; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_cnt : 9; + u64 raz1 : 5; + u64 cam_inval_state : 3; + u64 raz0 : 5; + u64 cam_inval_abort : 1; + u64 cam_rst_rdy : 1; + u64 done : 1; +#else + u64 done : 1; + u64 cam_rst_rdy : 1; + u64 cam_inval_abort : 1; + u64 raz0 : 5; + u64 cam_inval_state : 3; + u64 raz1 : 5; + u64 cam_clean_entry_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz3 : 23; +#endif + } s; +}; + +#endif /* __NITROX_CSR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h new file mode 100644 index 000000000000..2793a106b552 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -0,0 +1,175 @@ +#ifndef __NITROX_DEV_H +#define __NITROX_DEV_H + +#include +#include +#include + +#define VERSION_LEN 32 + +struct nitrox_cmdq { + /* command queue lock */ + spinlock_t cmdq_lock; + /* response list lock */ + spinlock_t response_lock; + /* backlog list lock */ + spinlock_t backlog_lock; + + /* request submitted to chip, in progress */ + struct list_head response_head; + /* hw queue full, hold in backlog list */ + struct list_head backlog_head; + + /* doorbell address */ + u8 __iomem *dbell_csr_addr; + /* base address of the queue */ + u8 *head; + + struct nitrox_device *ndev; + /* flush pending backlog commands */ + struct work_struct backlog_qflush; + + /* requests posted waiting for completion */ + atomic_t pending_count; + /* requests in backlog queues */ + atomic_t backlog_count; + + /* command size 32B/64B */ + u8 instr_size; + u8 qno; + u32 qsize; + + /* unaligned addresses */ + u8 *head_unaligned; + dma_addr_t dma_unaligned; + /* dma address of the base */ + dma_addr_t dma; +}; + +struct nitrox_hw { + /* firmware version */ + char fw_name[VERSION_LEN]; + + u16 vendor_id; + u16 device_id; + u8 revision_id; + + /* CNN55XX cores */ + u8 se_cores; + u8 ae_cores; + u8 zip_cores; +}; + +#define MAX_MSIX_VECTOR_NAME 20 +/** + * vectors for queues (64 AE, 64 SE and 64 ZIP) and + * error condition/mailbox. + */ +#define MAX_MSIX_VECTORS 192 + +struct nitrox_msix { + struct msix_entry *entries; + char **names; + DECLARE_BITMAP(irqs, MAX_MSIX_VECTORS); + u32 nr_entries; +}; + +struct bh_data { + /* slc port completion count address */ + u8 __iomem *completion_cnt_csr_addr; + + struct nitrox_cmdq *cmdq; + struct tasklet_struct resp_handler; +}; + +struct nitrox_bh { + struct bh_data *slc; +}; + +/* NITROX-5 driver state */ +#define NITROX_UCODE_LOADED 0 +#define NITROX_READY 1 + +/* command queue size */ +#define DEFAULT_CMD_QLEN 2048 +/* command timeout in milliseconds */ +#define CMD_TIMEOUT 2000 + +#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev)) +#define PF_MODE 0 + +#define NITROX_CSR_ADDR(ndev, offset) \ + ((ndev)->bar_addr + (offset)) + +/** + * struct nitrox_device - NITROX Device Information. + * @list: pointer to linked list of devices + * @bar_addr: iomap address + * @pdev: PCI device information + * @status: NITROX status + * @timeout: Request timeout in jiffies + * @refcnt: Device usage count + * @idx: device index (0..N) + * @node: NUMA node id attached + * @qlen: Command queue length + * @nr_queues: Number of command queues + * @ctx_pool: DMA pool for crypto context + * @pkt_cmdqs: SE Command queues + * @msix: MSI-X information + * @bh: post processing work + * @hw: hardware information + */ +struct nitrox_device { + struct list_head list; + + u8 __iomem *bar_addr; + struct pci_dev *pdev; + + unsigned long status; + unsigned long timeout; + refcount_t refcnt; + + u8 idx; + int node; + u16 qlen; + u16 nr_queues; + + struct dma_pool *ctx_pool; + struct nitrox_cmdq *pkt_cmdqs; + + struct nitrox_msix msix; + struct nitrox_bh bh; + + struct nitrox_hw hw; +}; + +/** + * nitrox_read_csr - Read from device register + * @ndev: NITROX device + * @offset: offset of the register to read + * + * Returns: value read + */ +static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset) +{ + return readq(ndev->bar_addr + offset); +} + +/** + * nitrox_write_csr - Write to device register + * @ndev: NITROX device + * @offset: offset of the register to write + * @value: value to write + */ +static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset, + u64 value) +{ + writeq(value, (ndev->bar_addr + offset)); +} + +static inline int nitrox_ready(struct nitrox_device *ndev) +{ + return test_bit(NITROX_READY, &ndev->status); +} + +#endif /* __NITROX_DEV_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c new file mode 100644 index 000000000000..f0655f82fa7d --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -0,0 +1,401 @@ +#include + +#include "nitrox_dev.h" +#include "nitrox_csr.h" + +/** + * emu_enable_cores - Enable EMU cluster cores. + * @ndev: N5 device + */ +static void emu_enable_cores(struct nitrox_device *ndev) +{ + union emu_se_enable emu_se; + union emu_ae_enable emu_ae; + int i; + + /* AE cores 20 per cluster */ + emu_ae.value = 0; + emu_ae.s.enable = 0xfffff; + + /* SE cores 16 per cluster */ + emu_se.value = 0; + emu_se.s.enable = 0xffff; + + /* enable per cluster cores */ + for (i = 0; i < NR_CLUSTERS; i++) { + nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value); + nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value); + } +} + +/** + * nitrox_config_emu_unit - configure EMU unit. + * @ndev: N5 device + */ +void nitrox_config_emu_unit(struct nitrox_device *ndev) +{ + union emu_wd_int_ena_w1s emu_wd_int; + union emu_ge_int_ena_w1s emu_ge_int; + u64 offset; + int i; + + /* enable cores */ + emu_enable_cores(ndev); + + /* enable general error and watch dog interrupts */ + emu_ge_int.value = 0; + emu_ge_int.s.se_ge = 0xffff; + emu_ge_int.s.ae_ge = 0xfffff; + emu_wd_int.value = 0; + emu_wd_int.s.se_wd = 1; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_WD_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_wd_int.value); + offset = EMU_GE_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_ge_int.value); + } +} + +static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + union nps_pkt_in_instr_baoff_dbell pkt_in_dbell; + union nps_pkt_in_done_cnts pkt_in_cnts; + u64 offset; + + offset = NPS_PKT_IN_INSTR_CTLX(ring); + /* disable the ring */ + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + usleep_range(100, 150); + + /* wait to clear [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + } while (pkt_in_ctl.s.enb); + + /* clear off door bell counts */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(ring); + pkt_in_dbell.value = 0; + pkt_in_dbell.s.dbell = 0xffffffff; + nitrox_write_csr(ndev, offset, pkt_in_dbell.value); + + /* clear done counts */ + offset = NPS_PKT_IN_DONE_CNTSX(ring); + pkt_in_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_in_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + u64 offset; + + /* 64-byte instruction size */ + offset = NPS_PKT_IN_INSTR_CTLX(ring); + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.is64b = 1; + pkt_in_ctl.s.enb = 1; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + + /* wait for set [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + } while (!pkt_in_ctl.s.enb); +} + +/** + * nitrox_config_pkt_input_rings - configure Packet Input Rings + * @ndev: N5 device + */ +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i]; + union nps_pkt_in_instr_rsize pkt_in_rsize; + u64 offset; + + reset_pkt_input_ring(ndev, i); + + /* configure ring base address 16-byte aligned, + * size and interrupt threshold. + */ + offset = NPS_PKT_IN_INSTR_BADDRX(i); + nitrox_write_csr(ndev, NPS_PKT_IN_INSTR_BADDRX(i), cmdq->dma); + + /* configure ring size */ + offset = NPS_PKT_IN_INSTR_RSIZEX(i); + pkt_in_rsize.value = 0; + pkt_in_rsize.s.rsize = ndev->qlen; + nitrox_write_csr(ndev, offset, pkt_in_rsize.value); + + /* set high threshold for pkt input ring interrupts */ + offset = NPS_PKT_IN_INT_LEVELSX(i); + nitrox_write_csr(ndev, offset, 0xffffffff); + + enable_pkt_input_ring(ndev, i); + } +} + +static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + union nps_pkt_slc_cnts pkt_slc_cnts; + u64 offset; + + /* disable slc port */ + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + pkt_slc_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + usleep_range(100, 150); + + /* wait to clear [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + } while (pkt_slc_ctl.s.enb); + + /* clear slc counters */ + offset = NPS_PKT_SLC_CNTSX(port); + pkt_slc_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_slc_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + u64 offset; + + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = 0; + pkt_slc_ctl.s.enb = 1; + + /* + * 8 trailing 0x00 bytes will be added + * to the end of the outgoing packet. + */ + pkt_slc_ctl.s.z = 1; + /* enable response header */ + pkt_slc_ctl.s.rh = 1; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + + /* wait to set [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + } while (!pkt_slc_ctl.s.enb); +} + +static void config_single_pkt_solicit_port(struct nitrox_device *ndev, + int port) +{ + union nps_pkt_slc_int_levels pkt_slc_int; + u64 offset; + + reset_pkt_solicit_port(ndev, port); + + offset = NPS_PKT_SLC_INT_LEVELSX(port); + pkt_slc_int.value = 0; + /* time interrupt threshold */ + pkt_slc_int.s.timet = 0x3fffff; + nitrox_write_csr(ndev, offset, pkt_slc_int.value); + + enable_pkt_solicit_port(ndev, port); +} + +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) + config_single_pkt_solicit_port(ndev, i); +} + +/** + * enable_nps_interrupts - enable NPS interrutps + * @ndev: N5 device. + * + * This includes NPS core, packet in and slc interrupts. + */ +static void enable_nps_interrupts(struct nitrox_device *ndev) +{ + union nps_core_int_ena_w1s core_int; + + /* NPS core interrutps */ + core_int.value = 0; + core_int.s.host_wr_err = 1; + core_int.s.host_wr_timeout = 1; + core_int.s.exec_wr_timeout = 1; + core_int.s.npco_dma_malform = 1; + core_int.s.host_nps_wr_err = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value); + + /* NPS packet in ring interrupts */ + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL)); + /* NPS packet slc port interrupts */ + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL)); +} + +void nitrox_config_nps_unit(struct nitrox_device *ndev) +{ + union nps_core_gbl_vfcfg core_gbl_vfcfg; + + /* endian control information */ + nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL); + + /* disable ILK interface */ + core_gbl_vfcfg.value = 0; + core_gbl_vfcfg.s.ilk_disable = 1; + core_gbl_vfcfg.s.cfg = PF_MODE; + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value); + /* config input and solicit ports */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* enable interrupts */ + enable_nps_interrupts(ndev); +} + +void nitrox_config_pom_unit(struct nitrox_device *ndev) +{ + union pom_int_ena_w1s pom_int; + int i; + + /* enable pom interrupts */ + pom_int.value = 0; + pom_int.s.illegal_dport = 1; + nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value); + + /* enable perf counters */ + for (i = 0; i < ndev->hw.se_cores; i++) + nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i)); +} + +/** + * nitrox_config_rand_unit - enable N5 random number unit + * @ndev: N5 device + */ +void nitrox_config_rand_unit(struct nitrox_device *ndev) +{ + union efl_rnm_ctl_status efl_rnm_ctl; + u64 offset; + + offset = EFL_RNM_CTL_STATUS; + efl_rnm_ctl.value = nitrox_read_csr(ndev, offset); + efl_rnm_ctl.s.ent_en = 1; + efl_rnm_ctl.s.rng_en = 1; + nitrox_write_csr(ndev, offset, efl_rnm_ctl.value); +} + +void nitrox_config_efl_unit(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int_ena_w1s efl_core_int; + u64 offset; + + /* EFL core interrupts */ + offset = EFL_CORE_INT_ENA_W1SX(i); + efl_core_int.value = 0; + efl_core_int.s.len_ovr = 1; + efl_core_int.s.d_left = 1; + efl_core_int.s.epci_decode_err = 1; + nitrox_write_csr(ndev, offset, efl_core_int.value); + + offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + } +} + +void nitrox_config_bmi_unit(struct nitrox_device *ndev) +{ + union bmi_ctl bmi_ctl; + union bmi_int_ena_w1s bmi_int_ena; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMI_CTL; + bmi_ctl.value = nitrox_read_csr(ndev, offset); + bmi_ctl.s.max_pkt_len = 0xff; + bmi_ctl.s.nps_free_thrsh = 0xff; + bmi_ctl.s.nps_hdrq_thrsh = 0x7a; + nitrox_write_csr(ndev, offset, bmi_ctl.value); + + /* enable interrupts */ + offset = BMI_INT_ENA_W1S; + bmi_int_ena.value = 0; + bmi_int_ena.s.max_len_err_nps = 1; + bmi_int_ena.s.pkt_rcv_err_nps = 1; + bmi_int_ena.s.fpf_undrrn = 1; + nitrox_write_csr(ndev, offset, bmi_int_ena.value); +} + +void nitrox_config_bmo_unit(struct nitrox_device *ndev) +{ + union bmo_ctl2 bmo_ctl2; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMO_CTL2; + bmo_ctl2.value = nitrox_read_csr(ndev, offset); + bmo_ctl2.s.nps_slc_buf_thrsh = 0xff; + nitrox_write_csr(ndev, offset, bmo_ctl2.value); +} + +void invalidate_lbc(struct nitrox_device *ndev) +{ + union lbc_inval_ctl lbc_ctl; + union lbc_inval_status lbc_stat; + u64 offset; + + /* invalidate LBC */ + offset = LBC_INVAL_CTL; + lbc_ctl.value = nitrox_read_csr(ndev, offset); + lbc_ctl.s.cam_inval_start = 1; + nitrox_write_csr(ndev, offset, lbc_ctl.value); + + offset = LBC_INVAL_STATUS; + + do { + lbc_stat.value = nitrox_read_csr(ndev, offset); + } while (!lbc_stat.s.done); +} + +void nitrox_config_lbc_unit(struct nitrox_device *ndev) +{ + union lbc_int_ena_w1s lbc_int_ena; + u64 offset; + + invalidate_lbc(ndev); + + /* enable interrupts */ + offset = LBC_INT_ENA_W1S; + lbc_int_ena.value = 0; + lbc_int_ena.s.dma_rd_err = 1; + lbc_int_ena.s.over_fetch_err = 1; + lbc_int_ena.s.cam_inval_abort = 1; + lbc_int_ena.s.cam_hard_err = 1; + nitrox_write_csr(ndev, offset, lbc_int_ena.value); + + offset = LBC_PLM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_PLM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + + offset = LBC_ELM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_ELM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c new file mode 100644 index 000000000000..71f934871a89 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -0,0 +1,467 @@ +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_csr.h" +#include "nitrox_common.h" + +#define NR_RING_VECTORS 3 +#define NPS_CORE_INT_ACTIVE_ENTRY 192 + +/** + * nps_pkt_slc_isr - IRQ handler for NPS solicit port + * @irq: irq number + * @data: argument + */ +static irqreturn_t nps_pkt_slc_isr(int irq, void *data) +{ + struct bh_data *slc = data; + union nps_pkt_slc_cnts pkt_slc_cnts; + + pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr); + /* New packet on SLC output port */ + if (pkt_slc_cnts.s.slc_int) + tasklet_hi_schedule(&slc->resp_handler); + + return IRQ_HANDLED; +} + +static void clear_nps_core_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + /* Write 1 to clear */ + value = nitrox_read_csr(ndev, NPS_CORE_INT); + nitrox_write_csr(ndev, NPS_CORE_INT, value); + + dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value); +} + +static void clear_nps_pkt_err_intr(struct nitrox_device *ndev) +{ + union nps_pkt_int pkt_int; + unsigned long value, offset; + int i; + + pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT); + dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n", + pkt_int.value); + + if (pkt_int.s.slc_err) { + offset = NPS_PKT_SLC_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the solicit ports */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_solicit_port(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value); + } + + if (pkt_int.s.in_err) { + offset = NPS_PKT_IN_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value); + offset = NPS_PKT_IN_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the input ring */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_input_ring(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_IN_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_HI 0x%016lx\n", value); + } +} + +static void clear_pom_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, POM_INT); + nitrox_write_csr(ndev, POM_INT, value); + dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value); +} + +static void clear_pem_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, PEM0_INT); + nitrox_write_csr(ndev, PEM0_INT, value); + dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value); +} + +static void clear_lbc_err_intr(struct nitrox_device *ndev) +{ + union lbc_int lbc_int; + u64 value, offset; + int i; + + lbc_int.value = nitrox_read_csr(ndev, LBC_INT); + dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value); + + if (lbc_int.s.dma_rd_err) { + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EFL_CORE_VF_ERR_INT0X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = EFL_CORE_VF_ERR_INT1X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } + + if (lbc_int.s.cam_soft_err) { + dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n"); + invalidate_lbc(ndev); + } + + if (lbc_int.s.pref_dat_len_mismatch_err) { + offset = LBC_PLM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_PLM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + + if (lbc_int.s.rd_dat_len_mismatch_err) { + offset = LBC_ELM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_ELM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + nitrox_write_csr(ndev, LBC_INT, lbc_int.value); +} + +static void clear_efl_err_intr(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int core_int; + u64 value, offset; + + offset = EFL_CORE_INTX(i); + core_int.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, core_int.value); + dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n", + i, core_int.value); + if (core_int.s.se_err) { + offset = EFL_CORE_SE_ERR_INTX(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } +} + +static void clear_bmi_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, BMI_INT); + nitrox_write_csr(ndev, BMI_INT, value); + dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value); +} + +/** + * clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts + * @ndev: NITROX device + */ +static void clear_nps_core_int_active(struct nitrox_device *ndev) +{ + union nps_core_int_active core_int_active; + + core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); + + if (core_int_active.s.nps_core) + clear_nps_core_err_intr(ndev); + + if (core_int_active.s.nps_pkt) + clear_nps_pkt_err_intr(ndev); + + if (core_int_active.s.pom) + clear_pom_err_intr(ndev); + + if (core_int_active.s.pem) + clear_pem_err_intr(ndev); + + if (core_int_active.s.lbc) + clear_lbc_err_intr(ndev); + + if (core_int_active.s.efl) + clear_efl_err_intr(ndev); + + if (core_int_active.s.bmi) + clear_bmi_err_intr(ndev); + + /* If more work callback the ISR, set resend */ + core_int_active.s.resend = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value); +} + +static irqreturn_t nps_core_int_isr(int irq, void *data) +{ + struct nitrox_device *ndev = data; + + clear_nps_core_int_active(ndev); + + return IRQ_HANDLED; +} + +static int nitrox_enable_msix(struct nitrox_device *ndev) +{ + struct msix_entry *entries; + char **names; + int i, nr_entries, ret; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * Entry 4: AQMQ ring 1 + * Entry 5: ZQM ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1; + entries = kzalloc_node(nr_entries * sizeof(struct msix_entry), + GFP_KERNEL, ndev->node); + if (!entries) + return -ENOMEM; + + names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL); + if (!names) { + kfree(entries); + return -ENOMEM; + } + + /* fill entires */ + for (i = 0; i < (nr_entries - 1); i++) + entries[i].entry = i; + + entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY; + + for (i = 0; i < nr_entries; i++) { + *(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL); + if (!(*(names + i))) { + ret = -ENOMEM; + goto msix_fail; + } + } + ndev->msix.entries = entries; + ndev->msix.names = names; + ndev->msix.nr_entries = nr_entries; + + ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries, + ndev->msix.nr_entries); + if (ret) { + dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n", + ret); + goto msix_fail; + } + return 0; + +msix_fail: + for (i = 0; i < nr_entries; i++) + kfree(*(names + i)); + + kfree(entries); + kfree(names); + return ret; +} + +static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev) +{ + int i; + + if (!ndev->bh.slc) + return; + + for (i = 0; i < ndev->nr_queues; i++) { + struct bh_data *bh = &ndev->bh.slc[i]; + + tasklet_disable(&bh->resp_handler); + tasklet_kill(&bh->resp_handler); + } + kfree(ndev->bh.slc); + ndev->bh.slc = NULL; +} + +static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev) +{ + u32 size; + int i; + + size = ndev->nr_queues * sizeof(struct bh_data); + ndev->bh.slc = kzalloc(size, GFP_KERNEL); + if (!ndev->bh.slc) + return -ENOMEM; + + for (i = 0; i < ndev->nr_queues; i++) { + struct bh_data *bh = &ndev->bh.slc[i]; + u64 offset; + + offset = NPS_PKT_SLC_CNTSX(i); + /* pre calculate completion count address */ + bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); + bh->cmdq = &ndev->pkt_cmdqs[i]; + + tasklet_init(&bh->resp_handler, pkt_slc_resp_handler, + (unsigned long)bh); + } + + return 0; +} + +static int nitrox_request_irqs(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + struct msix_entry *msix_ent = ndev->msix.entries; + int nr_ring_vectors, i = 0, ring, cpu, ret; + char *name; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS; + + /* request irq for pkt ring/ports only */ + while (i < nr_ring_vectors) { + name = *(ndev->msix.names + i); + ring = (i / NR_RING_VECTORS); + snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d", + ndev->idx, ring); + + ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0, + name, &ndev->bh.slc[ring]); + if (ret) { + dev_err(&pdev->dev, "failed to get irq %d for %s\n", + msix_ent[i].vector, name); + return ret; + } + cpu = ring % num_online_cpus(); + irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu)); + + set_bit(i, ndev->msix.irqs); + i += NR_RING_VECTORS; + } + + /* Request IRQ for NPS_CORE_INT_ACTIVE */ + name = *(ndev->msix.names + i); + snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx); + ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev); + if (ret) { + dev_err(&pdev->dev, "failed to get irq %d for %s\n", + msix_ent[i].vector, name); + return ret; + } + set_bit(i, ndev->msix.irqs); + + return 0; +} + +static void nitrox_disable_msix(struct nitrox_device *ndev) +{ + struct msix_entry *msix_ent = ndev->msix.entries; + char **names = ndev->msix.names; + int i = 0, ring, nr_ring_vectors; + + nr_ring_vectors = ndev->msix.nr_entries - 1; + + /* clear pkt ring irqs */ + while (i < nr_ring_vectors) { + if (test_and_clear_bit(i, ndev->msix.irqs)) { + ring = (i / NR_RING_VECTORS); + irq_set_affinity_hint(msix_ent[i].vector, NULL); + free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]); + } + i += NR_RING_VECTORS; + } + irq_set_affinity_hint(msix_ent[i].vector, NULL); + free_irq(msix_ent[i].vector, ndev); + clear_bit(i, ndev->msix.irqs); + + kfree(ndev->msix.entries); + for (i = 0; i < ndev->msix.nr_entries; i++) + kfree(*(names + i)); + + kfree(names); + pci_disable_msix(ndev->pdev); +} + +/** + * nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ + * @ndev: NITROX device + */ +void nitrox_pf_cleanup_isr(struct nitrox_device *ndev) +{ + nitrox_disable_msix(ndev); + nitrox_cleanup_pkt_slc_bh(ndev); +} + +/** + * nitrox_init_isr - Initialize PF MSI-X vectors and IRQ + * @ndev: NITROX device + * + * Return: 0 on success, a negative value on failure. + */ +int nitrox_pf_init_isr(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_setup_pkt_slc_bh(ndev); + if (err) + return err; + + err = nitrox_enable_msix(ndev); + if (err) + goto msix_fail; + + err = nitrox_request_irqs(ndev); + if (err) + goto irq_fail; + + return 0; + +irq_fail: + nitrox_disable_msix(ndev); +msix_fail: + nitrox_cleanup_pkt_slc_bh(ndev); + return err; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c new file mode 100644 index 000000000000..846e990d4f35 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -0,0 +1,172 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +#define CRYPTO_CTX_SIZE 256 + +/* command queue alignments */ +#define PKT_IN_ALIGN 16 + +static int cmdq_common_init(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + u32 qsize; + + qsize = (ndev->qlen) * cmdq->instr_size; + cmdq->head_unaligned = dma_zalloc_coherent(DEV(ndev), + (qsize + PKT_IN_ALIGN), + &cmdq->dma_unaligned, + GFP_KERNEL); + if (!cmdq->head_unaligned) + return -ENOMEM; + + cmdq->head = PTR_ALIGN(cmdq->head_unaligned, PKT_IN_ALIGN); + cmdq->dma = PTR_ALIGN(cmdq->dma_unaligned, PKT_IN_ALIGN); + cmdq->qsize = (qsize + PKT_IN_ALIGN); + + spin_lock_init(&cmdq->response_lock); + spin_lock_init(&cmdq->cmdq_lock); + spin_lock_init(&cmdq->backlog_lock); + + INIT_LIST_HEAD(&cmdq->response_head); +