diff options
author | Faisal Latif <faisal.latif@intel.com> | 2016-01-20 13:40:09 -0600 |
---|---|---|
committer | Doug Ledford <dledford@redhat.com> | 2016-02-29 17:10:53 -0500 |
commit | d37498417947cb2299fc749ae4af1d204c768cba (patch) | |
tree | 987296c0bdc391ff0e830fb19cf2858dfc625e4f /drivers | |
parent | 4e9042e647ff083239984c4051c6d1a4f927ecc3 (diff) |
i40iw: add files for iwarp interface
i40iw_verbs.[ch] are to handle iwarp interface.
Changes since v2:
Made infiniband interface changes for 4.5
removed i40iw_reg_phys_mr() for 4.5
made changes as made by Christoph Hellwig made for nes
in i40iw_get_dma_mr().
Changes since v1:
Following modification based on Christoph Hellwig's feedback
-remove kmap() calls and moved to i40iw_cm.c.
-cleanup some of casts
Acked-by: Anjali Singhai Jain <anjali.singhai@intel.com>
Acked-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Faisal Latif <faisal.latif@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/infiniband/hw/i40iw/i40iw_ucontext.h | 107 | ||||
-rw-r--r-- | drivers/infiniband/hw/i40iw/i40iw_verbs.c | 2434 | ||||
-rw-r--r-- | drivers/infiniband/hw/i40iw/i40iw_verbs.h | 173 |
3 files changed, 2714 insertions, 0 deletions
diff --git a/drivers/infiniband/hw/i40iw/i40iw_ucontext.h b/drivers/infiniband/hw/i40iw/i40iw_ucontext.h new file mode 100644 index 000000000000..12acd688def4 --- /dev/null +++ b/drivers/infiniband/hw/i40iw/i40iw_ucontext.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2006 - 2016 Intel Corporation. All rights reserved. + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Cisco Systems. All rights reserved. + * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef I40IW_USER_CONTEXT_H +#define I40IW_USER_CONTEXT_H + +#include <linux/types.h> + +#define I40IW_ABI_USERSPACE_VER 4 +#define I40IW_ABI_KERNEL_VER 4 +struct i40iw_alloc_ucontext_req { + __u32 reserved32; + __u8 userspace_ver; + __u8 reserved8[3]; +}; + +struct i40iw_alloc_ucontext_resp { + __u32 max_pds; /* maximum pds allowed for this user process */ + __u32 max_qps; /* maximum qps allowed for this user process */ + __u32 wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */ + __u8 kernel_ver; + __u8 reserved[3]; +}; + +struct i40iw_alloc_pd_resp { + __u32 pd_id; + __u8 reserved[4]; +}; + +struct i40iw_create_cq_req { + __u64 user_cq_buffer; + __u64 user_shadow_area; +}; + +struct i40iw_create_qp_req { + __u64 user_wqe_buffers; + __u64 user_compl_ctx; + + /* UDA QP PHB */ + __u64 user_sq_phb; /* place for VA of the sq phb buff */ + __u64 user_rq_phb; /* place for VA of the rq phb buff */ +}; + +enum i40iw_memreg_type { + IW_MEMREG_TYPE_MEM = 0x0000, + IW_MEMREG_TYPE_QP = 0x0001, + IW_MEMREG_TYPE_CQ = 0x0002, +}; + +struct i40iw_mem_reg_req { + __u16 reg_type; /* Memory, QP or CQ */ + __u16 cq_pages; + __u16 rq_pages; + __u16 sq_pages; +}; + +struct i40iw_create_cq_resp { + __u32 cq_id; + __u32 cq_size; + __u32 mmap_db_index; + __u32 reserved; +}; + +struct i40iw_create_qp_resp { + __u32 qp_id; + __u32 actual_sq_size; + __u32 actual_rq_size; + __u32 i40iw_drv_opt; + __u16 push_idx; + __u8 lsmm; + __u8 rsvd2; +}; + +#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c new file mode 100644 index 000000000000..c5c9805c9cb3 --- /dev/null +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -0,0 +1,2434 @@ +/******************************************************************************* +* +* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. +* +* This software is available to you under a choice of one of two +* licenses. You may choose to be licensed under the terms of the GNU +* General Public License (GPL) Version 2, available from the file +* COPYING in the main directory of this source tree, or the +* OpenFabrics.org BSD license below: +* +* Redistribution and use in source and binary forms, with or +* without modification, are permitted provided that the following +* conditions are met: +* +* - Redistributions of source code must retain the above +* copyright notice, this list of conditions and the following +* disclaimer. +* +* - Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials +* provided with the distribution. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +* +*******************************************************************************/ + +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/random.h> +#include <linux/highmem.h> +#include <linux/time.h> +#include <asm/byteorder.h> +#include <net/ip.h> +#include <rdma/ib_verbs.h> +#include <rdma/iw_cm.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_umem.h> +#include "i40iw.h" + +/** + * i40iw_query_device - get device attributes + * @ibdev: device pointer from stack + * @props: returning device attributes + * @udata: user data + */ +static int i40iw_query_device(struct ib_device *ibdev, + struct ib_device_attr *props, + struct ib_udata *udata) +{ + struct i40iw_device *iwdev = to_iwdev(ibdev); + + if (udata->inlen || udata->outlen) + return -EINVAL; + memset(props, 0, sizeof(*props)); + ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); + props->fw_ver = I40IW_FW_VERSION; + props->device_cap_flags = iwdev->device_cap_flags; + props->vendor_id = iwdev->vendor_id; + props->vendor_part_id = iwdev->vendor_part_id; + props->hw_ver = (u32)iwdev->sc_dev.hw_rev; + props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE; + props->max_qp = iwdev->max_qp; + props->max_qp_wr = (I40IW_MAX_WQ_ENTRIES >> 2) - 1; + props->max_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; + props->max_cq = iwdev->max_cq; + props->max_cqe = iwdev->max_cqe; + props->max_mr = iwdev->max_mr; + props->max_pd = iwdev->max_pd; + props->max_sge_rd = 1; + props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE; + props->max_qp_init_rd_atom = props->max_qp_rd_atom; + props->atomic_cap = IB_ATOMIC_NONE; + props->max_map_per_fmr = 1; + return 0; +} + +/** + * i40iw_query_port - get port attrubutes + * @ibdev: device pointer from stack + * @port: port number for query + * @props: returning device attributes + */ +static int i40iw_query_port(struct ib_device *ibdev, + u8 port, + struct ib_port_attr *props) +{ + struct i40iw_device *iwdev = to_iwdev(ibdev); + struct net_device *netdev = iwdev->netdev; + + memset(props, 0, sizeof(*props)); + + props->max_mtu = IB_MTU_4096; + if (netdev->mtu >= 4096) + props->active_mtu = IB_MTU_4096; + else if (netdev->mtu >= 2048) + props->active_mtu = IB_MTU_2048; + else if (netdev->mtu >= 1024) + props->active_mtu = IB_MTU_1024; + else if (netdev->mtu >= 512) + props->active_mtu = IB_MTU_512; + else + props->active_mtu = IB_MTU_256; + + props->lid = 1; + if (netif_carrier_ok(iwdev->netdev)) + props->state = IB_PORT_ACTIVE; + else + props->state = IB_PORT_DOWN; + props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | + IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; + props->gid_tbl_len = 1; + props->pkey_tbl_len = 1; + props->active_width = IB_WIDTH_4X; + props->active_speed = 1; + props->max_msg_sz = 0x80000000; + return 0; +} + +/** + * i40iw_alloc_ucontext - Allocate the user context data structure + * @ibdev: device pointer from stack + * @udata: user data + * + * This keeps track of all objects associated with a particular + * user-mode client. + */ +static struct ib_ucontext *i40iw_alloc_ucontext(struct ib_device *ibdev, + struct ib_udata *udata) +{ + struct i40iw_device *iwdev = to_iwdev(ibdev); + struct i40iw_alloc_ucontext_req req; + struct i40iw_alloc_ucontext_resp uresp; + struct i40iw_ucontext *ucontext; + + if (ib_copy_from_udata(&req, udata, sizeof(req))) + return ERR_PTR(-EINVAL); + + if (req.userspace_ver != I40IW_ABI_USERSPACE_VER) { + i40iw_pr_err("Invalid userspace driver version detected. Detected version %d, should be %d\n", + req.userspace_ver, I40IW_ABI_USERSPACE_VER); + return ERR_PTR(-EINVAL); + } + + memset(&uresp, 0, sizeof(uresp)); + uresp.max_qps = iwdev->max_qp; + uresp.max_pds = iwdev->max_pd; + uresp.wq_size = iwdev->max_qp_wr * 2; + uresp.kernel_ver = I40IW_ABI_KERNEL_VER; + + ucontext = kzalloc(sizeof(*ucontext), GFP_KERNEL); + if (!ucontext) + return ERR_PTR(-ENOMEM); + + ucontext->iwdev = iwdev; + + if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { + kfree(ucontext); + return ERR_PTR(-EFAULT); + } + + INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); + spin_lock_init(&ucontext->cq_reg_mem_list_lock); + INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); + spin_lock_init(&ucontext->qp_reg_mem_list_lock); + + return &ucontext->ibucontext; +} + +/** + * i40iw_dealloc_ucontext - deallocate the user context data structure + * @context: user context created during alloc + */ +static int i40iw_dealloc_ucontext(struct ib_ucontext *context) +{ + struct i40iw_ucontext *ucontext = to_ucontext(context); + unsigned long flags; + + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); + if (!list_empty(&ucontext->cq_reg_mem_list)) { + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + return -EBUSY; + } + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); + if (!list_empty(&ucontext->qp_reg_mem_list)) { + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + return -EBUSY; + } + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + + kfree(ucontext); + return 0; +} + +/** + * i40iw_mmap - user memory map + * @context: context created during alloc + * @vma: kernel info for user memory map + */ +static int i40iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) +{ + struct i40iw_ucontext *ucontext; + u64 db_addr_offset; + u64 push_offset; + + ucontext = to_ucontext(context); + if (ucontext->iwdev->sc_dev.is_pf) { + db_addr_offset = I40IW_DB_ADDR_OFFSET; + push_offset = I40IW_PUSH_OFFSET; + if (vma->vm_pgoff) + vma->vm_pgoff += I40IW_PF_FIRST_PUSH_PAGE_INDEX - 1; + } else { + db_addr_offset = I40IW_VF_DB_ADDR_OFFSET; + push_offset = I40IW_VF_PUSH_OFFSET; + if (vma->vm_pgoff) + vma->vm_pgoff += I40IW_VF_FIRST_PUSH_PAGE_INDEX - 1; + } + + vma->vm_pgoff += db_addr_offset >> PAGE_SHIFT; + + if (vma->vm_pgoff == (db_addr_offset >> PAGE_SHIFT)) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + vma->vm_private_data = ucontext; + } else { + if ((vma->vm_pgoff - (push_offset >> PAGE_SHIFT)) % 2) + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + else + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + } + + if (io_remap_pfn_range(vma, vma->vm_start, + vma->vm_pgoff + (pci_resource_start(ucontext->iwdev->ldev->pcidev, 0) >> PAGE_SHIFT), + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + + return 0; +} + +/** + * i40iw_alloc_push_page - allocate a push page for qp + * @iwdev: iwarp device + * @qp: hardware control qp + */ +static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp) +{ + struct i40iw_cqp_request *cqp_request; + struct cqp_commands_info *cqp_info; + struct i40iw_sc_dev *dev = &iwdev->sc_dev; + enum i40iw_status_code status; + + if (qp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX) + return; + + cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); + if (!cqp_request) + return; + + atomic_inc(&cqp_request->refcount); + + cqp_info = &cqp_request->info; + cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE; + cqp_info->post_sq = 1; + + cqp_info->in.u.manage_push_page.info.qs_handle = dev->qs_handle; + cqp_info->in.u.manage_push_page.info.free_page = 0; + cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp; + cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; + + status = i40iw_handle_cqp_op(iwdev, cqp_request); + if (!status) + qp->push_idx = cqp_request->compl_info.op_ret_val; + else + i40iw_pr_err("CQP-OP Push page fail"); + i40iw_put_cqp_request(&iwdev->cqp, cqp_request); +} + +/** + * i40iw_dealloc_push_page - free a push page for qp + * @iwdev: iwarp device + * @qp: hardware control qp + */ +static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp) +{ + struct i40iw_cqp_request *cqp_request; + struct cqp_commands_info *cqp_info; + struct i40iw_sc_dev *dev = &iwdev->sc_dev; + enum i40iw_status_code status; + + if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) + return; + + cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false); + if (!cqp_request) + return; + + cqp_info = &cqp_request->info; + cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE; + cqp_info->post_sq = 1; + + cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; + cqp_info->in.u.manage_push_page.info.qs_handle = dev->qs_handle; + cqp_info->in.u.manage_push_page.info.free_page = 1; + cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp; + cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; + + status = i40iw_handle_cqp_op(iwdev, cqp_request); + if (!status) + qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX; + else + i40iw_pr_err("CQP-OP Push page fail"); +} + +/** + * i40iw_alloc_pd - allocate protection domain + * @ibdev: device pointer from stack + * @context: user context created during alloc + * @udata: user data + */ +static struct ib_pd *i40iw_alloc_pd(struct ib_device *ibdev, + struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct i40iw_pd *iwpd; + struct i40iw_device *iwdev = to_iwdev(ibdev); + struct i40iw_sc_dev *dev = &iwdev->sc_dev; + struct i40iw_alloc_pd_resp uresp; + struct i40iw_sc_pd *sc_pd; + u32 pd_id = 0; + int err; + + err = i40iw_alloc_resource(iwdev, iwdev->allocated_pds, + iwdev->max_pd, &pd_id, &iwdev->next_pd); + if (err) { + i40iw_pr_err("alloc resource failed\n"); + return ERR_PTR(err); + } + + iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL); + if (!iwpd) { + err = -ENOMEM; + goto free_res; + } + + sc_pd = &iwpd->sc_pd; + dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id); + + if (context) { + memset(&uresp, 0, sizeof(uresp)); + uresp.pd_id = pd_id; + if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { + err = -EFAULT; + goto error; + } + } + + i40iw_add_pdusecount(iwpd); + return &iwpd->ibpd; +error: + kfree(iwpd); +free_res: + i40iw_free_resource(iwdev, iwdev->allocated_pds, pd_id); + return ERR_PTR(err); +} + +/** + * i40iw_dealloc_pd - deallocate pd + * @ibpd: ptr of pd to be deallocated + */ +static int i40iw_dealloc_pd(struct ib_pd *ibpd) +{ + struct i40iw_pd *iwpd = to_iwpd(ibpd); + struct i40iw_device *iwdev = to_iwdev(ibpd->device); + + i40iw_rem_pdusecount(iwpd, iwdev); + return 0; +} + +/** + * i40iw_qp_roundup - return round up qp ring size + * @wr_ring_size: ring size to round up + */ +static int i40iw_qp_roundup(u32 wr_ring_size) +{ + int scount = 1; + + if (wr_ring_size < I40IWQP_SW_MIN_WQSIZE) + wr_ring_size = I40IWQP_SW_MIN_WQSIZE; + + for (wr_ring_size--; scount <= 16; scount *= 2) + wr_ring_size |= wr_ring_size >> scount; + return ++wr_ring_size; +} + +/** + * i40iw_get_pbl - Retrieve pbl from a list given a virtual + * address + * @va: user virtual address + * @pbl_list: pbl list to search in (QP's or CQ's) + */ +static struct i40iw_pbl *i40iw_get_pbl(unsigned long va, + struct list_head *pbl_list) +{ + struct i40iw_pbl *iwpbl; + + list_for_each_entry(iwpbl, pbl_list, list) { + if (iwpbl->user_base == va) { + list_del(&iwpbl->list); + return iwpbl; + } + } + return NULL; +} + +/** + * i40iw_free_qp_resources - free up memory resources for qp + * @iwdev: iwarp device + * @iwqp: qp ptr (user or kernel) + * @qp_num: qp number assigned + */ +void i40iw_free_qp_resources(struct i40iw_device *iwdev, + struct i40iw_qp *iwqp, + u32 qp_num) +{ + i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp); + if (qp_num) + i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num); + i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->q2_ctx_mem); + i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->kqp.dma_mem); + kfree(iwqp->kqp.wrid_mem); + iwqp->kqp.wrid_mem = NULL; + kfree(iwqp->allocated_buffer); + iwqp->allocated_buffer = NULL; +} + +/** + * i40iw_clean_cqes - clean cq entries for qp + * @iwqp: qp ptr (user or kernel) + * @iwcq: cq ptr + */ +static void i40iw_clean_cqes(struct i40iw_qp *iwqp, struct i40iw_cq *iwcq) +{ + struct i40iw_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; + + ukcq->ops.iw_cq_clean(&iwqp->sc_qp.qp_uk, ukcq); +} + +/** + * i40iw_destroy_qp - destroy qp + * @ibqp: qp's ib pointer also to get to device's qp address + */ +static int i40iw_destroy_qp(struct ib_qp *ibqp) +{ + struct i40iw_qp *iwqp = to_iwqp(ibqp); + + iwqp->destroyed = 1; + + if (iwqp->ibqp_state >= IB_QPS_INIT && iwqp->ibqp_state < IB_QPS_RTS) + i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 0, 0, 0); + + if (!iwqp->user_mode) { + if (iwqp->iwscq) { + i40iw_clean_cqes(iwqp, iwqp->iwscq); + if (iwqp->iwrcq != iwqp->iwscq) + i40iw_clean_cqes(iwqp, iwqp->iwrcq); + } + } + + i40iw_rem_ref(&iwqp->ibqp); + return 0; +} + +/** + * i40iw_setup_virt_qp - setup for allocation of virtual qp + * @dev: iwarp device + * @qp: qp ptr + * @init_info: initialize info to return + */ +static int i40iw_setup_virt_qp(struct i40iw_device *iwdev, + struct i40iw_qp *iwqp, + struct i40iw_qp_init_info *init_info) +{ + struct i40iw_pbl *iwpbl = iwqp->iwpbl; + struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr; + + iwqp->page = qpmr->sq_page; + init_info->shadow_area_pa = cpu_to_le64(qpmr->shadow); + if (iwpbl->pbl_allocated) { + init_info->virtual_map = true; + init_info->sq_pa = qpmr->sq_pbl.idx; + init_info->rq_pa = qpmr->rq_pbl.idx; + } else { + init_info->sq_pa = qpmr->sq_pbl.addr; + init_info->rq_pa = qpmr->rq_pbl.addr; + } + return 0; +} + +/** + * i40iw_setup_kmode_qp - setup initialization for kernel mode qp + * @iwdev: iwarp device + * @iwqp: qp ptr (user or kernel) + * @info: initialize info to return + */ +static int i40iw_setup_kmode_qp(struct i40iw_device *iwdev, + struct i40iw_qp *iwqp, + struct i40iw_qp_init_info *info) +{ + struct i40iw_dma_mem *mem = &iwqp->kqp.dma_mem; + u32 sqdepth, rqdepth; + u32 sq_size, rq_size; + u8 sqshift, rqshift; + u32 size; + enum i40iw_status_code status; + struct i40iw_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; + + ukinfo->max_sq_frag_cnt = I40IW_MAX_WQ_FRAGMENT_COUNT; + + sq_size = i40iw_qp_roundup(ukinfo->sq_size + 1); + rq_size = i40iw_qp_roundup(ukinfo->rq_size + 1); + + status = i40iw_get_wqe_shift(sq_size, ukinfo->max_sq_frag_cnt, &sqshift); + if (!status) + status = i40iw_get_wqe_shift(rq_size, ukinfo->max_rq_frag_cnt, &rqshift); + + if (status) + return -ENOSYS; + + sqdepth = sq_size << sqshift; + rqdepth = rq_size << rqshift; + + size = sqdepth * sizeof(struct i40iw_sq_uk_wr_trk_info) + (rqdepth << 3); + iwqp->kqp.wrid_mem = kzalloc(size, GFP_KERNEL); + + ukinfo->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)iwqp->kqp.wrid_mem; + if (!ukinfo->sq_wrtrk_array) + return -ENOMEM; + + ukinfo->rq_wrid_array = (u64 *)&ukinfo->sq_wrtrk_array[sqdepth]; + + size = (sqdepth + rqdepth) * I40IW_QP_WQE_MIN_SIZE; + size += (I40IW_SHADOW_AREA_SIZE << 3); + + status = i40iw_allocate_dma_mem(iwdev->sc_dev.hw, mem, size, 256); + if (status) { + kfree(ukinfo->sq_wrtrk_array); + ukinfo->sq_wrtrk_array = NULL; + return -ENOMEM; + } + + ukinfo->sq = mem->va; + info->sq_pa = mem->pa; + + ukinfo->rq = &ukinfo->sq[sqdepth]; + info->rq_pa = info->sq_pa + (sqdepth * I40IW_QP_WQE_MIN_SIZE); + + ukinfo->shadow_area = ukinfo->rq[rqdepth].elem; + info->shadow_area_pa = info->rq_pa + (rqdepth * I40IW_QP_WQE_MIN_SIZE); + + ukinfo->sq_size = sq_size; + ukinfo->rq_size = rq_size; + ukinfo->qp_id = iwqp->ibqp.qp_num; + return 0; +} + +/** + * i40iw_create_qp - create qp + * @ibpd: ptr of pd + * @init_attr: attributes for qp + * @udata: user data for create qp + */ +static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct i40iw_pd *iwpd = to_iwpd(ibpd); + struct i40iw_device *iwdev = to_iwdev(ibpd->device); + struct i40iw_cqp *iwcqp = &iwdev->cqp; + struct i40iw_qp *iwqp; + struct i40iw_ucontext *ucontext; + struct i40iw_create_qp_req req; + struct i40iw_create_qp_resp uresp; + u32 qp_num = 0; + void *mem; + enum i40iw_status_code ret; + int err_code; + int sq_size; + int rq_size; + struct i40iw_sc_qp *qp; + struct i40iw_sc_dev *dev = &iwdev->sc_dev; + struct i40iw_qp_init_info init_info; + struct i40iw_create_qp_info *qp_info; + struct i40iw_cqp_request *cqp_request; + struct cqp_commands_info *cqp_info; + + struct i40iw_qp_host_ctx_info *ctx_info; + struct i40iwarp_offload_info *iwarp_info; + unsigned long flags; + + if (init_attr->create_flags) + return ERR_PTR(-EINVAL); + if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE) + init_attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE; + + memset(&init_info, 0, sizeof(init_info)); + + sq_size = init_attr->cap.max_send_wr; + rq_size = init_attr->cap.max_recv_wr; + + init_info.qp_uk_init_info.sq_size = sq_size; + init_info.qp_uk_init_info.rq_size = rq_size; + init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; + init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; + + mem = kzalloc(sizeof(*iwqp), GFP_KERNEL); + if (!mem) + return ERR_PTR(-ENOMEM); + + iwqp = (struct i40iw_qp *)mem; + qp = &iwqp->sc_qp; + qp->back_qp = (void *)iwqp; + qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX; + + iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info; + + if (i40iw_allocate_dma_mem(dev->hw, + &iwqp->q2_ctx_mem, + I40IW_Q2_BUFFER_SIZE + I40IW_QP_CTX_SIZE, + 256)) { + i40iw_pr_err("dma_mem failed\n"); + err_code = -ENOMEM; + goto error; + } + + init_info.q2 = iwqp->q2_ctx_mem.va; + init_info.q2_pa = iwqp->q2_ctx_mem.pa; + + init_info.host_ctx = (void *)init_info.q2 + I40IW_Q2_BUFFER_SIZE; + init_info.host_ctx_pa = init_info.q2_pa + I40IW_Q2_BUFFER_SIZE; + + err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_qps, iwdev->max_qp, + &qp_num, &iwdev->next_qp); + if (err_code) { + i40iw_pr_err("qp resource\n"); + goto error; + } + + iwqp->allocated_buffer = mem; + iwqp->iwdev = iwdev; + iwqp->iwpd = iwpd; + iwqp->ibqp.qp_num = qp_num; + qp = &iwqp->sc_qp; + iwqp->iwscq = to_iwcq(init_attr->send_cq); + iwqp->iwrcq = to_iwcq(init_attr->recv_cq); + + iwqp->host_ctx.va = init_info.host_ctx; + iwqp->host_ctx.pa = init_info.host_ctx_pa; + iwqp->host_ctx.size = I40IW_QP_CTX_SIZE; + + init_info.pd = &iwpd->sc_pd; + init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; + iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; + + if (init_attr->qp_type != IB_QPT_RC) { + err_code = -ENOSYS; + goto error; + } + if (iwdev->push_mode) + i40iw_alloc_push_page(iwdev, qp); + if (udata) { + err_code = ib_copy_from_udata(&req, udata, sizeof(req)); + if (err_code) { + i40iw_pr_err("ib_copy_from_data\n"); + goto error; + } + iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; + if (ibpd->uobject && ibpd->uobject->context) { + iwqp->user_mode = 1; + ucontext = to_ucontext(ibpd->uobject->context); + + if (req.user_wqe_buffers) { + spin_lock_irqsave( + &ucontext->qp_reg_mem_list_lock, flags); + iwqp->iwpbl = i40iw_get_pbl( + (unsigned long)req.user_wqe_buffers, + &ucontext->qp_reg_mem_list); + spin_unlock_irqrestore( + &ucontext->qp_reg_mem_list_lock, flags); + + if (!iwqp->iwpbl) { + err_code = -ENODATA; + i40iw_pr_err("no pbl info\n"); + goto error; + } + } + } + err_code = i40iw_setup_virt_qp(iwdev, iwqp, &init_info); + } else { + err_code = i40iw_setup_kmode_qp(iwdev, iwqp, &init_info); + } + + if (err_code) { + i40iw_pr_err("setup qp failed\n"); + goto error; + } + + init_info.type = I40IW_QP_TYPE_IWARP; + ret = dev->iw_priv_qp_ops->qp_init(qp, &init_info); + if (ret) { + err_code = -EPROTO; + i40iw_pr_err("qp_init fail\n"); + goto error; + } + ctx_info = &iwqp->ctx_info; + iwarp_info = &iwqp->iwarp_info; + iwarp_info->rd_enable = true; + iwarp_info->wr_rdresp_en = true; + if (!iwqp->user_mode) + iwarp_info->priv_mode_en = true; + iwarp_info->ddp_ver = 1; + iwarp_info->rdmap_ver = 1; + + ctx_info->iwarp_info_valid = true; + ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; + ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; + if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) { + ctx_info->push_mode_en = false; + } else { + ctx_info->push_mode_en = true; + ctx_info->push_idx = qp->push_idx; + } + + ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, + (u64 *)iwqp->host_ctx.va, + ctx_info); + ctx_info->iwarp_info_valid = false; + cqp_request = i40iw_get_cqp_request(iwcqp, true); + if (!cqp_request) { + err_code = -ENOMEM; + goto error; + } + cqp_info = &cqp_request->info; + qp_info = &cqp_request->info.in.u.qp_create.info; + + memset(qp_info, 0, sizeof(*qp_info)); + + qp_info->cq_num_valid = true; + qp_info->next_iwarp_state = I40IW_QP_STATE_IDLE; + + cqp_info->cqp_cmd = OP_QP_CREATE; + cqp_info->post_sq = 1; + cqp_info->in.u.qp_create.qp = qp; + cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; + ret = i40iw_handle_cqp_op(iwdev, cqp_request); + if (ret) { + i40iw_pr_err("CQP-OP QP create fail"); + err_code = -EACCES; + goto error; + } + + i40iw_add_ref(&iwqp->ibqp); + spin_lock_init(&iwqp->lock); + iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; + iwdev->qp_table[qp_num] = iwqp; + i40iw_add_pdusecount(iwqp->iwpd); + if (ibpd->uobject && udata) { + memset(&uresp, 0, sizeof(uresp)); + uresp.actual_sq_size = sq_size; + uresp.actual_rq_size = rq_size; + uresp.qp_id = qp_num; + uresp.push_idx = qp->push_idx; + err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + if (err_code) { + i40iw_pr_err("copy_to_udata failed\n"); + i40iw_destroy_qp(&iwqp->ibqp); + /* let the completion of the qp destroy free the qp */ + return ERR_PTR(err_code); + } + } + + return &iwqp->ibqp; +error: + i40iw_free_qp_resources(iwdev, iwqp, qp_num); + kfree(mem); + return ERR_PTR(err_code); +} + +/** + * i40iw_query - query qp attributes + * @ibqp: qp pointer + * @attr: attributes pointer + * @attr_mask: Not used + * @init_attr: qp attributes to return + */ +static int i40iw_query_qp(struct ib_qp *ibqp, + struct ib_qp_attr *attr, + int attr_mask, + struct ib_qp_init_attr *init_attr) +{ + struct i40iw_qp *iwqp = to_iwqp(ibqp); + struct i40iw_sc_qp *qp = &iwqp->sc_qp; + + attr->qp_access_flags = 0; + attr->cap.max_send_wr = qp->qp_uk.sq_size; + attr->cap.max_recv_wr = qp->qp_uk.rq_size; + attr->cap.max_recv_sge = 1; + attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE; + init_attr->event_handler = iwqp->ibqp.event_handler; + init_attr->qp_context = iwqp->ibqp.qp_context; + init_attr->send_cq = iwqp->ibqp.send_cq; + init_attr->recv_cq = iwqp->ibqp.recv_cq; + init_attr->srq = iwqp->ibqp.srq; + init_attr->cap = attr->cap; + return 0; +} + +/** + * i40iw_hw_modify_qp - setup cqp for modify qp + * @iwdev: iwarp device + * @iwqp: qp ptr (user or kernel) + * @info: info for modify qp + * @wait: flag to wait or not for modify qp completion + */ +void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp, + struct i40iw_modify_qp_info *info, bool wait) +{ + enum i40iw_status_code status; + struct i40iw_cqp_request *cqp_request; + struct cqp_commands_info *cqp_info; + struct i40iw_modify_qp_info *m_info; + + cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait); + if (!cqp_request) + return; + + cqp_info = &cqp_request->info; + m_info = &cqp_info->in.u.qp_modify.info; + memcpy(m_info, info, sizeof(*m_info)); + cqp_info->cqp_cmd = OP_QP_MODIFY; + cqp_info->post_sq = 1; + cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; + cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; + status = i40iw_handle_cqp_op(iwdev, cqp_request); + if (status) + i40iw_pr_err("CQP-OP Modify QP fail"); +} + +/** + * i40iw_modify_qp - modify qp request + * @ibqp: qp's pointer for modify + * @attr: access attributes + * @attr_mask: state mask + * @udata: user data + */ +int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct i40iw_qp *iwqp = to_iwqp(ibqp); + struct i40iw_device *iwdev = iwqp->iwdev; + struct i40iw_qp_host_ctx_info *ctx_info; + struct i40iwarp_offload_info *iwarp_info; + struct i40iw_modify_qp_info info; + u8 issue_modify_qp = 0; + u8 dont_wait = 0; + u32 err; + unsigned long flags; + + memset(&info, 0, sizeof(info)); + ctx_info = &iwqp->ctx_info; + iwarp_info = &iwqp->iwarp_info; + + spin_lock_irqsave(&iwqp->lock, flags); + + if (attr_mask & IB_QP_STATE) { + switch (attr->qp_state) { + case IB_QPS_INIT: + case IB_QPS_RTR: + if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_IDLE) { + err = -EINVAL; + goto exit; + } + if (iwqp->iwarp_state == I40IW_QP_STATE_INVALID) { + info.next_iwarp_state = I40IW_QP_STATE_IDLE; + issue_modify_qp = 1; + } + break; + case IB_QPS_RTS: + if ((iwqp->iwarp_state > (u32)I40IW_QP_STATE_RTS) || + (!iwqp->cm_id)) { + err = -EINVAL; + goto exit; + } + + issue_modify_qp = 1; + iwqp->hw_tcp_state = I40IW_TCP_STATE_ESTABLISHED; + iwqp->hte_added = 1; + info.next_iwarp_state = I40IW_QP_STATE_RTS; + info.tcp_ctx_valid = true; + info.ord_valid = true; + info.arp_cache_idx_valid = true; + info.cq_num_valid = true; + break; + case IB_QPS_SQD: + if (iwqp->hw_iwarp_state > (u32)I40IW_QP_STATE_RTS) { + err = 0; + goto exit; + } + if ((iwqp->iwarp_state == (u32)I40IW_QP_STATE_CLOSING) || + (iwqp->iwarp_state < (u32)I40IW_QP_STATE_RTS)) { + err = 0; + goto exit; + } + if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_CLOSING) { + err = -EINVAL; + goto exit; + } + info.next_iwarp_state = I40IW_QP_STATE_CLOSING; + issue_modify_qp = 1; + break; + case IB_QPS_SQE: + if (iwqp->iwarp_state >= (u32)I40IW_QP_STATE_TERMINATE) { + err = -EINVAL; + goto exit; + } + info.next_iwarp_state = I40IW_QP_STATE_TERMINATE; + issue_modify_qp = 1; + break; + case IB_QPS_ERR: + case IB_QPS_RESET: + if (iwqp->iwarp_state == (u32)I40IW_QP_STATE_ERROR) { + err = -EINVAL; + goto exit; + } + if (iwqp->sc_qp.term_flags) + del_timer(&iwqp->terminate_timer); + info.next_iwarp_state = I40IW_QP_STATE_ERROR; + if ((iwqp->hw_tcp_state > I40IW_TCP_STATE_CLOSED) && + iwdev->iw_status && + (iwqp->hw_tcp_state != I40IW_TCP_STATE_TIME_WAIT)) + info.reset_tcp_conn = true; + else + dont_wait = 1; + issue_modify_qp = 1; + info.next_iwarp_state = I40IW_QP_STATE_ERROR; + break; + default: + err = -EINVAL; + goto exit; + } + + iwqp->ibqp_state = attr->qp_state; + + if (issue_modify_qp) + iwqp->iwarp_state = info.next_iwarp_state; + else + info.next_iwarp_state = iwqp->iwarp_state; + } + if (attr_mask & IB_QP_ACCESS_FLAGS) { + ctx_info->iwarp_info_valid = true; + if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) + iwarp_info->wr_rdresp_en = true; + if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) + iwarp_info->wr_rdresp_en = true; + if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) + iwarp_info->rd_enable = true; + if (attr->qp_access_flags & IB_ACCESS_MW_BIND) + iwarp_info->bind_en = true; + + if (iwqp->user_mode) { + iwarp_info->rd_enable = true; + iwarp_info->wr_rdresp_en = true; + iwarp_info->priv_mode_en = false; + } + } + + if (ctx_info->iwarp_info_valid) { |