// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
/* Copyright (c) 2008-2019, IBM Corporation */
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/net.h>
#include <linux/scatterlist.h>
#include <linux/highmem.h>
#include <net/tcp.h>
#include <rdma/iw_cm.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_user_verbs.h>
#include "siw.h"
#include "siw_verbs.h"
#include "siw_mem.h"
#define MAX_HDR_INLINE \
(((uint32_t)(sizeof(struct siw_rreq_pkt) - \
sizeof(struct iwarp_send))) & 0xF8)
static struct page *siw_get_pblpage(struct siw_mem *mem, u64 addr, int *idx)
{
struct siw_pbl *pbl = mem->pbl;
u64 offset = addr - mem->va;
dma_addr_t paddr = siw_pbl_get_buffer(pbl, offset, NULL, idx);
if (paddr)
return virt_to_page(paddr);
return NULL;
}
/*
* Copy short payload at provided destination payload address
*/
static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr)
{
struct siw_wqe *wqe = &c_tx->wqe_active;
struct siw_sge *sge = &wqe->sqe.sge[0];
u32 bytes = sge->length;
if (bytes > MAX_HDR_INLINE || wqe->sqe.num_sge != 1)
return MAX_HDR_INLINE + 1;
if (!bytes)
return 0;
if (tx_flags(wqe) & SIW_WQE_INLINE) {
memcpy(paddr, &wqe->sqe.sge[1], bytes);
} else {
struct siw_mem *mem = wqe->mem[0];
if (!mem->mem_obj) {
/* Kernel client using kva */
memcpy(paddr,
(const void *)(uintptr_t)sge->laddr, bytes);
} else if (c_tx->in_syscall) {
if (copy_from_user(paddr, u64_to_user_ptr(sge->laddr),
bytes))
return -EFAULT;
} else {
unsigned int off = sge->laddr & ~PAGE_MASK;
struct page *p;
char *buffer;
int pbl_idx = 0;
if (!mem->is_pbl)
p = siw_get_upage(mem->umem, sge->laddr);
else
p = siw_get_pblpage(mem, sge->laddr, &pbl_idx);
if (unlik