// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2018 Intel Corporation. */
#include <linux/bpf_trace.h>
#include <net/xdp_sock.h>
#include <net/xdp.h>
#include "i40e.h"
#include "i40e_txrx_common.h"
#include "i40e_xsk.h"
/**
* i40e_alloc_xsk_umems - Allocate an array to store per ring UMEMs
* @vsi: Current VSI
*
* Returns 0 on success, <0 on failure
**/
static int i40e_alloc_xsk_umems(struct i40e_vsi *vsi)
{
if (vsi->xsk_umems)
return 0;
vsi->num_xsk_umems_used = 0;
vsi->num_xsk_umems = vsi->alloc_queue_pairs;
vsi->xsk_umems = kcalloc(vsi->num_xsk_umems, sizeof(*vsi->xsk_umems),
GFP_KERNEL);
if (!vsi->xsk_umems) {
vsi->num_xsk_umems = 0;
return -ENOMEM;
}
return 0;
}
/**
* i40e_add_xsk_umem - Store a UMEM for a certain ring/qid
* @vsi: Current VSI
* @umem: UMEM to store
* @qid: Ring/qid to associate with the UMEM
*
* Returns 0 on success, <0 on failure
**/
static int i40e_add_xsk_umem(struct i40e_vsi *vsi, struct xdp_umem *umem,
u16 qid)
{
int err;
err = i40e_alloc_xsk_umems(vsi);
if (err)
return err;
vsi->xsk_umems[qid] = umem;
vsi->num_xsk_umems_used++;
return 0;
}
/**
* i40e_remove_xsk_umem - Remove a UMEM for a certain ring/qid
* @vsi: Current VSI
* @qid: Ring/qid associated with the UMEM
**/
static void i40e_remove_xsk_umem(struct i40e_vsi *vsi, u16 qid)
{
vsi->xsk_umems[qid] = NULL;
vsi->num_xsk_umems_used--;
if (vsi->num_xsk_umems == 0) {
kfree(vsi->xsk_umems);
vsi->xsk_umems = NULL;
vsi->num_xsk_umems = 0;
}
}
/**
* i40e_xsk_umem_dma_map - DMA maps all UMEM memory for the netdev
* @vsi: Current VSI
* @umem: UMEM to DMA map
*
* Returns 0 on success, <0 on failure
**/
static int i40e_xsk_umem_dma_map(struct i40e_vsi *vsi, struct xdp_umem *umem)
{
struct i40e_pf *pf = vsi->back;
struct device *dev;
unsigned int i, j;
dma_addr_t dma;
dev = &pf->pdev->dev;
for (i = 0; i < umem->npgs; i++) {
dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE,
DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
if (dma_mapping_error(dev, dma))
goto out_unmap;
umem->pages[i].dma = dma;
}
return 0;
out_unmap:
for (j = 0; j < i; j++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
return -1;
}
/**
* i40e_xsk_umem_dma_unmap - DMA unmaps all UMEM memory for the netdev
* @vsi: Current VSI
* @umem: UMEM to DMA map
**/
static void i40e_xsk_umem_dma_unmap(struct i40e_vsi *vsi, struct xdp_umem *umem)
{
struct i40e_pf *pf = vsi->back;
struct device *dev;
unsigned int i;
dev = &pf->pdev->dev;
for (i = 0; i < umem->npgs; i++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
}
/**
* i40e_xsk_umem_enable - Enable/associate a UMEM to a certain ring/qid
* @vsi: Current VSI
* @umem: UMEM
* @qid: Rx ring to associate UMEM to
*
* Returns 0 on success, <0 on failure
**/
static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem,
u16 qid)
{
struct xdp_umem_fq_reuse *reuseq;
bool if_running;
int err;
if (vsi->type != I40E_VSI_MAIN)
return -EINVAL;
if (qid >= vsi->num_queue_pairs)
return -EINVAL;
if (vsi->xsk_umems) {
if (qid >= vsi->num_xsk_umems)
return -EINVAL;
if (vsi->xsk_umems[qid])
return -EBUSY;
}
reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);
if (!reuseq)
return -ENOMEM;
xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
err = i40e_xsk_umem_dma_map(vsi, umem);
if (err)
retu