// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2018 Intel Corporation. */
#include <linux/bpf_trace.h>
#include <net/xdp_sock.h>
#include <net/xdp.h>
#include "ixgbe.h"
#include "ixgbe_txrx_common.h"
struct xdp_umem *ixgbe_xsk_umem(struct ixgbe_adapter *adapter,
struct ixgbe_ring *ring)
{
bool xdp_on = READ_ONCE(adapter->xdp_prog);
int qid = ring->ring_idx;
if (!adapter->xsk_umems || !adapter->xsk_umems[qid] ||
qid >= adapter->num_xsk_umems || !xdp_on)
return NULL;
return adapter->xsk_umems[qid];
}
static int ixgbe_alloc_xsk_umems(struct ixgbe_adapter *adapter)
{
if (adapter->xsk_umems)
return 0;
adapter->num_xsk_umems_used = 0;
adapter->num_xsk_umems = adapter->num_rx_queues;
adapter->xsk_umems = kcalloc(adapter->num_xsk_umems,
sizeof(*adapter->xsk_umems),
GFP_KERNEL);
if (!adapter->xsk_umems) {
adapter->num_xsk_umems = 0;
return -ENOMEM;
}
return 0;
}
static int ixgbe_add_xsk_umem(struct ixgbe_adapter *adapter,
struct xdp_umem *umem,
u16 qid)
{
int err;
err = ixgbe_alloc_xsk_umems(adapter);
if (err)
return err;
adapter->xsk_umems[qid] = umem;
adapter->num_xsk_umems_used++;
return 0;
}
static void ixgbe_remove_xsk_umem(struct ixgbe_adapter *adapter, u16 qid)
{
adapter->xsk_umems[qid] = NULL;
adapter->num_xsk_umems_used--;
if (adapter->num_xsk_umems == 0) {
kfree(adapter->xsk_umems);
adapter->xsk_umems = NULL;
adapter->num_xsk_umems = 0;
}
}
static int ixgbe_xsk_umem_dma_map(struct ixgbe_adapter *adapter,
struct xdp_umem *umem)
{
struct device *dev = &adapter->pdev->dev;
unsigned int i, j;
dma_addr_t dma;
for (i = 0; i < umem->npgs; i++) {
dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE,
DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
if (dma_mapping_error(dev, dma))
goto out_unmap;
umem->pages[i].dma = dma;
}
return 0;
out_unmap:
for (j = 0; j < i; j++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
return -1;
}
static void ixgbe_xsk_umem_dma_unmap(struct ixgbe_adapter *adapter,
struct xdp_umem *umem)
{
struct device *dev = &adapter->pdev->dev;
unsigned int i;
for (i = 0; i < umem->npgs; i++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
}
static int ixgbe_xsk_umem_enable(struct ixgbe_adapter *adapter,
struct xdp_umem *umem,
u16 qid)
{
struct xdp_umem_fq_reuse *reuseq;
bool if_running;
int err;
if (qid >= adapter->num_rx_queues)
return -EINVAL;
if (adapter->xsk_umems) {
if (qid >= adapter->num_xsk_umems)
return -EINVAL;
if (adapter->xsk_umems[qid])
return -EBUSY;
}
reuseq = xsk_reuseq_prepare(adapter->rx_ring[0]->count);
if (!reuseq)
return -ENOMEM;
xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
err = ixgbe_xsk_umem_dma_map(adapter, umem);
if (err)
return err;
if_running = netif_running(adapter->netdev) &&
READ_ONCE(adapter->xdp_prog);
if (if_running)
ixgbe_txrx_ring_disable(adapter, qid);
err = ixgbe_add_xsk_umem(adapter, umem, qid);
if (err)
return err;
if (if_running) {
ixgbe_txrx_ring_enable(adapter, qid);
/* Kick start the NAPI context so that receiving will start */
err = ixgbe_xsk_async_xmit(adapter->netdev, qid);
if (err)
return err;
}
return 0;
}
static int ixgbe_xsk_umem_disable(struct ixgbe_adapter *adapter, u16 qid)
{
bool if_running;
if (!adapter->xsk_umems || qid >= adapter->num_xsk_umems ||
!adapter->xsk_umems[qid])
return -EINVAL;
if_running = netif_running(adapter->netdev) &&
READ_ONCE(adapter->xdp_prog);
if (if_running)
ixgbe_txrx_ring_disable(adapter,