// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019, Intel Corporation. */
#include <linux/bpf_trace.h>
#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "ice.h"
#include "ice_base.h"
#include "ice_type.h"
#include "ice_xsk.h"
#include "ice_txrx.h"
#include "ice_txrx_lib.h"
#include "ice_lib.h"
/**
* ice_qp_reset_stats - Resets all stats for rings of given index
* @vsi: VSI that contains rings of interest
* @q_idx: ring index in array
*/
static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx)
{
memset(&vsi->rx_rings[q_idx]->rx_stats, 0,
sizeof(vsi->rx_rings[q_idx]->rx_stats));
memset(&vsi->tx_rings[q_idx]->stats, 0,
sizeof(vsi->tx_rings[q_idx]->stats));
if (ice_is_xdp_ena_vsi(vsi))
memset(&vsi->xdp_rings[q_idx]->stats, 0,
sizeof(vsi->xdp_rings[q_idx]->stats));
}
/**
* ice_qp_clean_rings - Cleans all the rings of a given index
* @vsi: VSI that contains rings of interest
* @q_idx: ring index in array
*/
static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx)
{
ice_clean_tx_ring(vsi->tx_rings[q_idx]);
if (ice_is_xdp_ena_vsi(vsi))
ice_clean_tx_ring(vsi->xdp_rings[q_idx]);
ice_clean_rx_ring(vsi->rx_rings[q_idx]);
}
/**
* ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector
* @vsi: VSI that has netdev
* @q_vector: q_vector that has NAPI context
* @enable: true for enable, false for disable
*/
static void
ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector,
bool enable)
{
if (!vsi->netdev || !q_vector)
return;
if (enable)
napi_enable(&q_vector->napi);
else
napi_disable(&q_vector->napi);
}
/**
* ice_qvec_dis_irq - Mask off queue interrupt generation on given ring
* @vsi: the VSI that contains queue vector being un-configured
* @rx_ring: Rx ring that will have its IRQ disabled
* @q_vector: queue vector
*/
static void
ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_ring *rx_ring,
struct ice_q_vector *q_vector)
{
struct ice_pf *pf = vsi->back;
struct ice_hw *hw = &pf->hw;
int base = vsi->base_vector;
u16 reg;
u32 val;
/* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle
* here only QINT_RQCTL
*/
reg = rx_ring->reg_idx;
val = rd32(hw, QINT_RQCTL(reg));
val &= ~QINT_RQCTL_CAUSE_ENA_M;
wr32(hw, QINT_RQCTL(reg), val);
if (q_vector) {
u16 v_idx = q_vector->v_idx;
wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0);
ice_flush(hw);
synchronize_irq(pf->msix_entries[v_idx + base].vector);
}
}
/**
* ice_qvec_cfg_msix - Enable IRQ for given queue vector
* @vsi: the VSI that contains queue vector
* @q_vector: queue vector
*/
static void
ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
{
u16 reg_idx = q_vector->reg_idx;
struct ice_pf *pf = vsi->back;
struct ice_hw *hw = &pf->hw;
struct ice_ring *ring;
ice_cfg_itr(hw, q_vector);
wr32(hw, GLINT_RATE(reg_idx),
ice_intrl_usec_to_reg(q_vector->intrl, hw->intrl_gran));
ice_for_each_ring(ring, q_vector->tx)
ice_cfg_txq_interrupt(vsi, ring->reg_idx, reg_idx,
q_vector->tx.itr_idx);
ice_for_each_ring(ring, q_vector->rx)
ice_cfg_rxq_interrupt(vsi, ring->reg_idx, reg_idx,
q_vector->rx.itr_idx);
ice_flush(hw);
}
/**
* ice_qvec_ena_irq - Enable IRQ for given queue vector
* @vsi: the VSI that contains queue vector
* @q_vector: queue vector
*/
static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
{
struct ice_pf *pf = vsi->back;
struct ice_hw *hw = &pf->hw;
ice_irq_dynamic_ena(hw, vsi, q_vector);
ice_flush(hw);
}
/**
* ice_qp_dis - Disables a queue pair
* @vsi: VSI of interest
* @q_idx: ring index in array
*
* Returns 0 on success, negative on failure.
*/
static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx)
{
struct ice_txq_meta txq_meta = { };
struct ice_ring *tx_ring, *rx_ring;
struct ice_q_vector *q_vector;
int timeout = 50;
int err;
if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
return -EINVAL;
tx_ring = vsi->tx_rings[q_idx];
rx_ring = vsi->rx_rings[q_idx];
q_vector = rx_ring->q_vector;
while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state)) {
timeout--;
if (!timeout)
return -EBUSY;
usleep_range(1000, 2000);
}
netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
ice_qvec_dis_irq(vsi, rx_ring, q_vector);
ice_fill_txq_meta(vsi, tx_ring, &tx