diff options
author | Dmitry Kravkov <dmitry@broadcom.com> | 2010-07-27 12:34:34 +0000 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2010-07-27 20:35:41 -0700 |
commit | 9f6c925889ad9204c7d1f5ca116d2e5fd6036c72 (patch) | |
tree | ab84e3b050729a1a92b54c1b6ed526cb97f9ad7b /drivers/net/bnx2x | |
parent | b0efbb996e8554ed8fe59e3f79e0bc83218083ab (diff) |
bnx2x: Create bnx2x_cmn.* files
Newly created files have no functionality changes,
but includes some functionality from bnx2x_main.c which
is common for PF and coming in the future VF driver.
Signed-off-by: Dmitry Kravkov <dmitry@broadcom.com>
Signed-off-by: Eilon Greenstein <eilong@broadcom.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers/net/bnx2x')
-rw-r--r-- | drivers/net/bnx2x/Makefile | 2 | ||||
-rw-r--r-- | drivers/net/bnx2x/bnx2x.h | 1 | ||||
-rw-r--r-- | drivers/net/bnx2x/bnx2x_cmn.c | 2251 | ||||
-rw-r--r-- | drivers/net/bnx2x/bnx2x_cmn.h | 652 | ||||
-rw-r--r-- | drivers/net/bnx2x/bnx2x_main.c | 2637 |
5 files changed, 2943 insertions, 2600 deletions
diff --git a/drivers/net/bnx2x/Makefile b/drivers/net/bnx2x/Makefile index 46c853b6cc53..ef4eebb38663 100644 --- a/drivers/net/bnx2x/Makefile +++ b/drivers/net/bnx2x/Makefile @@ -4,4 +4,4 @@ obj-$(CONFIG_BNX2X) += bnx2x.o -bnx2x-objs := bnx2x_main.o bnx2x_link.o +bnx2x-objs := bnx2x_main.o bnx2x_link.o bnx2x_cmn.o diff --git a/drivers/net/bnx2x/bnx2x.h b/drivers/net/bnx2x/bnx2x.h index 4afd29201a5c..260507032d39 100644 --- a/drivers/net/bnx2x/bnx2x.h +++ b/drivers/net/bnx2x/bnx2x.h @@ -45,6 +45,7 @@ #endif #include <linux/mdio.h> +#include <linux/pci.h> #include "bnx2x_reg.h" #include "bnx2x_fw_defs.h" #include "bnx2x_hsi.h" diff --git a/drivers/net/bnx2x/bnx2x_cmn.c b/drivers/net/bnx2x/bnx2x_cmn.c new file mode 100644 index 000000000000..30d20c7fee0b --- /dev/null +++ b/drivers/net/bnx2x/bnx2x_cmn.c @@ -0,0 +1,2251 @@ +/* bnx2x_cmn.c: Broadcom Everest network driver. + * + * Copyright (c) 2007-2010 Broadcom Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Maintained by: Eilon Greenstein <eilong@broadcom.com> + * Written by: Eliezer Tamir + * Based on code from Michael Chan's bnx2 driver + * UDP CSUM errata workaround by Arik Gendelman + * Slowpath and fastpath rework by Vladislav Zolotarov + * Statistics and Link management by Yitchak Gertner + * + */ + + +#include <linux/etherdevice.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include "bnx2x_cmn.h" + +#ifdef BCM_VLAN +#include <linux/if_vlan.h> +#endif + +static int bnx2x_poll(struct napi_struct *napi, int budget); + +/* free skb in the packet ring at pos idx + * return idx of last bd freed + */ +static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp, + u16 idx) +{ + struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx]; + struct eth_tx_start_bd *tx_start_bd; + struct eth_tx_bd *tx_data_bd; + struct sk_buff *skb = tx_buf->skb; + u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons; + int nbd; + + /* prefetch skb end pointer to speedup dev_kfree_skb() */ + prefetch(&skb->end); + + DP(BNX2X_MSG_OFF, "pkt_idx %d buff @(%p)->skb %p\n", + idx, tx_buf, skb); + + /* unmap first bd */ + DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx); + tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd; + dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd), + BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE); + + nbd = le16_to_cpu(tx_start_bd->nbd) - 1; +#ifdef BNX2X_STOP_ON_ERROR + if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) { + BNX2X_ERR("BAD nbd!\n"); + bnx2x_panic(); + } +#endif + new_cons = nbd + tx_buf->first_bd; + + /* Get the next bd */ + bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); + + /* Skip a parse bd... */ + --nbd; + bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); + + /* ...and the TSO split header bd since they have no mapping */ + if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) { + --nbd; + bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); + } + + /* now free frags */ + while (nbd > 0) { + + DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx); + tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd; + dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd), + BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE); + if (--nbd) + bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); + } + + /* release skb */ + WARN_ON(!skb); + dev_kfree_skb(skb); + tx_buf->first_bd = 0; + tx_buf->skb = NULL; + + return new_cons; +} + +int bnx2x_tx_int(struct bnx2x_fastpath *fp) +{ + struct bnx2x *bp = fp->bp; + struct netdev_queue *txq; + u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons; + +#ifdef BNX2X_STOP_ON_ERROR + if (unlikely(bp->panic)) + return -1; +#endif + + txq = netdev_get_tx_queue(bp->dev, fp->index); + hw_cons = le16_to_cpu(*fp->tx_cons_sb); + sw_cons = fp->tx_pkt_cons; + + while (sw_cons != hw_cons) { + u16 pkt_cons; + + pkt_cons = TX_BD(sw_cons); + + /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */ + + DP(NETIF_MSG_TX_DONE, "hw_cons %u sw_cons %u pkt_cons %u\n", + hw_cons, sw_cons, pkt_cons); + +/* if (NEXT_TX_IDX(sw_cons) != hw_cons) { + rmb(); + prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb); + } +*/ + bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons); + sw_cons++; + } + + fp->tx_pkt_cons = sw_cons; + fp->tx_bd_cons = bd_cons; + + /* Need to make the tx_bd_cons update visible to start_xmit() + * before checking for netif_tx_queue_stopped(). Without the + * memory barrier, there is a small possibility that + * start_xmit() will miss it and cause the queue to be stopped + * forever. + */ + smp_mb(); + + /* TBD need a thresh? */ + if (unlikely(netif_tx_queue_stopped(txq))) { + /* Taking tx_lock() is needed to prevent reenabling the queue + * while it's empty. This could have happen if rx_action() gets + * suspended in bnx2x_tx_int() after the condition before + * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()): + * + * stops the queue->sees fresh tx_bd_cons->releases the queue-> + * sends some packets consuming the whole queue again-> + * stops the queue + */ + + __netif_tx_lock(txq, smp_processor_id()); + + if ((netif_tx_queue_stopped(txq)) && + (bp->state == BNX2X_STATE_OPEN) && + (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)) + netif_tx_wake_queue(txq); + + __netif_tx_unlock(txq); + } + return 0; +} + +static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp, + u16 idx) +{ + u16 last_max = fp->last_max_sge; + + if (SUB_S16(idx, last_max) > 0) + fp->last_max_sge = idx; +} + +static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp, + struct eth_fast_path_rx_cqe *fp_cqe) +{ + struct bnx2x *bp = fp->bp; + u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) - + le16_to_cpu(fp_cqe->len_on_bd)) >> + SGE_PAGE_SHIFT; + u16 last_max, last_elem, first_elem; + u16 delta = 0; + u16 i; + + if (!sge_len) + return; + + /* First mark all used pages */ + for (i = 0; i < sge_len; i++) + SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i]))); + + DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n", + sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1])); + + /* Here we assume that the last SGE index is the biggest */ + prefetch((void *)(fp->sge_mask)); + bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1])); + + last_max = RX_SGE(fp->last_max_sge); + last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT; + first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT; + + /* If ring is not full */ + if (last_elem + 1 != first_elem) + last_elem++; + + /* Now update the prod */ + for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) { + if (likely(fp->sge_mask[i])) + break; + + fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK; + delta += RX_SGE_MASK_ELEM_SZ; + } + + if (delta > 0) { + fp->rx_sge_prod += delta; + /* clear page-end entries */ + bnx2x_clear_sge_mask_next_elems(fp); + } + + DP(NETIF_MSG_RX_STATUS, + "fp->last_max_sge = %d fp->rx_sge_prod = %d\n", + fp->last_max_sge, fp->rx_sge_prod); +} + +static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue, + struct sk_buff *skb, u16 cons, u16 prod) +{ + struct bnx2x *bp = fp->bp; + struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons]; + struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod]; + struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod]; + dma_addr_t mapping; + + /* move empty skb from pool to prod and map it */ + prod_rx_buf->skb = fp->tpa_pool[queue].skb; + mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data, + bp->rx_buf_size, DMA_FROM_DEVICE); + dma_unmap_addr_set(prod_rx_buf, mapping, mapping); + + /* move partial skb from cons to pool (don't unmap yet) */ + fp->tpa_pool[queue] = *cons_rx_buf; + + /* mark bin state as start - print error if current state != stop */ + if (fp->tpa_state[queue] != BNX2X_TPA_STOP) + BNX2X_ERR("start of bin not in stop [%d]\n", queue); + + fp->tpa_state[queue] = BNX2X_TPA_START; + + /* point prod_bd to new skb */ + prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); + prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); + +#ifdef BNX2X_STOP_ON_ERROR + fp->tpa_queue_used |= (1 << queue); +#ifdef _ASM_GENERIC_INT_L64_H + DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n", +#else + DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n", +#endif + fp->tpa_queue_used); +#endif +} + +static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp, + struct sk_buff *skb, + struct eth_fast_path_rx_cqe *fp_cqe, + u16 cqe_idx) +{ + struct sw_rx_page *rx_pg, old_rx_pg; + u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd); + u32 i, frag_len, frag_size, pages; + int err; + int j; + + frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd; + pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT; + + /* This is needed in order to enable forwarding support */ + if (frag_size) + skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE, + max(frag_size, (u32)len_on_bd)); + +#ifdef BNX2X_STOP_ON_ERROR + if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) { + BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n", + pages, cqe_idx); + BNX2X_ERR("fp_cqe->pkt_len = %d fp_cqe->len_on_bd = %d\n", + fp_cqe->pkt_len, len_on_bd); + bnx2x_panic(); + return -EINVAL; + } +#endif + + /* Run through the SGL and compose the fragmented skb */ + for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) { + u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j])); + + /* FW gives the indices of the SGE as if the ring is an array + (meaning that "next" element will consume 2 indices) */ + frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE)); + rx_pg = &fp->rx_page_ring[sge_idx]; + old_rx_pg = *rx_pg; + + /* If we fail to allocate a substitute page, we simply stop + where we are and drop the whole packet */ + err = bnx2x_alloc_rx_sge(bp, fp, sge_idx); + if (unlikely(err)) { + fp->eth_q_stats.rx_skb_alloc_failed++; + return err; + } + + /* Unmap the page as we r going to pass it to the stack */ + dma_unmap_page(&bp->pdev->dev, + dma_unmap_addr(&old_rx_pg, mapping), + SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE); + + /* Add one frag and update the appropriate fields in the skb */ + skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len); + + skb->data_len += frag_len; + skb->truesize += frag_len; + skb->len += frag_len; + + frag_size -= frag_len; + } + + return 0; +} + +static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp, + u16 queue, int pad, int len, union eth_rx_cqe *cqe, + u16 cqe_idx) +{ + struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue]; + struct sk_buff *skb = rx_buf->skb; + /* alloc new skb */ + struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size); + + /* Unmap skb in the pool anyway, as we are going to change + pool entry status to BNX2X_TPA_STOP even if new skb allocation + fails. */ + dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping), + bp->rx_buf_size, DMA_FROM_DEVICE); + + if (likely(new_skb)) { + /* fix ip xsum and give it to the stack */ + /* (no need to map the new skb) */ +#ifdef BCM_VLAN + int is_vlan_cqe = + (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) & + PARSING_FLAGS_VLAN); + int is_not_hwaccel_vlan_cqe = + (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG))); +#endif + + prefetch(skb); + prefetch(((char *)(skb)) + 128); + +#ifdef BNX2X_STOP_ON_ERROR + if (pad + len > bp->rx_buf_size) { + BNX2X_ERR("skb_put is about to fail... " + "pad %d len %d rx_buf_size %d\n", + pad, len, bp->rx_buf_size); + bnx2x_panic(); + return; + } +#endif + + skb_reserve(skb, pad); + skb_put(skb, len); + + skb->protocol = eth_type_trans(skb, bp->dev); + skb->ip_summed = CHECKSUM_UNNECESSARY; + + { + struct iphdr *iph; + + iph = (struct iphdr *)skb->data; +#ifdef BCM_VLAN + /* If there is no Rx VLAN offloading - + take VLAN tag into an account */ + if (unlikely(is_not_hwaccel_vlan_cqe)) + iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN); +#endif + iph->check = 0; + iph->check = ip_fast_csum((u8 *)iph, iph->ihl); + } + + if (!bnx2x_fill_frag_skb(bp, fp, skb, + &cqe->fast_path_cqe, cqe_idx)) { +#ifdef BCM_VLAN + if ((bp->vlgrp != NULL) && is_vlan_cqe && + (!is_not_hwaccel_vlan_cqe)) + vlan_gro_receive(&fp->napi, bp->vlgrp, + le16_to_cpu(cqe->fast_path_cqe. + vlan_tag), skb); + else +#endif + napi_gro_receive(&fp->napi, skb); + } else { + DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages" + " - dropping packet!\n"); + dev_kfree_skb(skb); + } + + + /* put new skb in bin */ + fp->tpa_pool[queue].skb = new_skb; + + } else { + /* else drop the packet and keep the buffer in the bin */ + DP(NETIF_MSG_RX_STATUS, + "Failed to allocate new skb - dropping packet!\n"); + fp->eth_q_stats.rx_skb_alloc_failed++; + } + + fp->tpa_state[queue] = BNX2X_TPA_STOP; +} + +/* Set Toeplitz hash value in the skb using the value from the + * CQE (calculated by HW). + */ +static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe, + struct sk_buff *skb) +{ + /* Set Toeplitz hash from CQE */ + if ((bp->dev->features & NETIF_F_RXHASH) && + (cqe->fast_path_cqe.status_flags & + ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) + skb->rxhash = + le32_to_cpu(cqe->fast_path_cqe.rss_hash_result); +} + +int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget) +{ + struct bnx2x *bp = fp->bp; + u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons; + u16 hw_comp_cons, sw_comp_cons, sw_comp_prod; + int rx_pkt = 0; + +#ifdef BNX2X_STOP_ON_ERROR + if (unlikely(bp->panic)) + return 0; +#endif + + /* CQ "next element" is of the size of the regular element, + that's why it's ok here */ + hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb); + if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT) + hw_comp_cons++; + + bd_cons = fp->rx_bd_cons; + bd_prod = fp->rx_bd_prod; + bd_prod_fw = bd_prod; + sw_comp_cons = fp->rx_comp_cons; + sw_comp_prod = fp->rx_comp_prod; + + /* Memory barrier necessary as speculative reads of the rx + * buffer can be ahead of the index in the status block + */ + rmb(); + + DP(NETIF_MSG_RX_STATUS, + "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n", + fp->index, hw_comp_cons, sw_comp_cons); + + while (sw_comp_cons != hw_comp_cons) { + struct sw_rx_bd *rx_buf = NULL; + struct sk_buff *skb; + union eth_rx_cqe *cqe; + u8 cqe_fp_flags; + u16 len, pad; + + comp_ring_cons = RCQ_BD(sw_comp_cons); + bd_prod = RX_BD(bd_prod); + bd_cons = RX_BD(bd_cons); + + /* Prefetch the page containing the BD descriptor + at producer's index. It will be needed when new skb is + allocated */ + prefetch((void *)(PAGE_ALIGN((unsigned long) + (&fp->rx_desc_ring[bd_prod])) - + PAGE_SIZE + 1)); + + cqe = &fp->rx_comp_ring[comp_ring_cons]; + cqe_fp_flags = cqe->fast_path_cqe.type_error_flags; + + DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x" + " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags), + cqe_fp_flags, cqe->fast_path_cqe.status_flags, + le32_to_cpu(cqe->fast_path_cqe.rss_hash_result), + le16_to_cpu(cqe->fast_path_cqe.vlan_tag), + le16_to_cpu(cqe->fast_path_cqe.pkt_len)); + + /* is this a slowpath msg? */ + if (unlikely(CQE_TYPE(cqe_fp_flags))) { + bnx2x_sp_event(fp, cqe); + goto next_cqe; + + /* this is an rx packet */ + } else { + rx_buf = &fp->rx_buf_ring[bd_cons]; + skb = rx_buf->skb; + prefetch(skb); + len = le16_to_cpu(cqe->fast_path_cqe.pkt_len); + pad = cqe->fast_path_cqe.placement_offset; + + /* If CQE is marked both TPA_START and TPA_END + it is a non-TPA CQE */ + if ((!fp->disable_tpa) && + (TPA_TYPE(cqe_fp_flags) != + (TPA_TYPE_START | TPA_TYPE_END))) { + u16 queue = cqe->fast_path_cqe.queue_index; + + if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) { + DP(NETIF_MSG_RX_STATUS, + "calling tpa_start on queue %d\n", + queue); + + bnx2x_tpa_start(fp, queue, skb, + bd_cons, bd_prod); + + /* Set Toeplitz hash for an LRO skb */ + bnx2x_set_skb_rxhash(bp, cqe, skb); + + goto next_rx; + } + + if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) { + DP(NETIF_MSG_RX_STATUS, + "calling tpa_stop on queue %d\n", + queue); + + if (!BNX2X_RX_SUM_FIX(cqe)) + BNX2X_ERR("STOP on none TCP " + "data\n"); + + /* This is a size of the linear data + on this skb */ + len = le16_to_cpu(cqe->fast_path_cqe. + len_on_bd); + bnx2x_tpa_stop(bp, fp, queue, pad, + len, cqe, comp_ring_cons); +#ifdef BNX2X_STOP_ON_ERROR + if (bp->panic) + return 0; +#endif + + bnx2x_update_sge_prod(fp, + &cqe->fast_path_cqe); + goto next_cqe; + } + } + + dma_sync_single_for_device(&bp->pdev->dev, + dma_unmap_addr(rx_buf, mapping), + pad + RX_COPY_THRESH, + DMA_FROM_DEVICE); + prefetch(((char *)(skb)) + 128); + + /* is this an error packet? */ + if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) { + DP(NETIF_MSG_RX_ERR, + "ERROR flags %x rx packet %u\n", + cqe_fp_flags, sw_comp_cons); + fp->eth_q_stats.rx_err_discard_pkt++; + goto reuse_rx; + } + + /* Since we don't have a jumbo ring + * copy small packets if mtu > 1500 + */ + if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) && + (len <= RX_COPY_THRESH)) { + struct sk_buff *new_skb; + + new_skb = netdev_alloc_skb(bp->dev, + len + pad); + if (new_skb == NULL) { + DP(NETIF_MSG_RX_ERR, + "ERROR packet dropped " + "because of alloc failure\n"); + fp->eth_q_stats.rx_skb_alloc_failed++; + goto reuse_rx; + } + + /* aligned copy */ + skb_copy_from_linear_data_offset(skb, pad, + new_skb->data + pad, len); + skb_reserve(new_skb, pad); + skb_put(new_skb, len); + + bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod); + + skb = new_skb; + + } else + if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) { + dma_unmap_single(&bp->pdev->dev, + dma_unmap_addr(rx_buf, mapping), + bp->rx_buf_size, + DMA_FROM_DEVICE); + skb_reserve(skb, pad); + skb_put(skb, len); + + } else { + DP(NETIF_MSG_RX_ERR, + "ERROR packet dropped because " + "of alloc failure\n"); + fp->eth_q_stats.rx_skb_alloc_failed++; +reuse_rx: + bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod); + goto next_rx; + } + + skb->protocol = eth_type_trans(skb, bp->dev); + + /* Set Toeplitz hash for a none-LRO skb */ + bnx2x_set_skb_rxhash(bp, cqe, skb); + + skb->ip_summed = CHECKSUM_NONE; + if (bp->rx_csum) { + if (likely(BNX2X_RX_CSUM_OK(cqe))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + fp->eth_q_stats.hw_csum_err++; + } + } + + skb_record_rx_queue(skb, fp->index); + +#ifdef BCM_VLAN + if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) && + (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) & + PARSING_FLAGS_VLAN)) + vlan_gro_receive(&fp->napi, bp->vlgrp, + le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb); + else +#endif + napi_gro_receive(&fp->napi, skb); + + +next_rx: + rx_buf->skb = NULL; + + bd_cons = NEXT_RX_IDX(bd_cons); + bd_prod = NEXT_RX_IDX(bd_prod); + bd_prod_fw = NEXT_RX_IDX(bd_prod_fw); + rx_pkt++; +next_cqe: + sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod); + sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons); + + if (rx_pkt == budget) + break; + } /* while */ + + fp->rx_bd_cons = bd_cons; + fp->rx_bd_prod = bd_prod_fw; + fp->rx_comp_cons = sw_comp_cons; + fp->rx_comp_prod = sw_comp_prod; + + /* Update producers */ + bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod, + fp->rx_sge_prod); + + fp->rx_pkt += rx_pkt; + fp->rx_calls++; + + return rx_pkt; +} + +static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie) +{ + struct bnx2x_fastpath *fp = fp_cookie; + struct bnx2x *bp = fp->bp; + + /* Return here if interrupt is disabled */ + if (unlikely(atomic_read(&bp->intr_sem) != 0)) { + DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n"); + return IRQ_HANDLED; + } + + DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n", + fp->index, fp->sb_id); + bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0); + +#ifdef BNX2X_STOP_ON_ERROR + if (unlikely(bp->panic)) + return IRQ_HANDLED; +#endif + + /* Handle Rx and Tx according to MSI-X vector */ + prefetch(fp->rx_cons_sb); + prefetch(fp->tx_cons_sb); + prefetch(&fp->status_blk->u_status_block.status_block_index); + prefetch(&fp->status_blk->c_status_block.status_block_index); + napi_schedule(&bnx2x_fp(bp, fp->index, napi)); + + return IRQ_HANDLED; +} + + +/* HW Lock for shared dual port PHYs */ +void bnx2x_acquire_phy_lock(struct bnx2x *bp) +{ + mutex_lock(&bp->port.phy_mutex); + + if (bp->port.need_hw_lock) + bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); +} + +void bnx2x_release_phy_lock(struct bnx2x *bp) +{ + if (bp->port.need_hw_lock) + bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); + + mutex_unlock(&bp->port.phy_mutex); +} + +void bnx2x_link_report(struct bnx2x *bp) +{ + if (bp->flags & MF_FUNC_DIS) { + netif_carrier_off(bp->dev); + netdev_err(bp->dev, "NIC Link is Down\n"); + return; + } + + if (bp->link_vars.link_up) { + u16 line_speed; + + if (bp->state == BNX2X_STATE_OPEN) + netif_carrier_on(bp->dev); + netdev_info(bp->dev, "NIC Link is Up, "); + + line_speed = bp->link_vars.line_speed; + if (IS_E1HMF(bp)) { + u16 vn_max_rate; + + vn_max_rate = + ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >> + FUNC_MF_CFG_MAX_BW_SHIFT) * 100; + if (vn_max_rate < line_speed) + line_speed = vn_max_rate; + } + pr_cont("%d Mbps ", line_speed); + + if (bp->link_vars.duplex == DUPLEX_FULL) + pr_cont("full duplex"); + else + pr_cont("half duplex"); + + if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) { + if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) { + pr_cont(", receive "); + if (bp->link_vars.flow_ctrl & + BNX2X_FLOW_CTRL_TX) + pr_cont("& transmit "); + } else { + pr_cont(", transmit "); + } + pr_cont("flow control ON"); + } + pr_cont("\n"); + + } else { /* link_down */ + netif_carrier_off(bp->dev); + netdev_err(bp->dev, "NIC Link is Down\n"); + } +} + +void bnx2x_init_rx_rings(struct bnx2x *bp) +{ + int func = BP_FUNC(bp); + int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 : + ETH_MAX_AGGREGATION_QUEUES_E1H; + u16 ring_prod, cqe_ring_prod; + int i, j; + + bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN; + DP(NETIF_MSG_IFUP, + "mtu %d rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size); + + if (bp->flags & TPA_ENABLE_FLAG) { + + for_each_queue(bp, j) { + struct bnx2x_fastpath *fp = &bp->fp[j]; + + for (i = 0; i < max_agg_queues; i++) { + fp->tpa_pool[i].skb = + netdev_alloc_skb(bp->dev, bp->rx_buf_size); + if (!fp->tpa_pool[i].skb) { + BNX2X_ERR("Failed to allocate TPA " + "skb pool for queue[%d] - " + "disabling TPA on this " + "queue!\n", j); + bnx2x_free_tpa_pool(bp, fp, i); + fp->disable_tpa = 1; + break; + } + dma_unmap_addr_set((struct sw_rx_bd *) + &bp->fp->tpa_pool[i], + mapping, 0); + fp->tpa_state[i] = BNX2X_TPA_STOP; + } + } + } + + for_each_queue(bp, j) { + struct bnx2x_fastpath *fp = &bp->fp[j]; + + fp->rx_bd_cons = 0; + fp->rx_cons_sb = BNX2X_RX_SB_INDEX; + fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX; + + /* "next page" elements initialization */ + /* SGE ring */ + for (i = 1; i <= NUM_RX_SGE_PAGES; i++) { + struct eth_rx_sge *sge; + + sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2]; + sge->addr_hi = + cpu_to_le32(U64_HI(fp->rx_sge_mapping + + BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); + sge->addr_lo = + cpu_to_le32(U64_LO(fp->rx_sge_mapping + + BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); + } + + bnx2x_init_sge_ring_bit_mask(fp); + + /* RX BD ring */ + for (i = 1; i <= NUM_RX_RINGS; i++) { + struct eth_rx_bd *rx_bd; + + rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2]; + rx_bd->addr_hi = + cpu_to_le32(U64_HI(fp->rx_desc_mapping + + BCM_PAGE_SIZE*(i % NUM_RX_RINGS))); + rx_bd->addr_lo = + cpu_to_le32(U64_LO(fp->rx_desc_mapping + + BCM_PAGE_SIZE*(i % NUM_RX_RINGS))); + } + + /* CQ ring */ + for (i = 1; i <= NUM_RCQ_RINGS; i++) { + struct eth_rx_cqe_next_page *nextpg; + + nextpg = (struct eth_rx_cqe_next_page *) + &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1]; + nextpg->addr_hi = + cpu_to_le32(U64_HI(fp->rx_comp_mapping + + BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); + nextpg->addr_lo = + cpu_to_le32(U64_LO(fp->rx_comp_mapping + + BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); + } + + /* Allocate SGEs and initialize the ring elements */ + for (i = 0, ring_prod = 0; + i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) { + + if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) { + BNX2X_ERR("was only able to allocate " + "%d rx sges\n", i); + BNX2X_ERR("disabling TPA for queue[%d]\n", j); + /* Cleanup already allocated elements */ + bnx2x_free_rx_sge_range(bp, fp, ring_prod); + bnx2x_free_tpa_pool(bp, fp, max_agg_queues); + fp->disable_tpa = 1; + ring_prod = 0; + break; + } + ring_prod = NEXT_SGE_IDX(ring_prod); + } + fp->rx_sge_prod = ring_prod; + + /* Allocate BDs and initialize BD ring */ + fp->rx_comp_cons = 0; + cqe_ring_prod = ring_prod = 0; + for (i = 0; i < bp->rx_ring_size; i++) { + if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) { + BNX2X_ERR("was only able to allocate " + "%d rx skbs on queue[%d]\n", i, j); + fp->eth_q_stats.rx_skb_alloc_failed++; + break; + } + ring_prod = NEXT_RX_IDX(ring_prod); + cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod); + WARN_ON(ring_prod <= i); + } + + fp->rx_bd_prod = ring_prod; + /* must not have more available CQEs than BDs */ + fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT, + cqe_ring_prod); + fp->rx_pkt = fp->rx_calls = 0; + + /* Warning! + * this will generate an interrupt (to the TSTORM) + * must only be done after chip is initialized + */ + bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod, + fp->rx_sge_prod); + if (j != 0) + continue; + + REG_WR(bp, BAR_USTRORM_INTMEM + + USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func), + U64_LO(fp->rx_comp_mapping)); + REG_WR(bp, BAR_USTRORM_INTMEM + + USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4, + U64_HI(fp->rx_comp_mapping)); + } +} +static void bnx2x_free_tx_skbs(struct bnx2x *bp) +{ + int i; + + for_each_queue(bp, i) { + struct bnx2x_fastpath *fp = &bp->fp[i]; + + u16 bd_cons = fp->tx_bd_cons; + u16 sw_prod = fp->tx_pkt_prod; + u16 sw_cons = fp->tx_pkt_cons; + + while (sw_cons != sw_prod) { + bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons)); + sw_cons++; + } + } +} + +static void bnx2x_free_rx_skbs(struct bnx2x *bp) +{ + int i, j; + + for_each_queue(bp, j) { + struct bnx2x_fastpath *fp = &bp->fp[j]; + + for (i = 0; i < NUM_RX_BD; i++) { + struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i]; + struct sk_buff *skb = rx_buf->skb; + + if (skb == NULL) + continue; + + dma_unmap_single(&bp->pdev->dev, + dma_unmap_addr(rx_buf, mapping), + bp->rx_buf_size, DMA_FROM_DEVICE); + + rx_buf->skb = NULL; + dev_kfree_skb(skb); + } + if (!fp->disable_tpa) + bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ? + ETH_MAX_AGGREGATION_QUEUES_E1 : + ETH_MAX_AGGREGATION_QUEUES_E1H); + } +} + +void bnx2x_free_skbs(struct bnx2x *bp) +{ + bnx2x_free_tx_skbs(bp); + bnx2x_free_rx_skbs(bp); +} + +static void bnx2x_free_msix_irqs(struct bnx2x *bp) +{ + int i, offset = 1; + + free_irq(bp->msix_table[0].vector, bp->dev); + DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n", + bp->msix_table[0].vector); + +#ifdef BCM_CNIC + offset++; +#endif + for_each_queue(bp, i) { + DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq " + "state %x\n", i, bp->msix_table[i + offset].vector, + bnx2x_fp(bp, i, state)); + + free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]); + } +} + +void bnx2x_free_irq(struct bnx2x *bp, bool disable_only) +{ + if (bp->flags & USING_MSIX_FLAG) { + if (!disable_only) + bnx2x_free_msix_irqs(bp); + pci_disable_msix(bp->pdev); + bp->flags &= ~USING_MSIX_FLAG; + + } else if (bp->flags & USING_MSI_FLAG) { + if (!disable_only) + free_irq(bp->pdev->irq, bp->dev); + pci_disable_msi(bp->pdev); + bp->flags &= ~USING_MSI_FLAG; + + } else if (!disable_only) + free_irq(bp->pdev->irq, bp->dev); +} + +static int bnx2x_enable_msix(struct bnx2x *bp) +{ + int i, rc, offset = 1; + int igu_vec = 0; + + bp->msix_table[0].entry = igu_vec; + DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec); + +#ifdef BCM_CNIC + igu_vec = BP_L_ID(bp) + offset; + bp->msix_table[1].entry = igu_vec; + DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec); + offset++; +#endif + for_each_queue(bp, i) { + igu_vec = BP_L_ID(bp) + offset + i; + bp->msix_table[i + offset].entry = igu_vec; + DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d " + "(fastpath #%u)\n", i + offset, igu_vec, i); + } + + rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], + BNX2X_NUM_QUEUES(bp) + offset); + + /* + * reconfigure number of tx/rx queues according to available + * MSI-X vectors + */ + if (rc >= BNX2X_MIN_MSIX_VEC_CNT) { + /* vectors available for FP */ + int fp_vec = rc - BNX2X_MSIX_VEC_FP_START; + + DP(NETIF_MSG_IFUP, + "Trying to use less MSI-X vectors: %d\n", rc); + + rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc); + + if (rc) { + DP(NETIF_MSG_IFUP, + "MSI-X is not attainable rc %d\n", rc); + return rc; + } + + bp->num_queues = min(bp->num_queues, fp_vec); + + DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n", + bp->num_queues); + } else if (rc) { + DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc); + return rc; + } + + bp->flags |= USING_MSIX_FLAG; + + return 0; +} + +static int bnx2x_req_msix_irqs(struct bnx2x *bp) +{ + int i, rc, offset = 1; + + rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0, + bp->dev->name, bp->dev); + if (rc) { + BNX2X_ERR("request sp irq failed\n"); + return -EBUSY; + } + +#ifdef BCM_CNIC + offset++; +#endif + for_each_queue(bp, i) { + struct bnx2x_fastpath *fp = &bp->fp[i]; + snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", + bp->dev->name, i); + + rc = request_irq(bp->msix_table[i + offset].vector, + bnx2x_msix_fp_int, 0, fp->name, fp); + if (rc) { + BNX2X_ERR("request fp #%d irq failed rc %d\n", i, rc); + bnx2x_free_msix_irqs(bp); + return -EBUS |