From 4780b7cae60cf10af4ae75bc5d6643f41d4c2969 Mon Sep 17 00:00:00 2001 From: "Lendacky, Thomas" Date: Tue, 4 Nov 2014 16:06:26 -0600 Subject: amd-xgbe: Move ring allocation to device open Move the channel and ring tracking structures allocation to device open. This will allow for future support to vary the number of Tx/Rx queues without unloading the module. Signed-off-by: Tom Lendacky Signed-off-by: David S. Miller --- drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 93 ++++++++++++++++++++++++++-- drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c | 8 +-- drivers/net/ethernet/amd/xgbe/xgbe-main.c | 62 ------------------- 3 files changed, 93 insertions(+), 70 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 2349ea970255..07b00bdcd9f9 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -129,6 +129,80 @@ static int xgbe_poll(struct napi_struct *, int); static void xgbe_set_rx_mode(struct net_device *); +static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) +{ + struct xgbe_channel *channel_mem, *channel; + struct xgbe_ring *tx_ring, *rx_ring; + unsigned int count, i; + + count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count); + + channel_mem = kcalloc(count, sizeof(struct xgbe_channel), GFP_KERNEL); + if (!channel_mem) + goto err_channel; + + tx_ring = kcalloc(pdata->tx_ring_count, sizeof(struct xgbe_ring), + GFP_KERNEL); + if (!tx_ring) + goto err_tx_ring; + + rx_ring = kcalloc(pdata->rx_ring_count, sizeof(struct xgbe_ring), + GFP_KERNEL); + if (!rx_ring) + goto err_rx_ring; + + for (i = 0, channel = channel_mem; i < count; i++, channel++) { + snprintf(channel->name, sizeof(channel->name), "channel-%d", i); + channel->pdata = pdata; + channel->queue_index = i; + channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE + + (DMA_CH_INC * i); + + if (i < pdata->tx_ring_count) { + spin_lock_init(&tx_ring->lock); + channel->tx_ring = tx_ring++; + } + + if (i < pdata->rx_ring_count) { + spin_lock_init(&rx_ring->lock); + channel->rx_ring = rx_ring++; + } + + DBGPR(" %s - queue_index=%u, dma_regs=%p, tx=%p, rx=%p\n", + channel->name, channel->queue_index, channel->dma_regs, + channel->tx_ring, channel->rx_ring); + } + + pdata->channel = channel_mem; + pdata->channel_count = count; + + return 0; + +err_rx_ring: + kfree(tx_ring); + +err_tx_ring: + kfree(channel_mem); + +err_channel: + netdev_err(pdata->netdev, "channel allocation failed\n"); + + return -ENOMEM; +} + +static void xgbe_free_channels(struct xgbe_prv_data *pdata) +{ + if (!pdata->channel) + return; + + kfree(pdata->channel->rx_ring); + kfree(pdata->channel->tx_ring); + kfree(pdata->channel); + + pdata->channel = NULL; + pdata->channel_count = 0; +} + static inline unsigned int xgbe_tx_avail_desc(struct xgbe_ring *ring) { return (ring->rdesc_count - (ring->cur - ring->dirty)); @@ -1119,10 +1193,15 @@ static int xgbe_open(struct net_device *netdev) goto err_ptpclk; pdata->rx_buf_size = ret; + /* Allocate the channel and ring structures */ + ret = xgbe_alloc_channels(pdata); + if (ret) + goto err_ptpclk; + /* Allocate the ring descriptors and buffers */ ret = desc_if->alloc_ring_resources(pdata); if (ret) - goto err_ptpclk; + goto err_channels; /* Initialize the device restart and Tx timestamp work struct */ INIT_WORK(&pdata->restart_work, xgbe_restart); @@ -1134,7 +1213,7 @@ static int xgbe_open(struct net_device *netdev) if (ret) { netdev_alert(netdev, "error requesting irq %d\n", pdata->irq_number); - goto err_irq; + goto err_rings; } pdata->irq_number = netdev->irq; @@ -1152,9 +1231,12 @@ err_start: devm_free_irq(pdata->dev, pdata->irq_number, pdata); pdata->irq_number = 0; -err_irq: +err_rings: desc_if->free_ring_resources(pdata); +err_channels: + xgbe_free_channels(pdata); + err_ptpclk: clk_disable_unprepare(pdata->ptpclk); @@ -1181,9 +1263,12 @@ static int xgbe_close(struct net_device *netdev) /* Issue software reset to device */ hw_if->exit(pdata); - /* Free all the ring data */ + /* Free the ring descriptors and buffers */ desc_if->free_ring_resources(pdata); + /* Free the channel and ring structures */ + xgbe_free_channels(pdata); + /* Release the interrupt */ if (pdata->irq_number != 0) { devm_free_irq(pdata->dev, pdata->irq_number, pdata); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c index 49508ec98b72..47022fb00b79 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c @@ -452,9 +452,9 @@ static int xgbe_set_coalesce(struct net_device *netdev, rx_usecs); return -EINVAL; } - if (rx_frames > pdata->channel->rx_ring->rdesc_count) { + if (rx_frames > pdata->rx_desc_count) { netdev_alert(netdev, "rx-frames is limited to %d frames\n", - pdata->channel->rx_ring->rdesc_count); + pdata->rx_desc_count); return -EINVAL; } @@ -462,9 +462,9 @@ static int xgbe_set_coalesce(struct net_device *netdev, tx_frames = ec->tx_max_coalesced_frames; /* Check the bounds of values for Tx */ - if (tx_frames > pdata->channel->tx_ring->rdesc_count) { + if (tx_frames > pdata->tx_desc_count) { netdev_alert(netdev, "tx-frames is limited to %d frames\n", - pdata->channel->tx_ring->rdesc_count); + pdata->tx_desc_count); return -EINVAL; } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c index f5a8fa03921a..e5077fd5b012 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c @@ -133,60 +133,6 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_VERSION(XGBE_DRV_VERSION); MODULE_DESCRIPTION(XGBE_DRV_DESC); -static struct xgbe_channel *xgbe_alloc_rings(struct xgbe_prv_data *pdata) -{ - struct xgbe_channel *channel_mem, *channel; - struct xgbe_ring *tx_ring, *rx_ring; - unsigned int count, i; - - DBGPR("-->xgbe_alloc_rings\n"); - - count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count); - - channel_mem = devm_kcalloc(pdata->dev, count, - sizeof(struct xgbe_channel), GFP_KERNEL); - if (!channel_mem) - return NULL; - - tx_ring = devm_kcalloc(pdata->dev, pdata->tx_ring_count, - sizeof(struct xgbe_ring), GFP_KERNEL); - if (!tx_ring) - return NULL; - - rx_ring = devm_kcalloc(pdata->dev, pdata->rx_ring_count, - sizeof(struct xgbe_ring), GFP_KERNEL); - if (!rx_ring) - return NULL; - - for (i = 0, channel = channel_mem; i < count; i++, channel++) { - snprintf(channel->name, sizeof(channel->name), "channel-%d", i); - channel->pdata = pdata; - channel->queue_index = i; - channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE + - (DMA_CH_INC * i); - - if (i < pdata->tx_ring_count) { - spin_lock_init(&tx_ring->lock); - channel->tx_ring = tx_ring++; - } - - if (i < pdata->rx_ring_count) { - spin_lock_init(&rx_ring->lock); - channel->rx_ring = rx_ring++; - } - - DBGPR(" %s - queue_index=%u, dma_regs=%p, tx=%p, rx=%p\n", - channel->name, channel->queue_index, channel->dma_regs, - channel->tx_ring, channel->rx_ring); - } - - pdata->channel_count = count; - - DBGPR("<--xgbe_alloc_rings\n"); - - return channel_mem; -} - static void xgbe_default_config(struct xgbe_prv_data *pdata) { DBGPR("-->xgbe_default_config\n"); @@ -383,14 +329,6 @@ static int xgbe_probe(struct platform_device *pdev) goto err_io; } - /* Allocate the rings for the DMA channels */ - pdata->channel = xgbe_alloc_rings(pdata); - if (!pdata->channel) { - dev_err(dev, "ring allocation failed\n"); - ret = -ENOMEM; - goto err_io; - } - /* Prepare to regsiter with MDIO */ pdata->mii_bus_id = kasprintf(GFP_KERNEL, "%s", pdev->name); if (!pdata->mii_bus_id) { -- cgit v1.2.3 From a9d41981e95651143125352f0233138efc17378a Mon Sep 17 00:00:00 2001 From: "Lendacky, Thomas" Date: Tue, 4 Nov 2014 16:06:32 -0600 Subject: amd-xgbe: Rename pre_xmit function to dev_xmit The pre_xmit function name implies that it performs operations prior to transmitting the packet when in fact it is responsible for setting up the descriptors and initiating the transmit. Rename this to function from pre_xmit to dev_xmit, which is consistent with the name used during receive processing - dev_read. Signed-off-by: Tom Lendacky Signed-off-by: David S. Miller --- drivers/net/ethernet/amd/xgbe/xgbe-dev.c | 8 ++++---- drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 2 +- drivers/net/ethernet/amd/xgbe/xgbe.h | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index 9da3a03e8c07..7b97d3852091 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -1198,7 +1198,7 @@ static void xgbe_config_dcb_pfc(struct xgbe_prv_data *pdata) xgbe_config_flow_control(pdata); } -static void xgbe_pre_xmit(struct xgbe_channel *channel) +static void xgbe_dev_xmit(struct xgbe_channel *channel) { struct xgbe_prv_data *pdata = channel->pdata; struct xgbe_ring *ring = channel->tx_ring; @@ -1211,7 +1211,7 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) int start_index = ring->cur; int i; - DBGPR("-->xgbe_pre_xmit\n"); + DBGPR("-->xgbe_dev_xmit\n"); csum = XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, CSUM_ENABLE); @@ -1410,7 +1410,7 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) channel->name, start_index & (ring->rdesc_count - 1), (ring->cur - 1) & (ring->rdesc_count - 1)); - DBGPR("<--xgbe_pre_xmit\n"); + DBGPR("<--xgbe_dev_xmit\n"); } static int xgbe_dev_read(struct xgbe_channel *channel) @@ -2561,7 +2561,7 @@ void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *hw_if) hw_if->powerup_rx = xgbe_powerup_rx; hw_if->powerdown_rx = xgbe_powerdown_rx; - hw_if->pre_xmit = xgbe_pre_xmit; + hw_if->dev_xmit = xgbe_dev_xmit; hw_if->dev_read = xgbe_dev_read; hw_if->enable_int = xgbe_enable_int; hw_if->disable_int = xgbe_disable_int; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 07b00bdcd9f9..8cb2372f8fa9 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -1343,7 +1343,7 @@ static int xgbe_xmit(struct sk_buff *skb, struct net_device *netdev) xgbe_prep_tx_tstamp(pdata, skb, packet); /* Configure required descriptor fields for transmission */ - hw_if->pre_xmit(channel); + hw_if->dev_xmit(channel); #ifdef XGMAC_ENABLE_TX_PKT_DUMP xgbe_print_pkt(netdev, skb, true); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index 789957d43a13..0dc15d7c6ae1 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -456,7 +456,7 @@ struct xgbe_hw_if { int (*enable_int)(struct xgbe_channel *, enum xgbe_int); int (*disable_int)(struct xgbe_channel *, enum xgbe_int); - void (*pre_xmit)(struct xgbe_channel *); + void (*dev_xmit)(struct xgbe_channel *); int (*dev_read)(struct xgbe_channel *); void (*tx_desc_init)(struct xgbe_channel *); void (*rx_desc_init)(struct xgbe_channel *); -- cgit v1.2.3 From aa96bd3c9fda52b9f57128798b49d662e5d4659c Mon Sep 17 00:00:00 2001 From: "Lendacky, Thomas" Date: Tue, 4 Nov 2014 16:06:37 -0600 Subject: amd-xgbe: Use the u32 data type for descriptors The Tx and Rx descriptors are unsigned 32 bit values. Use the u32 type, rather than unsigned int, to map these descriptors. Signed-off-by: Tom Lendacky Signed-off-by: David S. Miller --- drivers/net/ethernet/amd/xgbe/xgbe.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index 0dc15d7c6ae1..19f1d9007a6d 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -234,10 +234,10 @@ struct xgbe_packet_data { /* Common Rx and Tx descriptor mapping */ struct xgbe_ring_desc { - unsigned int desc0; - unsigned int desc1; - unsigned int desc2; - unsigned int desc3; + u32 desc0; + u32 desc1; + u32 desc2; + u32 desc3; }; /* Structure used to hold information related to the descriptor -- cgit v1.2.3 From 08dcc47c06c79de31b9b2c0b4637f6119e5701fa Mon Sep 17 00:00:00 2001 From: "Lendacky, Thomas" Date: Tue, 4 Nov 2014 16:06:44 -0600 Subject: amd-xgbe: Use page allocations for Rx buffers Use page allocations for Rx buffers instead of pre-allocating skbs of a set size. Signed-off-by: Tom Lendacky Signed-off-by: David S. Miller --- drivers/net/ethernet/amd/xgbe/xgbe-desc.c | 143 +++++++++++++++++++++--------- drivers/net/ethernet/amd/xgbe/xgbe-dev.c | 60 +++---------- drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 95 +++++++++++++------- drivers/net/ethernet/amd/xgbe/xgbe.h | 25 +++++- 4 files changed, 196 insertions(+), 127 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c index 6fc5da01437d..99911f45f334 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c @@ -117,7 +117,7 @@ #include "xgbe.h" #include "xgbe-common.h" -static void xgbe_unmap_skb(struct xgbe_prv_data *, struct xgbe_ring_data *); +static void xgbe_unmap_rdata(struct xgbe_prv_data *, struct xgbe_ring_data *); static void xgbe_free_ring(struct xgbe_prv_data *pdata, struct xgbe_ring *ring) @@ -131,13 +131,24 @@ static void xgbe_free_ring(struct xgbe_prv_data *pdata, if (ring->rdata) { for (i = 0; i < ring->rdesc_count; i++) { rdata = XGBE_GET_DESC_DATA(ring, i); - xgbe_unmap_skb(pdata, rdata); + xgbe_unmap_rdata(pdata, rdata); } kfree(ring->rdata); ring->rdata = NULL; } + if (ring->rx_pa.pages) { + dma_unmap_page(pdata->dev, ring->rx_pa.pages_dma, + ring->rx_pa.pages_len, DMA_FROM_DEVICE); + put_page(ring->rx_pa.pages); + + ring->rx_pa.pages = NULL; + ring->rx_pa.pages_len = 0; + ring->rx_pa.pages_offset = 0; + ring->rx_pa.pages_dma = 0; + } + if (ring->rdesc) { dma_free_coherent(pdata->dev, (sizeof(struct xgbe_ring_desc) * @@ -233,6 +244,65 @@ err_ring: return ret; } +static int xgbe_map_rx_buffer(struct xgbe_prv_data *pdata, + struct xgbe_ring *ring, + struct xgbe_ring_data *rdata) +{ + if (!ring->rx_pa.pages) { + struct page *pages = NULL; + dma_addr_t pages_dma; + gfp_t gfp; + int order, ret; + + /* Try to obtain pages, decreasing order if necessary */ + gfp = GFP_ATOMIC | __GFP_COLD | __GFP_COMP; + order = max_t(int, PAGE_ALLOC_COSTLY_ORDER, 1); + while (--order >= 0) { + pages = alloc_pages(gfp, order); + if (pages) + break; + } + if (!pages) + return -ENOMEM; + + /* Map the pages */ + pages_dma = dma_map_page(pdata->dev, pages, 0, + PAGE_SIZE << order, DMA_FROM_DEVICE); + ret = dma_mapping_error(pdata->dev, pages_dma); + if (ret) { + put_page(pages); + return ret; + } + + /* Set the values for this ring */ + ring->rx_pa.pages = pages; + ring->rx_pa.pages_len = PAGE_SIZE << order; + ring->rx_pa.pages_offset = 0; + ring->rx_pa.pages_dma = pages_dma; + } + + get_page(ring->rx_pa.pages); + rdata->rx_pa = ring->rx_pa; + + rdata->rx_dma = ring->rx_pa.pages_dma + ring->rx_pa.pages_offset; + rdata->rx_dma_len = pdata->rx_buf_size; + + ring->rx_pa.pages_offset += pdata->rx_buf_size; + if ((ring->rx_pa.pages_offset + pdata->rx_buf_size) > + ring->rx_pa.pages_len) { + /* This data descriptor is responsible for unmapping page(s) */ + rdata->rx_unmap = ring->rx_pa; + + /* Get a new allocation next time */ + ring->rx_pa.pages = NULL; + ring->rx_pa.pages_len = 0; + ring->rx_pa.pages_offset = 0; + ring->rx_pa.pages_dma = 0; + } + + return 0; +} + static void xgbe_wrapper_tx_descriptor_init(struct xgbe_prv_data *pdata) { struct xgbe_hw_if *hw_if = &pdata->hw_if; @@ -281,8 +351,7 @@ static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata) struct xgbe_ring *ring; struct xgbe_ring_desc *rdesc; struct xgbe_ring_data *rdata; - dma_addr_t rdesc_dma, skb_dma; - struct sk_buff *skb = NULL; + dma_addr_t rdesc_dma; unsigned int i, j; DBGPR("-->xgbe_wrapper_rx_descriptor_init\n"); @@ -302,22 +371,8 @@ static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata) rdata->rdesc = rdesc; rdata->rdesc_dma = rdesc_dma; - /* Allocate skb & assign to each rdesc */ - skb = dev_alloc_skb(pdata->rx_buf_size); - if (skb == NULL) - break; - skb_dma = dma_map_single(pdata->dev, skb->data, - pdata->rx_buf_size, - DMA_FROM_DEVICE); - if (dma_mapping_error(pdata->dev, skb_dma)) { - netdev_alert(pdata->netdev, - "failed to do the dma map\n"); - dev_kfree_skb_any(skb); + if (xgbe_map_rx_buffer(pdata, ring, rdata)) break; - } - rdata->skb = skb; - rdata->skb_dma = skb_dma; - rdata->skb_dma_len = pdata->rx_buf_size; rdesc++; rdesc_dma += sizeof(struct xgbe_ring_desc); @@ -334,8 +389,8 @@ static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata) DBGPR("<--xgbe_wrapper_rx_descriptor_init\n"); } -static void xgbe_unmap_skb(struct xgbe_prv_data *pdata, - struct xgbe_ring_data *rdata) +static void xgbe_unmap_rdata(struct xgbe_prv_data *pdata, + struct xgbe_ring_data *rdata) { if (rdata->skb_dma) { if (rdata->mapped_as_page) { @@ -354,6 +409,21 @@ static void xgbe_unmap_skb(struct xgbe_prv_data *pdata, rdata->skb = NULL; } + if (rdata->rx_pa.pages) + put_page(rdata->rx_pa.pages); + + if (rdata->rx_unmap.pages) { + dma_unmap_page(pdata->dev, rdata->rx_unmap.pages_dma, + rdata->rx_unmap.pages_len, DMA_FROM_DEVICE); + put_page(rdata->rx_unmap.pages); + } + + memset(&rdata->rx_pa, 0, sizeof(rdata->rx_pa)); + memset(&rdata->rx_unmap, 0, sizeof(rdata->rx_unmap)); + + rdata->rx_dma = 0; + rdata->rx_dma_len = 0; + rdata->tso_header = 0; rdata->len = 0; rdata->interrupt = 0; @@ -494,7 +564,7 @@ static int xgbe_map_tx_skb(struct xgbe_channel *channel, struct sk_buff *skb) err_out: while (start_index < cur_index) { rdata = XGBE_GET_DESC_DATA(ring, start_index++); - xgbe_unmap_skb(pdata, rdata); + xgbe_unmap_rdata(pdata, rdata); } DBGPR("<--xgbe_map_tx_skb: count=0\n"); @@ -502,40 +572,25 @@ err_out: return 0; } -static void xgbe_realloc_skb(struct xgbe_channel *channel) +static void xgbe_realloc_rx_buffer(struct xgbe_channel *channel) { struct xgbe_prv_data *pdata = channel->pdata; struct xgbe_hw_if *hw_if = &pdata->hw_if; struct xgbe_ring *ring = channel->rx_ring; struct xgbe_ring_data *rdata; - struct sk_buff *skb = NULL; - dma_addr_t skb_dma; int i; - DBGPR("-->xgbe_realloc_skb: rx_ring->rx.realloc_index = %u\n", + DBGPR("-->xgbe_realloc_rx_buffer: rx_ring->rx.realloc_index = %u\n", ring->rx.realloc_index); for (i = 0; i < ring->dirty; i++) { rdata = XGBE_GET_DESC_DATA(ring, ring->rx.realloc_index); /* Reset rdata values */ - xgbe_unmap_skb(pdata, rdata); + xgbe_unmap_rdata(pdata, rdata); - /* Allocate skb & assign to each rdesc */ - skb = dev_alloc_skb(pdata->rx_buf_size); - if (skb == NULL) + if (xgbe_map_rx_buffer(pdata, ring, rdata)) break; - skb_dma = dma_map_single(pdata->dev, skb->data, - pdata->rx_buf_size, DMA_FROM_DEVICE); - if (dma_mapping_error(pdata->dev, skb_dma)) { - netdev_alert(pdata->netdev, - "failed to do the dma map\n"); - dev_kfree_skb_any(skb); - break; - } - rdata->skb = skb; - rdata->skb_dma = skb_dma; - rdata->skb_dma_len = pdata->rx_buf_size; hw_if->rx_desc_reset(rdata); @@ -543,7 +598,7 @@ static void xgbe_realloc_skb(struct xgbe_channel *channel) } ring->dirty = 0; - DBGPR("<--xgbe_realloc_skb\n"); + DBGPR("<--xgbe_realloc_rx_buffer\n"); } void xgbe_init_function_ptrs_desc(struct xgbe_desc_if *desc_if) @@ -553,8 +608,8 @@ void xgbe_init_function_ptrs_desc(struct xgbe_desc_if *desc_if) desc_if->alloc_ring_resources = xgbe_alloc_ring_resources; desc_if->free_ring_resources = xgbe_free_ring_resources; desc_if->map_tx_skb = xgbe_map_tx_skb; - desc_if->realloc_skb = xgbe_realloc_skb; - desc_if->unmap_skb = xgbe_unmap_skb; + desc_if->realloc_rx_buffer = xgbe_realloc_rx_buffer; + desc_if->unmap_rdata = xgbe_unmap_rdata; desc_if->wrapper_tx_desc_init = xgbe_wrapper_tx_descriptor_init; desc_if->wrapper_rx_desc_init = xgbe_wrapper_rx_descriptor_init; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index 7b97d3852091..7748b758baf6 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -880,13 +880,15 @@ static void xgbe_tx_desc_reset(struct xgbe_ring_data *rdata) rdesc->desc1 = 0; rdesc->desc2 = 0; rdesc->desc3 = 0; + + /* Make sure ownership is written to the descriptor */ + wmb(); } static void xgbe_tx_desc_init(struct xgbe_channel *channel) { struct xgbe_ring *ring = channel->tx_ring; struct xgbe_ring_data *rdata; - struct xgbe_ring_desc *rdesc; int i; int start_index = ring->cur; @@ -895,26 +897,11 @@ static void xgbe_tx_desc_init(struct xgbe_channel *channel) /* Initialze all descriptors */ for (i = 0; i < ring->rdesc_count; i++) { rdata = XGBE_GET_DESC_DATA(ring, i); - rdesc = rdata->rdesc; - /* Initialize Tx descriptor - * Set buffer 1 (lo) address to zero - * Set buffer 1 (hi) address to zero - * Reset all other control bits (IC, TTSE, B2L & B1L) - * Reset all other control bits (OWN, CTXT, FD, LD, CPC, CIC, - * etc) - */ - rdesc->desc0 = 0; - rdesc->desc1 = 0; - rdesc->desc2 = 0; - rdesc->desc3 = 0; + /* Initialize Tx descriptor */ + xgbe_tx_desc_reset(rdata); } - /* Make sure everything is written to the descriptor(s) before - * telling the device about them - */ - wmb(); - /* Update the total number of Tx descriptors */ XGMAC_DMA_IOWRITE(channel, DMA_CH_TDRLR, ring->rdesc_count - 1); @@ -939,8 +926,8 @@ static void xgbe_rx_desc_reset(struct xgbe_ring_data *rdata) * Set buffer 2 (hi) address to zero and set control bits * OWN and INTE */ - rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->skb_dma)); - rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->skb_dma)); + rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->rx_dma)); + rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->rx_dma)); rdesc->desc2 = 0; rdesc->desc3 = 0; @@ -964,7 +951,6 @@ static void xgbe_rx_desc_init(struct xgbe_channel *channel) struct xgbe_prv_data *pdata = channel->pdata; struct xgbe_ring *ring = channel->rx_ring; struct xgbe_ring_data *rdata; - struct xgbe_ring_desc *rdesc; unsigned int start_index = ring->cur; unsigned int rx_coalesce, rx_frames; unsigned int i; @@ -977,34 +963,16 @@ static void xgbe_rx_desc_init(struct xgbe_channel *channel) /* Initialize all descriptors */ for (i = 0; i < ring->rdesc_count; i++) { rdata = XGBE_GET_DESC_DATA(ring, i); - rdesc = rdata->rdesc; - /* Initialize Rx descriptor - * Set buffer 1 (lo) address to dma address (lo) - * Set buffer 1 (hi) address to dma address (hi) - * Set buffer 2 (lo) address to zero - * Set buffer 2 (hi) address to zero and set control - * bits OWN and INTE appropriateley - */ - rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->skb_dma)); - rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->skb_dma)); - rdesc->desc2 = 0; - rdesc->desc3 = 0; - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, OWN, 1); - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, 1); - rdata->interrupt = 1; - if (rx_coalesce && (!rx_frames || ((i + 1) % rx_frames))) { - /* Clear interrupt on completion bit */ - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, - 0); + /* Set interrupt on completion bit as appropriate */ + if (rx_coalesce && (!rx_frames || ((i + 1) % rx_frames))) rdata->interrupt = 0; - } - } + else + rdata->interrupt = 1; - /* Make sure everything is written to the descriptors before - * telling the device about them - */ - wmb(); + /* Initialize Rx descriptor */ + xgbe_rx_desc_reset(rdata); + } /* Update the total number of Rx descriptors */ XGMAC_DMA_IOWRITE(channel, DMA_CH_RDRLR, ring->rdesc_count - 1); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 8cb2372f8fa9..d65f5aa8fdce 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -218,8 +218,8 @@ static int xgbe_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu) } rx_buf_size = mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; - if (rx_buf_size < XGBE_RX_MIN_BUF_SIZE) - rx_buf_size = XGBE_RX_MIN_BUF_SIZE; + rx_buf_size = clamp_val(rx_buf_size, XGBE_RX_MIN_BUF_SIZE, PAGE_SIZE); + rx_buf_size = (rx_buf_size + XGBE_RX_BUF_ALIGN - 1) & ~(XGBE_RX_BUF_ALIGN - 1); @@ -546,7 +546,7 @@ void xgbe_init_rx_coalesce(struct xgbe_prv_data *pdata) DBGPR("<--xgbe_init_rx_coalesce\n"); } -static void xgbe_free_tx_skbuff(struct xgbe_prv_data *pdata) +static void xgbe_free_tx_data(struct xgbe_prv_data *pdata) { struct xgbe_desc_if *desc_if = &pdata->desc_if; struct xgbe_channel *channel; @@ -554,7 +554,7 @@ static void xgbe_free_tx_skbuff(struct xgbe_prv_data *pdata) struct xgbe_ring_data *rdata; unsigned int i, j; - DBGPR("-->xgbe_free_tx_skbuff\n"); + DBGPR("-->xgbe_free_tx_data\n"); channel = pdata->channel; for (i = 0; i < pdata->channel_count; i++, channel++) { @@ -564,14 +564,14 @@ static void xgbe_free_tx_skbuff(struct xgbe_prv_data *pdata) for (j = 0; j < ring->rdesc_count; j++) { rdata = XGBE_GET_DESC_DATA(ring, j); - desc_if->unmap_skb(pdata, rdata); + desc_if->unmap_rdata(pdata, rdata); } } - DBGPR("<--xgbe_free_tx_skbuff\n"); + DBGPR("<--xgbe_free_tx_data\n"); } -static void xgbe_free_rx_skbuff(struct xgbe_prv_data *pdata) +static void xgbe_free_rx_data(struct xgbe_prv_data *pdata) { struct xgbe_desc_if *desc_if = &pdata->desc_if; struct xgbe_channel *channel; @@ -579,7 +579,7 @@ static void xgbe_free_rx_skbuff(struct xgbe_prv_data *pdata) struct xgbe_ring_data *rdata; unsigned int i, j; - DBGPR("-->xgbe_free_rx_skbuff\n"); + DBGPR("-->xgbe_free_rx_data\n"); channel = pdata->channel; for (i = 0; i < pdata->channel_count; i++, channel++) { @@ -589,11 +589,11 @@ static void xgbe_free_rx_skbuff(struct xgbe_prv_data *pdata) for (j = 0; j < ring->rdesc_count; j++) { rdata = XGBE_GET_DESC_DATA(ring, j); - desc_if->unmap_skb(pdata, rdata); + desc_if->unmap_rdata(pdata, rdata); } } - DBGPR("<--xgbe_free_rx_skbuff\n"); + DBGPR("<--xgbe_free_rx_data\n"); } static void xgbe_adjust_link(struct net_device *netdev) @@ -839,8 +839,8 @@ static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset) xgbe_stop(pdata); synchronize_irq(pdata->irq_number); - xgbe_free_tx_skbuff(pdata); - xgbe_free_rx_skbuff(pdata); + xgbe_free_tx_data(pdata); + xgbe_free_rx_data(pdata); /* Issue software reset to device if requested */ if (reset) @@ -1609,7 +1609,7 @@ static void xgbe_rx_refresh(struct xgbe_channel *channel) struct xgbe_ring *ring = channel->rx_ring; struct xgbe_ring_data *rdata; - desc_if->realloc_skb(channel); + desc_if->realloc_rx_buffer(channel); /* Update the Rx Tail Pointer Register with address of * the last cleaned entry */ @@ -1618,6 +1618,37 @@ static void xgbe_rx_refresh(struct xgbe_channel *channel) lower_32_bits(rdata->rdesc_dma)); } +static struct sk_buff *xgbe_create_skb(struct xgbe_prv_data *pdata, + struct xgbe_ring_data *rdata, + unsigned int len) +{ + struct net_device *netdev = pdata->netdev; + struct sk_buff *skb; + u8 *packet; + unsigned int copy_len; + + skb = netdev_alloc_skb_ip_align(netdev, XGBE_SKB_ALLOC_SIZE); + if (!skb) + return NULL; + + packet = page_address(rdata->rx_pa.pages) + rdata->rx_pa.pages_offset; + copy_len = min_t(unsigned int, XGBE_SKB_ALLOC_SIZE, len); + skb_copy_to_linear_data(skb, packet, copy_len); + skb_put(skb, copy_len); + + rdata->rx_pa.pages_offset += copy_len; + len -= copy_len; + if (len) + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + rdata->rx_pa.pages, + rdata->rx_pa.pages_offset, + len, rdata->rx_dma_len); + else + put_page(rdata->rx_pa.pages); + + return skb; +} + static int xgbe_tx_poll(struct xgbe_channel *channel) { struct xgbe_prv_data *pdata = channel->pdata; @@ -1651,7 +1682,7 @@ static int xgbe_tx_poll(struct xgbe_channel *channel) #endif /* Free the SKB and reset the descriptor for re-use */ - desc_if->unmap_skb(pdata, rdata); + desc_if->unmap_rdata(pdata, rdata); hw_if->tx_desc_reset(rdata); processed++; @@ -1726,9 +1757,9 @@ read_again: ring->cur++; ring->dirty++; - dma_unmap_single(pdata->dev, rdata->skb_dma, - rdata->skb_dma_len, DMA_FROM_DEVICE); - rdata->skb_dma = 0; + dma_sync_single_for_cpu(pdata->dev, rdata->rx_dma, + rdata->rx_dma_len, + DMA_FROM_DEVICE); incomplete = XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, @@ -1753,26 +1784,22 @@ read_again: if (!context) { put_len = rdata->len - len; - if (skb) { - if (pskb_expand_head(skb, 0, put_len, - GFP_ATOMIC)) { - DBGPR("pskb_expand_head error\n"); - if (incomplete) { - error = 1; - goto read_again; - } - - dev_kfree_skb(skb); - goto next_packet; + len += put_len; + + if (!skb) { + skb = xgbe_create_skb(pdata, rdata, put_len); + if (!skb) { + error = 1; + goto read_again; } - memcpy(skb_tail_pointer(skb), rdata->skb->data, - put_len); } else { - skb = rdata->skb; - rdata->skb = NULL; + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + rdata->rx_pa.pages, + rdata->rx_pa.pages_offset, + put_len, rdata->rx_dma_len); } - skb_put(skb, put_len); - len += put_len; + + rdata->rx_pa.pages = NULL; } if (incomplete || context_next) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index 19f1d9007a6d..d3aa05501ee8 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -142,6 +142,7 @@ #define XGBE_RX_MIN_BUF_SIZE (ETH_FRAME_LEN + ETH_FCS_LEN + VLAN_HLEN) #define XGBE_RX_BUF_ALIGN 64 +#define XGBE_SKB_ALLOC_SIZE 256 #define XGBE_MAX_DMA_CHANNELS 16 #define XGBE_MAX_QUEUES 16 @@ -240,6 +241,15 @@ struct xgbe_ring_desc { u32 desc3; }; +/* Page allocation related values */ +struct xgbe_page_alloc { + struct page *pages; + unsigned int pages_len; + unsigned int pages_offset; + + dma_addr_t pages_dma; +}; + /* Structure used to hold information related to the descriptor * and the packet associated with the descriptor (always use * use the XGBE_GET_DESC_DATA macro to access this data from the ring) @@ -253,6 +263,12 @@ struct xgbe_ring_data { unsigned int skb_dma_len; /* Length of SKB DMA area */ unsigned int tso_header; /* TSO header indicator */ + struct xgbe_page_alloc rx_pa; /* Rx buffer page allocation */ + struct xgbe_page_alloc rx_unmap; + + dma_addr_t rx_dma; /* DMA address of Rx buffer */ + unsigned int rx_dma_len; /* Length of the Rx DMA buffer */ + unsigned short len; /* Length of received Rx packet */ unsigned int interrupt; /* Interrupt indicator */ @@ -291,6 +307,9 @@ struct xgbe_ring { */ struct xgbe_ring_data *rdata; + /* Page allocation for RX buffers */ + struct xgbe_page_alloc rx_pa; + /* Ring index values * cur - Tx: index of descriptor to be used for current transfer * Rx: index of descriptor to check for packet availability @@ -515,8 +534,8 @@ struct xgbe_desc_if { int (*alloc_ring_resources)(struct xgbe_prv_data *); void (*free_ring_resources)(struct xgbe_prv_data *); int (*map_tx_skb)(struct xgbe_channel *, struct sk_buff *); - void (*realloc_skb)(struct xgbe_channel *); - void (*unmap_skb)(struct xgbe_prv_data *, struct xgbe_ring_data *); + void (*realloc_rx_buffer)(struct xgbe_channel *); + void (*unmap_rdata)(struct xgbe_prv_data *, struct xgbe_ring_data *); void (*wrapper_tx_desc_init)(struct xgbe_prv_data *); void (*wrapper_rx_desc_init)(struct xgbe_prv_data *); }; @@ -624,7 +643,7 @@ struct xgbe_prv_data { unsigned int rx_riwt; unsigned int rx_frames; - /* Current MTU */ + /* Current Rx buffer size */ unsigned int rx_buf_size; /* Flow control settings */ -- cgit v1.2.3 From 174fd2597b0bd8c19fce6a97e8b0f753ef4ce7cb Mon Sep 17 00:00:00 2001 From: "Lendacky, Thomas" Date: Tue, 4 Nov 2014 16:06:50 -0600 Subject: amd-xgbe: Implement split header receive support Provide support for splitting IP packets so that the header and payload can be sent to different DMA addresses. This will allow the IP header to be put into the linear part of the skb while the payload can be added as frags. Signed-off-by: Tom Lendacky Signed-off-by: David S. Miller --- drivers/net/ethernet/amd/xgbe/xgbe-common.h | 8 ++ drivers/net/ethernet/amd/xgbe/xgbe-desc.c | 176 ++++++++++++++++++---------- drivers/net/ethernet/amd/xgbe/xgbe-dev.c | 44 +++++-- drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 63 +++++----- drivers/net/ethernet/amd/xgbe/xgbe.h | 21 +++- 5 files changed, 201 insertions(+), 111 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h index caade30820d5..39bcb1140198 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h @@ -207,6 +207,8 @@ /* DMA channel register entry bit positions and sizes */ #define DMA_CH_CR_PBLX8_INDEX 16 #define DMA_CH_CR_PBLX8_WIDTH 1 +#define DMA_CH_CR_SPH_INDEX 24 +#define DMA_CH_CR_SPH_WIDTH 1 #define DMA_CH_IER_AIE_INDEX 15 #define DMA_CH_IER_AIE_WIDTH 1 #define DMA_CH_IER_FBEE_INDEX 12 @@ -429,6 +431,8 @@ #define MAC_RCR_CST_WIDTH 1 #define MAC_RCR_DCRCC_INDEX 3 #define MAC_RCR_DCRCC_WIDTH 1 +#define MAC_RCR_HDSMS_INDEX 12 +#define MAC_RCR_HDSMS_WIDTH 3 #define MAC_RCR_IPC_INDEX 9 #define MAC_RCR_IPC_WIDTH 1 #define MAC_RCR_JE_INDEX 8 @@ -847,6 +851,8 @@ #define RX_NORMAL_DESC0_OVT_INDEX 0 #define RX_NORMAL_DESC0_OVT_WIDTH 16 +#define RX_NORMAL_DESC2_HL_INDEX 0 +#define RX_NORMAL_DESC2_HL_WIDTH 10 #define RX_NORMAL_DESC3_CDA_INDEX 27 #define RX_NORMAL_DESC3_CDA_WIDTH 1 #define RX_NORMAL_DESC3_CTXT_INDEX 30 @@ -855,6 +861,8 @@ #define RX_NORMAL_DESC3_ES_WIDTH 1 #define RX_NORMAL_DESC3_ETLT_INDEX 16 #define RX_NORMAL_DESC3_ETLT_WIDTH 4 +#define RX_NORMAL_DESC3_FD_INDEX 29 +#define RX_NORMAL_DESC3_FD_WIDTH 1 #define RX_NORMAL_DESC3_INTE_INDEX 30 #define RX_NORMAL_DESC3_INTE_WIDTH 1 #define RX_NORMAL_DESC3_LD_INDEX 28 diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c index 99911f45f334..e6b9f54b9697 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c @@ -138,15 +138,26 @@ static void xgbe_free_ring(struct xgbe_prv_data *pdata, ring->rdata = NULL; } - if (ring->rx_pa.pages) { - dma_unmap_page(pdata->dev, ring->rx_pa.pages_dma, - ring->rx_pa.pages_len, DMA_FROM_DEVICE); - put_page(ring->rx_pa.pages); - - ring->rx_pa.pages = NULL; - ring->rx_pa.pages_len = 0; - ring->rx_pa.pages_offset = 0; - ring->rx_pa.pages_dma = 0; + if (ring->rx_hdr_pa.pages) { + dma_unmap_page(pdata->dev, ring->rx_hdr_pa.pages_dma, + ring->rx_hdr_pa.pages_len, DMA_FROM_DEVICE); + put_page(ring->rx_hdr_pa.pages); + + ring->rx_hdr_pa.pages = NULL; + ring->rx_hdr_pa.pages_len = 0; + ring->rx_hdr_pa.pages_offset = 0; + ring->rx_hdr_pa.pages_dma = 0; + } + + if (ring->rx_buf_pa.pages) { + dma_unmap_page(pdata->dev, ring->rx_buf_pa.pages_dma, + ring->rx_buf_pa.pages_len, DMA_FROM_DEVICE); + put_page(ring->rx_buf_pa.pages); + + ring->rx_buf_pa.pages = NULL; + ring->rx_buf_pa.pages_len = 0; + ring->rx_buf_pa.pages_offset = 0; + ring->rx_buf_pa.pages_dma = 0; } if (ring->rdesc) { @@ -244,62 +255,93 @@ err_ring: return ret; } -static int xgbe_map_rx_buffer(struct xgbe_prv_data *pdata, - struct xgbe_ring *ring, - struct xgbe_ring_data *rdata) +static int xgbe_alloc_pages(struct xgbe_prv_data *pdata, + struct xgbe_page_alloc *pa, gfp_t gfp, int order) { - if (!ring->rx_pa.pages) { - struct page *pages = NULL; - dma_addr_t pages_dma; - gfp_t gfp; - int order, ret; - - /* Try to obtain pages, decreasing order if necessary */ - gfp = GFP_ATOMIC | __GFP_COLD | __GFP_COMP; - order = max_t(int, PAGE_ALLOC_COSTLY_ORDER, 1); - while (--order >= 0) { - pages = alloc_pages(gfp, order); - if (pages) - break; - } - if (!pages) - return -ENOMEM; + struct page *pages = NULL; + dma_addr_t pages_dma; + int ret; - /* Map the pages */ - pages_dma = dma_map_page(pdata->dev, pages, 0, - PAGE_SIZE << order, DMA_FROM_DEVICE); - ret = dma_mapping_error(pdata->dev, pages_dma); - if (ret) { - put_page(pages); - return ret; - } + /* Try to obtain pages, decreasing order if necessary */ + gfp |= __GFP_COLD | __GFP_COMP; + while (order >= 0) { + pages = alloc_pages(gfp, order); + if (pages) + break; - /* Set the values for this ring */ - ring->rx_pa.pages = pages; - ring->rx_pa.pages_len = PAGE_SIZE << order; - ring->rx_pa.pages_offset = 0; - ring->rx_pa.pages_dma = pages_dma; + order--; } + if (!pages) + return -ENOMEM; - get_page(ring->rx_pa.pages); - rdata->rx_pa = ring->rx_pa; + /* Map the pages */ + pages_dma = dma_map_page(pdata->dev, pages, 0, + PAGE_SIZE << order, DMA_FROM_DEVICE); + ret = dma_mapping_error(pdata->dev, pages_dma); + if (ret) { + put_page(pages); + return ret; + } - rdata->rx_dma = ring->rx_pa.pages_dma + ring->rx_pa.pages_offset; - rdata->rx_dma_len = pdata->rx_buf_size; + pa->pages = pages; + pa->pages_len = PAGE_SIZE << order; + pa->pages_offset = 0; + pa->pages_dma = pages_dma; - ring->rx_pa.pages_offset += pdata->rx_buf_size; - if ((ring->rx_pa.pages_offset + pdata->rx_buf_size) > - ring->rx_pa.pages_len) { + return 0; +} + +static void xgbe_set_buffer_data(struct xgbe_buffer_data *bd, + struct xgbe_page_alloc *pa, + unsigned int len) +{ + get_page(pa->pages); + bd->pa = *pa; + + bd->dma = pa->pages_dma + pa->pages_offset; + bd->dma_len = len; + + pa->pages_offset += len; + if ((pa->pages_offset + len) > pa->pages_len) { /* This data descriptor is responsible for unmapping page(s) */ - rdata->rx_unmap = ring->rx_pa; + bd->pa_unmap = *pa; /* Get a new allocation next time */ - ring->rx_pa.pages = NULL; - ring->rx_pa.pages_len = 0; - ring->rx_pa.pages_offset = 0; - ring->rx_pa.pages_dma = 0; + pa->pages = NULL; + pa->pages_len = 0; + pa->pages_offset = 0; + pa->pages_dma = 0; + } +} + +static int xgbe_map_rx_buffer(struct xgbe_prv_data *pdata, + struct xgbe_ring *ring, + struct xgbe_ring_data *rdata) +{ + int order, ret; + + if (!ring->rx_hdr_pa.pages) { + ret = xgbe_alloc_pages(pdata, &ring->rx_hdr_pa, GFP_ATOMIC, 0); + if (ret) + return ret; + } + + if (!ring->rx_buf_pa.pages) { + order = max_t(int, PAGE_ALLOC_COSTLY_ORDER - 1, 0); + ret = xgbe_alloc_pages(pdata, &ring->rx_buf_pa, GFP_ATOMIC, + order); + if (ret) + return ret; } + /* Set up the header page info */ + xgbe_set_buffer_data(&rdata->rx_hdr, &ring->rx_hdr_pa, + XGBE_SKB_ALLOC_SIZE); + + /* Set up the buffer page info */ + xgbe_set_buffer_data(&rdata->rx_buf, &ring->rx_buf_pa, + pdata->rx_buf_size); + return 0; } @@ -409,20 +451,28 @@ static void xgbe_unmap_rdata(struct xgbe_prv_data *pdata, rdata->skb = NULL; } - if (rdata->rx_pa.pages) - put_page(rdata->rx_pa.pages); + if (rdata->rx_hdr.pa.pages) + put_page(rdata->rx_hdr.pa.pages); - if (rdata->rx_unmap.pages) { - dma_unmap_page(pdata->dev, rdata->rx_unmap.pages_dma, - rdata->rx_unmap.pages_len, DMA_FROM_DEVICE); - put_page(rdata->rx_unmap.pages); + if (rdata->rx_hdr.pa_unmap.pages) { + dma_unmap_page(pdata->dev, rdata->rx_hdr.pa_unmap.pages_dma, + rdata->rx_hdr.pa_unmap.pages_len, + DMA_FROM_DEVICE); + put_page(rdata->rx_hdr.pa_unmap.pages); } - memset(&rdata->rx_pa, 0, sizeof(rdata->rx_pa)); - memset(&rdata->rx_unmap, 0, sizeof(rdata->rx_unmap)); + if (rdata->rx_buf.pa.pages) + put_page(rdata->rx_buf.pa.pages); + + if (rdata->rx_buf.pa_unmap.pages) { + dma_unmap_page(pdata->dev, rdata->rx_buf.pa_unmap.pages_dma, + rdata->rx_buf.pa_unmap.pages_len, + DMA_FROM_DEVICE); + put_page(rdata->rx_buf.pa_unmap.pages); + } - rdata->rx_dma = 0; - rdata->rx_dma_len = 0; + memset(&rdata->rx_hdr, 0, sizeof(rdata->rx_hdr)); + memset(&rdata->rx_buf, 0, sizeof(rdata->rx_buf)); rdata->tso_header = 0; rdata->len = 0; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index 7748b758baf6..b3719f154637 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -335,6 +335,22 @@ static void xgbe_config_tso_mode(struct xgbe_prv_data *pdata) } } +static void xgbe_config_sph_mode(struct xgbe_prv_data *pdata) +{ + struct xgbe_channel *channel; + unsigned int i; + + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + if (!channel->rx_ring) + break; + + XGMAC_DMA_IOWRITE_BITS(channel, DMA_CH_CR, SPH, 1); + } + + XGMAC_IOWRITE_BITS(pdata, MAC_RCR, HDSMS, XGBE_SPH_HDSMS_SIZE); +} + static int xgbe_disable_tx_flow_control(struct xgbe_prv_data *pdata) { unsigned int max_q_count, q_count; @@ -920,19 +936,19 @@ static void xgbe_rx_desc_reset(struct xgbe_ring_data *rdata) struct xgbe_ring_desc *rdesc = rdata->rdesc; /* Reset the Rx descriptor - * Set buffer 1 (lo) address to dma address (lo) - * Set buffer 1 (hi) address to dma address (hi) - * Set buffer 2 (lo) address to zero - * Set buffer 2 (hi) address to zero and set control bits - * OWN and INTE + * Set buffer 1 (lo) address to header dma address (lo) + * Set buffer 1 (hi) address to header dma address (hi) + * Set buffer 2 (lo) address to buffer dma address (lo) + * Set buffer 2 (hi) address to buffer dma address (hi) and + * set control bits OWN and INTE */ - rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->rx_dma)); - rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->rx_dma)); - rdesc->desc2 = 0; + rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->rx_hdr.dma)); + rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->rx_hdr.dma)); + rdesc->desc2 = cpu_to_le32(lower_32_bits(rdata->rx_buf.dma)); + rdesc->desc3 = cpu_to_le32(upper_32_bits(rdata->rx_buf.dma)); - rdesc->desc3 = 0; - if (rdata->interrupt) - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, 1); + XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, + rdata->interrupt ? 1 : 0); /* Since the Rx DMA engine is likely running, make sure everything * is written to the descriptor(s) before setting the OWN bit @@ -1422,6 +1438,11 @@ static int xgbe_dev_read(struct xgbe_channel *channel) XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, CONTEXT_NEXT, 1); + /* Get the header length */ + if (XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, FD)) + rdata->hdr_len = XGMAC_GET_BITS_LE(rdesc->desc2, + RX_NORMAL_DESC2, HL); + /* Get the packet length */ rdata->len = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, PL); @@ -2453,6 +2474,7 @@ static int xgbe_init(struct xgbe_prv_data *pdata) xgbe_config_tx_coalesce(pdata); xgbe_config_rx_buffer_size(pdata); xgbe_config_tso_mode(pdata); + xgbe_config_sph_mode(pdata); desc_if->wrapper_tx_desc_init(pdata); desc_if->wrapper_rx_desc_init(pdata); xgbe_enable_dma_interrupts(pdata); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index d65f5aa8fdce..07e2d216323a 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -1620,31 +1620,25 @@ static void xgbe_rx_refresh(struct xgbe_channel *channel) static struct sk_buff *xgbe_create_skb(struct xgbe_prv_data *pdata, struct xgbe_ring_data *rdata, - unsigned int len) + unsigned int *len) { struct net_device *netdev = pdata->netdev; struct sk_buff *skb; u8 *packet; unsigned int copy_len; - skb = netdev_alloc_skb_ip_align(netdev, XGBE_SKB_ALLOC_SIZE); + skb = netdev_alloc_skb_ip_align(netdev, rdata->rx_hdr.dma_len); if (!skb) return NULL; - packet = page_address(rdata->rx_pa.pages) + rdata->rx_pa.pages_offset; - copy_len = min_t(unsigned int, XGBE_SKB_ALLOC_SIZE, len); + packet = page_address(rdata->rx_hdr.pa.pages) + + rdata->rx_hdr.pa.pages_offset; + copy_len = (rdata->hdr_len) ? rdata->hdr_len : *len; + copy_len = min(rdata->rx_hdr.dma_len, copy_len); skb_copy_to_linear_data(skb, packet, copy_len); skb_put(skb, copy_len); - rdata->rx_pa.pages_offset += copy_len; - len -= copy_len; - if (len) - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - rdata->rx_pa.pages, - rdata->rx_pa.pages_offset, - len, rdata->rx_dma_len); - else - put_page(rdata->rx_pa.pages); + *len -= copy_len; return skb; } @@ -1757,10 +1751,6 @@ read_again: ring->cur++; ring->dirty++; - dma_sync_single_for_cpu(pdata->dev, rdata->rx_dma, - rdata->rx_dma_len, - DMA_FROM_DEVICE); - incomplete = XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, INCOMPLETE); @@ -1787,19 +1777,30 @@ read_again: len += put_len; if (!skb) { - skb = xgbe_create_skb(pdata, rdata, put_len); + dma_sync_single_for_cpu(pdata->dev, + rdata->rx_hdr.dma, + rdata->rx_hdr.dma_len, + DMA_FROM_DEVICE); + + skb = xgbe_create_skb(pdata, rdata, &put_len); if (!skb) { error = 1; goto read_again; } - } else { - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - rdata->rx_pa.pages, - rdata->rx_pa.pages_offset, - put_len, rdata->rx_dma_len); } - rdata->rx_pa.pages = NULL; + if (put_len) { + dma_sync_single_for_cpu(pdata->dev, + rdata->rx_buf.dma, + rdata->rx_buf.dma_len, + DMA_FROM_DEVICE); + + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + rdata->rx_buf.pa.pages, + rdata->rx_buf.pa.pages_offset, + put_len, rdata->rx_buf.dma_len); + rdata->rx_buf.pa.pages = NULL; + } } if (incomplete || context_next) @@ -1924,10 +1925,10 @@ void xgbe_dump_tx_desc(struct xgbe_ring *ring, unsigned int idx, while (count--) { rdata = XGBE_GET_DESC_DATA(ring, idx); rdesc = rdata->rdesc; - DBGPR("TX_NORMAL_DESC[%d %s] = %08x:%08x:%08x:%08x\n", idx, - (flag == 1) ? "QUEUED FOR TX" : "TX BY DEVICE", - le32_to_cpu(rdesc->desc0), le32_to_cpu(rdesc->desc1), - le32_to_cpu(rdesc->desc2), le32_to_cpu(rdesc->desc3)); + pr_alert("TX_NORMAL_DESC[%d %s] = %08x:%08x:%08x:%08x\n", idx, + (flag == 1) ? "QUEUED FOR TX" : "TX BY DEVICE", + le32_to_cpu(rdesc->desc0), le32_to_cpu(rdesc->desc1), + le32_to_cpu(rdesc->desc2), le32_to_cpu(rdesc->desc3)); idx++; } } @@ -1935,9 +1936,9 @@ void xgbe_dump_tx_desc(struct xgbe_ring *ring, unsigned int idx, void xgbe_dump_rx_desc(struct xgbe_ring *ring, struct xgbe_ring_desc *desc, unsigned int idx) { - DBGPR("RX_NORMAL_DESC[%d RX BY DEVICE] = %08x:%08x:%08x:%08x\n", idx, - le32_to_cpu(desc->desc0), le32_to_cpu(desc->desc1), - le32_to_cpu(desc->desc2), le32_to_cpu(desc->desc3)); + pr_alert("RX_NORMAL_DESC[%d RX BY DEVICE] = %08x:%08x:%08x:%08x\n", idx, + le32_to_cpu(desc->desc0), le32_to_cpu(desc->desc1), + le32_to_cpu(desc->desc2), le32_to_cpu(desc->desc3)); } void xgbe_print_pkt(struct net_device *netdev, struct sk_buff *skb, bool tx_rx) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index d3aa05501ee8..1480c9d41821 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -143,6 +143,7 @@ #define XGBE_RX_MIN_BUF_SIZE (ETH_FRAME_LEN + ETH_FCS_LEN + VLAN_HLEN) #define XGBE_RX_BUF_ALIGN 64 #define XGBE_SKB_ALLOC_SIZE 256 +#define XGBE_SPH_HDSMS_SIZE 2 /* Keep in sync with SKB_ALLOC_SIZE */ #define XGBE_MAX_DMA_CHANNELS 16 #define XGBE_MAX_QUEUES 16 @@ -250,6 +251,15 @@ struct xgbe_page_alloc { dma_addr_t pages_dma; }; +/* Ring entry buffer data */ +struct xgbe_buffer_data { + struct xgbe_page_alloc pa; + struct xgbe_page_alloc pa_unmap; + + dma_addr_t dma; + unsigned int dma_len; +}; + /* Structure used to hold information related to the descriptor * and the packet associated with the descriptor (always use * use the XGBE_GET_DESC_DATA macro to access this data from the ring) @@ -263,12 +273,10 @@ struct xgbe_ring_data { unsigned int skb_dma_len; /* Length of SKB DMA area */ unsigned int tso_header; /* TSO header indicator */ - struct xgbe_page_alloc rx_pa; /* Rx buffer page allocation */ - struct xgbe_page_alloc rx_unmap; - - dma_addr_t rx_dma; /* DMA address of Rx buffer */ - unsigned int rx_dma_len; /* Length of the Rx DMA buffer */ + struct xgbe_buffer_data rx_hdr; /* Header locations */ + struct xgbe_buffer_data rx_buf; /* Payload locations */ + unsigned short hdr_len; /* Length of received header */ unsigned short len; /* Length of received Rx packet */ unsigned int interrupt; /* Interrupt indicator */ @@ -308,7 +316,8 @@ struct xgbe_ring { struct xgbe_ring_data *rdata; /* Page allocation for RX buffers */ - struct xgbe_page_alloc rx_pa; + struct xgbe_page_alloc rx_hdr_pa; + struct xgbe_page_alloc rx_buf_pa; /* Ring index values * cur - Tx: index of descriptor to be used for current transfer -- cgit v1.2.3 From 9227dc5e579b6b2ef58ad0d3d0d23ddac77846ef Mon Sep 17 00:00:00 2001 From: "Lendacky, Thomas" Date: Tue, 4 Nov 2014 16:06:56 -0600 Subject: amd-xgbe: Add support for per DMA channel interrupts This patch provides support for interrupts that are generated by the Tx/Rx DMA channel pairs of the device. This allows for Tx and Rx processing to run across multiple processsors. Signed-off-by: Tom Lendacky Signed-off-by: David S. Miller --- Documentation/devicetree/bindings/net/amd-xgbe.txt | 12 +- drivers/net/ethernet/amd/xgbe/xgbe-dev.c | 12 +- drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 226 +++++++++++++++++---- drivers/net/ethernet/amd/xgbe/xgbe-main.c | 10 +- drivers/net/ethernet/amd/xgbe/xgbe.h | 10 +- 5 files changed, 219 insertions(+), 51 deletions(-) diff --git a/Documentation/devicetree/bindings/net/amd-xgbe.txt b/Documentation/devicetree/bindings/net/amd-xgbe.txt index 41354f730beb..26efd526d16c 100644 --- a/Documentation/devicetree/bindings/net/amd-xgbe.txt +++ b/Documentation/devicetree/bindings/net/amd-xgbe.txt @@ -7,7 +7,10 @@ Required properties: - PCS registers - interrupt-parent: Should be the phandle for the interrupt controller that services interrupts for this device -- interrupts: Should contain the amd-xgbe interrupt +- interrupts: Should contain the amd-xgbe interrupt(s). The first interrupt + listed is required and is the general device interrupt. If the optional + amd,per-channel-interrupt property is specified, then one additional + interrupt for each DMA channel supported by the device should be specified - clocks: - DMA clock for the amd-xgbe device (used for calculating the correct Rx interrupt watchdog timer value on a DMA channel @@ -23,6 +26,9 @@ Optional properties: - mac-address: mac address to be assigned to the device. Can be overridden by UEFI. - dma-coherent: Present if dma operations are coherent +- amd,per-channel-interrupt: Indicates that Rx and Tx complete will generate + a unique interrupt for each DMA channel - this requires an additional + interrupt be configured for each DMA channel Example: xgbe@e0700000 { @@ -30,7 +36,9 @@ Example: reg = <0 0xe0700000 0 0x80000>, <0 0xe0780000 0 0x80000>; interrupt-parent = <&gic>; - interrupts = <0 325 4>; + interrupts = <0 325 4>, + <0 326 1>, <0 327 1>, <0 328 1>, <0 329 1>; + amd,per-channel-interrupt; clocks = <&xgbe_dma_clk>, <&xgbe_ptp_clk>; clock-names = "dma_clk", "ptp_clk"; phy-handle = <&phy>; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index b3719f154637..ac3d319ffab3 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -481,17 +481,21 @@ static void xgbe_enable_dma_interrupts(struct xgbe_prv_data *pdata) if (channel->tx_ring) { /* Enable the following Tx interrupts - * TIE - Transmit Interrupt Enable (unless polling) + * TIE - Transmit Interrupt Enable (unless using + * per channel interrupts) */ - XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1); + if (!pdata->per_channel_irq) + XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1); } if (channel->rx_ring) { /* Enable following Rx interrupts * RBUE - Receive Buffer Unavailable Enable - * RIE - Receive Interrupt Enable + * RIE - Receive Interrupt Enable (unless using + * per channel interrupts) */ XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RBUE, 1); - XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1); + if (!pdata->per_channel_irq) + XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1); } XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, dma_ch_ier); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 07e2d216323a..c3533e104c61 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -114,6 +114,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include #include #include @@ -126,7 +127,8 @@ #include "xgbe.h" #include "xgbe-common.h" -static int xgbe_poll(struct napi_struct *, int); +static int xgbe_one_poll(struct napi_struct *, int); +static int xgbe_all_poll(struct napi_struct *, int); static void xgbe_set_rx_mode(struct net_device *); static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) @@ -134,6 +136,7 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) struct xgbe_channel *channel_mem, *channel; struct xgbe_ring *tx_ring, *rx_ring; unsigned int count, i; + int ret = -ENOMEM; count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count); @@ -158,6 +161,19 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE + (DMA_CH_INC * i); + if (pdata->per_channel_irq) { + /* Get the DMA interrupt (offset 1) */ + ret = platform_get_irq(pdata->pdev, i + 1); + if (ret < 0) { + netdev_err(pdata->netdev, + "platform_get_irq %u failed\n", + i + 1); + goto err_irq; + } + + channel->dma_irq = ret; + } + if (i < pdata->tx_ring_count) { spin_lock_init(&tx_ring->lock); channel->tx_ring = tx_ring++; @@ -168,9 +184,9 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) channel->rx_ring = rx_ring++; } - DBGPR(" %s - queue_index=%u, dma_regs=%p, tx=%p, rx=%p\n", + DBGPR(" %s: queue=%u, dma_regs=%p, dma_irq=%d, tx=%p, rx=%p\n", channel->name, channel->queue_index, channel->dma_regs, - channel->tx_ring, channel->rx_ring); + channel->dma_irq, channel->tx_ring, channel->rx_ring); } pdata->channel = channel_mem; @@ -178,6 +194,9 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) return 0; +err_irq: + kfree(rx_ring); + err_rx_ring: kfree(tx_ring); @@ -185,9 +204,7 @@ err_tx_ring: kfree(channel_mem); err_channel: - netdev_err(pdata->netdev, "channel allocation failed\n"); - - return -ENOMEM; + return ret; } static void xgbe_free_channels(struct xgbe_prv_data *pdata) @@ -287,11 +304,7 @@ static irqreturn_t xgbe_isr(int irq, void *data) if (!dma_isr) goto isr_done; - DBGPR("-->xgbe_isr\n"); - DBGPR(" DMA_ISR = %08x\n", dma_isr); - DBGPR(" DMA_DS0 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR0)); - DBGPR(" DMA_DS1 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR1)); for (i = 0; i < pdata->channel_count; i++) { if (!(dma_isr & (1 << i))) @@ -302,6 +315,10 @@ static irqreturn_t xgbe_isr(int irq, void *data) dma_ch_isr = XGMAC_DMA_IOREAD(channel, DMA_CH_SR); DBGPR(" DMA_CH%u_ISR = %08x\n", i, dma_ch_isr); + /* If we get a TI or RI interrupt that means per channel DMA + * interrupts are not enabled, so we use the private data napi + * structure, not the per channel napi structure + */ if (XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, TI) || XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RI)) { if (napi_schedule_prep(&pdata->napi)) { @@ -344,12 +361,28 @@ static irqreturn_t xgbe_isr(int irq, void *data) DBGPR(" DMA_ISR = %08x\n", XGMAC_IOREAD(pdata, DMA_ISR)); - DBGPR("<--xgbe_isr\n"); - isr_done: return IRQ_HANDLED; } +static irqreturn_t xgbe_dma_isr(int irq, void *data) +{ + struct xgbe_channel *channel = data; + + /* Per channel DMA interrupts are enabled, so we use the per + * channel napi structure and not the private data napi structure + */ + if (napi_schedule_prep(&channel->napi)) { + /* Disable Tx and Rx interrupts */ + disable_irq(channel->dma_irq); + + /* Turn on polling */ + __napi_schedule(&channel->napi); + } + + return IRQ_HANDLED; +} + static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer) { struct xgbe_channel *channel = container_of(timer, @@ -357,18 +390,24 @@ static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer) tx_timer); struct xgbe_ring *ring = channel->tx_ring; struct xgbe_prv_data *pdata = channel->pdata; + struct napi_struct *napi; unsigned long flags; DBGPR("-->xgbe_tx_timer\n"); + napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi; + spin_lock_irqsave(&ring->lock, flags); - if (napi_schedule_prep(&pdata->napi)) { + if (napi_schedule_prep(napi)) { /* Disable Tx and Rx interrupts */ - xgbe_disable_rx_tx_ints(pdata); + if (pdata->per_channel_irq) + disable_irq(channel->dma_irq); + else + xgbe_disable_rx_tx_ints(pdata); /* Turn on polling */ - __napi_schedule(&pdata->napi); + __napi_schedule(napi); } channel->tx_timer_active = 0; @@ -504,18 +543,46 @@ void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata) static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add) { - if (add) - netif_napi_add(pdata->netdev, &pdata->napi, xgbe_poll, - NAPI_POLL_WEIGHT); - napi_enable(&pdata->napi); + struct xgbe_channel *channel; + unsigned int i; + + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + if (add) + netif_napi_add(pdata->netdev, &channel->napi, + xgbe_one_poll, NAPI_POLL_WEIGHT); + + napi_enable(&channel->napi); + } + } else { + if (add) + netif_napi_add(pdata->netdev, &pdata->napi, + xgbe_all_poll, NAPI_POLL_WEIGHT); + + napi_enable(&pdata->napi); + } } static void xgbe_napi_disable(struct xgbe_prv_data *pdata, unsigned int del) { - napi_disable(&pdata->napi); + struct xgbe_channel *channel; + unsigned int i; + + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + napi_disable(&channel->napi); - if (del) - netif_napi_del(&pdata->napi); + if (del) + netif_napi_del(&channel->napi); + } + } else { + napi_disable(&pdata->napi); + + if (del) + netif_napi_del(&pdata->napi); + } } void xgbe_init_tx_coalesce(struct xgbe_prv_data *pdata) @@ -828,7 +895,9 @@ static void xgbe_stop(struct xgbe_prv_data *pdata) static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset) { + struct xgbe_channel *channel; struct xgbe_hw_if *hw_if = &pdata->hw_if; + unsigned int i; DBGPR("-->xgbe_restart_dev\n"); @@ -837,7 +906,12 @@ static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset) return; xgbe_stop(pdata); - synchronize_irq(pdata->irq_number); + synchronize_irq(pdata->dev_irq); + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) + synchronize_irq(channel->dma_irq); + } xgbe_free_tx_data(pdata); xgbe_free_rx_data(pdata); @@ -1165,6 +1239,9 @@ static int xgbe_open(struct net_device *netdev) struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; struct xgbe_desc_if *desc_if = &pdata->desc_if; + struct xgbe_channel *channel = NULL; + char dma_irq_name[IFNAMSIZ + 32]; + unsigned int i = 0; int ret; DBGPR("-->xgbe_open\n"); @@ -1208,14 +1285,32 @@ static int xgbe_open(struct net_device *netdev) INIT_WORK(&pdata->tx_tstamp_work, xgbe_tx_tstamp); /* Request interrupts */ - ret = devm_request_irq(pdata->dev, netdev->irq, xgbe_isr, 0, + ret = devm_request_irq(pdata->dev, pdata->dev_irq, xgbe_isr, 0, netdev->name, pdata); if (ret) { netdev_alert(netdev, "error requesting irq %d\n", - pdata->irq_number); + pdata->dev_irq); goto err_rings; } - pdata->irq_number = netdev->irq; + + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + snprintf(dma_irq_name, sizeof(dma_irq_name) - 1, + "%s-TxRx-%u", netdev_name(netdev), + channel->queue_index); + + ret = devm_request_irq(pdata->dev, channel->dma_irq, + xgbe_dma_isr, 0, dma_irq_name, + channel); + if (ret) { + netdev_alert(netdev, + "error requesting irq %d\n", + channel->dma_irq); + goto err_irq; + } + } + } ret = xgbe_start(pdata); if (ret) @@ -1228,8 +1323,14 @@ static int xgbe_open(struct net_device *netdev) err_start: hw_if->exit(pdata); - devm_free_irq(pdata->dev, pdata->irq_number, pdata); - pdata->irq_number = 0; +err_irq: + if (pdata->per_channel_irq) { + /* Using an unsigned int, 'i' will go to UINT_MAX and exit */ + for (i--, channel--; i < pdata->channel_count; i--, channel--) + devm_free_irq(pdata->dev, channel->dma_irq, channel); + } + + devm_free_irq(pdata->dev, pdata->dev_irq, pdata); err_rings: desc_if->free_ring_resources(pdata); @@ -1254,6 +1355,8 @@ static int xgbe_close(struct net_device *netdev) struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; struct xgbe_desc_if *desc_if = &pdata->desc_if; + struct xgbe_channel *channel; + unsigned int i; DBGPR("-->xgbe_close\n"); @@ -1269,10 +1372,12 @@ static int xgbe_close(struct net_device *netdev) /* Free the channel and ring structures */ xgbe_free_channels(pdata); - /* Release the interrupt */ - if (pdata->irq_number != 0) { - devm_free_irq(pdata->dev, pdata->irq_number, pdata); - pdata->irq_number = 0; + /* Release the interrupts */ + devm_free_irq(pdata->dev, pdata->dev_irq, pdata); + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) + devm_free_irq(pdata->dev, channel->dma_irq, channel); } /* Disable the clocks */ @@ -1505,14 +1610,20 @@ static int xgbe_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, static void xgbe_poll_controller(struct net_device *netdev) { struct xgbe_prv_data *pdata = netdev_priv(netdev); + struct xgbe_channel *channel; + unsigned int i; DBGPR("-->xgbe_poll_controller\n"); - disable_irq(pdata->irq_number); - - xgbe_isr(pdata->irq_number, pdata); - - enable_irq(pdata->irq_number); + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) + xgbe_dma_isr(channel->dma_irq, channel); + } else { + disable_irq(pdata->dev_irq); + xgbe_isr(pdata->dev_irq, pdata); + enable_irq(pdata->dev_irq); + } DBGPR("<--xgbe_poll_controller\n"); } @@ -1704,6 +1815,7 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) struct xgbe_ring_data *rdata; struct xgbe_packet_data *packet; struct net_device *netdev = pdata->netdev; + struct napi_struct *napi; struct sk_buff *skb; struct skb_shared_hwtstamps *hwtstamps; unsigned int incomplete, error, context_next, context; @@ -1717,6 +1829,8 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) if (!ring) return 0; + napi = (