summaryrefslogtreecommitdiffstats
path: root/drivers/net
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net')
-rw-r--r--drivers/net/virtio_net.c138
1 files changed, 63 insertions, 75 deletions
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index ca489e064f85..bf95016f442a 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -41,6 +41,8 @@ module_param(gso, bool, 0444);
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
#define GOOD_COPY_LEN 128
+#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
+
/* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
#define VIRTIO_XDP_HEADROOM 256
@@ -343,11 +345,10 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
static bool virtnet_xdp_xmit(struct virtnet_info *vi,
struct receive_queue *rq,
- struct xdp_buff *xdp,
- void *data)
+ struct xdp_buff *xdp)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
- unsigned int num_sg, len;
+ unsigned int len;
struct send_queue *sq;
unsigned int qp;
void *xdp_sent;
@@ -358,49 +359,23 @@ static bool virtnet_xdp_xmit(struct virtnet_info *vi,
/* Free up any pending old buffers before queueing new ones. */
while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (vi->mergeable_rx_bufs) {
- struct page *sent_page = virt_to_head_page(xdp_sent);
+ struct page *sent_page = virt_to_head_page(xdp_sent);
- put_page(sent_page);
- } else { /* small buffer */
- struct sk_buff *skb = xdp_sent;
-
- kfree_skb(skb);
- }
+ put_page(sent_page);
}
- if (vi->mergeable_rx_bufs) {
- xdp->data -= sizeof(struct virtio_net_hdr_mrg_rxbuf);
- /* Zero header and leave csum up to XDP layers */
- hdr = xdp->data;
- memset(hdr, 0, vi->hdr_len);
-
- num_sg = 1;
- sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
- } else { /* small buffer */
- struct sk_buff *skb = data;
+ xdp->data -= vi->hdr_len;
+ /* Zero header and leave csum up to XDP layers */
+ hdr = xdp->data;
+ memset(hdr, 0, vi->hdr_len);
- /* Zero header and leave csum up to XDP layers */
- hdr = skb_vnet_hdr(skb);
- memset(hdr, 0, vi->hdr_len);
+ sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
- num_sg = 2;
- sg_init_table(sq->sg, 2);
- sg_set_buf(sq->sg, hdr, vi->hdr_len);
- skb_to_sgvec(skb, sq->sg + 1,
- xdp->data - xdp->data_hard_start,
- xdp->data_end - xdp->data);
- }
- err = virtqueue_add_outbuf(sq->vq, sq->sg, num_sg,
- data, GFP_ATOMIC);
+ err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
if (unlikely(err)) {
- if (vi->mergeable_rx_bufs) {
- struct page *page = virt_to_head_page(xdp->data);
+ struct page *page = virt_to_head_page(xdp->data);
- put_page(page);
- } else /* small buffer */
- kfree_skb(data);
- /* On error abort to avoid unnecessary kick */
+ put_page(page);
return false;
}
@@ -408,39 +383,50 @@ static bool virtnet_xdp_xmit(struct virtnet_info *vi,
return true;
}
+static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
+{
+ return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
+}
+
static struct sk_buff *receive_small(struct net_device *dev,
struct virtnet_info *vi,
struct receive_queue *rq,
void *buf, unsigned int len)
{
- struct sk_buff * skb = buf;
+ struct sk_buff *skb;
struct bpf_prog *xdp_prog;
-
+ unsigned int xdp_headroom = virtnet_get_headroom(vi);
+ unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
+ unsigned int headroom = vi->hdr_len + header_offset;
+ unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+ unsigned int delta = 0;
len -= vi->hdr_len;
rcu_read_lock();
xdp_prog = rcu_dereference(rq->xdp_prog);
if (xdp_prog) {
- struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
+ struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
struct xdp_buff xdp;
+ void *orig_data;
u32 act;
if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
goto err_xdp;
- xdp.data_hard_start = skb->data;
- xdp.data = skb->data + VIRTIO_XDP_HEADROOM;
+ xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
+ xdp.data = xdp.data_hard_start + xdp_headroom;
xdp.data_end = xdp.data + len;
+ orig_data = xdp.data;
act = bpf_prog_run_xdp(xdp_prog, &xdp);
switch (act) {
case XDP_PASS:
/* Recalculate length in case bpf program changed it */
- __skb_pull(skb, xdp.data - xdp.data_hard_start);
- len = xdp.data_end - xdp.data;
+ delta = orig_data - xdp.data;
break;
case XDP_TX:
- if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp, skb)))
+ if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
trace_xdp_exception(vi->dev, xdp_prog, act);
rcu_read_unlock();
goto xdp_xmit;
@@ -454,13 +440,25 @@ static struct sk_buff *receive_small(struct net_device *dev,
}
rcu_read_unlock();
- skb_trim(skb, len);
+ skb = build_skb(buf, buflen);
+ if (!skb) {
+ put_page(virt_to_head_page(buf));
+ goto err;
+ }
+ skb_reserve(skb, headroom - delta);
+ skb_put(skb, len + delta);
+ if (!delta) {
+ buf += header_offset;
+ memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
+ } /* keep zeroed vnet hdr since packet was changed by bpf */
+
+err:
return skb;
err_xdp:
rcu_read_unlock();
dev->stats.rx_dropped++;
- kfree_skb(skb);
+ put_page(virt_to_head_page(buf));
xdp_xmit:
return NULL;
}
@@ -621,7 +619,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
}
break;
case XDP_TX:
- if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp, data)))
+ if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
trace_xdp_exception(vi->dev, xdp_prog, act);
ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
if (unlikely(xdp_page != page))
@@ -737,7 +735,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
} else if (vi->big_packets) {
give_pages(rq, buf);
} else {
- dev_kfree_skb(buf);
+ put_page(virt_to_head_page(buf));
}
return 0;
}
@@ -780,34 +778,28 @@ frame_err:
return 0;
}
-static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
-{
- return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
-}
-
static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
gfp_t gfp)
{
- int headroom = GOOD_PACKET_LEN + virtnet_get_headroom(vi);
+ struct page_frag *alloc_frag = &rq->alloc_frag;
+ char *buf;
unsigned int xdp_headroom = virtnet_get_headroom(vi);
- struct sk_buff *skb;
- struct virtio_net_hdr_mrg_rxbuf *hdr;
+ int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
int err;
- skb = __netdev_alloc_skb_ip_align(vi->dev, headroom, gfp);
- if (unlikely(!skb))
+ len = SKB_DATA_ALIGN(len) +
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+ if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
return -ENOMEM;
- skb_put(skb, headroom);
-
- hdr = skb_vnet_hdr(skb);
- sg_init_table(rq->sg, 2);
- sg_set_buf(rq->sg, hdr, vi->hdr_len);
- skb_to_sgvec(skb, rq->sg + 1, xdp_headroom, skb->len - xdp_headroom);
-
- err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp);
+ buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
+ get_page(alloc_frag->page);
+ alloc_frag->offset += len;
+ sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
+ vi->hdr_len + GOOD_PACKET_LEN);
+ err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
- dev_kfree_skb(skb);
+ put_page(virt_to_head_page(buf));
return err;
}
@@ -1994,10 +1986,6 @@ static void free_receive_page_frags(struct virtnet_info *vi)
static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
{
- /* For small receive mode always use kfree_skb variants */
- if (!vi->mergeable_rx_bufs)
- return false;
-
if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
return false;
else if (q < vi->curr_queue_pairs)
@@ -2032,7 +2020,7 @@ static void free_unused_bufs(struct virtnet_info *vi)
} else if (vi->big_packets) {
give_pages(&vi->rq[i], buf);
} else {
- dev_kfree_skb(buf);
+ put_page(virt_to_head_page(buf));
}
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-22 02:30:58 +0000