1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* A network driver using virtio.
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/ethtool.h>
10 #include <linux/module.h>
11 #include <linux/virtio.h>
12 #include <linux/virtio_net.h>
13 #include <linux/bpf.h>
14 #include <linux/bpf_trace.h>
15 #include <linux/scatterlist.h>
16 #include <linux/if_vlan.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/average.h>
20 #include <linux/filter.h>
21 #include <linux/kernel.h>
22 #include <linux/dim.h>
23 #include <net/route.h>
25 #include <net/net_failover.h>
26 #include <net/netdev_rx_queue.h>
27 #include <net/netdev_queues.h>
28 #include <net/xdp_sock_drv.h>
30 static int napi_weight = NAPI_POLL_WEIGHT;
31 module_param(napi_weight, int, 0444);
33 static bool csum = true, gso = true, napi_tx = true;
34 module_param(csum, bool, 0444);
35 module_param(gso, bool, 0444);
36 module_param(napi_tx, bool, 0644);
38 /* FIXME: MTU in config. */
39 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
40 #define GOOD_COPY_LEN 128
42 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
44 /* Separating two types of XDP xmit */
45 #define VIRTIO_XDP_TX BIT(0)
46 #define VIRTIO_XDP_REDIR BIT(1)
48 #define VIRTIO_XDP_FLAG BIT(0)
49 #define VIRTIO_ORPHAN_FLAG BIT(1)
51 /* RX packet size EWMA. The average packet size is used to determine the packet
52 * buffer size when refilling RX rings. As the entire RX ring may be refilled
53 * at once, the weight is chosen so that the EWMA will be insensitive to short-
54 * term, transient changes in packet size.
56 DECLARE_EWMA(pkt_len, 0, 64)
58 #define VIRTNET_DRIVER_VERSION "1.0.0"
60 static const unsigned long guest_offloads[] = {
61 VIRTIO_NET_F_GUEST_TSO4,
62 VIRTIO_NET_F_GUEST_TSO6,
63 VIRTIO_NET_F_GUEST_ECN,
64 VIRTIO_NET_F_GUEST_UFO,
65 VIRTIO_NET_F_GUEST_CSUM,
66 VIRTIO_NET_F_GUEST_USO4,
67 VIRTIO_NET_F_GUEST_USO6,
68 VIRTIO_NET_F_GUEST_HDRLEN
71 #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
72 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
73 (1ULL << VIRTIO_NET_F_GUEST_ECN) | \
74 (1ULL << VIRTIO_NET_F_GUEST_UFO) | \
75 (1ULL << VIRTIO_NET_F_GUEST_USO4) | \
76 (1ULL << VIRTIO_NET_F_GUEST_USO6))
78 struct virtnet_stat_desc {
79 char desc[ETH_GSTRING_LEN];
84 struct virtnet_sq_free_stats {
91 struct virtnet_sq_stats {
92 struct u64_stats_sync syncp;
96 u64_stats_t xdp_tx_drops;
98 u64_stats_t tx_timeouts;
103 struct virtnet_rq_stats {
104 struct u64_stats_sync syncp;
108 u64_stats_t xdp_packets;
110 u64_stats_t xdp_redirects;
111 u64_stats_t xdp_drops;
115 #define VIRTNET_SQ_STAT(name, m) {name, offsetof(struct virtnet_sq_stats, m), -1}
116 #define VIRTNET_RQ_STAT(name, m) {name, offsetof(struct virtnet_rq_stats, m), -1}
118 #define VIRTNET_SQ_STAT_QSTAT(name, m) \
121 offsetof(struct virtnet_sq_stats, m), \
122 offsetof(struct netdev_queue_stats_tx, m), \
125 #define VIRTNET_RQ_STAT_QSTAT(name, m) \
128 offsetof(struct virtnet_rq_stats, m), \
129 offsetof(struct netdev_queue_stats_rx, m), \
132 static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
133 VIRTNET_SQ_STAT("xdp_tx", xdp_tx),
134 VIRTNET_SQ_STAT("xdp_tx_drops", xdp_tx_drops),
135 VIRTNET_SQ_STAT("kicks", kicks),
136 VIRTNET_SQ_STAT("tx_timeouts", tx_timeouts),
139 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
140 VIRTNET_RQ_STAT("drops", drops),
141 VIRTNET_RQ_STAT("xdp_packets", xdp_packets),
142 VIRTNET_RQ_STAT("xdp_tx", xdp_tx),
143 VIRTNET_RQ_STAT("xdp_redirects", xdp_redirects),
144 VIRTNET_RQ_STAT("xdp_drops", xdp_drops),
145 VIRTNET_RQ_STAT("kicks", kicks),
148 static const struct virtnet_stat_desc virtnet_sq_stats_desc_qstat[] = {
149 VIRTNET_SQ_STAT_QSTAT("packets", packets),
150 VIRTNET_SQ_STAT_QSTAT("bytes", bytes),
151 VIRTNET_SQ_STAT_QSTAT("stop", stop),
152 VIRTNET_SQ_STAT_QSTAT("wake", wake),
155 static const struct virtnet_stat_desc virtnet_rq_stats_desc_qstat[] = {
156 VIRTNET_RQ_STAT_QSTAT("packets", packets),
157 VIRTNET_RQ_STAT_QSTAT("bytes", bytes),
160 #define VIRTNET_STATS_DESC_CQ(name) \
161 {#name, offsetof(struct virtio_net_stats_cvq, name), -1}
163 #define VIRTNET_STATS_DESC_RX(class, name) \
164 {#name, offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), -1}
166 #define VIRTNET_STATS_DESC_TX(class, name) \
167 {#name, offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), -1}
170 static const struct virtnet_stat_desc virtnet_stats_cvq_desc[] = {
171 VIRTNET_STATS_DESC_CQ(command_num),
172 VIRTNET_STATS_DESC_CQ(ok_num),
175 static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc[] = {
176 VIRTNET_STATS_DESC_RX(basic, packets),
177 VIRTNET_STATS_DESC_RX(basic, bytes),
179 VIRTNET_STATS_DESC_RX(basic, notifications),
180 VIRTNET_STATS_DESC_RX(basic, interrupts),
183 static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc[] = {
184 VIRTNET_STATS_DESC_TX(basic, packets),
185 VIRTNET_STATS_DESC_TX(basic, bytes),
187 VIRTNET_STATS_DESC_TX(basic, notifications),
188 VIRTNET_STATS_DESC_TX(basic, interrupts),
191 static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc[] = {
192 VIRTNET_STATS_DESC_RX(csum, needs_csum),
195 static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc[] = {
196 VIRTNET_STATS_DESC_TX(gso, gso_packets_noseg),
197 VIRTNET_STATS_DESC_TX(gso, gso_bytes_noseg),
200 static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc[] = {
201 VIRTNET_STATS_DESC_RX(speed, ratelimit_bytes),
204 static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc[] = {
205 VIRTNET_STATS_DESC_TX(speed, ratelimit_bytes),
208 #define VIRTNET_STATS_DESC_RX_QSTAT(class, name, qstat_field) \
211 offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), \
212 offsetof(struct netdev_queue_stats_rx, qstat_field), \
215 #define VIRTNET_STATS_DESC_TX_QSTAT(class, name, qstat_field) \
218 offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), \
219 offsetof(struct netdev_queue_stats_tx, qstat_field), \
222 static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc_qstat[] = {
223 VIRTNET_STATS_DESC_RX_QSTAT(basic, drops, hw_drops),
224 VIRTNET_STATS_DESC_RX_QSTAT(basic, drop_overruns, hw_drop_overruns),
227 static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc_qstat[] = {
228 VIRTNET_STATS_DESC_TX_QSTAT(basic, drops, hw_drops),
229 VIRTNET_STATS_DESC_TX_QSTAT(basic, drop_malformed, hw_drop_errors),
232 static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc_qstat[] = {
233 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_valid, csum_unnecessary),
234 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_none, csum_none),
235 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_bad, csum_bad),
238 static const struct virtnet_stat_desc virtnet_stats_tx_csum_desc_qstat[] = {
239 VIRTNET_STATS_DESC_TX_QSTAT(csum, csum_none, csum_none),
240 VIRTNET_STATS_DESC_TX_QSTAT(csum, needs_csum, needs_csum),
243 static const struct virtnet_stat_desc virtnet_stats_rx_gso_desc_qstat[] = {
244 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets, hw_gro_packets),
245 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes, hw_gro_bytes),
246 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets_coalesced, hw_gro_wire_packets),
247 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes_coalesced, hw_gro_wire_bytes),
250 static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc_qstat[] = {
251 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_packets, hw_gso_packets),
252 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_bytes, hw_gso_bytes),
253 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments, hw_gso_wire_packets),
254 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments_bytes, hw_gso_wire_bytes),
257 static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc_qstat[] = {
258 VIRTNET_STATS_DESC_RX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits),
261 static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc_qstat[] = {
262 VIRTNET_STATS_DESC_TX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits),
265 #define VIRTNET_Q_TYPE_RX 0
266 #define VIRTNET_Q_TYPE_TX 1
267 #define VIRTNET_Q_TYPE_CQ 2
269 struct virtnet_interrupt_coalesce {
274 /* The dma information of pages allocated at a time. */
275 struct virtnet_rq_dma {
282 /* Internal representation of a send virtqueue */
284 /* Virtqueue associated with this send _queue */
285 struct virtqueue *vq;
287 /* TX: fragments + linear part + virtio header */
288 struct scatterlist sg[MAX_SKB_FRAGS + 2];
290 /* Name of the send queue: output.$index */
293 struct virtnet_sq_stats stats;
295 struct virtnet_interrupt_coalesce intr_coal;
297 struct napi_struct napi;
299 /* Record whether sq is in reset state. */
303 /* Internal representation of a receive virtqueue */
304 struct receive_queue {
305 /* Virtqueue associated with this receive_queue */
306 struct virtqueue *vq;
308 struct napi_struct napi;
310 struct bpf_prog __rcu *xdp_prog;
312 struct virtnet_rq_stats stats;
314 /* The number of rx notifications */
317 /* Is dynamic interrupt moderation enabled? */
320 /* Used to protect dim_enabled and inter_coal */
321 struct mutex dim_lock;
323 /* Dynamic Interrupt Moderation */
328 struct virtnet_interrupt_coalesce intr_coal;
330 /* Chain pages by the private ptr. */
333 /* Average packet length for mergeable receive buffers. */
334 struct ewma_pkt_len mrg_avg_pkt_len;
336 /* Page frag for packet buffer allocation. */
337 struct page_frag alloc_frag;
339 /* RX: fragments + linear part + virtio header */
340 struct scatterlist sg[MAX_SKB_FRAGS + 2];
342 /* Min single buffer size for mergeable buffers case. */
343 unsigned int min_buf_len;
345 /* Name of this receive queue: input.$index */
348 struct xdp_rxq_info xdp_rxq;
350 /* Record the last dma info to free after new pages is allocated. */
351 struct virtnet_rq_dma *last_dma;
353 struct xsk_buff_pool *xsk_pool;
355 /* xdp rxq used by xsk */
356 struct xdp_rxq_info xsk_rxq_info;
358 struct xdp_buff **xsk_buffs;
364 /* This structure can contain rss message with maximum settings for indirection table and keysize
365 * Note, that default structure that describes RSS configuration virtio_net_rss_config
366 * contains same info but can't handle table values.
367 * In any case, structure would be passed to virtio hw through sg_buf split by parts
368 * because table sizes may be differ according to the device configuration.
370 #define VIRTIO_NET_RSS_MAX_KEY_SIZE 40
371 #define VIRTIO_NET_RSS_MAX_TABLE_LEN 128
372 struct virtio_net_ctrl_rss {
374 u16 indirection_table_mask;
375 u16 unclassified_queue;
376 u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
379 u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE];
382 /* Control VQ buffers: protected by the rtnl lock */
384 struct virtio_net_ctrl_hdr hdr;
385 virtio_net_ctrl_ack status;
388 struct virtnet_info {
389 struct virtio_device *vdev;
390 struct virtqueue *cvq;
391 struct net_device *dev;
392 struct send_queue *sq;
393 struct receive_queue *rq;
396 /* Max # of queue pairs supported by the device */
399 /* # of queue pairs currently used by the driver */
400 u16 curr_queue_pairs;
402 /* # of XDP queue pairs currently used by the driver */
405 /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
408 /* I like... big packets and I cannot lie! */
411 /* number of sg entries allocated for big packets */
412 unsigned int big_packets_num_skbfrags;
414 /* Host will merge rx buffers for big packets (shake it! shake it!) */
415 bool mergeable_rx_bufs;
417 /* Host supports rss and/or hash report */
419 bool has_rss_hash_report;
421 u16 rss_indir_table_size;
422 u32 rss_hash_types_supported;
423 u32 rss_hash_types_saved;
424 struct virtio_net_ctrl_rss rss;
426 /* Has control virtqueue */
429 /* Lock to protect the control VQ */
430 struct mutex cvq_lock;
432 /* Host can handle any s/g split between our header and packet data */
435 /* Packet virtio header size */
438 /* Work struct for delayed refilling if we run low on memory. */
439 struct delayed_work refill;
441 /* Is delayed refill enabled? */
444 /* The lock to synchronize the access to refill_enabled */
445 spinlock_t refill_lock;
447 /* Work struct for config space updates */
448 struct work_struct config_work;
450 /* Work struct for setting rx mode */
451 struct work_struct rx_mode_work;
453 /* OK to queue work setting RX mode? */
454 bool rx_mode_work_enabled;
456 /* Does the affinity hint is set for virtqueues? */
457 bool affinity_hint_set;
459 /* CPU hotplug instances for online & dead */
460 struct hlist_node node;
461 struct hlist_node node_dead;
463 struct control_buf *ctrl;
465 /* Ethtool settings */
469 /* Is rx dynamic interrupt moderation enabled? */
472 /* Interrupt coalescing settings */
473 struct virtnet_interrupt_coalesce intr_coal_tx;
474 struct virtnet_interrupt_coalesce intr_coal_rx;
476 unsigned long guest_offloads;
477 unsigned long guest_offloads_capable;
479 /* failover when STANDBY feature enabled */
480 struct failover *failover;
482 u64 device_stats_cap;
485 struct padded_vnet_hdr {
486 struct virtio_net_hdr_v1_hash hdr;
488 * hdr is in a separate sg buffer, and data sg buffer shares same page
489 * with this header sg. This padding makes next sg 16 byte aligned
495 struct virtio_net_common_hdr {
497 struct virtio_net_hdr hdr;
498 struct virtio_net_hdr_mrg_rxbuf mrg_hdr;
499 struct virtio_net_hdr_v1_hash hash_v1_hdr;
503 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
504 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
505 struct net_device *dev,
506 unsigned int *xdp_xmit,
507 struct virtnet_rq_stats *stats);
508 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq,
509 struct sk_buff *skb, u8 flags);
510 static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
511 struct sk_buff *curr_skb,
512 struct page *page, void *buf,
513 int len, int truesize);
515 static bool is_xdp_frame(void *ptr)
517 return (unsigned long)ptr & VIRTIO_XDP_FLAG;
520 static void *xdp_to_ptr(struct xdp_frame *ptr)
522 return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
525 static struct xdp_frame *ptr_to_xdp(void *ptr)
527 return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
530 static bool is_orphan_skb(void *ptr)
532 return (unsigned long)ptr & VIRTIO_ORPHAN_FLAG;
535 static void *skb_to_ptr(struct sk_buff *skb, bool orphan)
537 return (void *)((unsigned long)skb | (orphan ? VIRTIO_ORPHAN_FLAG : 0));
540 static struct sk_buff *ptr_to_skb(void *ptr)
542 return (struct sk_buff *)((unsigned long)ptr & ~VIRTIO_ORPHAN_FLAG);
545 static void __free_old_xmit(struct send_queue *sq, struct netdev_queue *txq,
546 bool in_napi, struct virtnet_sq_free_stats *stats)
551 while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
552 if (!is_xdp_frame(ptr)) {
553 struct sk_buff *skb = ptr_to_skb(ptr);
555 pr_debug("Sent skb %p\n", skb);
557 if (is_orphan_skb(ptr)) {
559 stats->bytes += skb->len;
561 stats->napi_packets++;
562 stats->napi_bytes += skb->len;
564 napi_consume_skb(skb, in_napi);
566 struct xdp_frame *frame = ptr_to_xdp(ptr);
569 stats->bytes += xdp_get_frame_len(frame);
570 xdp_return_frame(frame);
573 netdev_tx_completed_queue(txq, stats->napi_packets, stats->napi_bytes);
576 /* Converting between virtqueue no. and kernel tx/rx queue no.
577 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
579 static int vq2txq(struct virtqueue *vq)
581 return (vq->index - 1) / 2;
584 static int txq2vq(int txq)
589 static int vq2rxq(struct virtqueue *vq)
591 return vq->index / 2;
594 static int rxq2vq(int rxq)
599 static int vq_type(struct virtnet_info *vi, int qid)
601 if (qid == vi->max_queue_pairs * 2)
602 return VIRTNET_Q_TYPE_CQ;
605 return VIRTNET_Q_TYPE_TX;
607 return VIRTNET_Q_TYPE_RX;
610 static inline struct virtio_net_common_hdr *
611 skb_vnet_common_hdr(struct sk_buff *skb)
613 return (struct virtio_net_common_hdr *)skb->cb;
617 * private is used to chain pages for big packets, put the whole
618 * most recent used list in the beginning for reuse
620 static void give_pages(struct receive_queue *rq, struct page *page)
624 /* Find end of list, sew whole thing into vi->rq.pages. */
625 for (end = page; end->private; end = (struct page *)end->private);
626 end->private = (unsigned long)rq->pages;
630 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
632 struct page *p = rq->pages;
635 rq->pages = (struct page *)p->private;
636 /* clear private here, it is used to chain pages */
639 p = alloc_page(gfp_mask);
643 static void virtnet_rq_free_buf(struct virtnet_info *vi,
644 struct receive_queue *rq, void *buf)
646 if (vi->mergeable_rx_bufs)
647 put_page(virt_to_head_page(buf));
648 else if (vi->big_packets)
651 put_page(virt_to_head_page(buf));
654 static void enable_delayed_refill(struct virtnet_info *vi)
656 spin_lock_bh(&vi->refill_lock);
657 vi->refill_enabled = true;
658 spin_unlock_bh(&vi->refill_lock);
661 static void disable_delayed_refill(struct virtnet_info *vi)
663 spin_lock_bh(&vi->refill_lock);
664 vi->refill_enabled = false;
665 spin_unlock_bh(&vi->refill_lock);
668 static void enable_rx_mode_work(struct virtnet_info *vi)
671 vi->rx_mode_work_enabled = true;
675 static void disable_rx_mode_work(struct virtnet_info *vi)
678 vi->rx_mode_work_enabled = false;
682 static void virtqueue_napi_schedule(struct napi_struct *napi,
683 struct virtqueue *vq)
685 if (napi_schedule_prep(napi)) {
686 virtqueue_disable_cb(vq);
687 __napi_schedule(napi);
691 static bool virtqueue_napi_complete(struct napi_struct *napi,
692 struct virtqueue *vq, int processed)
696 opaque = virtqueue_enable_cb_prepare(vq);
697 if (napi_complete_done(napi, processed)) {
698 if (unlikely(virtqueue_poll(vq, opaque)))
699 virtqueue_napi_schedule(napi, vq);
703 virtqueue_disable_cb(vq);
709 static void skb_xmit_done(struct virtqueue *vq)
711 struct virtnet_info *vi = vq->vdev->priv;
712 struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
714 /* Suppress further interrupts. */
715 virtqueue_disable_cb(vq);
718 virtqueue_napi_schedule(napi, vq);
720 /* We were probably waiting for more output buffers. */
721 netif_wake_subqueue(vi->dev, vq2txq(vq));
724 #define MRG_CTX_HEADER_SHIFT 22
725 static void *mergeable_len_to_ctx(unsigned int truesize,
726 unsigned int headroom)
728 return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
731 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
733 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
736 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
738 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
741 static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen,
742 unsigned int headroom,
747 skb = build_skb(buf, buflen);
751 skb_reserve(skb, headroom);
757 /* Called from bottom half context */
758 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
759 struct receive_queue *rq,
760 struct page *page, unsigned int offset,
761 unsigned int len, unsigned int truesize,
762 unsigned int headroom)
765 struct virtio_net_common_hdr *hdr;
766 unsigned int copy, hdr_len, hdr_padded_len;
767 struct page *page_to_free = NULL;
768 int tailroom, shinfo_size;
769 char *p, *hdr_p, *buf;
771 p = page_address(page) + offset;
774 hdr_len = vi->hdr_len;
775 if (vi->mergeable_rx_bufs)
776 hdr_padded_len = hdr_len;
778 hdr_padded_len = sizeof(struct padded_vnet_hdr);
782 offset += hdr_padded_len;
784 tailroom = truesize - headroom - hdr_padded_len - len;
786 shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
788 if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
789 skb = virtnet_build_skb(buf, truesize, p - buf, len);
793 page = (struct page *)page->private;
795 give_pages(rq, page);
799 /* copy small packet so we can reuse these pages for small data */
800 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
804 /* Copy all frame if it fits skb->head, otherwise
805 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
807 if (len <= skb_tailroom(skb))
811 skb_put_data(skb, p, copy);
816 if (vi->mergeable_rx_bufs) {
818 skb_add_rx_frag(skb, 0, page, offset, len, truesize);
825 * Verify that we can indeed put this data into a skb.
826 * This is here to handle cases when the device erroneously
827 * tries to receive more than is possible. This is usually
828 * the case of a broken device.
830 if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
831 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
835 BUG_ON(offset >= PAGE_SIZE);
837 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
838 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
839 frag_size, truesize);
841 page = (struct page *)page->private;
846 give_pages(rq, page);
849 hdr = skb_vnet_common_hdr(skb);
850 memcpy(hdr, hdr_p, hdr_len);
852 put_page(page_to_free);
857 static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
859 struct page *page = virt_to_head_page(buf);
860 struct virtnet_rq_dma *dma;
864 head = page_address(page);
870 if (dma->need_sync && len) {
871 offset = buf - (head + sizeof(*dma));
873 virtqueue_dma_sync_single_range_for_cpu(rq->vq, dma->addr,
881 virtqueue_dma_unmap_single_attrs(rq->vq, dma->addr, dma->len,
882 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
886 static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
890 buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
891 if (buf && rq->do_dma)
892 virtnet_rq_unmap(rq, buf, *len);
897 static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
899 struct virtnet_rq_dma *dma;
905 sg_init_one(rq->sg, buf, len);
909 head = page_address(rq->alloc_frag.page);
915 addr = dma->addr - sizeof(*dma) + offset;
917 sg_init_table(rq->sg, 1);
918 rq->sg[0].dma_address = addr;
919 rq->sg[0].length = len;
922 static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp)
924 struct page_frag *alloc_frag = &rq->alloc_frag;
925 struct virtnet_rq_dma *dma;
929 if (unlikely(!skb_page_frag_refill(size, alloc_frag, gfp)))
932 head = page_address(alloc_frag->page);
938 if (!alloc_frag->offset) {
940 /* Now, the new page is allocated, the last dma
941 * will not be used. So the dma can be unmapped
944 virtnet_rq_unmap(rq, rq->last_dma, 0);
948 dma->len = alloc_frag->size - sizeof(*dma);
950 addr = virtqueue_dma_map_single_attrs(rq->vq, dma + 1,
951 dma->len, DMA_FROM_DEVICE, 0);
952 if (virtqueue_dma_mapping_error(rq->vq, addr))
956 dma->need_sync = virtqueue_dma_need_sync(rq->vq, addr);
958 /* Add a reference to dma to prevent the entire dma from
959 * being released during error handling. This reference
960 * will be freed after the pages are no longer used.
962 get_page(alloc_frag->page);
964 alloc_frag->offset = sizeof(*dma);
972 buf = head + alloc_frag->offset;
974 get_page(alloc_frag->page);
975 alloc_frag->offset += size;
980 static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf)
982 struct virtnet_info *vi = vq->vdev->priv;
983 struct receive_queue *rq;
989 xsk_buff_free((struct xdp_buff *)buf);
994 virtnet_rq_unmap(rq, buf, 0);
996 virtnet_rq_free_buf(vi, rq, buf);
999 static void free_old_xmit(struct send_queue *sq, struct netdev_queue *txq,
1002 struct virtnet_sq_free_stats stats = {0};
1004 __free_old_xmit(sq, txq, in_napi, &stats);
1006 /* Avoid overhead when no packets have been processed
1007 * happens when called speculatively from start_xmit.
1009 if (!stats.packets && !stats.napi_packets)
1012 u64_stats_update_begin(&sq->stats.syncp);
1013 u64_stats_add(&sq->stats.bytes, stats.bytes + stats.napi_bytes);
1014 u64_stats_add(&sq->stats.packets, stats.packets + stats.napi_packets);
1015 u64_stats_update_end(&sq->stats.syncp);
1018 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
1020 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
1022 else if (q < vi->curr_queue_pairs)
1028 static void check_sq_full_and_disable(struct virtnet_info *vi,
1029 struct net_device *dev,
1030 struct send_queue *sq)
1032 bool use_napi = sq->napi.weight;
1037 /* If running out of space, stop queue to avoid getting packets that we
1038 * are then unable to transmit.
1039 * An alternative would be to force queuing layer to requeue the skb by
1040 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1041 * returned in a normal path of operation: it means that driver is not
1042 * maintaining the TX queue stop/start state properly, and causes
1043 * the stack to do a non-trivial amount of useless work.
1044 * Since most packets only take 1 or 2 ring slots, stopping the queue
1045 * early means 16 slots are typically wasted.
1047 if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1048 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1050 netif_tx_stop_queue(txq);
1051 u64_stats_update_begin(&sq->stats.syncp);
1052 u64_stats_inc(&sq->stats.stop);
1053 u64_stats_update_end(&sq->stats.syncp);
1055 if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
1056 virtqueue_napi_schedule(&sq->napi, sq->vq);
1057 } else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1058 /* More just got used, free them then recheck. */
1059 free_old_xmit(sq, txq, false);
1060 if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1061 netif_start_subqueue(dev, qnum);
1062 u64_stats_update_begin(&sq->stats.syncp);
1063 u64_stats_inc(&sq->stats.wake);
1064 u64_stats_update_end(&sq->stats.syncp);
1065 virtqueue_disable_cb(sq->vq);
1071 static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
1073 sg->dma_address = addr;
1077 static struct xdp_buff *buf_to_xdp(struct virtnet_info *vi,
1078 struct receive_queue *rq, void *buf, u32 len)
1080 struct xdp_buff *xdp;
1083 xdp = (struct xdp_buff *)buf;
1085 bufsize = xsk_pool_get_rx_frame_size(rq->xsk_pool) + vi->hdr_len;
1087 if (unlikely(len > bufsize)) {
1088 pr_debug("%s: rx error: len %u exceeds truesize %u\n",
1089 vi->dev->name, len, bufsize);
1090 DEV_STATS_INC(vi->dev, rx_length_errors);
1095 xsk_buff_set_size(xdp, len);
1096 xsk_buff_dma_sync_for_cpu(xdp);
1101 static struct sk_buff *xsk_construct_skb(struct receive_queue *rq,
1102 struct xdp_buff *xdp)
1104 unsigned int metasize = xdp->data - xdp->data_meta;
1105 struct sk_buff *skb;
1108 size = xdp->data_end - xdp->data_hard_start;
1109 skb = napi_alloc_skb(&rq->napi, size);
1110 if (unlikely(!skb)) {
1115 skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);
1117 size = xdp->data_end - xdp->data_meta;
1118 memcpy(__skb_put(skb, size), xdp->data_meta, size);
1121 __skb_pull(skb, metasize);
1122 skb_metadata_set(skb, metasize);
1130 static struct sk_buff *virtnet_receive_xsk_small(struct net_device *dev, struct virtnet_info *vi,
1131 struct receive_queue *rq, struct xdp_buff *xdp,
1132 unsigned int *xdp_xmit,
1133 struct virtnet_rq_stats *stats)
1135 struct bpf_prog *prog;
1140 prog = rcu_dereference(rq->xdp_prog);
1142 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats);
1147 return xsk_construct_skb(rq, xdp);
1156 u64_stats_inc(&stats->drops);
1161 static void xsk_drop_follow_bufs(struct net_device *dev,
1162 struct receive_queue *rq,
1164 struct virtnet_rq_stats *stats)
1166 struct xdp_buff *xdp;
1169 while (num_buf-- > 1) {
1170 xdp = virtqueue_get_buf(rq->vq, &len);
1171 if (unlikely(!xdp)) {
1172 pr_debug("%s: rx error: %d buffers missing\n",
1173 dev->name, num_buf);
1174 DEV_STATS_INC(dev, rx_length_errors);
1177 u64_stats_add(&stats->bytes, len);
1182 static int xsk_append_merge_buffer(struct virtnet_info *vi,
1183 struct receive_queue *rq,
1184 struct sk_buff *head_skb,
1186 struct virtio_net_hdr_mrg_rxbuf *hdr,
1187 struct virtnet_rq_stats *stats)
1189 struct sk_buff *curr_skb;
1190 struct xdp_buff *xdp;
1195 curr_skb = head_skb;
1198 buf = virtqueue_get_buf(rq->vq, &len);
1199 if (unlikely(!buf)) {
1200 pr_debug("%s: rx error: %d buffers out of %d missing\n",
1201 vi->dev->name, num_buf,
1202 virtio16_to_cpu(vi->vdev,
1204 DEV_STATS_INC(vi->dev, rx_length_errors);
1208 u64_stats_add(&stats->bytes, len);
1210 xdp = buf_to_xdp(vi, rq, buf, len);
1214 buf = napi_alloc_frag(len);
1220 memcpy(buf, xdp->data - vi->hdr_len, len);
1224 page = virt_to_page(buf);
1228 curr_skb = virtnet_skb_append_frag(head_skb, curr_skb, page,
1229 buf, len, truesize);
1239 xsk_drop_follow_bufs(vi->dev, rq, num_buf, stats);
1243 static struct sk_buff *virtnet_receive_xsk_merge(struct net_device *dev, struct virtnet_info *vi,
1244 struct receive_queue *rq, struct xdp_buff *xdp,
1245 unsigned int *xdp_xmit,
1246 struct virtnet_rq_stats *stats)
1248 struct virtio_net_hdr_mrg_rxbuf *hdr;
1249 struct bpf_prog *prog;
1250 struct sk_buff *skb;
1253 hdr = xdp->data - vi->hdr_len;
1254 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
1258 prog = rcu_dereference(rq->xdp_prog);
1259 /* TODO: support multi buffer. */
1260 if (prog && num_buf == 1)
1261 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats);
1266 skb = xsk_construct_skb(rq, xdp);
1270 if (xsk_append_merge_buffer(vi, rq, skb, num_buf, hdr, stats)) {
1287 xsk_drop_follow_bufs(dev, rq, num_buf, stats);
1290 u64_stats_inc(&stats->drops);
1294 static void virtnet_receive_xsk_buf(struct virtnet_info *vi, struct receive_queue *rq,
1296 unsigned int *xdp_xmit,
1297 struct virtnet_rq_stats *stats)
1299 struct net_device *dev = vi->dev;
1300 struct sk_buff *skb = NULL;
1301 struct xdp_buff *xdp;
1306 u64_stats_add(&stats->bytes, len);
1308 xdp = buf_to_xdp(vi, rq, buf, len);
1312 if (unlikely(len < ETH_HLEN)) {
1313 pr_debug("%s: short packet %i\n", dev->name, len);
1314 DEV_STATS_INC(dev, rx_length_errors);
1319 flags = ((struct virtio_net_common_hdr *)(xdp->data - vi->hdr_len))->hdr.flags;
1321 if (!vi->mergeable_rx_bufs)
1322 skb = virtnet_receive_xsk_small(dev, vi, rq, xdp, xdp_xmit, stats);
1324 skb = virtnet_receive_xsk_merge(dev, vi, rq, xdp, xdp_xmit, stats);
1327 virtnet_receive_done(vi, rq, skb, flags);
1330 static int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct receive_queue *rq,
1331 struct xsk_buff_pool *pool, gfp_t gfp)
1333 struct xdp_buff **xsk_buffs;
1339 xsk_buffs = rq->xsk_buffs;
1341 num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free);
1345 len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
1347 for (i = 0; i < num; ++i) {
1348 /* Use the part of XDP_PACKET_HEADROOM as the virtnet hdr space.
1349 * We assume XDP_PACKET_HEADROOM is larger than hdr->len.
1350 * (see function virtnet_xsk_pool_enable)
1352 addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
1354 sg_init_table(rq->sg, 1);
1355 sg_fill_dma(rq->sg, addr, len);
1357 err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
1365 for (; i < num; ++i)
1366 xsk_buff_free(xsk_buffs[i]);
1371 static int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag)
1373 struct virtnet_info *vi = netdev_priv(dev);
1374 struct send_queue *sq;
1376 if (!netif_running(dev))
1379 if (qid >= vi->curr_queue_pairs)
1384 if (napi_if_scheduled_mark_missed(&sq->napi))
1388 virtqueue_napi_schedule(&sq->napi, sq->vq);
1394 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
1395 struct send_queue *sq,
1396 struct xdp_frame *xdpf)
1398 struct virtio_net_hdr_mrg_rxbuf *hdr;
1399 struct skb_shared_info *shinfo;
1403 if (unlikely(xdpf->headroom < vi->hdr_len))
1406 if (unlikely(xdp_frame_has_frags(xdpf))) {
1407 shinfo = xdp_get_shared_info_from_frame(xdpf);
1408 nr_frags = shinfo->nr_frags;
1411 /* In wrapping function virtnet_xdp_xmit(), we need to free
1412 * up the pending old buffers, where we need to calculate the
1413 * position of skb_shared_info in xdp_get_frame_len() and
1414 * xdp_return_frame(), which will involve to xdpf->data and
1415 * xdpf->headroom. Therefore, we need to update the value of
1416 * headroom synchronously here.
1418 xdpf->headroom -= vi->hdr_len;
1419 xdpf->data -= vi->hdr_len;
1420 /* Zero header and leave csum up to XDP layers */
1422 memset(hdr, 0, vi->hdr_len);
1423 xdpf->len += vi->hdr_len;
1425 sg_init_table(sq->sg, nr_frags + 1);
1426 sg_set_buf(sq->sg, xdpf->data, xdpf->len);
1427 for (i = 0; i < nr_frags; i++) {
1428 skb_frag_t *frag = &shinfo->frags[i];
1430 sg_set_page(&sq->sg[i + 1], skb_frag_page(frag),
1431 skb_frag_size(frag), skb_frag_off(frag));
1434 err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
1435 xdp_to_ptr(xdpf), GFP_ATOMIC);
1437 return -ENOSPC; /* Caller handle free/refcnt */
1442 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on
1443 * the current cpu, so it does not need to be locked.
1445 * Here we use marco instead of inline functions because we have to deal with
1446 * three issues at the same time: 1. the choice of sq. 2. judge and execute the
1447 * lock/unlock of txq 3. make sparse happy. It is difficult for two inline
1448 * functions to perfectly solve these three problems at the same time.
1450 #define virtnet_xdp_get_sq(vi) ({ \
1451 int cpu = smp_processor_id(); \
1452 struct netdev_queue *txq; \
1453 typeof(vi) v = (vi); \
1456 if (v->curr_queue_pairs > nr_cpu_ids) { \
1457 qp = v->curr_queue_pairs - v->xdp_queue_pairs; \
1459 txq = netdev_get_tx_queue(v->dev, qp); \
1460 __netif_tx_acquire(txq); \
1462 qp = cpu % v->curr_queue_pairs; \
1463 txq = netdev_get_tx_queue(v->dev, qp); \
1464 __netif_tx_lock(txq, cpu); \
1469 #define virtnet_xdp_put_sq(vi, q) { \
1470 struct netdev_queue *txq; \
1471 typeof(vi) v = (vi); \
1473 txq = netdev_get_tx_queue(v->dev, (q) - v->sq); \
1474 if (v->curr_queue_pairs > nr_cpu_ids) \
1475 __netif_tx_release(txq); \
1477 __netif_tx_unlock(txq); \
1480 static int virtnet_xdp_xmit(struct net_device *dev,
1481 int n, struct xdp_frame **frames, u32 flags)
1483 struct virtnet_info *vi = netdev_priv(dev);
1484 struct virtnet_sq_free_stats stats = {0};
1485 struct receive_queue *rq = vi->rq;
1486 struct bpf_prog *xdp_prog;
1487 struct send_queue *sq;
1493 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
1494 * indicate XDP resources have been successfully allocated.
1496 xdp_prog = rcu_access_pointer(rq->xdp_prog);
1500 sq = virtnet_xdp_get_sq(vi);
1502 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
1507 /* Free up any pending old buffers before queueing new ones. */
1508 __free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq),
1511 for (i = 0; i < n; i++) {
1512 struct xdp_frame *xdpf = frames[i];
1514 if (__virtnet_xdp_xmit_one(vi, sq, xdpf))
1520 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq))
1521 check_sq_full_and_disable(vi, dev, sq);
1523 if (flags & XDP_XMIT_FLUSH) {
1524 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
1528 u64_stats_update_begin(&sq->stats.syncp);
1529 u64_stats_add(&sq->stats.bytes, stats.bytes);
1530 u64_stats_add(&sq->stats.packets, stats.packets);
1531 u64_stats_add(&sq->stats.xdp_tx, n);
1532 u64_stats_add(&sq->stats.xdp_tx_drops, n - nxmit);
1533 u64_stats_add(&sq->stats.kicks, kicks);
1534 u64_stats_update_end(&sq->stats.syncp);
1536 virtnet_xdp_put_sq(vi, sq);
1540 static void put_xdp_frags(struct xdp_buff *xdp)
1542 struct skb_shared_info *shinfo;
1543 struct page *xdp_page;
1546 if (xdp_buff_has_frags(xdp)) {
1547 shinfo = xdp_get_shared_info_from_buff(xdp);
1548 for (i = 0; i < shinfo->nr_frags; i++) {
1549 xdp_page = skb_frag_page(&shinfo->frags[i]);
1555 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
1556 struct net_device *dev,
1557 unsigned int *xdp_xmit,
1558 struct virtnet_rq_stats *stats)
1560 struct xdp_frame *xdpf;
1564 act = bpf_prog_run_xdp(xdp_prog, xdp);
1565 u64_stats_inc(&stats->xdp_packets);
1572 u64_stats_inc(&stats->xdp_tx);
1573 xdpf = xdp_convert_buff_to_frame(xdp);
1574 if (unlikely(!xdpf)) {
1575 netdev_dbg(dev, "convert buff to frame failed for xdp\n");
1579 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
1580 if (unlikely(!err)) {
1581 xdp_return_frame_rx_napi(xdpf);
1582 } else if (unlikely(err < 0)) {
1583 trace_xdp_exception(dev, xdp_prog, act);
1586 *xdp_xmit |= VIRTIO_XDP_TX;
1590 u64_stats_inc(&stats->xdp_redirects);
1591 err = xdp_do_redirect(dev, xdp, xdp_prog);
1595 *xdp_xmit |= VIRTIO_XDP_REDIR;
1599 bpf_warn_invalid_xdp_action(dev, xdp_prog, act);
1602 trace_xdp_exception(dev, xdp_prog, act);
1609 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
1611 return vi->xdp_enabled ? XDP_PACKET_HEADROOM : 0;
1614 /* We copy the packet for XDP in the following cases:
1616 * 1) Packet is scattered across multiple rx buffers.
1617 * 2) Headroom space is insufficient.
1619 * This is inefficient but it's a temporary condition that
1620 * we hit right after XDP is enabled and until queue is refilled
1621 * with large buffers with sufficient headroom - so it should affect
1622 * at most queue size packets.
1623 * Afterwards, the conditions to enable
1624 * XDP should preclude the underlying device from sending packets
1625 * across multiple buffers (num_buf > 1), and we make sure buffers
1626 * have enough headroom.
1628 static struct page *xdp_linearize_page(struct receive_queue *rq,
1635 int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1638 if (page_off + *len + tailroom > PAGE_SIZE)
1641 page = alloc_page(GFP_ATOMIC);
1645 memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
1648 while (--*num_buf) {
1649 unsigned int buflen;
1653 buf = virtnet_rq_get_buf(rq, &buflen, NULL);
1657 p = virt_to_head_page(buf);
1658 off = buf - page_address(p);
1660 /* guard against a misconfigured or uncooperative backend that
1661 * is sending packet larger than the MTU.
1663 if ((page_off + buflen + tailroom) > PAGE_SIZE) {
1668 memcpy(page_address(page) + page_off,
1669 page_address(p) + off, buflen);
1674 /* Headroom does not contribute to packet length */
1675 *len = page_off - XDP_PACKET_HEADROOM;
1678 __free_pages(page, 0);
1682 static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi,
1683 unsigned int xdp_headroom,
1687 unsigned int header_offset;
1688 unsigned int headroom;
1689 unsigned int buflen;
1690 struct sk_buff *skb;
1692 header_offset = VIRTNET_RX_PAD + xdp_headroom;
1693 headroom = vi->hdr_len + header_offset;
1694 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1695 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1697 skb = virtnet_build_skb(buf, buflen, headroom, len);
1701 buf += header_offset;
1702 memcpy(skb_vnet_common_hdr(skb), buf, vi->hdr_len);
1707 static struct sk_buff *receive_small_xdp(struct net_device *dev,
1708 struct virtnet_info *vi,
1709 struct receive_queue *rq,
1710 struct bpf_prog *xdp_prog,
1712 unsigned int xdp_headroom,
1714 unsigned int *xdp_xmit,
1715 struct virtnet_rq_stats *stats)
1717 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
1718 unsigned int headroom = vi->hdr_len + header_offset;
1719 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
1720 struct page *page = virt_to_head_page(buf);
1721 struct page *xdp_page;
1722 unsigned int buflen;
1723 struct xdp_buff xdp;
1724 struct sk_buff *skb;
1725 unsigned int metasize = 0;
1728 if (unlikely(hdr->hdr.gso_type))
1731 /* Partially checksummed packets must be dropped. */
1732 if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM))
1735 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1736 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1738 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
1739 int offset = buf - page_address(page) + header_offset;
1740 unsigned int tlen = len + vi->hdr_len;
1743 xdp_headroom = virtnet_get_headroom(vi);
1744 header_offset = VIRTNET_RX_PAD + xdp_headroom;
1745 headroom = vi->hdr_len + header_offset;
1746 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1747 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1748 xdp_page = xdp_linearize_page(rq, &num_buf, page,
1749 offset, header_offset,
1754 buf = page_address(xdp_page);
1759 xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
1760 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
1761 xdp_headroom, len, true);
1763 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
1767 /* Recalculate length in case bpf program changed it */
1768 len = xdp.data_end - xdp.data;
1769 metasize = xdp.data - xdp.data_meta;
1780 skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len);
1785 skb_metadata_set(skb, metasize);
1790 u64_stats_inc(&stats->xdp_drops);
1792 u64_stats_inc(&stats->drops);
1798 static struct sk_buff *receive_small(struct net_device *dev,
1799 struct virtnet_info *vi,
1800 struct receive_queue *rq,
1801 void *buf, void *ctx,
1803 unsigned int *xdp_xmit,
1804 struct virtnet_rq_stats *stats)
1806 unsigned int xdp_headroom = (unsigned long)ctx;
1807 struct page *page = virt_to_head_page(buf);
1808 struct sk_buff *skb;
1810 /* We passed the address of virtnet header to virtio-core,
1811 * so truncate the padding.
1813 buf -= VIRTNET_RX_PAD + xdp_headroom;
1816 u64_stats_add(&stats->bytes, len);
1818 if (unlikely(len > GOOD_PACKET_LEN)) {
1819 pr_debug("%s: rx error: len %u exceeds max size %d\n",
1820 dev->name, len, GOOD_PACKET_LEN);
1821 DEV_STATS_INC(dev, rx_length_errors);
1825 if (unlikely(vi->xdp_enabled)) {
1826 struct bpf_prog *xdp_prog;
1829 xdp_prog = rcu_dereference(rq->xdp_prog);
1831 skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf,
1832 xdp_headroom, len, xdp_xmit,
1840 skb = receive_small_build_skb(vi, xdp_headroom, buf, len);
1845 u64_stats_inc(&stats->drops);
1850 static struct sk_buff *receive_big(struct net_device *dev,
1851 struct virtnet_info *vi,
1852 struct receive_queue *rq,
1855 struct virtnet_rq_stats *stats)
1857 struct page *page = buf;
1858 struct sk_buff *skb =
1859 page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0);
1861 u64_stats_add(&stats->bytes, len - vi->hdr_len);
1868 u64_stats_inc(&stats->drops);
1869 give_pages(rq, page);
1873 static void mergeable_buf_free(struct receive_queue *rq, int num_buf,
1874 struct net_device *dev,
1875 struct virtnet_rq_stats *stats)
1881 while (num_buf-- > 1) {
1882 buf = virtnet_rq_get_buf(rq, &len, NULL);
1883 if (unlikely(!buf)) {
1884 pr_debug("%s: rx error: %d buffers missing\n",
1885 dev->name, num_buf);
1886 DEV_STATS_INC(dev, rx_length_errors);
1889 u64_stats_add(&stats->bytes, len);
1890 page = virt_to_head_page(buf);
1895 /* Why not use xdp_build_skb_from_frame() ?
1896 * XDP core assumes that xdp frags are PAGE_SIZE in length, while in
1897 * virtio-net there are 2 points that do not match its requirements:
1898 * 1. The size of the prefilled buffer is not fixed before xdp is set.
1899 * 2. xdp_build_skb_from_frame() does more checks that we don't need,
1900 * like eth_type_trans() (which virtio-net does in receive_buf()).
1902 static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev,
1903 struct virtnet_info *vi,
1904 struct xdp_buff *xdp,
1905 unsigned int xdp_frags_truesz)
1907 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
1908 unsigned int headroom, data_len;
1909 struct sk_buff *skb;
1913 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {
1914 pr_debug("Error building skb as missing reserved tailroom for xdp");
1918 if (unlikely(xdp_buff_has_frags(xdp)))
1919 nr_frags = sinfo->nr_frags;
1921 skb = build_skb(xdp->data_hard_start, xdp->frame_sz);
1925 headroom = xdp->data - xdp->data_hard_start;
1926 data_len = xdp->data_end - xdp->data;
1927 skb_reserve(skb, headroom);
1928 __skb_put(skb, data_len);
1930 metasize = xdp->data - xdp->data_meta;
1931 metasize = metasize > 0 ? metasize : 0;
1933 skb_metadata_set(skb, metasize);
1935 if (unlikely(xdp_buff_has_frags(xdp)))
1936 xdp_update_skb_shared_info(skb, nr_frags,
1937 sinfo->xdp_frags_size,
1939 xdp_buff_is_frag_pfmemalloc(xdp));
1944 /* TODO: build xdp in big mode */
1945 static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
1946 struct virtnet_info *vi,
1947 struct receive_queue *rq,
1948 struct xdp_buff *xdp,
1951 unsigned int frame_sz,
1953 unsigned int *xdp_frags_truesize,
1954 struct virtnet_rq_stats *stats)
1956 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
1957 unsigned int headroom, tailroom, room;
1958 unsigned int truesize, cur_frag_size;
1959 struct skb_shared_info *shinfo;
1960 unsigned int xdp_frags_truesz = 0;
1966 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq);
1967 xdp_prepare_buff(xdp, buf - XDP_PACKET_HEADROOM,
1968 XDP_PACKET_HEADROOM + vi->hdr_len, len - vi->hdr_len, true);
1974 /* If we want to build multi-buffer xdp, we need
1975 * to specify that the flags of xdp_buff have the
1976 * XDP_FLAGS_HAS_FRAG bit.
1978 if (!xdp_buff_has_frags(xdp))
1979 xdp_buff_set_frags_flag(xdp);
1981 shinfo = xdp_get_shared_info_from_buff(xdp);
1982 shinfo->nr_frags = 0;
1983 shinfo->xdp_frags_size = 0;
1986 if (*num_buf > MAX_SKB_FRAGS + 1)
1989 while (--*num_buf > 0) {
1990 buf = virtnet_rq_get_buf(rq, &len, &ctx);
1991 if (unlikely(!buf)) {
1992 pr_debug("%s: rx error: %d buffers out of %d missing\n",
1993 dev->name, *num_buf,
1994 virtio16_to_cpu(vi->vdev, hdr->num_buffers));
1995 DEV_STATS_INC(dev, rx_length_errors);
1999 u64_stats_add(&stats->bytes, len);
2000 page = virt_to_head_page(buf);
2001 offset = buf - page_address(page);
2003 truesize = mergeable_ctx_to_truesize(ctx);
2004 headroom = mergeable_ctx_to_headroom(ctx);
2005 tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2006 room = SKB_DATA_ALIGN(headroom + tailroom);
2008 cur_frag_size = truesize;
2009 xdp_frags_truesz += cur_frag_size;
2010 if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) {
2012 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
2013 dev->name, len, (unsigned long)(truesize - room));
2014 DEV_STATS_INC(dev, rx_length_errors);
2018 frag = &shinfo->frags[shinfo->nr_frags++];
2019 skb_frag_fill_page_desc(frag, page, offset, len);
2020 if (page_is_pfmemalloc(page))
2021 xdp_buff_set_frag_pfmemalloc(xdp);
2023 shinfo->xdp_frags_size += len;
2026 *xdp_frags_truesize = xdp_frags_truesz;
2034 static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
2035 struct receive_queue *rq,
2036 struct bpf_prog *xdp_prog,
2038 unsigned int *frame_sz,
2043 struct virtio_net_hdr_mrg_rxbuf *hdr)
2045 unsigned int truesize = mergeable_ctx_to_truesize(ctx);
2046 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
2047 struct page *xdp_page;
2048 unsigned int xdp_room;
2050 /* Transient failure which in theory could occur if
2051 * in-flight packets from before XDP was enabled reach
2052 * the receive path after XDP is loaded.
2054 if (unlikely(hdr->hdr.gso_type))
2057 /* Partially checksummed packets must be dropped. */
2058 if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM))
2061 /* Now XDP core assumes frag size is PAGE_SIZE, but buffers
2062 * with headroom may add hole in truesize, which
2063 * make their length exceed PAGE_SIZE. So we disabled the
2064 * hole mechanism for xdp. See add_recvbuf_mergeable().
2066 *frame_sz = truesize;
2068 if (likely(headroom >= virtnet_get_headroom(vi) &&
2069 (*num_buf == 1 || xdp_prog->aux->xdp_has_frags))) {
2070 return page_address(*page) + offset;
2073 /* This happens when headroom is not enough because
2074 * of the buffer was prefilled before XDP is set.
2075 * This should only happen for the first several packets.
2076 * In fact, vq reset can be used here to help us clean up
2077 * the prefilled buffers, but many existing devices do not
2078 * support it, and we don't want to bother users who are
2079 * using xdp normally.
2081 if (!xdp_prog->aux->xdp_has_frags) {
2082 /* linearize data for XDP */
2083 xdp_page = xdp_linearize_page(rq, num_buf,
2085 XDP_PACKET_HEADROOM,
2090 xdp_room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM +
2091 sizeof(struct skb_shared_info));
2092 if (*len + xdp_room > PAGE_SIZE)
2095 xdp_page = alloc_page(GFP_ATOMIC);
2099 memcpy(page_address(xdp_page) + XDP_PACKET_HEADROOM,
2100 page_address(*page) + offset, *len);
2103 *frame_sz = PAGE_SIZE;
2109 return page_address(*page) + XDP_PACKET_HEADROOM;
2112 static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
2113 struct virtnet_info *vi,
2114 struct receive_queue *rq,
2115 struct bpf_prog *xdp_prog,
2119 unsigned int *xdp_xmit,
2120 struct virtnet_rq_stats *stats)
2122 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
2123 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
2124 struct page *page = virt_to_head_page(buf);
2125 int offset = buf - page_address(page);
2126 unsigned int xdp_frags_truesz = 0;
2127 struct sk_buff *head_skb;
2128 unsigned int frame_sz;
2129 struct xdp_buff xdp;
2134 data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz, &num_buf, &page,
2136 if (unlikely(!data))
2139 err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,
2140 &num_buf, &xdp_frags_truesz, stats);
2144 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
2148 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
2149 if (unlikely(!head_skb))
2161 put_xdp_frags(&xdp);
2165 mergeable_buf_free(rq, num_buf, dev, stats);
2167 u64_stats_inc(&stats->xdp_drops);
2168 u64_stats_inc(&stats->drops);
2172 static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
2173 struct sk_buff *curr_skb,
2174 struct page *page, void *buf,
2175 int len, int truesize)
2180 num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
2181 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
2182 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
2184 if (unlikely(!nskb))
2187 if (curr_skb == head_skb)
2188 skb_shinfo(curr_skb)->frag_list = nskb;
2190 curr_skb->next = nskb;
2192 head_skb->truesize += nskb->truesize;
2196 if (curr_skb != head_skb) {
2197 head_skb->data_len += len;
2198 head_skb->len += len;
2199 head_skb->truesize += truesize;
2202 offset = buf - page_address(page);
2203 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
2205 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
2208 skb_add_rx_frag(curr_skb, num_skb_frags, page,
2209 offset, len, truesize);
2215 static struct sk_buff *receive_mergeable(struct net_device *dev,
2216 struct virtnet_info *vi,
2217 struct receive_queue *rq,
2221 unsigned int *xdp_xmit,
2222 struct virtnet_rq_stats *stats)
2224 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
2225 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
2226 struct page *page = virt_to_head_page(buf);
2227 int offset = buf - page_address(page);
2228 struct sk_buff *head_skb, *curr_skb;
2229 unsigned int truesize = mergeable_ctx_to_truesize(ctx);
2230 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
2231 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2232 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
2235 u64_stats_add(&stats->bytes, len - vi->hdr_len);
2237 if (unlikely(len > truesize - room)) {
2238 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
2239 dev->name, len, (unsigned long)(truesize - room));
2240 DEV_STATS_INC(dev, rx_length_errors);
2244 if (unlikely(vi->xdp_enabled)) {
2245 struct bpf_prog *xdp_prog;
2248 xdp_prog = rcu_dereference(rq->xdp_prog);
2250 head_skb = receive_mergeable_xdp(dev, vi, rq, xdp_prog, buf, ctx,
2251 len, xdp_xmit, stats);
2258 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom);
2259 curr_skb = head_skb;
2261 if (unlikely(!curr_skb))
2264 buf = virtnet_rq_get_buf(rq, &len, &ctx);
2265 if (unlikely(!buf)) {
2266 pr_debug("%s: rx error: %d buffers out of %d missing\n",
2268 virtio16_to_cpu(vi->vdev,
2270 DEV_STATS_INC(dev, rx_length_errors);
2274 u64_stats_add(&stats->bytes, len);
2275 page = virt_to_head_page(buf);
2277 truesize = mergeable_ctx_to_truesize(ctx);
2278 headroom = mergeable_ctx_to_headroom(ctx);
2279 tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2280 room = SKB_DATA_ALIGN(headroom + tailroom);
2281 if (unlikely(len > truesize - room)) {
2282 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
2283 dev->name, len, (unsigned long)(truesize - room));
2284 DEV_STATS_INC(dev, rx_length_errors);
2288 curr_skb = virtnet_skb_append_frag(head_skb, curr_skb, page,
2289 buf, len, truesize);
2294 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
2299 mergeable_buf_free(rq, num_buf, dev, stats);
2302 u64_stats_inc(&stats->drops);
2303 dev_kfree_skb(head_skb);
2307 static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
2308 struct sk_buff *skb)
2310 enum pkt_hash_types rss_hash_type;
2312 if (!hdr_hash || !skb)
2315 switch (__le16_to_cpu(hdr_hash->hash_report)) {
2316 case VIRTIO_NET_HASH_REPORT_TCPv4:
2317 case VIRTIO_NET_HASH_REPORT_UDPv4:
2318 case VIRTIO_NET_HASH_REPORT_TCPv6:
2319 case VIRTIO_NET_HASH_REPORT_UDPv6:
2320 case VIRTIO_NET_HASH_REPORT_TCPv6_EX:
2321 case VIRTIO_NET_HASH_REPORT_UDPv6_EX:
2322 rss_hash_type = PKT_HASH_TYPE_L4;
2324 case VIRTIO_NET_HASH_REPORT_IPv4:
2325 case VIRTIO_NET_HASH_REPORT_IPv6:
2326 case VIRTIO_NET_HASH_REPORT_IPv6_EX:
2327 rss_hash_type = PKT_HASH_TYPE_L3;
2329 case VIRTIO_NET_HASH_REPORT_NONE:
2331 rss_hash_type = PKT_HASH_TYPE_NONE;
2333 skb_set_hash(skb, __le32_to_cpu(hdr_hash->hash_value), rss_hash_type);
2336 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq,
2337 struct sk_buff *skb, u8 flags)
2339 struct virtio_net_common_hdr *hdr;
2340 struct net_device *dev = vi->dev;
2342 hdr = skb_vnet_common_hdr(skb);
2343 if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report)
2344 virtio_skb_set_hash(&hdr->hash_v1_hdr, skb);
2346 if (flags & VIRTIO_NET_HDR_F_DATA_VALID)
2347 skb->ip_summed = CHECKSUM_UNNECESSARY;
2349 if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
2350 virtio_is_little_endian(vi->vdev))) {
2351 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
2352 dev->name, hdr->hdr.gso_type,
2357 skb_record_rx_queue(skb, vq2rxq(rq->vq));
2358 skb->protocol = eth_type_trans(skb, dev);
2359 pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
2360 ntohs(skb->protocol), skb->len, skb->pkt_type);
2362 napi_gro_receive(&rq->napi, skb);
2366 DEV_STATS_INC(dev, rx_frame_errors);
2370 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
2371 void *buf, unsigned int len, void **ctx,
2372 unsigned int *xdp_xmit,
2373 struct virtnet_rq_stats *stats)
2375 struct net_device *dev = vi->dev;
2376 struct sk_buff *skb;
2379 if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
2380 pr_debug("%s: short packet %i\n", dev->name, len);
2381 DEV_STATS_INC(dev, rx_length_errors);
2382 virtnet_rq_free_buf(vi, rq, buf);
2386 /* 1. Save the flags early, as the XDP program might overwrite them.
2387 * These flags ensure packets marked as VIRTIO_NET_HDR_F_DATA_VALID
2388 * stay valid after XDP processing.
2389 * 2. XDP doesn't work with partially checksummed packets (refer to
2390 * virtnet_xdp_set()), so packets marked as
2391 * VIRTIO_NET_HDR_F_NEEDS_CSUM get dropped during XDP processing.
2393 flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags;
2395 if (vi->mergeable_rx_bufs)
2396 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
2398 else if (vi->big_packets)
2399 skb = receive_big(dev, vi, rq, buf, len, stats);
2401 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);
2406 virtnet_receive_done(vi, rq, skb, flags);
2409 /* Unlike mergeable buffers, all buffers are allocated to the
2410 * same size, except for the headroom. For this reason we do
2411 * not need to use mergeable_len_to_ctx here - it is enough
2412 * to store the headroom as the context ignoring the truesize.
2414 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
2418 unsigned int xdp_headroom = virtnet_get_headroom(vi);
2419 void *ctx = (void *)(unsigned long)xdp_headroom;
2420 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
2423 len = SKB_DATA_ALIGN(len) +
2424 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2426 buf = virtnet_rq_alloc(rq, len, gfp);
2430 buf += VIRTNET_RX_PAD + xdp_headroom;
2432 virtnet_rq_init_one_sg(rq, buf, vi->hdr_len + GOOD_PACKET_LEN);
2434 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
2437 virtnet_rq_unmap(rq, buf, 0);
2438 put_page(virt_to_head_page(buf));
2444 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
2447 struct page *first, *list = NULL;
2451 sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2);
2453 /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */
2454 for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) {
2455 first = get_a_page(rq, gfp);
2458 give_pages(rq, list);
2461 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
2463 /* chain new page in list head to match sg */
2464 first->private = (unsigned long)list;
2468 first = get_a_page(rq, gfp);
2470 give_pages(rq, list);
2473 p = page_address(first);
2475 /* rq->sg[0], rq->sg[1] share the same page */
2476 /* a separated rq->sg[0] for header - required in case !any_header_sg */
2477 sg_set_buf(&rq->sg[0], p, vi->hdr_len);
2479 /* rq->sg[1] for data packet, from offset */
2480 offset = sizeof(struct padded_vnet_hdr);
2481 sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
2483 /* chain first in list head */
2484 first->private = (unsigned long)list;
2485 err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2,
2488 give_pages(rq, first);
2493 static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
2494 struct ewma_pkt_len *avg_pkt_len,
2497 struct virtnet_info *vi = rq->vq->vdev->priv;
2498 const size_t hdr_len = vi->hdr_len;
2502 return PAGE_SIZE - room;
2504 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
2505 rq->min_buf_len, PAGE_SIZE - hdr_len);
2507 return ALIGN(len, L1_CACHE_BYTES);
2510 static int add_recvbuf_mergeable(struct virtnet_info *vi,
2511 struct receive_queue *rq, gfp_t gfp)
2513 struct page_frag *alloc_frag = &rq->alloc_frag;
2514 unsigned int headroom = virtnet_get_headroom(vi);
2515 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2516 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
2517 unsigned int len, hole;
2522 /* Extra tailroom is needed to satisfy XDP's assumption. This
2523 * means rx frags coalescing won't work, but consider we've
2524 * disabled GSO for XDP, it won't be a big issue.
2526 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
2528 buf = virtnet_rq_alloc(rq, len + room, gfp);
2532 buf += headroom; /* advance address leaving hole at front of pkt */
2533 hole = alloc_frag->size - alloc_frag->offset;
2534 if (hole < len + room) {
2535 /* To avoid internal fragmentation, if there is very likely not
2536 * enough space for another buffer, add the remaining space to
2537 * the current buffer.
2538 * XDP core assumes that frame_size of xdp_buff and the length
2539 * of the frag are PAGE_SIZE, so we disable the hole mechanism.
2543 alloc_frag->offset += hole;
2546 virtnet_rq_init_one_sg(rq, buf, len);
2548 ctx = mergeable_len_to_ctx(len + room, headroom);
2549 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
2552 virtnet_rq_unmap(rq, buf, 0);
2553 put_page(virt_to_head_page(buf));
2560 * Returns false if we couldn't fill entirely (OOM).
2562 * Normally run in the receive path, but can also be run from ndo_open
2563 * before we're receiving packets, or from refill_work which is
2564 * careful to disable receiving (using napi_disable).
2566 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
2572 err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk_pool, gfp);
2577 if (vi->mergeable_rx_bufs)
2578 err = add_recvbuf_mergeable(vi, rq, gfp);
2579 else if (vi->big_packets)
2580 err = add_recvbuf_big(vi, rq, gfp);
2582 err = add_recvbuf_small(vi, rq, gfp);
2586 } while (rq->vq->num_free);
2589 if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
2590 unsigned long flags;
2592 flags = u64_stats_update_begin_irqsave(&rq->stats.syncp);
2593 u64_stats_inc(&rq->stats.kicks);
2594 u64_stats_update_end_irqrestore(&rq->stats.syncp, flags);
2597 return err != -ENOMEM;
2600 static void skb_recv_done(struct virtqueue *rvq)
2602 struct virtnet_info *vi = rvq->vdev->priv;
2603 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
2606 virtqueue_napi_schedule(&rq->napi, rvq);
2609 static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
2613 /* If all buffers were filled by other side before we napi_enabled, we
2614 * won't get another interrupt, so process any outstanding packets now.
2615 * Call local_bh_enable after to trigger softIRQ processing.
2618 virtqueue_napi_schedule(napi, vq);
2622 static void virtnet_napi_tx_enable(struct virtnet_info *vi,
2623 struct virtqueue *vq,
2624 struct napi_struct *napi)
2629 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
2630 * enable the feature if this is likely affine with the transmit path.
2632 if (!vi->affinity_hint_set) {
2637 return virtnet_napi_enable(vq, napi);
2640 static void virtnet_napi_tx_disable(struct napi_struct *napi)
2646 static void refill_work(struct work_struct *work)
2648 struct virtnet_info *vi =
2649 container_of(work, struct virtnet_info, refill.work);
2653 for (i = 0; i < vi->curr_queue_pairs; i++) {
2654 struct receive_queue *rq = &vi->rq[i];
2656 napi_disable(&rq->napi);
2657 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
2658 virtnet_napi_enable(rq->vq, &rq->napi);
2660 /* In theory, this can happen: if we don't get any buffers in
2661 * we will *never* try to fill again.
2664 schedule_delayed_work(&vi->refill, HZ/2);
2668 static int virtnet_receive_xsk_bufs(struct virtnet_info *vi,
2669 struct receive_queue *rq,
2671 unsigned int *xdp_xmit,
2672 struct virtnet_rq_stats *stats)
2678 while (packets < budget) {
2679 buf = virtqueue_get_buf(rq->vq, &len);
2683 virtnet_receive_xsk_buf(vi, rq, buf, len, xdp_xmit, stats);
2690 static int virtnet_receive_packets(struct virtnet_info *vi,
2691 struct receive_queue *rq,
2693 unsigned int *xdp_xmit,
2694 struct virtnet_rq_stats *stats)
2700 if (!vi->big_packets || vi->mergeable_rx_bufs) {
2702 while (packets < budget &&
2703 (buf = virtnet_rq_get_buf(rq, &len, &ctx))) {
2704 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, stats);
2708 while (packets < budget &&
2709 (buf = virtnet_rq_get_buf(rq, &len, NULL)) != NULL) {
2710 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, stats);
2718 static int virtnet_receive(struct receive_queue *rq, int budget,
2719 unsigned int *xdp_xmit)
2721 struct virtnet_info *vi = rq->vq->vdev->priv;
2722 struct virtnet_rq_stats stats = {};
2726 packets = virtnet_receive_xsk_bufs(vi, rq, budget, xdp_xmit, &stats);
2728 packets = virtnet_receive_packets(vi, rq, budget, xdp_xmit, &stats);
2730 if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
2731 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) {
2732 spin_lock(&vi->refill_lock);
2733 if (vi->refill_enabled)
2734 schedule_delayed_work(&vi->refill, 0);
2735 spin_unlock(&vi->refill_lock);
2739 u64_stats_set(&stats.packets, packets);
2740 u64_stats_update_begin(&rq->stats.syncp);
2741 for (i = 0; i < ARRAY_SIZE(virtnet_rq_stats_desc); i++) {
2742 size_t offset = virtnet_rq_stats_desc[i].offset;
2743 u64_stats_t *item, *src;
2745 item = (u64_stats_t *)((u8 *)&rq->stats + offset);
2746 src = (u64_stats_t *)((u8 *)&stats + offset);
2747 u64_stats_add(item, u64_stats_read(src));
2750 u64_stats_add(&rq->stats.packets, u64_stats_read(&stats.packets));
2751 u64_stats_add(&rq->stats.bytes, u64_stats_read(&stats.bytes));
2753 u64_stats_update_end(&rq->stats.syncp);
2758 static void virtnet_poll_cleantx(struct receive_queue *rq, int budget)
2760 struct virtnet_info *vi = rq->vq->vdev->priv;
2761 unsigned int index = vq2rxq(rq->vq);
2762 struct send_queue *sq = &vi->sq[index];
2763 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
2765 if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
2768 if (__netif_tx_trylock(txq)) {
2770 __netif_tx_unlock(txq);
2775 virtqueue_disable_cb(sq->vq);
2776 free_old_xmit(sq, txq, !!budget);
2777 } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
2779 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
2780 if (netif_tx_queue_stopped(txq)) {
2781 u64_stats_update_begin(&sq->stats.syncp);
2782 u64_stats_inc(&sq->stats.wake);
2783 u64_stats_update_end(&sq->stats.syncp);
2785 netif_tx_wake_queue(txq);
2788 __netif_tx_unlock(txq);
2792 static void virtnet_rx_dim_update(struct virtnet_info *vi, struct receive_queue *rq)
2794 struct dim_sample cur_sample = {};
2796 if (!rq->packets_in_napi)
2799 /* Don't need protection when fetching stats, since fetcher and
2800 * updater of the stats are in same context
2802 dim_update_sample(rq->calls,
2803 u64_stats_read(&rq->stats.packets),
2804 u64_stats_read(&rq->stats.bytes),
2807 net_dim(&rq->dim, cur_sample);
2808 rq->packets_in_napi = 0;
2811 static int virtnet_poll(struct napi_struct *napi, int budget)
2813 struct receive_queue *rq =
2814 container_of(napi, struct receive_queue, napi);
2815 struct virtnet_info *vi = rq->vq->vdev->priv;
2816 struct send_queue *sq;
2817 unsigned int received;
2818 unsigned int xdp_xmit = 0;
2821 virtnet_poll_cleantx(rq, budget);
2823 received = virtnet_receive(rq, budget, &xdp_xmit);
2824 rq->packets_in_napi += received;
2826 if (xdp_xmit & VIRTIO_XDP_REDIR)
2829 /* Out of packets? */
2830 if (received < budget) {
2831 napi_complete = virtqueue_napi_complete(napi, rq->vq, received);
2832 /* Intentionally not taking dim_lock here. This may result in a
2833 * spurious net_dim call. But if that happens virtnet_rx_dim_work
2834 * will not act on the scheduled work.
2836 if (napi_complete && rq->dim_enabled)
2837 virtnet_rx_dim_update(vi, rq);
2840 if (xdp_xmit & VIRTIO_XDP_TX) {
2841 sq = virtnet_xdp_get_sq(vi);
2842 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
2843 u64_stats_update_begin(&sq->stats.syncp);
2844 u64_stats_inc(&sq->stats.kicks);
2845 u64_stats_update_end(&sq->stats.syncp);
2847 virtnet_xdp_put_sq(vi, sq);
2853 static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index)
2855 virtnet_napi_tx_disable(&vi->sq[qp_index].napi);
2856 napi_disable(&vi->rq[qp_index].napi);
2857 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq);
2860 static int virtnet_enable_queue_pair(struct virtnet_info *vi, int qp_index)
2862 struct net_device *dev = vi->dev;
2865 err = xdp_rxq_info_reg(&vi->rq[qp_index].xdp_rxq, dev, qp_index,
2866 vi->rq[qp_index].napi.napi_id);
2870 err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq,
2871 MEM_TYPE_PAGE_SHARED, NULL);
2873 goto err_xdp_reg_mem_model;
2875 netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, qp_index));
2876 virtnet_napi_enable(vi->rq[qp_index].vq, &vi->rq[qp_index].napi);
2877 virtnet_napi_tx_enable(vi, vi->sq[qp_index].vq, &vi->sq[qp_index].napi);
2881 err_xdp_reg_mem_model:
2882 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq);
2886 static void virtnet_cancel_dim(struct virtnet_info *vi, struct dim *dim)
2888 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
2890 net_dim_work_cancel(dim);
2893 static void virtnet_update_settings(struct virtnet_info *vi)
2898 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
2901 virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed);
2903 if (ethtool_validate_speed(speed))
2906 virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex);
2908 if (ethtool_validate_duplex(duplex))
2909 vi->duplex = duplex;
2912 static int virtnet_open(struct net_device *dev)
2914 struct virtnet_info *vi = netdev_priv(dev);
2917 enable_delayed_refill(vi);
2919 for (i = 0; i < vi->max_queue_pairs; i++) {
2920 if (i < vi->curr_queue_pairs)
2921 /* Make sure we have some buffers: if oom use wq. */
2922 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
2923 schedule_delayed_work(&vi->refill, 0);
2925 err = virtnet_enable_queue_pair(vi, i);
2930 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
2931 if (vi->status & VIRTIO_NET_S_LINK_UP)
2932 netif_carrier_on(vi->dev);
2933 virtio_config_driver_enable(vi->vdev);
2935 vi->status = VIRTIO_NET_S_LINK_UP;
2936 netif_carrier_on(dev);
2942 disable_delayed_refill(vi);
2943 cancel_delayed_work_sync(&vi->refill);
2945 for (i--; i >= 0; i--) {
2946 virtnet_disable_queue_pair(vi, i);
2947 virtnet_cancel_dim(vi, &vi->rq[i].dim);
2953 static int virtnet_poll_tx(struct napi_struct *napi, int budget)
2955 struct send_queue *sq = container_of(napi, struct send_queue, napi);
2956 struct virtnet_info *vi = sq->vq->vdev->priv;
2957 unsigned int index = vq2txq(sq->vq);
2958 struct netdev_queue *txq;
2962 if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
2963 /* We don't need to enable cb for XDP */
2964 napi_complete_done(napi, 0);
2968 txq = netdev_get_tx_queue(vi->dev, index);
2969 __netif_tx_lock(txq, raw_smp_processor_id());
2970 virtqueue_disable_cb(sq->vq);
2971 free_old_xmit(sq, txq, !!budget);
2973 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
2974 if (netif_tx_queue_stopped(txq)) {
2975 u64_stats_update_begin(&sq->stats.syncp);
2976 u64_stats_inc(&sq->stats.wake);
2977 u64_stats_update_end(&sq->stats.syncp);
2979 netif_tx_wake_queue(txq);
2982 opaque = virtqueue_enable_cb_prepare(sq->vq);
2984 done = napi_complete_done(napi, 0);
2987 virtqueue_disable_cb(sq->vq);
2989 __netif_tx_unlock(txq);
2992 if (unlikely(virtqueue_poll(sq->vq, opaque))) {
2993 if (napi_schedule_prep(napi)) {
2994 __netif_tx_lock(txq, raw_smp_processor_id());
2995 virtqueue_disable_cb(sq->vq);
2996 __netif_tx_unlock(txq);
2997 __napi_schedule(napi);
3005 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan)
3007 struct virtio_net_hdr_mrg_rxbuf *hdr;
3008 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
3009 struct virtnet_info *vi = sq->vq->vdev->priv;
3011 unsigned hdr_len = vi->hdr_len;
3014 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
3016 can_push = vi->any_header_sg &&
3017 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
3018 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
3019 /* Even if we can, don't push here yet as this would skew
3020 * csum_start offset below. */
3022 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
3024 hdr = &skb_vnet_common_hdr(skb)->mrg_hdr;
3026 if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
3027 virtio_is_little_endian(vi->vdev), false,
3031 if (vi->mergeable_rx_bufs)
3032 hdr->num_buffers = 0;
3034 sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
3036 __skb_push(skb, hdr_len);
3037 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
3038 if (unlikely(num_sg < 0))
3040 /* Pull header back to avoid skew in tx bytes calculations. */
3041 __skb_pull(skb, hdr_len);
3043 sg_set_buf(sq->sg, hdr, hdr_len);
3044 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
3045 if (unlikely(num_sg < 0))
3049 return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg,
3050 skb_to_ptr(skb, orphan), GFP_ATOMIC);
3053 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
3055 struct virtnet_info *vi = netdev_priv(dev);
3056 int qnum = skb_get_queue_mapping(skb);
3057 struct send_queue *sq = &vi->sq[qnum];
3059 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
3060 bool xmit_more = netdev_xmit_more();
3061 bool use_napi = sq->napi.weight;
3064 /* Free up any pending old buffers before queueing new ones. */
3067 virtqueue_disable_cb(sq->vq);
3069 free_old_xmit(sq, txq, false);
3071 } while (use_napi && !xmit_more &&
3072 unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
3074 /* timestamp packet in software */
3075 skb_tx_timestamp(skb);
3077 /* Try to transmit */
3078 err = xmit_skb(sq, skb, !use_napi);
3080 /* This should not happen! */
3081 if (unlikely(err)) {
3082 DEV_STATS_INC(dev, tx_fifo_errors);
3083 if (net_ratelimit())
3085 "Unexpected TXQ (%d) queue failure: %d\n",
3087 DEV_STATS_INC(dev, tx_dropped);
3088 dev_kfree_skb_any(skb);
3089 return NETDEV_TX_OK;
3092 /* Don't wait up for transmitted skbs to be freed. */
3098 check_sq_full_and_disable(vi, dev, sq);
3100 kick = use_napi ? __netdev_tx_sent_queue(txq, skb->len, xmit_more) :
3101 !xmit_more || netif_xmit_stopped(txq);
3103 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
3104 u64_stats_update_begin(&sq->stats.syncp);
3105 u64_stats_inc(&sq->stats.kicks);
3106 u64_stats_update_end(&sq->stats.syncp);
3110 return NETDEV_TX_OK;
3113 static void virtnet_rx_pause(struct virtnet_info *vi, struct receive_queue *rq)
3115 bool running = netif_running(vi->dev);
3118 napi_disable(&rq->napi);
3119 virtnet_cancel_dim(vi, &rq->dim);
3123 static void virtnet_rx_resume(struct virtnet_info *vi, struct receive_queue *rq)
3125 bool running = netif_running(vi->dev);
3127 if (!try_fill_recv(vi, rq, GFP_KERNEL))
3128 schedule_delayed_work(&vi->refill, 0);
3131 virtnet_napi_enable(rq->vq, &rq->napi);
3134 static int virtnet_rx_resize(struct virtnet_info *vi,
3135 struct receive_queue *rq, u32 ring_num)
3139 qindex = rq - vi->rq;
3141 virtnet_rx_pause(vi, rq);
3143 err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_unmap_free_buf);
3145 netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
3147 virtnet_rx_resume(vi, rq);
3151 static void virtnet_tx_pause(struct virtnet_info *vi, struct send_queue *sq)
3153 bool running = netif_running(vi->dev);
3154 struct netdev_queue *txq;
3157 qindex = sq - vi->sq;
3160 virtnet_napi_tx_disable(&sq->napi);
3162 txq = netdev_get_tx_queue(vi->dev, qindex);
3164 /* 1. wait all ximt complete
3165 * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue()
3167 __netif_tx_lock_bh(txq);
3169 /* Prevent rx poll from accessing sq. */
3172 /* Prevent the upper layer from trying to send packets. */
3173 netif_stop_subqueue(vi->dev, qindex);
3175 __netif_tx_unlock_bh(txq);
3178 static void virtnet_tx_resume(struct virtnet_info *vi, struct send_queue *sq)
3180 bool running = netif_running(vi->dev);
3181 struct netdev_queue *txq;
3184 qindex = sq - vi->sq;
3186 txq = netdev_get_tx_queue(vi->dev, qindex);
3188 __netif_tx_lock_bh(txq);
3190 netif_tx_wake_queue(txq);
3191 __netif_tx_unlock_bh(txq);
3194 virtnet_napi_tx_enable(vi, sq->vq, &sq->napi);
3197 static int virtnet_tx_resize(struct virtnet_info *vi, struct send_queue *sq,
3202 qindex = sq - vi->sq;
3204 virtnet_tx_pause(vi, sq);
3206 err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
3208 netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
3210 virtnet_tx_resume(vi, sq);
3216 * Send command via the control virtqueue and check status. Commands
3217 * supported by the hypervisor, as indicated by feature bits, should
3218 * never fail unless improperly formatted.
3220 static bool virtnet_send_command_reply(struct virtnet_info *vi, u8 class, u8 cmd,
3221 struct scatterlist *out,
3222 struct scatterlist *in)
3224 struct scatterlist *sgs[5], hdr, stat;
3225 u32 out_num = 0, tmp, in_num = 0;
3229 /* Caller should know better */
3230 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
3232 mutex_lock(&vi->cvq_lock);
3233 vi->ctrl->status = ~0;
3234 vi->ctrl->hdr.class = class;
3235 vi->ctrl->hdr.cmd = cmd;
3237 sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
3238 sgs[out_num++] = &hdr;
3241 sgs[out_num++] = out;
3243 /* Add return status. */
3244 sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
3245 sgs[out_num + in_num++] = &stat;
3248 sgs[out_num + in_num++] = in;
3250 BUG_ON(out_num + in_num > ARRAY_SIZE(sgs));
3251 ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, in_num, vi, GFP_ATOMIC);
3253 dev_warn(&vi->vdev->dev,
3254 "Failed to add sgs for command vq: %d\n.", ret);
3255 mutex_unlock(&vi->cvq_lock);
3259 if (unlikely(!virtqueue_kick(vi->cvq)))
3262 /* Spin for a response, the kick causes an ioport write, trapping
3263 * into the hypervisor, so the request should be handled immediately.
3265 while (!virtqueue_get_buf(vi->cvq, &tmp) &&
3266 !virtqueue_is_broken(vi->cvq)) {
3272 ok = vi->ctrl->status == VIRTIO_NET_OK;
3273 mutex_unlock(&vi->cvq_lock);
3277 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
3278 struct scatterlist *out)
3280 return virtnet_send_command_reply(vi, class, cmd, out, NULL);
3283 static int virtnet_set_mac_address(struct net_device *dev, void *p)
3285 struct virtnet_info *vi = netdev_priv(dev);
3286 struct virtio_device *vdev = vi->vdev;
3288 struct sockaddr *addr;
3289 struct scatterlist sg;
3291 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
3294 addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
3298 ret = eth_prepare_mac_addr_change(dev, addr);
3302 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
3303 sg_init_one(&sg, addr->sa_data, dev->addr_len);
3304 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
3305 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
3306 dev_warn(&vdev->dev,
3307 "Failed to set mac address by vq command.\n");
3311 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
3312 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3315 /* Naturally, this has an atomicity problem. */
3316 for (i = 0; i < dev->addr_len; i++)
3317 virtio_cwrite8(vdev,
3318 offsetof(struct virtio_net_config, mac) +
3319 i, addr->sa_data[i]);
3322 eth_commit_mac_addr_change(dev, p);
3330 static void virtnet_stats(struct net_device *dev,
3331 struct rtnl_link_stats64 *tot)
3333 struct virtnet_info *vi = netdev_priv(dev);
3337 for (i = 0; i < vi->max_queue_pairs; i++) {
3338 u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops;
3339 struct receive_queue *rq = &vi->rq[i];
3340 struct send_queue *sq = &vi->sq[i];
3343 start = u64_stats_fetch_begin(&sq->stats.syncp);
3344 tpackets = u64_stats_read(&sq->stats.packets);
3345 tbytes = u64_stats_read(&sq->stats.bytes);
3346 terrors = u64_stats_read(&sq->stats.tx_timeouts);
3347 } while (u64_stats_fetch_retry(&sq->stats.syncp, start));
3350 start = u64_stats_fetch_begin(&rq->stats.syncp);
3351 rpackets = u64_stats_read(&rq->stats.packets);
3352 rbytes = u64_stats_read(&rq->stats.bytes);
3353 rdrops = u64_stats_read(&rq->stats.drops);
3354 } while (u64_stats_fetch_retry(&rq->stats.syncp, start));
3356 tot->rx_packets += rpackets;
3357 tot->tx_packets += tpackets;
3358 tot->rx_bytes += rbytes;
3359 tot->tx_bytes += tbytes;
3360 tot->rx_dropped += rdrops;
3361 tot->tx_errors += terrors;
3364 tot->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
3365 tot->tx_fifo_errors = DEV_STATS_READ(dev, tx_fifo_errors);
3366 tot->rx_length_errors = DEV_STATS_READ(dev, rx_length_errors);
3367 tot->rx_frame_errors = DEV_STATS_READ(dev, rx_frame_errors);
3370 static void virtnet_ack_link_announce(struct virtnet_info *vi)
3372 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
3373 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
3374 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
3377 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
3379 struct virtio_net_ctrl_mq *mq __free(kfree) = NULL;
3380 struct scatterlist sg;
3381 struct net_device *dev = vi->dev;
3383 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
3386 mq = kzalloc(sizeof(*mq), GFP_KERNEL);
3390 mq->virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
3391 sg_init_one(&sg, mq, sizeof(*mq));
3393 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
3394 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
3395 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
3399 vi->curr_queue_pairs = queue_pairs;
3400 /* virtnet_open() will refill when device is going to up. */
3401 if (dev->flags & IFF_UP)
3402 schedule_delayed_work(&vi->refill, 0);
3408 static int virtnet_close(struct net_device *dev)
3410 struct virtnet_info *vi = netdev_priv(dev);
3413 /* Make sure NAPI doesn't schedule refill work */
3414 disable_delayed_refill(vi);
3415 /* Make sure refill_work doesn't re-enable napi! */
3416 cancel_delayed_work_sync(&vi->refill);
3417 /* Prevent the config change callback from changing carrier
3420 virtio_config_driver_disable(vi->vdev);
3421 /* Stop getting status/speed updates: we don't care until next
3424 cancel_work_sync(&vi->config_work);
3426 for (i = 0; i < vi->max_queue_pairs; i++) {
3427 virtnet_disable_queue_pair(vi, i);
3428 virtnet_cancel_dim(vi, &vi->rq[i].dim);
3431 netif_carrier_off(dev);
3436 static void virtnet_rx_mode_work(struct work_struct *work)
3438 struct virtnet_info *vi =
3439 container_of(work, struct virtnet_info, rx_mode_work);
3440 u8 *promisc_allmulti __free(kfree) = NULL;
3441 struct net_device *dev = vi->dev;
3442 struct scatterlist sg[2];
3443 struct virtio_net_ctrl_mac *mac_data;
3444 struct netdev_hw_addr *ha;
3450 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
3451 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
3454 promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_KERNEL);
3455 if (!promisc_allmulti) {
3456 dev_warn(&dev->dev, "Failed to set RX mode, no memory.\n");
3462 *promisc_allmulti = !!(dev->flags & IFF_PROMISC);
3463 sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti));
3465 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
3466 VIRTIO_NET_CTRL_RX_PROMISC, sg))
3467 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
3468 *promisc_allmulti ? "en" : "dis");
3470 *promisc_allmulti = !!(dev->flags & IFF_ALLMULTI);
3471 sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti));
3473 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
3474 VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
3475 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
3476 *promisc_allmulti ? "en" : "dis");
3478 netif_addr_lock_bh(dev);
3480 uc_count = netdev_uc_count(dev);
3481 mc_count = netdev_mc_count(dev);
3482 /* MAC filter - use one buffer for both lists */
3483 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
3484 (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
3487 netif_addr_unlock_bh(dev);
3492 sg_init_table(sg, 2);
3494 /* Store the unicast list and count in the front of the buffer */
3495 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
3497 netdev_for_each_uc_addr(ha, dev)
3498 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
3500 sg_set_buf(&sg[0], mac_data,
3501 sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
3503 /* multicast list and count fill the end */
3504 mac_data = (void *)&mac_data->macs[uc_count][0];
3506 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
3508 netdev_for_each_mc_addr(ha, dev)
3509 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
3511 netif_addr_unlock_bh(dev);
3513 sg_set_buf(&sg[1], mac_data,
3514 sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
3516 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
3517 VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
3518 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
3525 static void virtnet_set_rx_mode(struct net_device *dev)
3527 struct virtnet_info *vi = netdev_priv(dev);
3529 if (vi->rx_mode_work_enabled)
3530 schedule_work(&vi->rx_mode_work);
3533 static int virtnet_vlan_rx_add_vid(struct net_device *dev,
3534 __be16 proto, u16 vid)
3536 struct virtnet_info *vi = netdev_priv(dev);
3537 __virtio16 *_vid __free(kfree) = NULL;
3538 struct scatterlist sg;
3540 _vid = kzalloc(sizeof(*_vid), GFP_KERNEL);
3544 *_vid = cpu_to_virtio16(vi->vdev, vid);
3545 sg_init_one(&sg, _vid, sizeof(*_vid));
3547 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
3548 VIRTIO_NET_CTRL_VLAN_ADD, &sg))
3549 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
3553 static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
3554 __be16 proto, u16 vid)
3556 struct virtnet_info *vi = netdev_priv(dev);
3557 __virtio16 *_vid __free(kfree) = NULL;
3558 struct scatterlist sg;
3560 _vid = kzalloc(sizeof(*_vid), GFP_KERNEL);
3564 *_vid = cpu_to_virtio16(vi->vdev, vid);
3565 sg_init_one(&sg, _vid, sizeof(*_vid));
3567 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
3568 VIRTIO_NET_CTRL_VLAN_DEL, &sg))
3569 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
3573 static void virtnet_clean_affinity(struct virtnet_info *vi)
3577 if (vi->affinity_hint_set) {
3578 for (i = 0; i < vi->max_queue_pairs; i++) {
3579 virtqueue_set_affinity(vi->rq[i].vq, NULL);
3580 virtqueue_set_affinity(vi->sq[i].vq, NULL);
3583 vi->affinity_hint_set = false;
3587 static void virtnet_set_affinity(struct virtnet_info *vi)
3596 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
3597 virtnet_clean_affinity(vi);
3601 num_cpu = num_online_cpus();
3602 stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);
3603 stragglers = num_cpu >= vi->curr_queue_pairs ?
3604 num_cpu % vi->curr_queue_pairs :
3606 cpu = cpumask_first(cpu_online_mask);
3608 for (i = 0; i < vi->curr_queue_pairs; i++) {
3609 group_size = stride + (i < stragglers ? 1 : 0);
3611 for (j = 0; j < group_size; j++) {
3612 cpumask_set_cpu(cpu, mask);
3613 cpu = cpumask_next_wrap(cpu, cpu_online_mask,
3616 virtqueue_set_affinity(vi->rq[i].vq, mask);
3617 virtqueue_set_affinity(vi->sq[i].vq, mask);
3618 __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS);
3619 cpumask_clear(mask);
3622 vi->affinity_hint_set = true;
3623 free_cpumask_var(mask);
3626 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
3628 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
3630 virtnet_set_affinity(vi);
3634 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
3636 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
3638 virtnet_set_affinity(vi);
3642 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
3644 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
3647 virtnet_clean_affinity(vi);
3651 static enum cpuhp_state virtionet_online;
3653 static int virtnet_cpu_notif_add(struct virtnet_info *vi)
3657 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
3660 ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
3664 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
3668 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
3670 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
3671 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
3675 static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi,
3676 u16 vqn, u32 max_usecs, u32 max_packets)
3678 struct virtio_net_ctrl_coal_vq *coal_vq __free(kfree) = NULL;
3679 struct scatterlist sgs;
3681 coal_vq = kzalloc(sizeof(*coal_vq), GFP_KERNEL);
3685 coal_vq->vqn = cpu_to_le16(vqn);
3686 coal_vq->coal.max_usecs = cpu_to_le32(max_usecs);
3687 coal_vq->coal.max_packets = cpu_to_le32(max_packets);
3688 sg_init_one(&sgs, coal_vq, sizeof(*coal_vq));
3690 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
3691 VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET,
3698 static int virtnet_send_rx_ctrl_coal_vq_cmd(struct virtnet_info *vi,
3699 u16 queue, u32 max_usecs,
3704 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
3707 err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(queue),
3708 max_usecs, max_packets);
3712 vi->rq[queue].intr_coal.max_usecs = max_usecs;
3713 vi->rq[queue].intr_coal.max_packets = max_packets;
3718 static int virtnet_send_tx_ctrl_coal_vq_cmd(struct virtnet_info *vi,
3719 u16 queue, u32 max_usecs,
3724 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
3727 err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(queue),
3728 max_usecs, max_packets);
3732 vi->sq[queue].intr_coal.max_usecs = max_usecs;
3733 vi->sq[queue].intr_coal.max_packets = max_packets;
3738 static void virtnet_get_ringparam(struct net_device *dev,
3739 struct ethtool_ringparam *ring,
3740 struct kernel_ethtool_ringparam *kernel_ring,
3741 struct netlink_ext_ack *extack)
3743 struct virtnet_info *vi = netdev_priv(dev);
3745 ring->rx_max_pending = vi->rq[0].vq->num_max;
3746 ring->tx_max_pending = vi->sq[0].vq->num_max;
3747 ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
3748 ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
3751 static int virtnet_set_ringparam(struct net_device *dev,
3752 struct ethtool_ringparam *ring,
3753 struct kernel_ethtool_ringparam *kernel_ring,
3754 struct netlink_ext_ack *extack)
3756 struct virtnet_info *vi = netdev_priv(dev);
3757 u32 rx_pending, tx_pending;
3758 struct receive_queue *rq;
3759 struct send_queue *sq;
3762 if (ring->rx_mini_pending || ring->rx_jumbo_pending)
3765 rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
3766 tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
3768 if (ring->rx_pending == rx_pending &&
3769 ring->tx_pending == tx_pending)
3772 if (ring->rx_pending > vi->rq[0].vq->num_max)
3775 if (ring->tx_pending > vi->sq[0].vq->num_max)
3778 for (i = 0; i < vi->max_queue_pairs; i++) {
3782 if (ring->tx_pending != tx_pending) {
3783 err = virtnet_tx_resize(vi, sq, ring->tx_pending);
3787 /* Upon disabling and re-enabling a transmit virtqueue, the device must
3788 * set the coalescing parameters of the virtqueue to those configured
3789 * through the VIRTIO_NET_CTRL_NOTF_COAL_TX_SET command, or, if the driver
3790 * did not set any TX coalescing parameters, to 0.
3792 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, i,
3793 vi->intr_coal_tx.max_usecs,
3794 vi->intr_coal_tx.max_packets);
3796 /* Don't break the tx resize action if the vq coalescing is not
3797 * supported. The same is true for rx resize below.
3799 if (err && err != -EOPNOTSUPP)
3803 if (ring->rx_pending != rx_pending) {
3804 err = virtnet_rx_resize(vi, rq, ring->rx_pending);
3808 /* The reason is same as the transmit virtqueue reset */
3809 mutex_lock(&vi->rq[i].dim_lock);
3810 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, i,
3811 vi->intr_coal_rx.max_usecs,
3812 vi->intr_coal_rx.max_packets);
3813 mutex_unlock(&vi->rq[i].dim_lock);
3814 if (err && err != -EOPNOTSUPP)
3822 static bool virtnet_commit_rss_command(struct virtnet_info *vi)
3824 struct net_device *dev = vi->dev;
3825 struct scatterlist sgs[4];
3826 unsigned int sg_buf_size;
3829 sg_init_table(sgs, 4);
3831 sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table);
3832 sg_set_buf(&sgs[0], &vi->rss, sg_buf_size);
3834 sg_buf_size = sizeof(uint16_t) * (vi->rss.indirection_table_mask + 1);
3835 sg_set_buf(&sgs[1], vi->rss.indirection_table, sg_buf_size);
3837 sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key)
3838 - offsetof(struct virtio_net_ctrl_rss, max_tx_vq);
3839 sg_set_buf(&sgs[2], &vi->rss.max_tx_vq, sg_buf_size);
3841 sg_buf_size = vi->rss_key_size;
3842 sg_set_buf(&sgs[3], vi->rss.key, sg_buf_size);
3844 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
3845 vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG
3846 : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs))
3852 dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n");
3857 static void virtnet_init_default_rss(struct virtnet_info *vi)
3862 vi->rss.hash_types = vi->rss_hash_types_supported;
3863 vi->rss_hash_types_saved = vi->rss_hash_types_supported;
3864 vi->rss.indirection_table_mask = vi->rss_indir_table_size
3865 ? vi->rss_indir_table_size - 1 : 0;
3866 vi->rss.unclassified_queue = 0;
3868 for (; i < vi->rss_indir_table_size; ++i) {
3869 indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
3870 vi->rss.indirection_table[i] = indir_val;
3873 vi->rss.max_tx_vq = vi->has_rss ? vi->curr_queue_pairs : 0;
3874 vi->rss.hash_key_length = vi->rss_key_size;
3876 netdev_rss_key_fill(vi->rss.key, vi->rss_key_size);
3879 static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info)
3882 switch (info->flow_type) {
3884 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {
3885 info->data = RXH_IP_SRC | RXH_IP_DST |
3886 RXH_L4_B_0_1 | RXH_L4_B_2_3;
3887 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
3888 info->data = RXH_IP_SRC | RXH_IP_DST;
3892 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {
3893 info->data = RXH_IP_SRC | RXH_IP_DST |
3894 RXH_L4_B_0_1 | RXH_L4_B_2_3;
3895 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
3896 info->data = RXH_IP_SRC | RXH_IP_DST;
3900 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {
3901 info->data = RXH_IP_SRC | RXH_IP_DST |
3902 RXH_L4_B_0_1 | RXH_L4_B_2_3;
3903 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
3904 info->data = RXH_IP_SRC | RXH_IP_DST;
3908 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {
3909 info->data = RXH_IP_SRC | RXH_IP_DST |
3910 RXH_L4_B_0_1 | RXH_L4_B_2_3;
3911 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
3912 info->data = RXH_IP_SRC | RXH_IP_DST;
3916 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4)
3917 info->data = RXH_IP_SRC | RXH_IP_DST;
3921 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6)
3922 info->data = RXH_IP_SRC | RXH_IP_DST;
3931 static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info)
3933 u32 new_hashtypes = vi->rss_hash_types_saved;
3934 bool is_disable = info->data & RXH_DISCARD;
3935 bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3);
3937 /* supports only 'sd', 'sdfn' and 'r' */
3938 if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable))
3941 switch (info->flow_type) {
3943 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4);
3945 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
3946 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0);
3949 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4);
3951 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
3952 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0);
3955 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4;
3957 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4;
3960 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6);
3962 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
3963 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0);
3966 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6);
3968 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
3969 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0);
3972 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6;
3974 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6;
3977 /* unsupported flow */
3981 /* if unsupported hashtype was set */
3982 if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported))
3985 if (new_hashtypes != vi->rss_hash_types_saved) {
3986 vi->rss_hash_types_saved = new_hashtypes;
3987 vi->rss.hash_types = vi->rss_hash_types_saved;
3988 if (vi->dev->features & NETIF_F_RXHASH)
3989 return virtnet_commit_rss_command(vi);
3995 static void virtnet_get_drvinfo(struct net_device *dev,
3996 struct ethtool_drvinfo *info)
3998 struct virtnet_info *vi = netdev_priv(dev);
3999 struct virtio_device *vdev = vi->vdev;
4001 strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
4002 strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
4003 strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
4007 /* TODO: Eliminate OOO packets during switching */
4008 static int virtnet_set_channels(struct net_device *dev,
4009 struct ethtool_channels *channels)
4011 struct virtnet_info *vi = netdev_priv(dev);
4012 u16 queue_pairs = channels->combined_count;
4015 /* We don't support separate rx/tx channels.
4016 * We don't allow setting 'other' channels.
4018 if (channels->rx_count || channels->tx_count || channels->other_count)
4021 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
4024 /* For now we don't support modifying channels while XDP is loaded
4025 * also when XDP is loaded all RX queues have XDP programs so we only
4026 * need to check a single RX queue.
4028 if (vi->rq[0].xdp_prog)
4032 err = virtnet_set_queues(vi, queue_pairs);
4037 virtnet_set_affinity(vi);
4040 netif_set_real_num_tx_queues(dev, queue_pairs);
4041 netif_set_real_num_rx_queues(dev, queue_pairs);
4046 static void virtnet_stats_sprintf(u8 **p, const char *fmt, const char *noq_fmt,
4047 int num, int qid, const struct virtnet_stat_desc *desc)
4052 for (i = 0; i < num; ++i)
4053 ethtool_sprintf(p, noq_fmt, desc[i].desc);
4055 for (i = 0; i < num; ++i)
4056 ethtool_sprintf(p, fmt, qid, desc[i].desc);
4060 /* qid == -1: for rx/tx queue total field */
4061 static void virtnet_get_stats_string(struct virtnet_info *vi, int type, int qid, u8 **data)
4063 const struct virtnet_stat_desc *desc;
4064 const char *fmt, *noq_fmt;
4068 if (type == VIRTNET_Q_TYPE_CQ && qid >= 0) {
4069 noq_fmt = "cq_hw_%s";
4071 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) {
4072 desc = &virtnet_stats_cvq_desc[0];
4073 num = ARRAY_SIZE(virtnet_stats_cvq_desc);
4075 virtnet_stats_sprintf(&p, NULL, noq_fmt, num, -1, desc);
4079 if (type == VIRTNET_Q_TYPE_RX) {
4083 desc = &virtnet_rq_stats_desc[0];
4084 num = ARRAY_SIZE(virtnet_rq_stats_desc);
4086 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4089 noq_fmt = "rx_hw_%s";
4091 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4092 desc = &virtnet_stats_rx_basic_desc[0];
4093 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc);
4095 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4098 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4099 desc = &virtnet_stats_rx_csum_desc[0];
4100 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc);
4102 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4105 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4106 desc = &virtnet_stats_rx_speed_desc[0];
4107 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc);
4109 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4113 if (type == VIRTNET_Q_TYPE_TX) {
4117 desc = &virtnet_sq_stats_desc[0];
4118 num = ARRAY_SIZE(virtnet_sq_stats_desc);
4120 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4123 noq_fmt = "tx_hw_%s";
4125 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4126 desc = &virtnet_stats_tx_basic_desc[0];
4127 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc);
4129 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4132 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4133 desc = &virtnet_stats_tx_gso_desc[0];
4134 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc);
4136 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4139 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4140 desc = &virtnet_stats_tx_speed_desc[0];
4141 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc);
4143 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4150 struct virtnet_stats_ctx {
4151 /* The stats are write to qstats or ethtool -S */
4154 /* Used to calculate the offset inside the output buffer. */
4157 /* The actual supported stat types. */
4160 /* Used to calculate the reply buffer size. */
4163 /* Record the output buffer. */
4167 static void virtnet_stats_ctx_init(struct virtnet_info *vi,
4168 struct virtnet_stats_ctx *ctx,
4169 u64 *data, bool to_qstat)
4174 ctx->to_qstat = to_qstat;
4177 ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc_qstat);
4178 ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc_qstat);
4180 queue_type = VIRTNET_Q_TYPE_RX;
4182 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4183 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_BASIC;
4184 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat);
4185 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_basic);
4188 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4189 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_CSUM;
4190 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat);
4191 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_csum);
4194 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) {
4195 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_GSO;
4196 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat);
4197 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_gso);
4200 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4201 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_SPEED;
4202 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat);
4203 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_speed);
4206 queue_type = VIRTNET_Q_TYPE_TX;
4208 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4209 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_BASIC;
4210 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat);
4211 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_basic);
4214 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) {
4215 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_CSUM;
4216 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat);
4217 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_csum);
4220 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4221 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_GSO;
4222 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat);
4223 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_gso);
4226 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4227 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_SPEED;
4228 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat);
4229 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_speed);
4235 ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc);
4236 ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc);
4238 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) {
4239 queue_type = VIRTNET_Q_TYPE_CQ;
4241 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_CVQ;
4242 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_cvq_desc);
4243 ctx->size[queue_type] += sizeof(struct virtio_net_stats_cvq);
4246 queue_type = VIRTNET_Q_TYPE_RX;
4248 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4249 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_BASIC;
4250 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc);
4251 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_basic);
4254 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4255 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_CSUM;
4256 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc);
4257 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_csum);
4260 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4261 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_SPEED;
4262 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc);
4263 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_speed);
4266 queue_type = VIRTNET_Q_TYPE_TX;
4268 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4269 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_BASIC;
4270 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc);
4271 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_basic);
4274 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4275 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_GSO;
4276 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc);
4277 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_gso);
4280 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4281 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_SPEED;
4282 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc);
4283 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_speed);
4287 /* stats_sum_queue - Calculate the sum of the same fields in sq or rq.
4288 * @sum: the position to store the sum values
4290 * @q_value: the first queue fields
4291 * @q_num: number of the queues
4293 static void stats_sum_queue(u64 *sum, u32 num, u64 *q_value, u32 q_num)
4299 for (i = 0; i < num; ++i) {
4303 for (j = 0; j < q_num; ++j)
4304 *p += *(q_value + i + j * step);
4308 static void virtnet_fill_total_fields(struct virtnet_info *vi,
4309 struct virtnet_stats_ctx *ctx)
4311 u64 *data, *first_rx_q, *first_tx_q;
4312 u32 num_cq, num_rx, num_tx;
4314 num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ];
4315 num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX];
4316 num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX];
4318 first_rx_q = ctx->data + num_rx + num_tx + num_cq;
4319 first_tx_q = first_rx_q + vi->curr_queue_pairs * num_rx;
4323 stats_sum_queue(data, num_rx, first_rx_q, vi->curr_queue_pairs);
4325 data = ctx->data + num_rx;
4327 stats_sum_queue(data, num_tx, first_tx_q, vi->curr_queue_pairs);
4330 static void virtnet_fill_stats_qstat(struct virtnet_info *vi, u32 qid,
4331 struct virtnet_stats_ctx *ctx,
4332 const u8 *base, bool drv_stats, u8 reply_type)
4334 const struct virtnet_stat_desc *desc;
4335 const u64_stats_t *v_stat;
4341 queue_type = vq_type(vi, qid);
4342 bitmap = ctx->bitmap[queue_type];
4345 if (queue_type == VIRTNET_Q_TYPE_RX) {
4346 desc = &virtnet_rq_stats_desc_qstat[0];
4347 num = ARRAY_SIZE(virtnet_rq_stats_desc_qstat);
4349 desc = &virtnet_sq_stats_desc_qstat[0];
4350 num = ARRAY_SIZE(virtnet_sq_stats_desc_qstat);
4353 for (i = 0; i < num; ++i) {
4354 offset = desc[i].qstat_offset / sizeof(*ctx->data);
4355 v_stat = (const u64_stats_t *)(base + desc[i].offset);
4356 ctx->data[offset] = u64_stats_read(v_stat);
4361 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4362 desc = &virtnet_stats_rx_basic_desc_qstat[0];
4363 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat);
4364 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC)
4368 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4369 desc = &virtnet_stats_rx_csum_desc_qstat[0];
4370 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat);
4371 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM)
4375 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_GSO) {
4376 desc = &virtnet_stats_rx_gso_desc_qstat[0];
4377 num = ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat);
4378 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_GSO)
4382 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4383 desc = &virtnet_stats_rx_speed_desc_qstat[0];
4384 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat);
4385 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED)
4389 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4390 desc = &virtnet_stats_tx_basic_desc_qstat[0];
4391 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat);
4392 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC)
4396 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_CSUM) {
4397 desc = &virtnet_stats_tx_csum_desc_qstat[0];
4398 num = ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat);
4399 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_CSUM)
4403 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4404 desc = &virtnet_stats_tx_gso_desc_qstat[0];
4405 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat);
4406 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO)
4410 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4411 desc = &virtnet_stats_tx_speed_desc_qstat[0];
4412 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat);
4413 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED)
4420 for (i = 0; i < num; ++i) {
4421 offset = desc[i].qstat_offset / sizeof(*ctx->data);
4422 v = (const __le64 *)(base + desc[i].offset);
4423 ctx->data[offset] = le64_to_cpu(*v);
4427 /* virtnet_fill_stats - copy the stats to qstats or ethtool -S
4428 * The stats source is the device or the driver.
4430 * @vi: virtio net info
4432 * @ctx: stats ctx (initiated by virtnet_stats_ctx_init())
4433 * @base: pointer to the device reply or the driver stats structure.
4434 * @drv_stats: designate the base type (device reply, driver stats)
4435 * @type: the type of the device reply (if drv_stats is true, this must be zero)
4437 static void virtnet_fill_stats(struct virtnet_info *vi, u32 qid,
4438 struct virtnet_stats_ctx *ctx,
4439 const u8 *base, bool drv_stats, u8 reply_type)
4441 u32 queue_type, num_rx, num_tx, num_cq;
4442 const struct virtnet_stat_desc *desc;
4443 const u64_stats_t *v_stat;
4449 return virtnet_fill_stats_qstat(vi, qid, ctx, base, drv_stats, reply_type);
4451 num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ];
4452 num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX];
4453 num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX];
4455 queue_type = vq_type(vi, qid);
4456 bitmap = ctx->bitmap[queue_type];
4458 /* skip the total fields of pairs */
4459 offset = num_rx + num_tx;
4461 if (queue_type == VIRTNET_Q_TYPE_TX) {
4462 offset += num_cq + num_rx * vi->curr_queue_pairs + num_tx * (qid / 2);
4464 num = ARRAY_SIZE(virtnet_sq_stats_desc);
4466 desc = &virtnet_sq_stats_desc[0];
4472 } else if (queue_type == VIRTNET_Q_TYPE_RX) {
4473 offset += num_cq + num_rx * (qid / 2);
4475 num = ARRAY_SIZE(virtnet_rq_stats_desc);
4477 desc = &virtnet_rq_stats_desc[0];
4484 if (bitmap & VIRTIO_NET_STATS_TYPE_CVQ) {
4485 desc = &virtnet_stats_cvq_desc[0];
4486 num = ARRAY_SIZE(virtnet_stats_cvq_desc);
4487 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_CVQ)
4493 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4494 desc = &virtnet_stats_rx_basic_desc[0];
4495 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc);
4496 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC)
4502 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4503 desc = &virtnet_stats_rx_csum_desc[0];
4504 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc);
4505 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM)
4511 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4512 desc = &virtnet_stats_rx_speed_desc[0];
4513 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc);
4514 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED)
4520 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4521 desc = &virtnet_stats_tx_basic_desc[0];
4522 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc);
4523 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC)
4529 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4530 desc = &virtnet_stats_tx_gso_desc[0];
4531 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc);
4532 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO)
4538 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4539 desc = &virtnet_stats_tx_speed_desc[0];
4540 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc);
4541 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED)
4550 for (i = 0; i < num; ++i) {
4551 v = (const __le64 *)(base + desc[i].offset);
4552 ctx->data[offset + i] = le64_to_cpu(*v);
4558 for (i = 0; i < num; ++i) {
4559 v_stat = (const u64_stats_t *)(base + desc[i].offset);
4560 ctx->data[offset + i] = u64_stats_read(v_stat);
4564 static int __virtnet_get_hw_stats(struct virtnet_info *vi,
4565 struct virtnet_stats_ctx *ctx,
4566 struct virtio_net_ctrl_queue_stats *req,
4567 int req_size, void *reply, int res_size)
4569 struct virtio_net_stats_reply_hdr *hdr;
4570 struct scatterlist sgs_in, sgs_out;
4575 sg_init_one(&sgs_out, req, req_size);
4576 sg_init_one(&sgs_in, reply, res_size);
4578 ok = virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS,
4579 VIRTIO_NET_CTRL_STATS_GET,
4585 for (p = reply; p - reply < res_size; p += le16_to_cpu(hdr->size)) {
4587 qid = le16_to_cpu(hdr->vq_index);
4588 virtnet_fill_stats(vi, qid, ctx, p, false, hdr->type);
4594 static void virtnet_make_stat_req(struct virtnet_info *vi,
4595 struct virtnet_stats_ctx *ctx,
4596 struct virtio_net_ctrl_queue_stats *req,
4599 int qtype = vq_type(vi, qid);
4600 u64 bitmap = ctx->bitmap[qtype];
4605 req->stats[*idx].vq_index = cpu_to_le16(qid);
4606 req->stats[*idx].types_bitmap[0] = cpu_to_le64(bitmap);
4610 /* qid: -1: get stats of all vq.
4611 * > 0: get the stats for the special vq. This must not be cvq.
4613 static int virtnet_get_hw_stats(struct virtnet_info *vi,
4614 struct virtnet_stats_ctx *ctx, int qid)
4616 int qnum, i, j, res_size, qtype, last_vq, first_vq;
4617 struct virtio_net_ctrl_queue_stats *req;
4622 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS))
4626 last_vq = vi->curr_queue_pairs * 2 - 1;
4637 for (i = first_vq; i <= last_vq ; ++i) {
4638 qtype = vq_type(vi, i);
4639 if (ctx->bitmap[qtype]) {
4641 res_size += ctx->size[qtype];
4645 if (enable_cvq && ctx->bitmap[VIRTNET_Q_TYPE_CQ]) {
4646 res_size += ctx->size[VIRTNET_Q_TYPE_CQ];
4650 req = kcalloc(qnum, sizeof(*req), GFP_KERNEL);
4654 reply = kmalloc(res_size, GFP_KERNEL);
4661 for (i = first_vq; i <= last_vq ; ++i)
4662 virtnet_make_stat_req(vi, ctx, req, i, &j);
4665 virtnet_make_stat_req(vi, ctx, req, vi->max_queue_pairs * 2, &j);
4667 ok = __virtnet_get_hw_stats(vi, ctx, req, sizeof(*req) * j, reply, res_size);
4675 static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
4677 struct virtnet_info *vi = netdev_priv(dev);
4681 switch (stringset) {
4683 /* Generate the total field names. */
4684 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, -1, &p);
4685 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, -1, &p);
4687 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_CQ, 0, &p);
4689 for (i = 0; i < vi->curr_queue_pairs; ++i)
4690 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, i, &p);
4692 for (i = 0; i < vi->curr_queue_pairs; ++i)
4693 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, i, &p);
4698 static int virtnet_get_sset_count(struct net_device *dev, int sset)
4700 struct virtnet_info *vi = netdev_priv(dev);
4701 struct virtnet_stats_ctx ctx = {0};
4706 virtnet_stats_ctx_init(vi, &ctx, NULL, false);
4708 pair_count = ctx.desc_num[VIRTNET_Q_TYPE_RX] + ctx.desc_num[VIRTNET_Q_TYPE_TX];
4710 return pair_count + ctx.desc_num[VIRTNET_Q_TYPE_CQ] +
4711 vi->curr_queue_pairs * pair_count;
4717 static void virtnet_get_ethtool_stats(struct net_device *dev,
4718 struct ethtool_stats *stats, u64 *data)
4720 struct virtnet_info *vi = netdev_priv(dev);
4721 struct virtnet_stats_ctx ctx = {0};
4722 unsigned int start, i;
4723 const u8 *stats_base;
4725 virtnet_stats_ctx_init(vi, &ctx, data, false);
4726 if (virtnet_get_hw_stats(vi, &ctx, -1))
4727 dev_warn(&vi->dev->dev, "Failed to get hw stats.\n");
4729 for (i = 0; i < vi->curr_queue_pairs; i++) {
4730 struct receive_queue *rq = &vi->rq[i];
4731 struct send_queue *sq = &vi->sq[i];
4733 stats_base = (const u8 *)&rq->stats;
4735 start = u64_stats_fetch_begin(&rq->stats.syncp);
4736 virtnet_fill_stats(vi, i * 2, &ctx, stats_base, true, 0);
4737 } while (u64_stats_fetch_retry(&rq->stats.syncp, start));
4739 stats_base = (const u8 *)&sq->stats;
4741 start = u64_stats_fetch_begin(&sq->stats.syncp);
4742 virtnet_fill_stats(vi, i * 2 + 1, &ctx, stats_base, true, 0);
4743 } while (u64_stats_fetch_retry(&sq->stats.syncp, start));
4746 virtnet_fill_total_fields(vi, &ctx);
4749 static void virtnet_get_channels(struct net_device *dev,
4750 struct ethtool_channels *channels)
4752 struct virtnet_info *vi = netdev_priv(dev);
4754 channels->combined_count = vi->curr_queue_pairs;
4755 channels->max_combined = vi->max_queue_pairs;
4756 channels->max_other = 0;
4757 channels->rx_count = 0;
4758 channels->tx_count = 0;
4759 channels->other_count = 0;
4762 static int virtnet_set_link_ksettings(struct net_device *dev,
4763 const struct ethtool_link_ksettings *cmd)
4765 struct virtnet_info *vi = netdev_priv(dev);
4767 return ethtool_virtdev_set_link_ksettings(dev, cmd,
4768 &vi->speed, &vi->duplex);
4771 static int virtnet_get_link_ksettings(struct net_device *dev,
4772 struct ethtool_link_ksettings *cmd)
4774 struct virtnet_info *vi = netdev_priv(dev);
4776 cmd->base.speed = vi->speed;
4777 cmd->base.duplex = vi->duplex;
4778 cmd->base.port = PORT_OTHER;
4783 static int virtnet_send_tx_notf_coal_cmds(struct virtnet_info *vi,
4784 struct ethtool_coalesce *ec)
4786 struct virtio_net_ctrl_coal_tx *coal_tx __free(kfree) = NULL;
4787 struct scatterlist sgs_tx;
4790 coal_tx = kzalloc(sizeof(*coal_tx), GFP_KERNEL);
4794 coal_tx->tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs);
4795 coal_tx->tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames);
4796 sg_init_one(&sgs_tx, coal_tx, sizeof(*coal_tx));
4798 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
4799 VIRTIO_NET_CTRL_NOTF_COAL_TX_SET,
4803 vi->intr_coal_tx.max_usecs = ec->tx_coalesce_usecs;
4804 vi->intr_coal_tx.max_packets = ec->tx_max_coalesced_frames;
4805 for (i = 0; i < vi->max_queue_pairs; i++) {
4806 vi->sq[i].intr_coal.max_usecs = ec->tx_coalesce_usecs;
4807 vi->sq[i].intr_coal.max_packets = ec->tx_max_coalesced_frames;
4813 static int virtnet_send_rx_notf_coal_cmds(struct virtnet_info *vi,
4814 struct ethtool_coalesce *ec)
4816 struct virtio_net_ctrl_coal_rx *coal_rx __free(kfree) = NULL;
4817 bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce;
4818 struct scatterlist sgs_rx;
4821 if (rx_ctrl_dim_on && !virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
4824 if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != vi->intr_coal_rx.max_usecs ||
4825 ec->rx_max_coalesced_frames != vi->intr_coal_rx.max_packets))
4828 if (rx_ctrl_dim_on && !vi->rx_dim_enabled) {
4829 vi->rx_dim_enabled = true;
4830 for (i = 0; i < vi->max_queue_pairs; i++) {
4831 mutex_lock(&vi->rq[i].dim_lock);
4832 vi->rq[i].dim_enabled = true;
4833 mutex_unlock(&vi->rq[i].dim_lock);
4838 coal_rx = kzalloc(sizeof(*coal_rx), GFP_KERNEL);
4842 if (!rx_ctrl_dim_on && vi->rx_dim_enabled) {
4843 vi->rx_dim_enabled = false;
4844 for (i = 0; i < vi->max_queue_pairs; i++) {
4845 mutex_lock(&vi->rq[i].dim_lock);
4846 vi->rq[i].dim_enabled = false;
4847 mutex_unlock(&vi->rq[i].dim_lock);
4851 /* Since the per-queue coalescing params can be set,
4852 * we need apply the global new params even if they
4855 coal_rx->rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs);
4856 coal_rx->rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames);
4857 sg_init_one(&sgs_rx, coal_rx, sizeof(*coal_rx));
4859 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
4860 VIRTIO_NET_CTRL_NOTF_COAL_RX_SET,
4864 vi->intr_coal_rx.max_usecs = ec->rx_coalesce_usecs;
4865 vi->intr_coal_rx.max_packets = ec->rx_max_coalesced_frames;
4866 for (i = 0; i < vi->max_queue_pairs; i++) {
4867 mutex_lock(&vi->rq[i].dim_lock);
4868 vi->rq[i].intr_coal.max_usecs = ec->rx_coalesce_usecs;
4869 vi->rq[i].intr_coal.max_packets = ec->rx_max_coalesced_frames;
4870 mutex_unlock(&vi->rq[i].dim_lock);
4876 static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi,
4877 struct ethtool_coalesce *ec)
4881 err = virtnet_send_tx_notf_coal_cmds(vi, ec);
4885 err = virtnet_send_rx_notf_coal_cmds(vi, ec);
4892 static int virtnet_send_rx_notf_coal_vq_cmds(struct virtnet_info *vi,
4893 struct ethtool_coalesce *ec,
4896 bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce;
4897 u32 max_usecs, max_packets;
4901 mutex_lock(&vi->rq[queue].dim_lock);
4902 cur_rx_dim = vi->rq[queue].dim_enabled;
4903 max_usecs = vi->rq[queue].intr_coal.max_usecs;
4904 max_packets = vi->rq[queue].intr_coal.max_packets;
4906 if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != max_usecs ||
4907 ec->rx_max_coalesced_frames != max_packets)) {
4908 mutex_unlock(&vi->rq[queue].dim_lock);
4912 if (rx_ctrl_dim_on && !cur_rx_dim) {
4913 vi->rq[queue].dim_enabled = true;
4914 mutex_unlock(&vi->rq[queue].dim_lock);
4918 if (!rx_ctrl_dim_on && cur_rx_dim)
4919 vi->rq[queue].dim_enabled = false;
4921 /* If no params are updated, userspace ethtool will
4922 * reject the modification.
4924 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, queue,
4925 ec->rx_coalesce_usecs,
4926 ec->rx_max_coalesced_frames);
4927 mutex_unlock(&vi->rq[queue].dim_lock);
4931 static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi,
4932 struct ethtool_coalesce *ec,
4937 err = virtnet_send_rx_notf_coal_vq_cmds(vi, ec, queue);
4941 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, queue,
4942 ec->tx_coalesce_usecs,
4943 ec->tx_max_coalesced_frames);
4950 static void virtnet_rx_dim_work(struct work_struct *work)
4952 struct dim *dim = container_of(work, struct dim, work);
4953 struct receive_queue *rq = container_of(dim,
4954 struct receive_queue, dim);
4955 struct virtnet_info *vi = rq->vq->vdev->priv;
4956 struct net_device *dev = vi->dev;
4957 struct dim_cq_moder update_moder;
4962 mutex_lock(&rq->dim_lock);
4963 if (!rq->dim_enabled)
4966 update_moder = net_dim_get_rx_irq_moder(dev, dim);
4967 if (update_moder.usec != rq->intr_coal.max_usecs ||
4968 update_moder.pkts != rq->intr_coal.max_packets) {
4969 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, qnum,
4973 pr_debug("%s: Failed to send dim parameters on rxq%d\n",
4977 dim->state = DIM_START_MEASURE;
4978 mutex_unlock(&rq->dim_lock);
4981 static int virtnet_coal_params_supported(struct ethtool_coalesce *ec)
4983 /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL
4984 * or VIRTIO_NET_F_VQ_NOTF_COAL feature is negotiated.
4986 if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs)
4989 if (ec->tx_max_coalesced_frames > 1 ||
4990 ec->rx_max_coalesced_frames != 1)
4996 static int virtnet_should_update_vq_weight(int dev_flags, int weight,
4997 int vq_weight, bool *should_update)
4999 if (weight ^ vq_weight) {
5000 if (dev_flags & IFF_UP)
5002 *should_update = true;
5008 static int virtnet_set_coalesce(struct net_device *dev,
5009 struct ethtool_coalesce *ec,
5010 struct kernel_ethtool_coalesce *kernel_coal,
5011 struct netlink_ext_ack *extack)
5013 struct virtnet_info *vi = netdev_priv(dev);
5014 int ret, queue_number, napi_weight;
5015 bool update_napi = false;
5017 /* Can't change NAPI weight if the link is up */
5018 napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
5019 for (queue_number = 0; queue_number < vi->max_queue_pairs; queue_number++) {
5020 ret = virtnet_should_update_vq_weight(dev->flags, napi_weight,
5021 vi->sq[queue_number].napi.weight,
5027 /* All queues that belong to [queue_number, vi->max_queue_pairs] will be
5028 * updated for the sake of simplicity, which might not be necessary
5034 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL))
5035 ret = virtnet_send_notf_coal_cmds(vi, ec);
5037 ret = virtnet_coal_params_supported(ec);
5043 for (; queue_number < vi->max_queue_pairs; queue_number++)
5044 vi->sq[queue_number].napi.weight = napi_weight;
5050 static int virtnet_get_coalesce(struct net_device *dev,
5051 struct ethtool_coalesce *ec,
5052 struct kernel_ethtool_coalesce *kernel_coal,
5053 struct netlink_ext_ack *extack)
5055 struct virtnet_info *vi = netdev_priv(dev);
5057 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
5058 ec->rx_coalesce_usecs = vi->intr_coal_rx.max_usecs;
5059 ec->tx_coalesce_usecs = vi->intr_coal_tx.max_usecs;
5060 ec->tx_max_coalesced_frames = vi->intr_coal_tx.max_packets;
5061 ec->rx_max_coalesced_frames = vi->intr_coal_rx.max_packets;
5062 ec->use_adaptive_rx_coalesce = vi->rx_dim_enabled;
5064 ec->rx_max_coalesced_frames = 1;
5066 if (vi->sq[0].napi.weight)
5067 ec->tx_max_coalesced_frames = 1;
5073 static int virtnet_set_per_queue_coalesce(struct net_device *dev,
5075 struct ethtool_coalesce *ec)
5077 struct virtnet_info *vi = netdev_priv(dev);
5078 int ret, napi_weight;
5079 bool update_napi = false;
5081 if (queue >= vi->max_queue_pairs)
5084 /* Can't change NAPI weight if the link is up */
5085 napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
5086 ret = virtnet_should_update_vq_weight(dev->flags, napi_weight,
5087 vi->sq[queue].napi.weight,
5092 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
5093 ret = virtnet_send_notf_coal_vq_cmds(vi, ec, queue);
5095 ret = virtnet_coal_params_supported(ec);
5101 vi->sq[queue].napi.weight = napi_weight;
5106 static int virtnet_get_per_queue_coalesce(struct net_device *dev,
5108 struct ethtool_coalesce *ec)
5110 struct virtnet_info *vi = netdev_priv(dev);
5112 if (queue >= vi->max_queue_pairs)
5115 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) {
5116 mutex_lock(&vi->rq[queue].dim_lock);
5117 ec->rx_coalesce_usecs = vi->rq[queue].intr_coal.max_usecs;
5118 ec->tx_coalesce_usecs = vi->sq[queue].intr_coal.max_usecs;
5119 ec->tx_max_coalesced_frames = vi->sq[queue].intr_coal.max_packets;
5120 ec->rx_max_coalesced_frames = vi->rq[queue].intr_coal.max_packets;
5121 ec->use_adaptive_rx_coalesce = vi->rq[queue].dim_enabled;
5122 mutex_unlock(&vi->rq[queue].dim_lock);
5124 ec->rx_max_coalesced_frames = 1;
5126 if (vi->sq[queue].napi.weight)
5127 ec->tx_max_coalesced_frames = 1;
5133 static void virtnet_init_settings(struct net_device *dev)
5135 struct virtnet_info *vi = netdev_priv(dev);
5137 vi->speed = SPEED_UNKNOWN;
5138 vi->duplex = DUPLEX_UNKNOWN;
5141 static u32 virtnet_get_rxfh_key_size(struct net_device *dev)
5143 return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size;
5146 static u32 virtnet_get_rxfh_indir_size(struct net_device *dev)
5148 return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size;
5151 static int virtnet_get_rxfh(struct net_device *dev,
5152 struct ethtool_rxfh_param *rxfh)
5154 struct virtnet_info *vi = netdev_priv(dev);
5158 for (i = 0; i < vi->rss_indir_table_size; ++i)
5159 rxfh->indir[i] = vi->rss.indirection_table[i];
5163 memcpy(rxfh->key, vi->rss.key, vi->rss_key_size);
5165 rxfh->hfunc = ETH_RSS_HASH_TOP;
5170 static int virtnet_set_rxfh(struct net_device *dev,
5171 struct ethtool_rxfh_param *rxfh,
5172 struct netlink_ext_ack *extack)
5174 struct virtnet_info *vi = netdev_priv(dev);
5175 bool update = false;
5178 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
5179 rxfh->hfunc != ETH_RSS_HASH_TOP)
5186 for (i = 0; i < vi->rss_indir_table_size; ++i)
5187 vi->rss.indirection_table[i] = rxfh->indir[i];
5192 /* If either _F_HASH_REPORT or _F_RSS are negotiated, the
5193 * device provides hash calculation capabilities, that is,
5194 * hash_key is configured.
5196 if (!vi->has_rss && !vi->has_rss_hash_report)
5199 memcpy(vi->rss.key, rxfh->key, vi->rss_key_size);
5204 virtnet_commit_rss_command(vi);
5209 static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs)
5211 struct virtnet_info *vi = netdev_priv(dev);
5214 switch (info->cmd) {
5215 case ETHTOOL_GRXRINGS:
5216 info->data = vi->curr_queue_pairs;
5219 virtnet_get_hashflow(vi, info);
5228 static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info)
5230 struct virtnet_info *vi = netdev_priv(dev);
5233 switch (info->cmd) {
5235 if (!virtnet_set_hashflow(vi, info))
5246 static const struct ethtool_ops virtnet_ethtool_ops = {
5247 .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES |
5248 ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
5249 .get_drvinfo = virtnet_get_drvinfo,
5250 .get_link = ethtool_op_get_link,
5251 .get_ringparam = virtnet_get_ringparam,
5252 .set_ringparam = virtnet_set_ringparam,
5253 .get_strings = virtnet_get_strings,
5254 .get_sset_count = virtnet_get_sset_count,
5255 .get_ethtool_stats = virtnet_get_ethtool_stats,
5256 .set_channels = virtnet_set_channels,
5257 .get_channels = virtnet_get_channels,
5258 .get_ts_info = ethtool_op_get_ts_info,
5259 .get_link_ksettings = virtnet_get_link_ksettings,
5260 .set_link_ksettings = virtnet_set_link_ksettings,
5261 .set_coalesce = virtnet_set_coalesce,
5262 .get_coalesce = virtnet_get_coalesce,
5263 .set_per_queue_coalesce = virtnet_set_per_queue_coalesce,
5264 .get_per_queue_coalesce = virtnet_get_per_queue_coalesce,
5265 .get_rxfh_key_size = virtnet_get_rxfh_key_size,
5266 .get_rxfh_indir_size = virtnet_get_rxfh_indir_size,
5267 .get_rxfh = virtnet_get_rxfh,
5268 .set_rxfh = virtnet_set_rxfh,
5269 .get_rxnfc = virtnet_get_rxnfc,
5270 .set_rxnfc = virtnet_set_rxnfc,
5273 static void virtnet_get_queue_stats_rx(struct net_device *dev, int i,
5274 struct netdev_queue_stats_rx *stats)
5276 struct virtnet_info *vi = netdev_priv(dev);
5277 struct receive_queue *rq = &vi->rq[i];
5278 struct virtnet_stats_ctx ctx = {0};
5280 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true);
5282 virtnet_get_hw_stats(vi, &ctx, i * 2);
5283 virtnet_fill_stats(vi, i * 2, &ctx, (void *)&rq->stats, true, 0);
5286 static void virtnet_get_queue_stats_tx(struct net_device *dev, int i,
5287 struct netdev_queue_stats_tx *stats)
5289 struct virtnet_info *vi = netdev_priv(dev);
5290 struct send_queue *sq = &vi->sq[i];
5291 struct virtnet_stats_ctx ctx = {0};
5293 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true);
5295 virtnet_get_hw_stats(vi, &ctx, i * 2 + 1);
5296 virtnet_fill_stats(vi, i * 2 + 1, &ctx, (void *)&sq->stats, true, 0);
5299 static void virtnet_get_base_stats(struct net_device *dev,
5300 struct netdev_queue_stats_rx *rx,
5301 struct netdev_queue_stats_tx *tx)
5303 struct virtnet_info *vi = netdev_priv(dev);
5305 /* The queue stats of the virtio-net will not be reset. So here we
5311 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
5313 rx->hw_drop_overruns = 0;
5316 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
5317 rx->csum_unnecessary = 0;
5322 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) {
5323 rx->hw_gro_packets = 0;
5324 rx->hw_gro_bytes = 0;
5325 rx->hw_gro_wire_packets = 0;
5326 rx->hw_gro_wire_bytes = 0;
5329 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED)
5330 rx->hw_drop_ratelimits = 0;
5337 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
5339 tx->hw_drop_errors = 0;
5342 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) {
5347 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
5348 tx->hw_gso_packets = 0;
5349 tx->hw_gso_bytes = 0;
5350 tx->hw_gso_wire_packets = 0;
5351 tx->hw_gso_wire_bytes = 0;
5354 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED)
5355 tx->hw_drop_ratelimits = 0;
5358 static const struct netdev_stat_ops virtnet_stat_ops = {
5359 .get_queue_stats_rx = virtnet_get_queue_stats_rx,
5360 .get_queue_stats_tx = virtnet_get_queue_stats_tx,
5361 .get_base_stats = virtnet_get_base_stats,
5364 static void virtnet_freeze_down(struct virtio_device *vdev)
5366 struct virtnet_info *vi = vdev->priv;
5368 /* Make sure no work handler is accessing the device */
5369 flush_work(&vi->config_work);
5370 disable_rx_mode_work(vi);
5371 flush_work(&vi->rx_mode_work);
5373 netif_tx_lock_bh(vi->dev);
5374 netif_device_detach(vi->dev);
5375 netif_tx_unlock_bh(vi->dev);
5376 if (netif_running(vi->dev))
5377 virtnet_close(vi->dev);
5380 static int init_vqs(struct virtnet_info *vi);
5382 static int virtnet_restore_up(struct virtio_device *vdev)
5384 struct virtnet_info *vi = vdev->priv;
5391 virtio_device_ready(vdev);
5393 enable_delayed_refill(vi);
5394 enable_rx_mode_work(vi);
5396 if (netif_running(vi->dev)) {
5397 err = virtnet_open(vi->dev);
5402 netif_tx_lock_bh(vi->dev);
5403 netif_device_attach(vi->dev);
5404 netif_tx_unlock_bh(vi->dev);
5408 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
5410 __virtio64 *_offloads __free(kfree) = NULL;
5411 struct scatterlist sg;
5413 _offloads = kzalloc(sizeof(*_offloads), GFP_KERNEL);
5417 *_offloads = cpu_to_virtio64(vi->vdev, offloads);
5419 sg_init_one(&sg, _offloads, sizeof(*_offloads));
5421 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
5422 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
5423 dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");
5430 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
5434 if (!vi->guest_offloads)
5437 return virtnet_set_guest_offloads(vi, offloads);
5440 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
5442 u64 offloads = vi->guest_offloads;
5444 if (!vi->guest_offloads)
5447 return virtnet_set_guest_offloads(vi, offloads);
5450 static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq,
5451 struct xsk_buff_pool *pool)
5455 qindex = rq - vi->rq;
5458 err = xdp_rxq_info_reg(&rq->xsk_rxq_info, vi->dev, qindex, rq->napi.napi_id);
5462 err = xdp_rxq_info_reg_mem_model(&rq->xsk_rxq_info,
5463 MEM_TYPE_XSK_BUFF_POOL, NULL);
5467 xsk_pool_set_rxq_info(pool, &rq->xsk_rxq_info);
5470 virtnet_rx_pause(vi, rq);
5472 err = virtqueue_reset(rq->vq, virtnet_rq_unmap_free_buf);
5474 netdev_err(vi->dev, "reset rx fail: rx queue index: %d err: %d\n", qindex, err);
5479 rq->xsk_pool = pool;
5481 virtnet_rx_resume(vi, rq);
5487 xdp_rxq_info_unreg(&rq->xsk_rxq_info);
5491 static int virtnet_xsk_pool_enable(struct net_device *dev,
5492 struct xsk_buff_pool *pool,
5495 struct virtnet_info *vi = netdev_priv(dev);
5496 struct receive_queue *rq;
5497 struct device *dma_dev;
5498 struct send_queue *sq;
5501 if (vi->hdr_len > xsk_pool_get_headroom(pool))
5504 /* In big_packets mode, xdp cannot work, so there is no need to
5505 * initialize xsk of rq.
5507 if (vi->big_packets && !vi->mergeable_rx_bufs)
5510 if (qid >= vi->curr_queue_pairs)
5516 /* xsk assumes that tx and rx must have the same dma device. The af-xdp
5517 * may use one buffer to receive from the rx and reuse this buffer to
5518 * send by the tx. So the dma dev of sq and rq must be the same one.
5520 * But vq->dma_dev allows every vq has the respective dma dev. So I
5521 * check the dma dev of vq and sq is the same dev.
5523 if (virtqueue_dma_dev(rq->vq) != virtqueue_dma_dev(sq->vq))
5526 dma_dev = virtqueue_dma_dev(rq->vq);
5530 size = virtqueue_get_vring_size(rq->vq);
5532 rq->xsk_buffs = kvcalloc(size, sizeof(*rq->xsk_buffs), GFP_KERNEL);
5536 err = xsk_pool_dma_map(pool, dma_dev, 0);
5540 err = virtnet_rq_bind_xsk_pool(vi, rq, pool);
5547 xsk_pool_dma_unmap(pool, 0);
5552 static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
5554 struct virtnet_info *vi = netdev_priv(dev);
5555 struct xsk_buff_pool *pool;
5556 struct receive_queue *rq;
5559 if (qid >= vi->curr_queue_pairs)
5564 pool = rq->xsk_pool;
5566 err = virtnet_rq_bind_xsk_pool(vi, rq, NULL);
5568 xsk_pool_dma_unmap(pool, 0);
5570 kvfree(rq->xsk_buffs);
5575 static int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp)
5578 return virtnet_xsk_pool_enable(dev, xdp->xsk.pool,
5581 return virtnet_xsk_pool_disable(dev, xdp->xsk.queue_id);
5584 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
5585 struct netlink_ext_ack *extack)
5587 unsigned int room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM +
5588 sizeof(struct skb_shared_info));
5589 unsigned int max_sz = PAGE_SIZE - room - ETH_HLEN;
5590 struct virtnet_info *vi = netdev_priv(dev);
5591 struct bpf_prog *old_prog;
5592 u16 xdp_qp = 0, curr_qp;
5595 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
5596 && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
5597 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
5598 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
5599 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
5600 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) ||
5601 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) ||
5602 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) {
5603 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first");
5607 if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
5608 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
5612 if (prog && !prog->aux->xdp_has_frags && dev->mtu > max_sz) {
5613 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP without frags");
5614 netdev_warn(dev, "single-buffer XDP requires MTU less than %u\n", max_sz);
5618 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
5620 xdp_qp = nr_cpu_ids;
5622 /* XDP requires extra queues for XDP_TX */
5623 if (curr_qp + xdp_qp > vi->max_queue_pairs) {
5624 netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n",
5625 curr_qp + xdp_qp, vi->max_queue_pairs);
5629 old_prog = rtnl_dereference(vi->rq[0].xdp_prog);
5630 if (!prog && !old_prog)
5634 bpf_prog_add(prog, vi->max_queue_pairs - 1);
5636 /* Make sure NAPI is not using any XDP TX queues for RX. */
5637 if (netif_running(dev)) {
5638 for (i = 0; i < vi->max_queue_pairs; i++) {
5639 napi_disable(&vi->rq[i].napi);
5640 virtnet_napi_tx_disable(&vi->sq[i].napi);
5645 for (i = 0; i < vi->max_queue_pairs; i++) {
5646 rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
5648 virtnet_restore_guest_offloads(vi);
5653 err = virtnet_set_queues(vi, curr_qp + xdp_qp);
5656 netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
5657 vi->xdp_queue_pairs = xdp_qp;
5660 vi->xdp_enabled = true;
5661 for (i = 0; i < vi->max_queue_pairs; i++) {
5662 rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
5663 if (i == 0 && !old_prog)
5664 virtnet_clear_guest_offloads(vi);
5667 xdp_features_set_redirect_target(dev, true);
5669 xdp_features_clear_redirect_target(dev);
5670 vi->xdp_enabled = false;
5673 for (i = 0; i < vi->max_queue_pairs; i++) {
5675 bpf_prog_put(old_prog);
5676 if (netif_running(dev)) {
5677 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
5678 virtnet_napi_tx_enable(vi, vi->sq[i].vq,
5687 virtnet_clear_guest_offloads(vi);
5688 for (i = 0; i < vi->max_queue_pairs; i++)
5689 rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);
5692 if (netif_running(dev)) {
5693 for (i = 0; i < vi->max_queue_pairs; i++) {
5694 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
5695 virtnet_napi_tx_enable(vi, vi->sq[i].vq,
5700 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
5704 static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
5706 switch (xdp->command) {
5707 case XDP_SETUP_PROG:
5708 return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
5709 case XDP_SETUP_XSK_POOL:
5710 return virtnet_xsk_pool_setup(dev, xdp);
5716 static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
5719 struct virtnet_info *vi = netdev_priv(dev);
5722 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
5725 ret = snprintf(buf, len, "sby");
5732 static int virtnet_set_features(struct net_device *dev,
5733 netdev_features_t features)
5735 struct virtnet_info *vi = netdev_priv(dev);
5739 if ((dev->features ^ features) & NETIF_F_GRO_HW) {
5740 if (vi->xdp_enabled)
5743 if (features & NETIF_F_GRO_HW)
5744 offloads = vi->guest_offloads_capable;
5746 offloads = vi->guest_offloads_capable &
5747 ~GUEST_OFFLOAD_GRO_HW_MASK;
5749 err = virtnet_set_guest_offloads(vi, offloads);
5752 vi->guest_offloads = offloads;
5755 if ((dev->features ^ features) & NETIF_F_RXHASH) {
5756 if (features & NETIF_F_RXHASH)
5757 vi->rss.hash_types = vi->rss_hash_types_saved;
5759 vi->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE;
5761 if (!virtnet_commit_rss_command(vi))
5768 static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue)
5770 struct virtnet_info *priv = netdev_priv(dev);
5771 struct send_queue *sq = &priv->sq[txqueue];
5772 struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
5774 u64_stats_update_begin(&sq->stats.syncp);
5775 u64_stats_inc(&sq->stats.tx_timeouts);
5776 u64_stats_update_end(&sq->stats.syncp);
5778 netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n",
5779 txqueue, sq->name, sq->vq->index, sq->vq->name,
5780 jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start)));
5783 static int virtnet_init_irq_moder(struct virtnet_info *vi)
5785 u8 profile_flags = 0, coal_flags = 0;
5788 profile_flags |= DIM_PROFILE_RX;
5789 coal_flags |= DIM_COALESCE_USEC | DIM_COALESCE_PKTS;
5790 ret = net_dim_init_irq_moder(vi->dev, profile_flags, coal_flags,
5791 DIM_CQ_PERIOD_MODE_START_FROM_EQE,
5792 0, virtnet_rx_dim_work, NULL);
5797 for (i = 0; i < vi->max_queue_pairs; i++)
5798 net_dim_setting(vi->dev, &vi->rq[i].dim, false);
5803 static void virtnet_free_irq_moder(struct virtnet_info *vi)
5805 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
5809 net_dim_free_irq_moder(vi->dev);
5813 static const struct net_device_ops virtnet_netdev = {
5814 .ndo_open = virtnet_open,
5815 .ndo_stop = virtnet_close,
5816 .ndo_start_xmit = start_xmit,
5817 .ndo_validate_addr = eth_validate_addr,
5818 .ndo_set_mac_address = virtnet_set_mac_address,
5819 .ndo_set_rx_mode = virtnet_set_rx_mode,
5820 .ndo_get_stats64 = virtnet_stats,
5821 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
5822 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
5823 .ndo_bpf = virtnet_xdp,
5824 .ndo_xdp_xmit = virtnet_xdp_xmit,
5825 .ndo_xsk_wakeup = virtnet_xsk_wakeup,
5826 .ndo_features_check = passthru_features_check,
5827 .ndo_get_phys_port_name = virtnet_get_phys_port_name,
5828 .ndo_set_features = virtnet_set_features,
5829 .ndo_tx_timeout = virtnet_tx_timeout,
5832 static void virtnet_config_changed_work(struct work_struct *work)
5834 struct virtnet_info *vi =
5835 container_of(work, struct virtnet_info, config_work);
5838 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
5839 struct virtio_net_config, status, &v) < 0)
5842 if (v & VIRTIO_NET_S_ANNOUNCE) {
5843 netdev_notify_peers(vi->dev);
5844 virtnet_ack_link_announce(vi);
5847 /* Ignore unknown (future) status bits */
5848 v &= VIRTIO_NET_S_LINK_UP;
5850 if (vi->status == v)
5855 if (vi->status & VIRTIO_NET_S_LINK_UP) {
5856 virtnet_update_settings(vi);
5857 netif_carrier_on(vi->dev);
5858 netif_tx_wake_all_queues(vi->dev);
5860 netif_carrier_off(vi->dev);
5861 netif_tx_stop_all_queues(vi->dev);
5865 static void virtnet_config_changed(struct virtio_device *vdev)
5867 struct virtnet_info *vi = vdev->priv;
5869 schedule_work(&vi->config_work);
5872 static void virtnet_free_queues(struct virtnet_info *vi)
5876 for (i = 0; i < vi->max_queue_pairs; i++) {
5877 __netif_napi_del(&vi->rq[i].napi);
5878 __netif_napi_del(&vi->sq[i].napi);
5881 /* We called __netif_napi_del(),
5882 * we need to respect an RCU grace period before freeing vi->rq
5891 static void _free_receive_bufs(struct virtnet_info *vi)
5893 struct bpf_prog *old_prog;
5896 for (i = 0; i < vi->max_queue_pairs; i++) {
5897 while (vi->rq[i].pages)
5898 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
5900 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
5901 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
5903 bpf_prog_put(old_prog);
5907 static void free_receive_bufs(struct virtnet_info *vi)
5910 _free_receive_bufs(vi);
5914 static void free_receive_page_frags(struct virtnet_info *vi)
5917 for (i = 0; i < vi->max_queue_pairs; i++)
5918 if (vi->rq[i].alloc_frag.page) {
5919 if (vi->rq[i].do_dma && vi->rq[i].last_dma)
5920 virtnet_rq_unmap(&vi->rq[i], vi->rq[i].last_dma, 0);
5921 put_page(vi->rq[i].alloc_frag.page);
5925 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
5927 if (!is_xdp_frame(buf))
5930 xdp_return_frame(ptr_to_xdp(buf));
5933 static void free_unused_bufs(struct virtnet_info *vi)
5938 for (i = 0; i < vi->max_queue_pairs; i++) {
5939 struct virtqueue *vq = vi->sq[i].vq;
5940 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
5941 virtnet_sq_free_unused_buf(vq, buf);
5945 for (i = 0; i < vi->max_queue_pairs; i++) {
5946 struct virtqueue *vq = vi->rq[i].vq;
5948 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
5949 virtnet_rq_unmap_free_buf(vq, buf);
5954 static void virtnet_del_vqs(struct virtnet_info *vi)
5956 struct virtio_device *vdev = vi->vdev;
5958 virtnet_clean_affinity(vi);
5960 vdev->config->del_vqs(vdev);
5962 virtnet_free_queues(vi);
5965 /* How large should a single buffer be so a queue full of these can fit at
5966 * least one full packet?
5967 * Logic below assumes the mergeable buffer header is used.
5969 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
5971 const unsigned int hdr_len = vi->hdr_len;
5972 unsigned int rq_size = virtqueue_get_vring_size(vq);
5973 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
5974 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
5975 unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
5977 return max(max(min_buf_len, hdr_len) - hdr_len,
5978 (unsigned int)GOOD_PACKET_LEN);
5981 static int virtnet_find_vqs(struct virtnet_info *vi)
5983 struct virtqueue_info *vqs_info;
5984 struct virtqueue **vqs;
5990 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
5991 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
5992 * possible control vq.
5994 total_vqs = vi->max_queue_pairs * 2 +
5995 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
5997 /* Allocate space for find_vqs parameters */
5998 vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
6001 vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL);
6004 if (!vi->big_packets || vi->mergeable_rx_bufs) {
6005 ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
6012 /* Parameters for control virtqueue, if any */
6014 vqs_info[total_vqs - 1].name = "control";
6017 /* Allocate/initialize parameters for send/receive virtqueues */
6018 for (i = 0; i < vi->max_queue_pairs; i++) {
6019 vqs_info[rxq2vq(i)].callback = skb_recv_done;
6020 vqs_info[txq2vq(i)].callback = skb_xmit_done;
6021 sprintf(vi->rq[i].name, "input.%u", i);
6022 sprintf(vi->sq[i].name, "output.%u", i);
6023 vqs_info[rxq2vq(i)].name = vi->rq[i].name;
6024 vqs_info[txq2vq(i)].name = vi->sq[i].name;
6026 vqs_info[rxq2vq(i)].ctx = true;
6029 ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL);
6034 vi->cvq = vqs[total_vqs - 1];
6035 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
6036 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
6039 for (i = 0; i < vi->max_queue_pairs; i++) {
6040 vi->rq[i].vq = vqs[rxq2vq(i)];
6041 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
6042 vi->sq[i].vq = vqs[txq2vq(i)];
6045 /* run here: ret == 0. */
6058 static int virtnet_alloc_queues(struct virtnet_info *vi)
6063 vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
6069 vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);
6072 vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);
6076 INIT_DELAYED_WORK(&vi->refill, refill_work);
6077 for (i = 0; i < vi->max_queue_pairs; i++) {
6078 vi->rq[i].pages = NULL;
6079 netif_napi_add_weight(vi->dev, &vi->rq[i].napi, virtnet_poll,
6081 netif_napi_add_tx_weight(vi->dev, &vi->sq[i].napi,
6083 napi_tx ? napi_weight : 0);
6085 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
6086 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
6087 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
6089 u64_stats_init(&vi->rq[i].stats.syncp);
6090 u64_stats_init(&vi->sq[i].stats.syncp);
6091 mutex_init(&vi->rq[i].dim_lock);
6104 static int init_vqs(struct virtnet_info *vi)
6108 /* Allocate send & receive queues */
6109 ret = virtnet_alloc_queues(vi);
6113 ret = virtnet_find_vqs(vi);
6118 virtnet_set_affinity(vi);
6124 virtnet_free_queues(vi);
6130 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
6133 struct virtnet_info *vi = netdev_priv(queue->dev);
6134 unsigned int queue_index = get_netdev_rx_queue_index(queue);
6135 unsigned int headroom = virtnet_get_headroom(vi);
6136 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
6137 struct ewma_pkt_len *avg;
6139 BUG_ON(queue_index >= vi->max_queue_pairs);
6140 avg = &vi->rq[queue_index].mrg_avg_pkt_len;
6141 return sprintf(buf, "%u\n",
6142 get_mergeable_buf_len(&vi->rq[queue_index], avg,
6143 SKB_DATA_ALIGN(headroom + tailroom)));
6146 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
6147 __ATTR_RO(mergeable_rx_buffer_size);
6149 static struct attribute *virtio_net_mrg_rx_attrs[] = {
6150 &mergeable_rx_buffer_size_attribute.attr,
6154 static const struct attribute_group virtio_net_mrg_rx_group = {
6155 .name = "virtio_net",
6156 .attrs = virtio_net_mrg_rx_attrs
6160 static bool virtnet_fail_on_feature(struct virtio_device *vdev,
6162 const char *fname, const char *dname)
6164 if (!virtio_has_feature(vdev, fbit))
6167 dev_err(&vdev->dev, "device advertises feature %s but not %s",
6173 #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \
6174 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
6176 static bool virtnet_validate_features(struct virtio_device *vdev)
6178 if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
6179 (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
6180 "VIRTIO_NET_F_CTRL_VQ") ||
6181 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
6182 "VIRTIO_NET_F_CTRL_VQ") ||
6183 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
6184 "VIRTIO_NET_F_CTRL_VQ") ||
6185 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
6186 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
6187 "VIRTIO_NET_F_CTRL_VQ") ||
6188 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
6189 "VIRTIO_NET_F_CTRL_VQ") ||
6190 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
6191 "VIRTIO_NET_F_CTRL_VQ") ||
6192 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL,
6193 "VIRTIO_NET_F_CTRL_VQ") ||
6194 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_VQ_NOTF_COAL,
6195 "VIRTIO_NET_F_CTRL_VQ"))) {
6202 #define MIN_MTU ETH_MIN_MTU
6203 #define MAX_MTU ETH_MAX_MTU
6205 static int virtnet_validate(struct virtio_device *vdev)
6207 if (!vdev->config->get) {
6208 dev_err(&vdev->dev, "%s failure: config access disabled\n",
6213 if (!virtnet_validate_features(vdev))
6216 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
6217 int mtu = virtio_cread16(vdev,
6218 offsetof(struct virtio_net_config,
6221 __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
6224 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY) &&
6225 !virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
6226 dev_warn(&vdev->dev, "device advertises feature VIRTIO_NET_F_STANDBY but not VIRTIO_NET_F_MAC, disabling standby");
6227 __virtio_clear_bit(vdev, VIRTIO_NET_F_STANDBY);
6233 static bool virtnet_check_guest_gso(const struct virtnet_info *vi)
6235 return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
6236 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
6237 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
6238 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
6239 (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) &&
6240 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6));
6243 static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu)
6245 bool guest_gso = virtnet_check_guest_gso(vi);
6247 /* If device can receive ANY guest GSO packets, regardless of mtu,
6248 * allocate packets of maximum size, otherwise limit it to only
6249 * mtu size worth only.
6251 if (mtu > ETH_DATA_LEN || guest_gso) {
6252 vi->big_packets = true;
6253 vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE);
6257 #define VIRTIO_NET_HASH_REPORT_MAX_TABLE 10
6258 static enum xdp_rss_hash_type
6259 virtnet_xdp_rss_type[VIRTIO_NET_HASH_REPORT_MAX_TABLE] = {
6260 [VIRTIO_NET_HASH_REPORT_NONE] = XDP_RSS_TYPE_NONE,
6261 [VIRTIO_NET_HASH_REPORT_IPv4] = XDP_RSS_TYPE_L3_IPV4,
6262 [VIRTIO_NET_HASH_REPORT_TCPv4] = XDP_RSS_TYPE_L4_IPV4_TCP,
6263 [VIRTIO_NET_HASH_REPORT_UDPv4] = XDP_RSS_TYPE_L4_IPV4_UDP,
6264 [VIRTIO_NET_HASH_REPORT_IPv6] = XDP_RSS_TYPE_L3_IPV6,
6265 [VIRTIO_NET_HASH_REPORT_TCPv6] = XDP_RSS_TYPE_L4_IPV6_TCP,
6266 [VIRTIO_NET_HASH_REPORT_UDPv6] = XDP_RSS_TYPE_L4_IPV6_UDP,
6267 [VIRTIO_NET_HASH_REPORT_IPv6_EX] = XDP_RSS_TYPE_L3_IPV6_EX,
6268 [VIRTIO_NET_HASH_REPORT_TCPv6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX,
6269 [VIRTIO_NET_HASH_REPORT_UDPv6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX
6272 static int virtnet_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
6273 enum xdp_rss_hash_type *rss_type)
6275 const struct xdp_buff *xdp = (void *)_ctx;
6276 struct virtio_net_hdr_v1_hash *hdr_hash;
6277 struct virtnet_info *vi;
6280 if (!(xdp->rxq->dev->features & NETIF_F_RXHASH))
6283 vi = netdev_priv(xdp->rxq->dev);
6284 hdr_hash = (struct virtio_net_hdr_v1_hash *)(xdp->data - vi->hdr_len);
6285 hash_report = __le16_to_cpu(hdr_hash->hash_report);
6287 if (hash_report >= VIRTIO_NET_HASH_REPORT_MAX_TABLE)
6288 hash_report = VIRTIO_NET_HASH_REPORT_NONE;
6290 *rss_type = virtnet_xdp_rss_type[hash_report];
6291 *hash = __le32_to_cpu(hdr_hash->hash_value);
6295 static const struct xdp_metadata_ops virtnet_xdp_metadata_ops = {
6296 .xmo_rx_hash = virtnet_xdp_rx_hash,
6299 static int virtnet_probe(struct virtio_device *vdev)
6301 int i, err = -ENOMEM;
6302 struct net_device *dev;
6303 struct virtnet_info *vi;
6304 u16 max_queue_pairs;
6307 /* Find if host supports multiqueue/rss virtio_net device */
6308 max_queue_pairs = 1;
6309 if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
6311 virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs));
6313 /* We need at least 2 queue's */
6314 if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
6315 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
6316 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
6317 max_queue_pairs = 1;
6319 /* Allocate ourselves a network device with room for our info */
6320 dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
6324 /* Set up network device as normal. */
6325 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE |
6326 IFF_TX_SKB_NO_LINEAR;
6327 dev->netdev_ops = &virtnet_netdev;
6328 dev->stat_ops = &virtnet_stat_ops;
6329 dev->features = NETIF_F_HIGHDMA;
6331 dev->ethtool_ops = &virtnet_ethtool_ops;
6332 SET_NETDEV_DEV(dev, &vdev->dev);
6334 /* Do we support "hardware" checksums? */
6335 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
6336 /* This opens up the world of extra features. */
6337 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
6339 dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
6341 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
6342 dev->hw_features |= NETIF_F_TSO
6343 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
6345 /* Individual feature bits: what can host handle? */
6346 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
6347 dev->hw_features |= NETIF_F_TSO;
6348 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
6349 dev->hw_features |= NETIF_F_TSO6;
6350 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
6351 dev->hw_features |= NETIF_F_TSO_ECN;
6352 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_USO))
6353 dev->hw_features |= NETIF_F_GSO_UDP_L4;
6355 dev->features |= NETIF_F_GSO_ROBUST;
6358 dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
6359 /* (!csum && gso) case will be fixed by register_netdev() */
6362 /* 1. With VIRTIO_NET_F_GUEST_CSUM negotiation, the driver doesn't
6363 * need to calculate checksums for partially checksummed packets,
6364 * as they're considered valid by the upper layer.
6365 * 2. Without VIRTIO_NET_F_GUEST_CSUM negotiation, the driver only
6366 * receives fully checksummed packets. The device may assist in
6367 * validating these packets' checksums, so the driver won't have to.
6369 dev->features |= NETIF_F_RXCSUM;
6371 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
6372 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6))
6373 dev->features |= NETIF_F_GRO_HW;
6374 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))
6375 dev->hw_features |= NETIF_F_GRO_HW;
6377 dev->vlan_features = dev->features;
6378 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
6380 /* MTU range: 68 - 65535 */
6381 dev->min_mtu = MIN_MTU;
6382 dev->max_mtu = MAX_MTU;
6384 /* Configuration may specify what MAC to use. Otherwise random. */
6385 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
6388 virtio_cread_bytes(vdev,
6389 offsetof(struct virtio_net_config, mac),
6391 eth_hw_addr_set(dev, addr);
6393 eth_hw_addr_random(dev);
6394 dev_info(&vdev->dev, "Assigned random MAC address %pM\n",
6398 /* Set up our device-specific information */
6399 vi = netdev_priv(dev);
6404 INIT_WORK(&vi->config_work, virtnet_config_changed_work);
6405 INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work);
6406 spin_lock_init(&vi->refill_lock);
6408 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) {
6409 vi->mergeable_rx_bufs = true;
6410 dev->xdp_features |= NETDEV_XDP_ACT_RX_SG;
6413 if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
6414 vi->has_rss_hash_report = true;
6416 if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) {
6419 vi->rss_indir_table_size =
6420 virtio_cread16(vdev, offsetof(struct virtio_net_config,
6421 rss_max_indirection_table_length));
6424 if (vi->has_rss || vi->has_rss_hash_report) {
6426 virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size));
6428 vi->rss_hash_types_supported =
6429 virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types));
6430 vi->rss_hash_types_supported &=
6431 ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX |
6432 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX |
6433 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX);
6435 dev->hw_features |= NETIF_F_RXHASH;
6436 dev->xdp_metadata_ops = &virtnet_xdp_metadata_ops;
6439 if (vi->has_rss_hash_report)
6440 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash);
6441 else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
6442 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
6443 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
6445 vi->hdr_len = sizeof(struct virtio_net_hdr);
6447 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
6448 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
6449 vi->any_header_sg = true;
6451 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
6454 mutex_init(&vi->cvq_lock);
6456 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
6457 mtu = virtio_cread16(vdev,
6458 offsetof(struct virtio_net_config,
6460 if (mtu < dev->min_mtu) {
6461 /* Should never trigger: MTU was previously validated
6462 * in virtnet_validate.
6465 "device MTU appears to have changed it is now %d < %d",
6475 virtnet_set_big_packets(vi, mtu);
6477 if (vi->any_header_sg)
6478 dev->needed_headroom = vi->hdr_len;
6480 /* Enable multiqueue by default */
6481 if (num_online_cpus() >= max_queue_pairs)
6482 vi->curr_queue_pairs = max_queue_pairs;
6484 vi->curr_queue_pairs = num_online_cpus();
6485 vi->max_queue_pairs = max_queue_pairs;
6487 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
6492 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
6493 vi->intr_coal_rx.max_usecs = 0;
6494 vi->intr_coal_tx.max_usecs = 0;
6495 vi->intr_coal_rx.max_packets = 0;
6497 /* Keep the default values of the coalescing parameters
6498 * aligned with the default napi_tx state.
6500 if (vi->sq[0].napi.weight)
6501 vi->intr_coal_tx.max_packets = 1;
6503 vi->intr_coal_tx.max_packets = 0;
6506 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) {
6507 /* The reason is the same as VIRTIO_NET_F_NOTF_COAL. */
6508 for (i = 0; i < vi->max_queue_pairs; i++)
6509 if (vi->sq[i].napi.weight)
6510 vi->sq[i].intr_coal.max_packets = 1;
6512 err = virtnet_init_irq_moder(vi);
6518 if (vi->mergeable_rx_bufs)
6519 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
6521 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
6522 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
6524 virtnet_init_settings(dev);
6526 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
6527 vi->failover = net_failover_create(vi->dev);
6528 if (IS_ERR(vi->failover)) {
6529 err = PTR_ERR(vi->failover);
6534 if (vi->has_rss || vi->has_rss_hash_report)
6535 virtnet_init_default_rss(vi);
6537 enable_rx_mode_work(vi);
6539 /* serialize netdev register + virtio_device_ready() with ndo_open() */
6542 err = register_netdevice(dev);
6544 pr_debug("virtio_net: registering device failed\n");
6549 /* Disable config change notification until ndo_open. */
6550 virtio_config_driver_disable(vi->vdev);
6552 virtio_device_ready(vdev);
6554 virtnet_set_queues(vi, vi->curr_queue_pairs);
6556 /* a random MAC address has been assigned, notify the device.
6557 * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there
6558 * because many devices work fine without getting MAC explicitly
6560 if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
6561 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
6562 struct scatterlist sg;
6564 sg_init_one(&sg, dev->dev_addr, dev->addr_len);
6565 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
6566 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
6567 pr_debug("virtio_net: setting MAC address failed\n");
6570 goto free_unregister_netdev;
6574 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) {
6575 struct virtio_net_stats_capabilities *stats_cap __free(kfree) = NULL;
6576 struct scatterlist sg;
6579 stats_cap = kzalloc(sizeof(*stats_cap), GFP_KERNEL);
6583 goto free_unregister_netdev;
6586 sg_init_one(&sg, stats_cap, sizeof(*stats_cap));
6588 if (!virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS,
6589 VIRTIO_NET_CTRL_STATS_QUERY,
6591 pr_debug("virtio_net: fail to get stats capability\n");
6594 goto free_unregister_netdev;
6597 v = stats_cap->supported_stats_types[0];
6598 vi->device_stats_cap = le64_to_cpu(v);
6601 /* Assume link up if device can't report link status,
6602 otherwise get link status from config. */
6603 netif_carrier_off(dev);
6604 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
6605 virtnet_config_changed_work(&vi->config_work);
6607 vi->status = VIRTIO_NET_S_LINK_UP;
6608 virtnet_update_settings(vi);
6609 netif_carrier_on(dev);
6612 for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
6613 if (virtio_has_feature(vi->vdev, guest_offloads[i]))
6614 set_bit(guest_offloads[i], &vi->guest_offloads);
6615 vi->guest_offloads_capable = vi->guest_offloads;
6619 err = virtnet_cpu_notif_add(vi);
6621 pr_debug("virtio_net: registering cpu notifier failed\n");
6622 goto free_unregister_netdev;
6625 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
6626 dev->name, max_queue_pairs);
6630 free_unregister_netdev:
6631 unregister_netdev(dev);
6633 net_failover_destroy(vi->failover);
6635 virtio_reset_device(vdev);
6636 cancel_delayed_work_sync(&vi->refill);
6637 free_receive_page_frags(vi);
6638 virtnet_del_vqs(vi);
6644 static void remove_vq_common(struct virtnet_info *vi)
6646 virtio_reset_device(vi->vdev);
6648 /* Free unused buffers in both send and recv, if any. */
6649 free_unused_bufs(vi);
6651 free_receive_bufs(vi);
6653 free_receive_page_frags(vi);
6655 virtnet_del_vqs(vi);
6658 static void virtnet_remove(struct virtio_device *vdev)
6660 struct virtnet_info *vi = vdev->priv;
6662 virtnet_cpu_notif_remove(vi);
6664 /* Make sure no work handler is accessing the device. */
6665 flush_work(&vi->config_work);
6666 disable_rx_mode_work(vi);
6667 flush_work(&vi->rx_mode_work);
6669 virtnet_free_irq_moder(vi);
6671 unregister_netdev(vi->dev);
6673 net_failover_destroy(vi->failover);
6675 remove_vq_common(vi);
6677 free_netdev(vi->dev);
6680 static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
6682 struct virtnet_info *vi = vdev->priv;
6684 virtnet_cpu_notif_remove(vi);
6685 virtnet_freeze_down(vdev);
6686 remove_vq_common(vi);
6691 static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
6693 struct virtnet_info *vi = vdev->priv;
6696 err = virtnet_restore_up(vdev);
6699 virtnet_set_queues(vi, vi->curr_queue_pairs);
6701 err = virtnet_cpu_notif_add(vi);
6703 virtnet_freeze_down(vdev);
6704 remove_vq_common(vi);
6711 static struct virtio_device_id id_table[] = {
6712 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
6716 #define VIRTNET_FEATURES \
6717 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
6719 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
6720 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
6721 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
6722 VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_GUEST_USO4, VIRTIO_NET_F_GUEST_USO6, \
6723 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
6724 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
6725 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
6726 VIRTIO_NET_F_CTRL_MAC_ADDR, \
6727 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
6728 VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
6729 VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL, \
6730 VIRTIO_NET_F_VQ_NOTF_COAL, \
6731 VIRTIO_NET_F_GUEST_HDRLEN, VIRTIO_NET_F_DEVICE_STATS
6733 static unsigned int features[] = {
6737 static unsigned int features_legacy[] = {
6740 VIRTIO_F_ANY_LAYOUT,
6743 static struct virtio_driver virtio_net_driver = {
6744 .feature_table = features,
6745 .feature_table_size = ARRAY_SIZE(features),
6746 .feature_table_legacy = features_legacy,
6747 .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
6748 .driver.name = KBUILD_MODNAME,
6749 .id_table = id_table,
6750 .validate = virtnet_validate,
6751 .probe = virtnet_probe,
6752 .remove = virtnet_remove,
6753 .config_changed = virtnet_config_changed,
6754 #ifdef CONFIG_PM_SLEEP
6755 .freeze = virtnet_freeze,
6756 .restore = virtnet_restore,
6760 static __init int virtio_net_driver_init(void)
6764 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
6766 virtnet_cpu_down_prep);
6769 virtionet_online = ret;
6770 ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
6771 NULL, virtnet_cpu_dead);
6774 ret = register_virtio_driver(&virtio_net_driver);
6779 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
6781 cpuhp_remove_multi_state(virtionet_online);
6785 module_init(virtio_net_driver_init);
6787 static __exit void virtio_net_driver_exit(void)
6789 unregister_virtio_driver(&virtio_net_driver);
6790 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
6791 cpuhp_remove_multi_state(virtionet_online);
6793 module_exit(virtio_net_driver_exit);
6795 MODULE_DEVICE_TABLE(virtio, id_table);
6796 MODULE_DESCRIPTION("Virtio network driver");
6797 MODULE_LICENSE("GPL");