drivers/net/virtio_net.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* A network driver using virtio.
   3  *
   4  * Copyright 2007 Rusty Russell <[email protected]> IBM Corporation
   5  */
   6 //#define DEBUG
   7 #include <linux/netdevice.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/ethtool.h>
  10 #include <linux/module.h>
  11 #include <linux/virtio.h>
  12 #include <linux/virtio_net.h>
  13 #include <linux/bpf.h>
  14 #include <linux/bpf_trace.h>
  15 #include <linux/scatterlist.h>
  16 #include <linux/if_vlan.h>
  17 #include <linux/slab.h>
  18 #include <linux/cpu.h>
  19 #include <linux/average.h>
  20 #include <linux/filter.h>
  21 #include <linux/kernel.h>
  22 #include <linux/dim.h>
  23 #include <net/route.h>
  24 #include <net/xdp.h>
  25 #include <net/net_failover.h>
  26 #include <net/netdev_rx_queue.h>
  27 #include <net/netdev_queues.h>
  28 #include <net/xdp_sock_drv.h>
  29
  30 static int napi_weight = NAPI_POLL_WEIGHT;
  31 module_param(napi_weight, int, 0444);
  32
  33 static bool csum = true, gso = true, napi_tx = true;
  34 module_param(csum, bool, 0444);
  35 module_param(gso, bool, 0444);
  36 module_param(napi_tx, bool, 0644);
  37
  38 /* FIXME: MTU in config. */
  39 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  40 #define GOOD_COPY_LEN   128
  41
  42 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
  43
  44 /* Separating two types of XDP xmit */
  45 #define VIRTIO_XDP_TX           BIT(0)
  46 #define VIRTIO_XDP_REDIR        BIT(1)
  47
  48 #define VIRTIO_XDP_FLAG         BIT(0)
  49 #define VIRTIO_ORPHAN_FLAG      BIT(1)
  50
  51 /* RX packet size EWMA. The average packet size is used to determine the packet
  52  * buffer size when refilling RX rings. As the entire RX ring may be refilled
  53  * at once, the weight is chosen so that the EWMA will be insensitive to short-
  54  * term, transient changes in packet size.
  55  */
  56 DECLARE_EWMA(pkt_len, 0, 64)
  57
  58 #define VIRTNET_DRIVER_VERSION "1.0.0"
  59
  60 static const unsigned long guest_offloads[] = {
  61         VIRTIO_NET_F_GUEST_TSO4,
  62         VIRTIO_NET_F_GUEST_TSO6,
  63         VIRTIO_NET_F_GUEST_ECN,
  64         VIRTIO_NET_F_GUEST_UFO,
  65         VIRTIO_NET_F_GUEST_CSUM,
  66         VIRTIO_NET_F_GUEST_USO4,
  67         VIRTIO_NET_F_GUEST_USO6,
  68         VIRTIO_NET_F_GUEST_HDRLEN
  69 };
  70
  71 #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
  72                                 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
  73                                 (1ULL << VIRTIO_NET_F_GUEST_ECN)  | \
  74                                 (1ULL << VIRTIO_NET_F_GUEST_UFO)  | \
  75                                 (1ULL << VIRTIO_NET_F_GUEST_USO4) | \
  76                                 (1ULL << VIRTIO_NET_F_GUEST_USO6))
  77
  78 struct virtnet_stat_desc {
  79         char desc[ETH_GSTRING_LEN];
  80         size_t offset;
  81         size_t qstat_offset;
  82 };
  83
  84 struct virtnet_sq_free_stats {
  85         u64 packets;
  86         u64 bytes;
  87         u64 napi_packets;
  88         u64 napi_bytes;
  89 };
  90
  91 struct virtnet_sq_stats {
  92         struct u64_stats_sync syncp;
  93         u64_stats_t packets;
  94         u64_stats_t bytes;
  95         u64_stats_t xdp_tx;
  96         u64_stats_t xdp_tx_drops;
  97         u64_stats_t kicks;
  98         u64_stats_t tx_timeouts;
  99         u64_stats_t stop;
 100         u64_stats_t wake;
 101 };
 102
 103 struct virtnet_rq_stats {
 104         struct u64_stats_sync syncp;
 105         u64_stats_t packets;
 106         u64_stats_t bytes;
 107         u64_stats_t drops;
 108         u64_stats_t xdp_packets;
 109         u64_stats_t xdp_tx;
 110         u64_stats_t xdp_redirects;
 111         u64_stats_t xdp_drops;
 112         u64_stats_t kicks;
 113 };
 114
 115 #define VIRTNET_SQ_STAT(name, m) {name, offsetof(struct virtnet_sq_stats, m), -1}
 116 #define VIRTNET_RQ_STAT(name, m) {name, offsetof(struct virtnet_rq_stats, m), -1}
 117
 118 #define VIRTNET_SQ_STAT_QSTAT(name, m)                          \
 119         {                                                       \
 120                 name,                                           \
 121                 offsetof(struct virtnet_sq_stats, m),           \
 122                 offsetof(struct netdev_queue_stats_tx, m),      \
 123         }
 124
 125 #define VIRTNET_RQ_STAT_QSTAT(name, m)                          \
 126         {                                                       \
 127                 name,                                           \
 128                 offsetof(struct virtnet_rq_stats, m),           \
 129                 offsetof(struct netdev_queue_stats_rx, m),      \
 130         }
 131
 132 static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
 133         VIRTNET_SQ_STAT("xdp_tx",       xdp_tx),
 134         VIRTNET_SQ_STAT("xdp_tx_drops", xdp_tx_drops),
 135         VIRTNET_SQ_STAT("kicks",        kicks),
 136         VIRTNET_SQ_STAT("tx_timeouts",  tx_timeouts),
 137 };
 138
 139 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
 140         VIRTNET_RQ_STAT("drops",         drops),
 141         VIRTNET_RQ_STAT("xdp_packets",   xdp_packets),
 142         VIRTNET_RQ_STAT("xdp_tx",        xdp_tx),
 143         VIRTNET_RQ_STAT("xdp_redirects", xdp_redirects),
 144         VIRTNET_RQ_STAT("xdp_drops",     xdp_drops),
 145         VIRTNET_RQ_STAT("kicks",         kicks),
 146 };
 147
 148 static const struct virtnet_stat_desc virtnet_sq_stats_desc_qstat[] = {
 149         VIRTNET_SQ_STAT_QSTAT("packets", packets),
 150         VIRTNET_SQ_STAT_QSTAT("bytes",   bytes),
 151         VIRTNET_SQ_STAT_QSTAT("stop",    stop),
 152         VIRTNET_SQ_STAT_QSTAT("wake",    wake),
 153 };
 154
 155 static const struct virtnet_stat_desc virtnet_rq_stats_desc_qstat[] = {
 156         VIRTNET_RQ_STAT_QSTAT("packets", packets),
 157         VIRTNET_RQ_STAT_QSTAT("bytes",   bytes),
 158 };
 159
 160 #define VIRTNET_STATS_DESC_CQ(name) \
 161         {#name, offsetof(struct virtio_net_stats_cvq, name), -1}
 162
 163 #define VIRTNET_STATS_DESC_RX(class, name) \
 164         {#name, offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), -1}
 165
 166 #define VIRTNET_STATS_DESC_TX(class, name) \
 167         {#name, offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), -1}
 168
 169
 170 static const struct virtnet_stat_desc virtnet_stats_cvq_desc[] = {
 171         VIRTNET_STATS_DESC_CQ(command_num),
 172         VIRTNET_STATS_DESC_CQ(ok_num),
 173 };
 174
 175 static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc[] = {
 176         VIRTNET_STATS_DESC_RX(basic, packets),
 177         VIRTNET_STATS_DESC_RX(basic, bytes),
 178
 179         VIRTNET_STATS_DESC_RX(basic, notifications),
 180         VIRTNET_STATS_DESC_RX(basic, interrupts),
 181 };
 182
 183 static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc[] = {
 184         VIRTNET_STATS_DESC_TX(basic, packets),
 185         VIRTNET_STATS_DESC_TX(basic, bytes),
 186
 187         VIRTNET_STATS_DESC_TX(basic, notifications),
 188         VIRTNET_STATS_DESC_TX(basic, interrupts),
 189 };
 190
 191 static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc[] = {
 192         VIRTNET_STATS_DESC_RX(csum, needs_csum),
 193 };
 194
 195 static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc[] = {
 196         VIRTNET_STATS_DESC_TX(gso, gso_packets_noseg),
 197         VIRTNET_STATS_DESC_TX(gso, gso_bytes_noseg),
 198 };
 199
 200 static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc[] = {
 201         VIRTNET_STATS_DESC_RX(speed, ratelimit_bytes),
 202 };
 203
 204 static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc[] = {
 205         VIRTNET_STATS_DESC_TX(speed, ratelimit_bytes),
 206 };
 207
 208 #define VIRTNET_STATS_DESC_RX_QSTAT(class, name, qstat_field)                   \
 209         {                                                                       \
 210                 #name,                                                          \
 211                 offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name),    \
 212                 offsetof(struct netdev_queue_stats_rx, qstat_field),            \
 213         }
 214
 215 #define VIRTNET_STATS_DESC_TX_QSTAT(class, name, qstat_field)                   \
 216         {                                                                       \
 217                 #name,                                                          \
 218                 offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name),    \
 219                 offsetof(struct netdev_queue_stats_tx, qstat_field),            \
 220         }
 221
 222 static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc_qstat[] = {
 223         VIRTNET_STATS_DESC_RX_QSTAT(basic, drops,         hw_drops),
 224         VIRTNET_STATS_DESC_RX_QSTAT(basic, drop_overruns, hw_drop_overruns),
 225 };
 226
 227 static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc_qstat[] = {
 228         VIRTNET_STATS_DESC_TX_QSTAT(basic, drops,          hw_drops),
 229         VIRTNET_STATS_DESC_TX_QSTAT(basic, drop_malformed, hw_drop_errors),
 230 };
 231
 232 static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc_qstat[] = {
 233         VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_valid, csum_unnecessary),
 234         VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_none,  csum_none),
 235         VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_bad,   csum_bad),
 236 };
 237
 238 static const struct virtnet_stat_desc virtnet_stats_tx_csum_desc_qstat[] = {
 239         VIRTNET_STATS_DESC_TX_QSTAT(csum, csum_none,  csum_none),
 240         VIRTNET_STATS_DESC_TX_QSTAT(csum, needs_csum, needs_csum),
 241 };
 242
 243 static const struct virtnet_stat_desc virtnet_stats_rx_gso_desc_qstat[] = {
 244         VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets,           hw_gro_packets),
 245         VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes,             hw_gro_bytes),
 246         VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets_coalesced, hw_gro_wire_packets),
 247         VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes_coalesced,   hw_gro_wire_bytes),
 248 };
 249
 250 static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc_qstat[] = {
 251         VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_packets,        hw_gso_packets),
 252         VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_bytes,          hw_gso_bytes),
 253         VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments,       hw_gso_wire_packets),
 254         VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments_bytes, hw_gso_wire_bytes),
 255 };
 256
 257 static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc_qstat[] = {
 258         VIRTNET_STATS_DESC_RX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits),
 259 };
 260
 261 static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc_qstat[] = {
 262         VIRTNET_STATS_DESC_TX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits),
 263 };
 264
 265 #define VIRTNET_Q_TYPE_RX 0
 266 #define VIRTNET_Q_TYPE_TX 1
 267 #define VIRTNET_Q_TYPE_CQ 2
 268
 269 struct virtnet_interrupt_coalesce {
 270         u32 max_packets;
 271         u32 max_usecs;
 272 };
 273
 274 /* The dma information of pages allocated at a time. */
 275 struct virtnet_rq_dma {
 276         dma_addr_t addr;
 277         u32 ref;
 278         u16 len;
 279         u16 need_sync;
 280 };
 281
 282 /* Internal representation of a send virtqueue */
 283 struct send_queue {
 284         /* Virtqueue associated with this send _queue */
 285         struct virtqueue *vq;
 286
 287         /* TX: fragments + linear part + virtio header */
 288         struct scatterlist sg[MAX_SKB_FRAGS + 2];
 289
 290         /* Name of the send queue: output.$index */
 291         char name[16];
 292
 293         struct virtnet_sq_stats stats;
 294
 295         struct virtnet_interrupt_coalesce intr_coal;
 296
 297         struct napi_struct napi;
 298
 299         /* Record whether sq is in reset state. */
 300         bool reset;
 301 };
 302
 303 /* Internal representation of a receive virtqueue */
 304 struct receive_queue {
 305         /* Virtqueue associated with this receive_queue */
 306         struct virtqueue *vq;
 307
 308         struct napi_struct napi;
 309
 310         struct bpf_prog __rcu *xdp_prog;
 311
 312         struct virtnet_rq_stats stats;
 313
 314         /* The number of rx notifications */
 315         u16 calls;
 316
 317         /* Is dynamic interrupt moderation enabled? */
 318         bool dim_enabled;
 319
 320         /* Used to protect dim_enabled and inter_coal */
 321         struct mutex dim_lock;
 322
 323         /* Dynamic Interrupt Moderation */
 324         struct dim dim;
 325
 326         u32 packets_in_napi;
 327
 328         struct virtnet_interrupt_coalesce intr_coal;
 329
 330         /* Chain pages by the private ptr. */
 331         struct page *pages;
 332
 333         /* Average packet length for mergeable receive buffers. */
 334         struct ewma_pkt_len mrg_avg_pkt_len;
 335
 336         /* Page frag for packet buffer allocation. */
 337         struct page_frag alloc_frag;
 338
 339         /* RX: fragments + linear part + virtio header */
 340         struct scatterlist sg[MAX_SKB_FRAGS + 2];
 341
 342         /* Min single buffer size for mergeable buffers case. */
 343         unsigned int min_buf_len;
 344
 345         /* Name of this receive queue: input.$index */
 346         char name[16];
 347
 348         struct xdp_rxq_info xdp_rxq;
 349
 350         /* Record the last dma info to free after new pages is allocated. */
 351         struct virtnet_rq_dma *last_dma;
 352
 353         struct xsk_buff_pool *xsk_pool;
 354
 355         /* xdp rxq used by xsk */
 356         struct xdp_rxq_info xsk_rxq_info;
 357
 358         struct xdp_buff **xsk_buffs;
 359
 360         /* Do dma by self */
 361         bool do_dma;
 362 };
 363
 364 /* This structure can contain rss message with maximum settings for indirection table and keysize
 365  * Note, that default structure that describes RSS configuration virtio_net_rss_config
 366  * contains same info but can't handle table values.
 367  * In any case, structure would be passed to virtio hw through sg_buf split by parts
 368  * because table sizes may be differ according to the device configuration.
 369  */
 370 #define VIRTIO_NET_RSS_MAX_KEY_SIZE     40
 371 #define VIRTIO_NET_RSS_MAX_TABLE_LEN    128
 372 struct virtio_net_ctrl_rss {
 373         u32 hash_types;
 374         u16 indirection_table_mask;
 375         u16 unclassified_queue;
 376         u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
 377         u16 max_tx_vq;
 378         u8 hash_key_length;
 379         u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE];
 380 };
 381
 382 /* Control VQ buffers: protected by the rtnl lock */
 383 struct control_buf {
 384         struct virtio_net_ctrl_hdr hdr;
 385         virtio_net_ctrl_ack status;
 386 };
 387
 388 struct virtnet_info {
 389         struct virtio_device *vdev;
 390         struct virtqueue *cvq;
 391         struct net_device *dev;
 392         struct send_queue *sq;
 393         struct receive_queue *rq;
 394         unsigned int status;
 395
 396         /* Max # of queue pairs supported by the device */
 397         u16 max_queue_pairs;
 398
 399         /* # of queue pairs currently used by the driver */
 400         u16 curr_queue_pairs;
 401
 402         /* # of XDP queue pairs currently used by the driver */
 403         u16 xdp_queue_pairs;
 404
 405         /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
 406         bool xdp_enabled;
 407
 408         /* I like... big packets and I cannot lie! */
 409         bool big_packets;
 410
 411         /* number of sg entries allocated for big packets */
 412         unsigned int big_packets_num_skbfrags;
 413
 414         /* Host will merge rx buffers for big packets (shake it! shake it!) */
 415         bool mergeable_rx_bufs;
 416
 417         /* Host supports rss and/or hash report */
 418         bool has_rss;
 419         bool has_rss_hash_report;
 420         u8 rss_key_size;
 421         u16 rss_indir_table_size;
 422         u32 rss_hash_types_supported;
 423         u32 rss_hash_types_saved;
 424         struct virtio_net_ctrl_rss rss;
 425
 426         /* Has control virtqueue */
 427         bool has_cvq;
 428
 429         /* Lock to protect the control VQ */
 430         struct mutex cvq_lock;
 431
 432         /* Host can handle any s/g split between our header and packet data */
 433         bool any_header_sg;
 434
 435         /* Packet virtio header size */
 436         u8 hdr_len;
 437
 438         /* Work struct for delayed refilling if we run low on memory. */
 439         struct delayed_work refill;
 440
 441         /* Is delayed refill enabled? */
 442         bool refill_enabled;
 443
 444         /* The lock to synchronize the access to refill_enabled */
 445         spinlock_t refill_lock;
 446
 447         /* Work struct for config space updates */
 448         struct work_struct config_work;
 449
 450         /* Work struct for setting rx mode */
 451         struct work_struct rx_mode_work;
 452
 453         /* OK to queue work setting RX mode? */
 454         bool rx_mode_work_enabled;
 455
 456         /* Does the affinity hint is set for virtqueues? */
 457         bool affinity_hint_set;
 458
 459         /* CPU hotplug instances for online & dead */
 460         struct hlist_node node;
 461         struct hlist_node node_dead;
 462
 463         struct control_buf *ctrl;
 464
 465         /* Ethtool settings */
 466         u8 duplex;
 467         u32 speed;
 468
 469         /* Is rx dynamic interrupt moderation enabled? */
 470         bool rx_dim_enabled;
 471
 472         /* Interrupt coalescing settings */
 473         struct virtnet_interrupt_coalesce intr_coal_tx;
 474         struct virtnet_interrupt_coalesce intr_coal_rx;
 475
 476         unsigned long guest_offloads;
 477         unsigned long guest_offloads_capable;
 478
 479         /* failover when STANDBY feature enabled */
 480         struct failover *failover;
 481
 482         u64 device_stats_cap;
 483 };
 484
 485 struct padded_vnet_hdr {
 486         struct virtio_net_hdr_v1_hash hdr;
 487         /*
 488          * hdr is in a separate sg buffer, and data sg buffer shares same page
 489          * with this header sg. This padding makes next sg 16 byte aligned
 490          * after the header.
 491          */
 492         char padding[12];
 493 };
 494
 495 struct virtio_net_common_hdr {
 496         union {
 497                 struct virtio_net_hdr hdr;
 498                 struct virtio_net_hdr_mrg_rxbuf mrg_hdr;
 499                 struct virtio_net_hdr_v1_hash hash_v1_hdr;
 500         };
 501 };
 502
 503 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
 504 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
 505                                struct net_device *dev,
 506                                unsigned int *xdp_xmit,
 507                                struct virtnet_rq_stats *stats);
 508 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq,
 509                                  struct sk_buff *skb, u8 flags);
 510 static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
 511                                                struct sk_buff *curr_skb,
 512                                                struct page *page, void *buf,
 513                                                int len, int truesize);
 514
 515 static bool is_xdp_frame(void *ptr)
 516 {
 517         return (unsigned long)ptr & VIRTIO_XDP_FLAG;
 518 }
 519
 520 static void *xdp_to_ptr(struct xdp_frame *ptr)
 521 {
 522         return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
 523 }
 524
 525 static struct xdp_frame *ptr_to_xdp(void *ptr)
 526 {
 527         return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
 528 }
 529
 530 static bool is_orphan_skb(void *ptr)
 531 {
 532         return (unsigned long)ptr & VIRTIO_ORPHAN_FLAG;
 533 }
 534
 535 static void *skb_to_ptr(struct sk_buff *skb, bool orphan)
 536 {
 537         return (void *)((unsigned long)skb | (orphan ? VIRTIO_ORPHAN_FLAG : 0));
 538 }
 539
 540 static struct sk_buff *ptr_to_skb(void *ptr)
 541 {
 542         return (struct sk_buff *)((unsigned long)ptr & ~VIRTIO_ORPHAN_FLAG);
 543 }
 544
 545 static void __free_old_xmit(struct send_queue *sq, struct netdev_queue *txq,
 546                             bool in_napi, struct virtnet_sq_free_stats *stats)
 547 {
 548         unsigned int len;
 549         void *ptr;
 550
 551         while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
 552                 if (!is_xdp_frame(ptr)) {
 553                         struct sk_buff *skb = ptr_to_skb(ptr);
 554
 555                         pr_debug("Sent skb %p\n", skb);
 556
 557                         if (is_orphan_skb(ptr)) {
 558                                 stats->packets++;
 559                                 stats->bytes += skb->len;
 560                         } else {
 561                                 stats->napi_packets++;
 562                                 stats->napi_bytes += skb->len;
 563                         }
 564                         napi_consume_skb(skb, in_napi);
 565                 } else {
 566                         struct xdp_frame *frame = ptr_to_xdp(ptr);
 567
 568                         stats->packets++;
 569                         stats->bytes += xdp_get_frame_len(frame);
 570                         xdp_return_frame(frame);
 571                 }
 572         }
 573         netdev_tx_completed_queue(txq, stats->napi_packets, stats->napi_bytes);
 574 }
 575
 576 /* Converting between virtqueue no. and kernel tx/rx queue no.
 577  * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 578  */
 579 static int vq2txq(struct virtqueue *vq)
 580 {
 581         return (vq->index - 1) / 2;
 582 }
 583
 584 static int txq2vq(int txq)
 585 {
 586         return txq * 2 + 1;
 587 }
 588
 589 static int vq2rxq(struct virtqueue *vq)
 590 {
 591         return vq->index / 2;
 592 }
 593
 594 static int rxq2vq(int rxq)
 595 {
 596         return rxq * 2;
 597 }
 598
 599 static int vq_type(struct virtnet_info *vi, int qid)
 600 {
 601         if (qid == vi->max_queue_pairs * 2)
 602                 return VIRTNET_Q_TYPE_CQ;
 603
 604         if (qid % 2)
 605                 return VIRTNET_Q_TYPE_TX;
 606
 607         return VIRTNET_Q_TYPE_RX;
 608 }
 609
 610 static inline struct virtio_net_common_hdr *
 611 skb_vnet_common_hdr(struct sk_buff *skb)
 612 {
 613         return (struct virtio_net_common_hdr *)skb->cb;
 614 }
 615
 616 /*
 617  * private is used to chain pages for big packets, put the whole
 618  * most recent used list in the beginning for reuse
 619  */
 620 static void give_pages(struct receive_queue *rq, struct page *page)
 621 {
 622         struct page *end;
 623
 624         /* Find end of list, sew whole thing into vi->rq.pages. */
 625         for (end = page; end->private; end = (struct page *)end->private);
 626         end->private = (unsigned long)rq->pages;
 627         rq->pages = page;
 628 }
 629
 630 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
 631 {
 632         struct page *p = rq->pages;
 633
 634         if (p) {
 635                 rq->pages = (struct page *)p->private;
 636                 /* clear private here, it is used to chain pages */
 637                 p->private = 0;
 638         } else
 639                 p = alloc_page(gfp_mask);
 640         return p;
 641 }
 642
 643 static void virtnet_rq_free_buf(struct virtnet_info *vi,
 644                                 struct receive_queue *rq, void *buf)
 645 {
 646         if (vi->mergeable_rx_bufs)
 647                 put_page(virt_to_head_page(buf));
 648         else if (vi->big_packets)
 649                 give_pages(rq, buf);
 650         else
 651                 put_page(virt_to_head_page(buf));
 652 }
 653
 654 static void enable_delayed_refill(struct virtnet_info *vi)
 655 {
 656         spin_lock_bh(&vi->refill_lock);
 657         vi->refill_enabled = true;
 658         spin_unlock_bh(&vi->refill_lock);
 659 }
 660
 661 static void disable_delayed_refill(struct virtnet_info *vi)
 662 {
 663         spin_lock_bh(&vi->refill_lock);
 664         vi->refill_enabled = false;
 665         spin_unlock_bh(&vi->refill_lock);
 666 }
 667
 668 static void enable_rx_mode_work(struct virtnet_info *vi)
 669 {
 670         rtnl_lock();
 671         vi->rx_mode_work_enabled = true;
 672         rtnl_unlock();
 673 }
 674
 675 static void disable_rx_mode_work(struct virtnet_info *vi)
 676 {
 677         rtnl_lock();
 678         vi->rx_mode_work_enabled = false;
 679         rtnl_unlock();
 680 }
 681
 682 static void virtqueue_napi_schedule(struct napi_struct *napi,
 683                                     struct virtqueue *vq)
 684 {
 685         if (napi_schedule_prep(napi)) {
 686                 virtqueue_disable_cb(vq);
 687                 __napi_schedule(napi);
 688         }
 689 }
 690
 691 static bool virtqueue_napi_complete(struct napi_struct *napi,
 692                                     struct virtqueue *vq, int processed)
 693 {
 694         int opaque;
 695
 696         opaque = virtqueue_enable_cb_prepare(vq);
 697         if (napi_complete_done(napi, processed)) {
 698                 if (unlikely(virtqueue_poll(vq, opaque)))
 699                         virtqueue_napi_schedule(napi, vq);
 700                 else
 701                         return true;
 702         } else {
 703                 virtqueue_disable_cb(vq);
 704         }
 705
 706         return false;
 707 }
 708
 709 static void skb_xmit_done(struct virtqueue *vq)
 710 {
 711         struct virtnet_info *vi = vq->vdev->priv;
 712         struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
 713
 714         /* Suppress further interrupts. */
 715         virtqueue_disable_cb(vq);
 716
 717         if (napi->weight)
 718                 virtqueue_napi_schedule(napi, vq);
 719         else
 720                 /* We were probably waiting for more output buffers. */
 721                 netif_wake_subqueue(vi->dev, vq2txq(vq));
 722 }
 723
 724 #define MRG_CTX_HEADER_SHIFT 22
 725 static void *mergeable_len_to_ctx(unsigned int truesize,
 726                                   unsigned int headroom)
 727 {
 728         return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
 729 }
 730
 731 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
 732 {
 733         return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
 734 }
 735
 736 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
 737 {
 738         return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
 739 }
 740
 741 static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen,
 742                                          unsigned int headroom,
 743                                          unsigned int len)
 744 {
 745         struct sk_buff *skb;
 746
 747         skb = build_skb(buf, buflen);
 748         if (unlikely(!skb))
 749                 return NULL;
 750
 751         skb_reserve(skb, headroom);
 752         skb_put(skb, len);
 753
 754         return skb;
 755 }
 756
 757 /* Called from bottom half context */
 758 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 759                                    struct receive_queue *rq,
 760                                    struct page *page, unsigned int offset,
 761                                    unsigned int len, unsigned int truesize,
 762                                    unsigned int headroom)
 763 {
 764         struct sk_buff *skb;
 765         struct virtio_net_common_hdr *hdr;
 766         unsigned int copy, hdr_len, hdr_padded_len;
 767         struct page *page_to_free = NULL;
 768         int tailroom, shinfo_size;
 769         char *p, *hdr_p, *buf;
 770
 771         p = page_address(page) + offset;
 772         hdr_p = p;
 773
 774         hdr_len = vi->hdr_len;
 775         if (vi->mergeable_rx_bufs)
 776                 hdr_padded_len = hdr_len;
 777         else
 778                 hdr_padded_len = sizeof(struct padded_vnet_hdr);
 779
 780         buf = p - headroom;
 781         len -= hdr_len;
 782         offset += hdr_padded_len;
 783         p += hdr_padded_len;
 784         tailroom = truesize - headroom  - hdr_padded_len - len;
 785
 786         shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 787
 788         if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
 789                 skb = virtnet_build_skb(buf, truesize, p - buf, len);
 790                 if (unlikely(!skb))
 791                         return NULL;
 792
 793                 page = (struct page *)page->private;
 794                 if (page)
 795                         give_pages(rq, page);
 796                 goto ok;
 797         }
 798
 799         /* copy small packet so we can reuse these pages for small data */
 800         skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
 801         if (unlikely(!skb))
 802                 return NULL;
 803
 804         /* Copy all frame if it fits skb->head, otherwise
 805          * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
 806          */
 807         if (len <= skb_tailroom(skb))
 808                 copy = len;
 809         else
 810                 copy = ETH_HLEN;
 811         skb_put_data(skb, p, copy);
 812
 813         len -= copy;
 814         offset += copy;
 815
 816         if (vi->mergeable_rx_bufs) {
 817                 if (len)
 818                         skb_add_rx_frag(skb, 0, page, offset, len, truesize);
 819                 else
 820                         page_to_free = page;
 821                 goto ok;
 822         }
 823
 824         /*
 825          * Verify that we can indeed put this data into a skb.
 826          * This is here to handle cases when the device erroneously
 827          * tries to receive more than is possible. This is usually
 828          * the case of a broken device.
 829          */
 830         if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
 831                 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
 832                 dev_kfree_skb(skb);
 833                 return NULL;
 834         }
 835         BUG_ON(offset >= PAGE_SIZE);
 836         while (len) {
 837                 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
 838                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
 839                                 frag_size, truesize);
 840                 len -= frag_size;
 841                 page = (struct page *)page->private;
 842                 offset = 0;
 843         }
 844
 845         if (page)
 846                 give_pages(rq, page);
 847
 848 ok:
 849         hdr = skb_vnet_common_hdr(skb);
 850         memcpy(hdr, hdr_p, hdr_len);
 851         if (page_to_free)
 852                 put_page(page_to_free);
 853
 854         return skb;
 855 }
 856
 857 static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
 858 {
 859         struct page *page = virt_to_head_page(buf);
 860         struct virtnet_rq_dma *dma;
 861         void *head;
 862         int offset;
 863
 864         head = page_address(page);
 865
 866         dma = head;
 867
 868         --dma->ref;
 869
 870         if (dma->need_sync && len) {
 871                 offset = buf - (head + sizeof(*dma));
 872
 873                 virtqueue_dma_sync_single_range_for_cpu(rq->vq, dma->addr,
 874                                                         offset, len,
 875                                                         DMA_FROM_DEVICE);
 876         }
 877
 878         if (dma->ref)
 879                 return;
 880
 881         virtqueue_dma_unmap_single_attrs(rq->vq, dma->addr, dma->len,
 882                                          DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
 883         put_page(page);
 884 }
 885
 886 static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
 887 {
 888         void *buf;
 889
 890         buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
 891         if (buf && rq->do_dma)
 892                 virtnet_rq_unmap(rq, buf, *len);
 893
 894         return buf;
 895 }
 896
 897 static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
 898 {
 899         struct virtnet_rq_dma *dma;
 900         dma_addr_t addr;
 901         u32 offset;
 902         void *head;
 903
 904         if (!rq->do_dma) {
 905                 sg_init_one(rq->sg, buf, len);
 906                 return;
 907         }
 908
 909         head = page_address(rq->alloc_frag.page);
 910
 911         offset = buf - head;
 912
 913         dma = head;
 914
 915         addr = dma->addr - sizeof(*dma) + offset;
 916
 917         sg_init_table(rq->sg, 1);
 918         rq->sg[0].dma_address = addr;
 919         rq->sg[0].length = len;
 920 }
 921
 922 static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp)
 923 {
 924         struct page_frag *alloc_frag = &rq->alloc_frag;
 925         struct virtnet_rq_dma *dma;
 926         void *buf, *head;
 927         dma_addr_t addr;
 928
 929         if (unlikely(!skb_page_frag_refill(size, alloc_frag, gfp)))
 930                 return NULL;
 931
 932         head = page_address(alloc_frag->page);
 933
 934         if (rq->do_dma) {
 935                 dma = head;
 936
 937                 /* new pages */
 938                 if (!alloc_frag->offset) {
 939                         if (rq->last_dma) {
 940                                 /* Now, the new page is allocated, the last dma
 941                                  * will not be used. So the dma can be unmapped
 942                                  * if the ref is 0.
 943                                  */
 944                                 virtnet_rq_unmap(rq, rq->last_dma, 0);
 945                                 rq->last_dma = NULL;
 946                         }
 947
 948                         dma->len = alloc_frag->size - sizeof(*dma);
 949
 950                         addr = virtqueue_dma_map_single_attrs(rq->vq, dma + 1,
 951                                                               dma->len, DMA_FROM_DEVICE, 0);
 952                         if (virtqueue_dma_mapping_error(rq->vq, addr))
 953                                 return NULL;
 954
 955                         dma->addr = addr;
 956                         dma->need_sync = virtqueue_dma_need_sync(rq->vq, addr);
 957
 958                         /* Add a reference to dma to prevent the entire dma from
 959                          * being released during error handling. This reference
 960                          * will be freed after the pages are no longer used.
 961                          */
 962                         get_page(alloc_frag->page);
 963                         dma->ref = 1;
 964                         alloc_frag->offset = sizeof(*dma);
 965
 966                         rq->last_dma = dma;
 967                 }
 968
 969                 ++dma->ref;
 970         }
 971
 972         buf = head + alloc_frag->offset;
 973
 974         get_page(alloc_frag->page);
 975         alloc_frag->offset += size;
 976
 977         return buf;
 978 }
 979
 980 static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf)
 981 {
 982         struct virtnet_info *vi = vq->vdev->priv;
 983         struct receive_queue *rq;
 984         int i = vq2rxq(vq);
 985
 986         rq = &vi->rq[i];
 987
 988         if (rq->xsk_pool) {
 989                 xsk_buff_free((struct xdp_buff *)buf);
 990                 return;
 991         }
 992
 993         if (rq->do_dma)
 994                 virtnet_rq_unmap(rq, buf, 0);
 995
 996         virtnet_rq_free_buf(vi, rq, buf);
 997 }
 998
 999 static void free_old_xmit(struct send_queue *sq, struct netdev_queue *txq,
1000                           bool in_napi)
1001 {
1002         struct virtnet_sq_free_stats stats = {0};
1003
1004         __free_old_xmit(sq, txq, in_napi, &stats);
1005
1006         /* Avoid overhead when no packets have been processed
1007          * happens when called speculatively from start_xmit.
1008          */
1009         if (!stats.packets && !stats.napi_packets)
1010                 return;
1011
1012         u64_stats_update_begin(&sq->stats.syncp);
1013         u64_stats_add(&sq->stats.bytes, stats.bytes + stats.napi_bytes);
1014         u64_stats_add(&sq->stats.packets, stats.packets + stats.napi_packets);
1015         u64_stats_update_end(&sq->stats.syncp);
1016 }
1017
1018 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
1019 {
1020         if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
1021                 return false;
1022         else if (q < vi->curr_queue_pairs)
1023                 return true;
1024         else
1025                 return false;
1026 }
1027
1028 static void check_sq_full_and_disable(struct virtnet_info *vi,
1029                                       struct net_device *dev,
1030                                       struct send_queue *sq)
1031 {
1032         bool use_napi = sq->napi.weight;
1033         int qnum;
1034
1035         qnum = sq - vi->sq;
1036
1037         /* If running out of space, stop queue to avoid getting packets that we
1038          * are then unable to transmit.
1039          * An alternative would be to force queuing layer to requeue the skb by
1040          * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1041          * returned in a normal path of operation: it means that driver is not
1042          * maintaining the TX queue stop/start state properly, and causes
1043          * the stack to do a non-trivial amount of useless work.
1044          * Since most packets only take 1 or 2 ring slots, stopping the queue
1045          * early means 16 slots are typically wasted.
1046          */
1047         if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1048                 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1049
1050                 netif_tx_stop_queue(txq);
1051                 u64_stats_update_begin(&sq->stats.syncp);
1052                 u64_stats_inc(&sq->stats.stop);
1053                 u64_stats_update_end(&sq->stats.syncp);
1054                 if (use_napi) {
1055                         if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
1056                                 virtqueue_napi_schedule(&sq->napi, sq->vq);
1057                 } else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1058                         /* More just got used, free them then recheck. */
1059                         free_old_xmit(sq, txq, false);
1060                         if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1061                                 netif_start_subqueue(dev, qnum);
1062                                 u64_stats_update_begin(&sq->stats.syncp);
1063                                 u64_stats_inc(&sq->stats.wake);
1064                                 u64_stats_update_end(&sq->stats.syncp);
1065                                 virtqueue_disable_cb(sq->vq);
1066                         }
1067                 }
1068         }
1069 }
1070
1071 static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
1072 {
1073         sg->dma_address = addr;
1074         sg->length = len;
1075 }
1076
1077 static struct xdp_buff *buf_to_xdp(struct virtnet_info *vi,
1078                                    struct receive_queue *rq, void *buf, u32 len)
1079 {
1080         struct xdp_buff *xdp;
1081         u32 bufsize;
1082
1083         xdp = (struct xdp_buff *)buf;
1084
1085         bufsize = xsk_pool_get_rx_frame_size(rq->xsk_pool) + vi->hdr_len;
1086
1087         if (unlikely(len > bufsize)) {
1088                 pr_debug("%s: rx error: len %u exceeds truesize %u\n",
1089                          vi->dev->name, len, bufsize);
1090                 DEV_STATS_INC(vi->dev, rx_length_errors);
1091                 xsk_buff_free(xdp);
1092                 return NULL;
1093         }
1094
1095         xsk_buff_set_size(xdp, len);
1096         xsk_buff_dma_sync_for_cpu(xdp);
1097
1098         return xdp;
1099 }
1100
1101 static struct sk_buff *xsk_construct_skb(struct receive_queue *rq,
1102                                          struct xdp_buff *xdp)
1103 {
1104         unsigned int metasize = xdp->data - xdp->data_meta;
1105         struct sk_buff *skb;
1106         unsigned int size;
1107
1108         size = xdp->data_end - xdp->data_hard_start;
1109         skb = napi_alloc_skb(&rq->napi, size);
1110         if (unlikely(!skb)) {
1111                 xsk_buff_free(xdp);
1112                 return NULL;
1113         }
1114
1115         skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);
1116
1117         size = xdp->data_end - xdp->data_meta;
1118         memcpy(__skb_put(skb, size), xdp->data_meta, size);
1119
1120         if (metasize) {
1121                 __skb_pull(skb, metasize);
1122                 skb_metadata_set(skb, metasize);
1123         }
1124
1125         xsk_buff_free(xdp);
1126
1127         return skb;
1128 }
1129
1130 static struct sk_buff *virtnet_receive_xsk_small(struct net_device *dev, struct virtnet_info *vi,
1131                                                  struct receive_queue *rq, struct xdp_buff *xdp,
1132                                                  unsigned int *xdp_xmit,
1133                                                  struct virtnet_rq_stats *stats)
1134 {
1135         struct bpf_prog *prog;
1136         u32 ret;
1137
1138         ret = XDP_PASS;
1139         rcu_read_lock();
1140         prog = rcu_dereference(rq->xdp_prog);
1141         if (prog)
1142                 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats);
1143         rcu_read_unlock();
1144
1145         switch (ret) {
1146         case XDP_PASS:
1147                 return xsk_construct_skb(rq, xdp);
1148
1149         case XDP_TX:
1150         case XDP_REDIRECT:
1151                 return NULL;
1152
1153         default:
1154                 /* drop packet */
1155                 xsk_buff_free(xdp);
1156                 u64_stats_inc(&stats->drops);
1157                 return NULL;
1158         }
1159 }
1160
1161 static void xsk_drop_follow_bufs(struct net_device *dev,
1162                                  struct receive_queue *rq,
1163                                  u32 num_buf,
1164                                  struct virtnet_rq_stats *stats)
1165 {
1166         struct xdp_buff *xdp;
1167         u32 len;
1168
1169         while (num_buf-- > 1) {
1170                 xdp = virtqueue_get_buf(rq->vq, &len);
1171                 if (unlikely(!xdp)) {
1172                         pr_debug("%s: rx error: %d buffers missing\n",
1173                                  dev->name, num_buf);
1174                         DEV_STATS_INC(dev, rx_length_errors);
1175                         break;
1176                 }
1177                 u64_stats_add(&stats->bytes, len);
1178                 xsk_buff_free(xdp);
1179         }
1180 }
1181
1182 static int xsk_append_merge_buffer(struct virtnet_info *vi,
1183                                    struct receive_queue *rq,
1184                                    struct sk_buff *head_skb,
1185                                    u32 num_buf,
1186                                    struct virtio_net_hdr_mrg_rxbuf *hdr,
1187                                    struct virtnet_rq_stats *stats)
1188 {
1189         struct sk_buff *curr_skb;
1190         struct xdp_buff *xdp;
1191         u32 len, truesize;
1192         struct page *page;
1193         void *buf;
1194
1195         curr_skb = head_skb;
1196
1197         while (--num_buf) {
1198                 buf = virtqueue_get_buf(rq->vq, &len);
1199                 if (unlikely(!buf)) {
1200                         pr_debug("%s: rx error: %d buffers out of %d missing\n",
1201                                  vi->dev->name, num_buf,
1202                                  virtio16_to_cpu(vi->vdev,
1203                                                  hdr->num_buffers));
1204                         DEV_STATS_INC(vi->dev, rx_length_errors);
1205                         return -EINVAL;
1206                 }
1207
1208                 u64_stats_add(&stats->bytes, len);
1209
1210                 xdp = buf_to_xdp(vi, rq, buf, len);
1211                 if (!xdp)
1212                         goto err;
1213
1214                 buf = napi_alloc_frag(len);
1215                 if (!buf) {
1216                         xsk_buff_free(xdp);
1217                         goto err;
1218                 }
1219
1220                 memcpy(buf, xdp->data - vi->hdr_len, len);
1221
1222                 xsk_buff_free(xdp);
1223
1224                 page = virt_to_page(buf);
1225
1226                 truesize = len;
1227
1228                 curr_skb  = virtnet_skb_append_frag(head_skb, curr_skb, page,
1229                                                     buf, len, truesize);
1230                 if (!curr_skb) {
1231                         put_page(page);
1232                         goto err;
1233                 }
1234         }
1235
1236         return 0;
1237
1238 err:
1239         xsk_drop_follow_bufs(vi->dev, rq, num_buf, stats);
1240         return -EINVAL;
1241 }
1242
1243 static struct sk_buff *virtnet_receive_xsk_merge(struct net_device *dev, struct virtnet_info *vi,
1244                                                  struct receive_queue *rq, struct xdp_buff *xdp,
1245                                                  unsigned int *xdp_xmit,
1246                                                  struct virtnet_rq_stats *stats)
1247 {
1248         struct virtio_net_hdr_mrg_rxbuf *hdr;
1249         struct bpf_prog *prog;
1250         struct sk_buff *skb;
1251         u32 ret, num_buf;
1252
1253         hdr = xdp->data - vi->hdr_len;
1254         num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
1255
1256         ret = XDP_PASS;
1257         rcu_read_lock();
1258         prog = rcu_dereference(rq->xdp_prog);
1259         /* TODO: support multi buffer. */
1260         if (prog && num_buf == 1)
1261                 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats);
1262         rcu_read_unlock();
1263
1264         switch (ret) {
1265         case XDP_PASS:
1266                 skb = xsk_construct_skb(rq, xdp);
1267                 if (!skb)
1268                         goto drop_bufs;
1269
1270                 if (xsk_append_merge_buffer(vi, rq, skb, num_buf, hdr, stats)) {
1271                         dev_kfree_skb(skb);
1272                         goto drop;
1273                 }
1274
1275                 return skb;
1276
1277         case XDP_TX:
1278         case XDP_REDIRECT:
1279                 return NULL;
1280
1281         default:
1282                 /* drop packet */
1283                 xsk_buff_free(xdp);
1284         }
1285
1286 drop_bufs:
1287         xsk_drop_follow_bufs(dev, rq, num_buf, stats);
1288
1289 drop:
1290         u64_stats_inc(&stats->drops);
1291         return NULL;
1292 }
1293
1294 static void virtnet_receive_xsk_buf(struct virtnet_info *vi, struct receive_queue *rq,
1295                                     void *buf, u32 len,
1296                                     unsigned int *xdp_xmit,
1297                                     struct virtnet_rq_stats *stats)
1298 {
1299         struct net_device *dev = vi->dev;
1300         struct sk_buff *skb = NULL;
1301         struct xdp_buff *xdp;
1302         u8 flags;
1303
1304         len -= vi->hdr_len;
1305
1306         u64_stats_add(&stats->bytes, len);
1307
1308         xdp = buf_to_xdp(vi, rq, buf, len);
1309         if (!xdp)
1310                 return;
1311
1312         if (unlikely(len < ETH_HLEN)) {
1313                 pr_debug("%s: short packet %i\n", dev->name, len);
1314                 DEV_STATS_INC(dev, rx_length_errors);
1315                 xsk_buff_free(xdp);
1316                 return;
1317         }
1318
1319         flags = ((struct virtio_net_common_hdr *)(xdp->data - vi->hdr_len))->hdr.flags;
1320
1321         if (!vi->mergeable_rx_bufs)
1322                 skb = virtnet_receive_xsk_small(dev, vi, rq, xdp, xdp_xmit, stats);
1323         else
1324                 skb = virtnet_receive_xsk_merge(dev, vi, rq, xdp, xdp_xmit, stats);
1325
1326         if (skb)
1327                 virtnet_receive_done(vi, rq, skb, flags);
1328 }
1329
1330 static int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct receive_queue *rq,
1331                                    struct xsk_buff_pool *pool, gfp_t gfp)
1332 {
1333         struct xdp_buff **xsk_buffs;
1334         dma_addr_t addr;
1335         int err = 0;
1336         u32 len, i;
1337         int num;
1338
1339         xsk_buffs = rq->xsk_buffs;
1340
1341         num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free);
1342         if (!num)
1343                 return -ENOMEM;
1344
1345         len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
1346
1347         for (i = 0; i < num; ++i) {
1348                 /* Use the part of XDP_PACKET_HEADROOM as the virtnet hdr space.
1349                  * We assume XDP_PACKET_HEADROOM is larger than hdr->len.
1350                  * (see function virtnet_xsk_pool_enable)
1351                  */
1352                 addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
1353
1354                 sg_init_table(rq->sg, 1);
1355                 sg_fill_dma(rq->sg, addr, len);
1356
1357                 err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
1358                 if (err)
1359                         goto err;
1360         }
1361
1362         return num;
1363
1364 err:
1365         for (; i < num; ++i)
1366                 xsk_buff_free(xsk_buffs[i]);
1367
1368         return err;
1369 }
1370
1371 static int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag)
1372 {
1373         struct virtnet_info *vi = netdev_priv(dev);
1374         struct send_queue *sq;
1375
1376         if (!netif_running(dev))
1377                 return -ENETDOWN;
1378
1379         if (qid >= vi->curr_queue_pairs)
1380                 return -EINVAL;
1381
1382         sq = &vi->sq[qid];
1383
1384         if (napi_if_scheduled_mark_missed(&sq->napi))
1385                 return 0;
1386
1387         local_bh_disable();
1388         virtqueue_napi_schedule(&sq->napi, sq->vq);
1389         local_bh_enable();
1390
1391         return 0;
1392 }
1393
1394 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
1395                                    struct send_queue *sq,
1396                                    struct xdp_frame *xdpf)
1397 {
1398         struct virtio_net_hdr_mrg_rxbuf *hdr;
1399         struct skb_shared_info *shinfo;
1400         u8 nr_frags = 0;
1401         int err, i;
1402
1403         if (unlikely(xdpf->headroom < vi->hdr_len))
1404                 return -EOVERFLOW;
1405
1406         if (unlikely(xdp_frame_has_frags(xdpf))) {
1407                 shinfo = xdp_get_shared_info_from_frame(xdpf);
1408                 nr_frags = shinfo->nr_frags;
1409         }
1410
1411         /* In wrapping function virtnet_xdp_xmit(), we need to free
1412          * up the pending old buffers, where we need to calculate the
1413          * position of skb_shared_info in xdp_get_frame_len() and
1414          * xdp_return_frame(), which will involve to xdpf->data and
1415          * xdpf->headroom. Therefore, we need to update the value of
1416          * headroom synchronously here.
1417          */
1418         xdpf->headroom -= vi->hdr_len;
1419         xdpf->data -= vi->hdr_len;
1420         /* Zero header and leave csum up to XDP layers */
1421         hdr = xdpf->data;
1422         memset(hdr, 0, vi->hdr_len);
1423         xdpf->len   += vi->hdr_len;
1424
1425         sg_init_table(sq->sg, nr_frags + 1);
1426         sg_set_buf(sq->sg, xdpf->data, xdpf->len);
1427         for (i = 0; i < nr_frags; i++) {
1428                 skb_frag_t *frag = &shinfo->frags[i];
1429
1430                 sg_set_page(&sq->sg[i + 1], skb_frag_page(frag),
1431                             skb_frag_size(frag), skb_frag_off(frag));
1432         }
1433
1434         err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
1435                                    xdp_to_ptr(xdpf), GFP_ATOMIC);
1436         if (unlikely(err))
1437                 return -ENOSPC; /* Caller handle free/refcnt */
1438
1439         return 0;
1440 }
1441
1442 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on
1443  * the current cpu, so it does not need to be locked.
1444  *
1445  * Here we use marco instead of inline functions because we have to deal with
1446  * three issues at the same time: 1. the choice of sq. 2. judge and execute the
1447  * lock/unlock of txq 3. make sparse happy. It is difficult for two inline
1448  * functions to perfectly solve these three problems at the same time.
1449  */
1450 #define virtnet_xdp_get_sq(vi) ({                                       \
1451         int cpu = smp_processor_id();                                   \
1452         struct netdev_queue *txq;                                       \
1453         typeof(vi) v = (vi);                                            \
1454         unsigned int qp;                                                \
1455                                                                         \
1456         if (v->curr_queue_pairs > nr_cpu_ids) {                         \
1457                 qp = v->curr_queue_pairs - v->xdp_queue_pairs;          \
1458                 qp += cpu;                                              \
1459                 txq = netdev_get_tx_queue(v->dev, qp);                  \
1460                 __netif_tx_acquire(txq);                                \
1461         } else {                                                        \
1462                 qp = cpu % v->curr_queue_pairs;                         \
1463                 txq = netdev_get_tx_queue(v->dev, qp);                  \
1464                 __netif_tx_lock(txq, cpu);                              \
1465         }                                                               \
1466         v->sq + qp;                                                     \
1467 })
1468
1469 #define virtnet_xdp_put_sq(vi, q) {                                     \
1470         struct netdev_queue *txq;                                       \
1471         typeof(vi) v = (vi);                                            \
1472                                                                         \
1473         txq = netdev_get_tx_queue(v->dev, (q) - v->sq);                 \
1474         if (v->curr_queue_pairs > nr_cpu_ids)                           \
1475                 __netif_tx_release(txq);                                \
1476         else                                                            \
1477                 __netif_tx_unlock(txq);                                 \
1478 }
1479
1480 static int virtnet_xdp_xmit(struct net_device *dev,
1481                             int n, struct xdp_frame **frames, u32 flags)
1482 {
1483         struct virtnet_info *vi = netdev_priv(dev);
1484         struct virtnet_sq_free_stats stats = {0};
1485         struct receive_queue *rq = vi->rq;
1486         struct bpf_prog *xdp_prog;
1487         struct send_queue *sq;
1488         int nxmit = 0;
1489         int kicks = 0;
1490         int ret;
1491         int i;
1492
1493         /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
1494          * indicate XDP resources have been successfully allocated.
1495          */
1496         xdp_prog = rcu_access_pointer(rq->xdp_prog);
1497         if (!xdp_prog)
1498                 return -ENXIO;
1499
1500         sq = virtnet_xdp_get_sq(vi);
1501
1502         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
1503                 ret = -EINVAL;
1504                 goto out;
1505         }
1506
1507         /* Free up any pending old buffers before queueing new ones. */
1508         __free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq),
1509                         false, &stats);
1510
1511         for (i = 0; i < n; i++) {
1512                 struct xdp_frame *xdpf = frames[i];
1513
1514                 if (__virtnet_xdp_xmit_one(vi, sq, xdpf))
1515                         break;
1516                 nxmit++;
1517         }
1518         ret = nxmit;
1519
1520         if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq))
1521                 check_sq_full_and_disable(vi, dev, sq);
1522
1523         if (flags & XDP_XMIT_FLUSH) {
1524                 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
1525                         kicks = 1;
1526         }
1527 out:
1528         u64_stats_update_begin(&sq->stats.syncp);
1529         u64_stats_add(&sq->stats.bytes, stats.bytes);
1530         u64_stats_add(&sq->stats.packets, stats.packets);
1531         u64_stats_add(&sq->stats.xdp_tx, n);
1532         u64_stats_add(&sq->stats.xdp_tx_drops, n - nxmit);
1533         u64_stats_add(&sq->stats.kicks, kicks);
1534         u64_stats_update_end(&sq->stats.syncp);
1535
1536         virtnet_xdp_put_sq(vi, sq);
1537         return ret;
1538 }
1539
1540 static void put_xdp_frags(struct xdp_buff *xdp)
1541 {
1542         struct skb_shared_info *shinfo;
1543         struct page *xdp_page;
1544         int i;
1545
1546         if (xdp_buff_has_frags(xdp)) {
1547                 shinfo = xdp_get_shared_info_from_buff(xdp);
1548                 for (i = 0; i < shinfo->nr_frags; i++) {
1549                         xdp_page = skb_frag_page(&shinfo->frags[i]);
1550                         put_page(xdp_page);
1551                 }
1552         }
1553 }
1554
1555 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
1556                                struct net_device *dev,
1557                                unsigned int *xdp_xmit,
1558                                struct virtnet_rq_stats *stats)
1559 {
1560         struct xdp_frame *xdpf;
1561         int err;
1562         u32 act;
1563
1564         act = bpf_prog_run_xdp(xdp_prog, xdp);
1565         u64_stats_inc(&stats->xdp_packets);
1566
1567         switch (act) {
1568         case XDP_PASS:
1569                 return act;
1570
1571         case XDP_TX:
1572                 u64_stats_inc(&stats->xdp_tx);
1573                 xdpf = xdp_convert_buff_to_frame(xdp);
1574                 if (unlikely(!xdpf)) {
1575                         netdev_dbg(dev, "convert buff to frame failed for xdp\n");
1576                         return XDP_DROP;
1577                 }
1578
1579                 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
1580                 if (unlikely(!err)) {
1581                         xdp_return_frame_rx_napi(xdpf);
1582                 } else if (unlikely(err < 0)) {
1583                         trace_xdp_exception(dev, xdp_prog, act);
1584                         return XDP_DROP;
1585                 }
1586                 *xdp_xmit |= VIRTIO_XDP_TX;
1587                 return act;
1588
1589         case XDP_REDIRECT:
1590                 u64_stats_inc(&stats->xdp_redirects);
1591                 err = xdp_do_redirect(dev, xdp, xdp_prog);
1592                 if (err)
1593                         return XDP_DROP;
1594
1595                 *xdp_xmit |= VIRTIO_XDP_REDIR;
1596                 return act;
1597
1598         default:
1599                 bpf_warn_invalid_xdp_action(dev, xdp_prog, act);
1600                 fallthrough;
1601         case XDP_ABORTED:
1602                 trace_xdp_exception(dev, xdp_prog, act);
1603                 fallthrough;
1604         case XDP_DROP:
1605                 return XDP_DROP;
1606         }
1607 }
1608
1609 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
1610 {
1611         return vi->xdp_enabled ? XDP_PACKET_HEADROOM : 0;
1612 }
1613
1614 /* We copy the packet for XDP in the following cases:
1615  *
1616  * 1) Packet is scattered across multiple rx buffers.
1617  * 2) Headroom space is insufficient.
1618  *
1619  * This is inefficient but it's a temporary condition that
1620  * we hit right after XDP is enabled and until queue is refilled
1621  * with large buffers with sufficient headroom - so it should affect
1622  * at most queue size packets.
1623  * Afterwards, the conditions to enable
1624  * XDP should preclude the underlying device from sending packets
1625  * across multiple buffers (num_buf > 1), and we make sure buffers
1626  * have enough headroom.
1627  */
1628 static struct page *xdp_linearize_page(struct receive_queue *rq,
1629                                        int *num_buf,
1630                                        struct page *p,
1631                                        int offset,
1632                                        int page_off,
1633                                        unsigned int *len)
1634 {
1635         int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1636         struct page *page;
1637
1638         if (page_off + *len + tailroom > PAGE_SIZE)
1639                 return NULL;
1640
1641         page = alloc_page(GFP_ATOMIC);
1642         if (!page)
1643                 return NULL;
1644
1645         memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
1646         page_off += *len;
1647
1648         while (--*num_buf) {
1649                 unsigned int buflen;
1650                 void *buf;
1651                 int off;
1652
1653                 buf = virtnet_rq_get_buf(rq, &buflen, NULL);
1654                 if (unlikely(!buf))
1655                         goto err_buf;
1656
1657                 p = virt_to_head_page(buf);
1658                 off = buf - page_address(p);
1659
1660                 /* guard against a misconfigured or uncooperative backend that
1661                  * is sending packet larger than the MTU.
1662                  */
1663                 if ((page_off + buflen + tailroom) > PAGE_SIZE) {
1664                         put_page(p);
1665                         goto err_buf;
1666                 }
1667
1668                 memcpy(page_address(page) + page_off,
1669                        page_address(p) + off, buflen);
1670                 page_off += buflen;
1671                 put_page(p);
1672         }
1673
1674         /* Headroom does not contribute to packet length */
1675         *len = page_off - XDP_PACKET_HEADROOM;
1676         return page;
1677 err_buf:
1678         __free_pages(page, 0);
1679         return NULL;
1680 }
1681
1682 static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi,
1683                                                unsigned int xdp_headroom,
1684                                                void *buf,
1685                                                unsigned int len)
1686 {
1687         unsigned int header_offset;
1688         unsigned int headroom;
1689         unsigned int buflen;
1690         struct sk_buff *skb;
1691
1692         header_offset = VIRTNET_RX_PAD + xdp_headroom;
1693         headroom = vi->hdr_len + header_offset;
1694         buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1695                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1696
1697         skb = virtnet_build_skb(buf, buflen, headroom, len);
1698         if (unlikely(!skb))
1699                 return NULL;
1700
1701         buf += header_offset;
1702         memcpy(skb_vnet_common_hdr(skb), buf, vi->hdr_len);
1703
1704         return skb;
1705 }
1706
1707 static struct sk_buff *receive_small_xdp(struct net_device *dev,
1708                                          struct virtnet_info *vi,
1709                                          struct receive_queue *rq,
1710                                          struct bpf_prog *xdp_prog,
1711                                          void *buf,
1712                                          unsigned int xdp_headroom,
1713                                          unsigned int len,
1714                                          unsigned int *xdp_xmit,
1715                                          struct virtnet_rq_stats *stats)
1716 {
1717         unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
1718         unsigned int headroom = vi->hdr_len + header_offset;
1719         struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
1720         struct page *page = virt_to_head_page(buf);
1721         struct page *xdp_page;
1722         unsigned int buflen;
1723         struct xdp_buff xdp;
1724         struct sk_buff *skb;
1725         unsigned int metasize = 0;
1726         u32 act;
1727
1728         if (unlikely(hdr->hdr.gso_type))
1729                 goto err_xdp;
1730
1731         /* Partially checksummed packets must be dropped. */
1732         if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM))
1733                 goto err_xdp;
1734
1735         buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1736                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1737
1738         if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
1739                 int offset = buf - page_address(page) + header_offset;
1740                 unsigned int tlen = len + vi->hdr_len;
1741                 int num_buf = 1;
1742
1743                 xdp_headroom = virtnet_get_headroom(vi);
1744                 header_offset = VIRTNET_RX_PAD + xdp_headroom;
1745                 headroom = vi->hdr_len + header_offset;
1746                 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1747                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1748                 xdp_page = xdp_linearize_page(rq, &num_buf, page,
1749                                               offset, header_offset,
1750                                               &tlen);
1751                 if (!xdp_page)
1752                         goto err_xdp;
1753
1754                 buf = page_address(xdp_page);
1755                 put_page(page);
1756                 page = xdp_page;
1757         }
1758
1759         xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
1760         xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
1761                          xdp_headroom, len, true);
1762
1763         act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
1764
1765         switch (act) {
1766         case XDP_PASS:
1767                 /* Recalculate length in case bpf program changed it */
1768                 len = xdp.data_end - xdp.data;
1769                 metasize = xdp.data - xdp.data_meta;
1770                 break;
1771
1772         case XDP_TX:
1773         case XDP_REDIRECT:
1774                 goto xdp_xmit;
1775
1776         default:
1777                 goto err_xdp;
1778         }
1779
1780         skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len);
1781         if (unlikely(!skb))
1782                 goto err;
1783
1784         if (metasize)
1785                 skb_metadata_set(skb, metasize);
1786
1787         return skb;
1788
1789 err_xdp:
1790         u64_stats_inc(&stats->xdp_drops);
1791 err:
1792         u64_stats_inc(&stats->drops);
1793         put_page(page);
1794 xdp_xmit:
1795         return NULL;
1796 }
1797
1798 static struct sk_buff *receive_small(struct net_device *dev,
1799                                      struct virtnet_info *vi,
1800                                      struct receive_queue *rq,
1801                                      void *buf, void *ctx,
1802                                      unsigned int len,
1803                                      unsigned int *xdp_xmit,
1804                                      struct virtnet_rq_stats *stats)
1805 {
1806         unsigned int xdp_headroom = (unsigned long)ctx;
1807         struct page *page = virt_to_head_page(buf);
1808         struct sk_buff *skb;
1809
1810         /* We passed the address of virtnet header to virtio-core,
1811          * so truncate the padding.
1812          */
1813         buf -= VIRTNET_RX_PAD + xdp_headroom;
1814
1815         len -= vi->hdr_len;
1816         u64_stats_add(&stats->bytes, len);
1817
1818         if (unlikely(len > GOOD_PACKET_LEN)) {
1819                 pr_debug("%s: rx error: len %u exceeds max size %d\n",
1820                          dev->name, len, GOOD_PACKET_LEN);
1821                 DEV_STATS_INC(dev, rx_length_errors);
1822                 goto err;
1823         }
1824
1825         if (unlikely(vi->xdp_enabled)) {
1826                 struct bpf_prog *xdp_prog;
1827
1828                 rcu_read_lock();
1829                 xdp_prog = rcu_dereference(rq->xdp_prog);
1830                 if (xdp_prog) {
1831                         skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf,
1832                                                 xdp_headroom, len, xdp_xmit,
1833                                                 stats);
1834                         rcu_read_unlock();
1835                         return skb;
1836                 }
1837                 rcu_read_unlock();
1838         }
1839
1840         skb = receive_small_build_skb(vi, xdp_headroom, buf, len);
1841         if (likely(skb))
1842                 return skb;
1843
1844 err:
1845         u64_stats_inc(&stats->drops);
1846         put_page(page);
1847         return NULL;
1848 }
1849
1850 static struct sk_buff *receive_big(struct net_device *dev,
1851                                    struct virtnet_info *vi,
1852                                    struct receive_queue *rq,
1853                                    void *buf,
1854                                    unsigned int len,
1855                                    struct virtnet_rq_stats *stats)
1856 {
1857         struct page *page = buf;
1858         struct sk_buff *skb =
1859                 page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0);
1860
1861         u64_stats_add(&stats->bytes, len - vi->hdr_len);
1862         if (unlikely(!skb))
1863                 goto err;
1864
1865         return skb;
1866
1867 err:
1868         u64_stats_inc(&stats->drops);
1869         give_pages(rq, page);
1870         return NULL;
1871 }
1872
1873 static void mergeable_buf_free(struct receive_queue *rq, int num_buf,
1874                                struct net_device *dev,
1875                                struct virtnet_rq_stats *stats)
1876 {
1877         struct page *page;
1878         void *buf;
1879         int len;
1880
1881         while (num_buf-- > 1) {
1882                 buf = virtnet_rq_get_buf(rq, &len, NULL);
1883                 if (unlikely(!buf)) {
1884                         pr_debug("%s: rx error: %d buffers missing\n",
1885                                  dev->name, num_buf);
1886                         DEV_STATS_INC(dev, rx_length_errors);
1887                         break;
1888                 }
1889                 u64_stats_add(&stats->bytes, len);
1890                 page = virt_to_head_page(buf);
1891                 put_page(page);
1892         }
1893 }
1894
1895 /* Why not use xdp_build_skb_from_frame() ?
1896  * XDP core assumes that xdp frags are PAGE_SIZE in length, while in
1897  * virtio-net there are 2 points that do not match its requirements:
1898  *  1. The size of the prefilled buffer is not fixed before xdp is set.
1899  *  2. xdp_build_skb_from_frame() does more checks that we don't need,
1900  *     like eth_type_trans() (which virtio-net does in receive_buf()).
1901  */
1902 static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev,
1903                                                struct virtnet_info *vi,
1904                                                struct xdp_buff *xdp,
1905                                                unsigned int xdp_frags_truesz)
1906 {
1907         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
1908         unsigned int headroom, data_len;
1909         struct sk_buff *skb;
1910         int metasize;
1911         u8 nr_frags;
1912
1913         if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {
1914                 pr_debug("Error building skb as missing reserved tailroom for xdp");
1915                 return NULL;
1916         }
1917
1918         if (unlikely(xdp_buff_has_frags(xdp)))
1919                 nr_frags = sinfo->nr_frags;
1920
1921         skb = build_skb(xdp->data_hard_start, xdp->frame_sz);
1922         if (unlikely(!skb))
1923                 return NULL;
1924
1925         headroom = xdp->data - xdp->data_hard_start;
1926         data_len = xdp->data_end - xdp->data;
1927         skb_reserve(skb, headroom);
1928         __skb_put(skb, data_len);
1929
1930         metasize = xdp->data - xdp->data_meta;
1931         metasize = metasize > 0 ? metasize : 0;
1932         if (metasize)
1933                 skb_metadata_set(skb, metasize);
1934
1935         if (unlikely(xdp_buff_has_frags(xdp)))
1936                 xdp_update_skb_shared_info(skb, nr_frags,
1937                                            sinfo->xdp_frags_size,
1938                                            xdp_frags_truesz,
1939                                            xdp_buff_is_frag_pfmemalloc(xdp));
1940
1941         return skb;
1942 }
1943
1944 /* TODO: build xdp in big mode */
1945 static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
1946                                       struct virtnet_info *vi,
1947                                       struct receive_queue *rq,
1948                                       struct xdp_buff *xdp,
1949                                       void *buf,
1950                                       unsigned int len,
1951                                       unsigned int frame_sz,
1952                                       int *num_buf,
1953                                       unsigned int *xdp_frags_truesize,
1954                                       struct virtnet_rq_stats *stats)
1955 {
1956         struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
1957         unsigned int headroom, tailroom, room;
1958         unsigned int truesize, cur_frag_size;
1959         struct skb_shared_info *shinfo;
1960         unsigned int xdp_frags_truesz = 0;
1961         struct page *page;
1962         skb_frag_t *frag;
1963         int offset;
1964         void *ctx;
1965
1966         xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq);
1967         xdp_prepare_buff(xdp, buf - XDP_PACKET_HEADROOM,
1968                          XDP_PACKET_HEADROOM + vi->hdr_len, len - vi->hdr_len, true);
1969
1970         if (!*num_buf)
1971                 return 0;
1972
1973         if (*num_buf > 1) {
1974                 /* If we want to build multi-buffer xdp, we need
1975                  * to specify that the flags of xdp_buff have the
1976                  * XDP_FLAGS_HAS_FRAG bit.
1977                  */
1978                 if (!xdp_buff_has_frags(xdp))
1979                         xdp_buff_set_frags_flag(xdp);
1980
1981                 shinfo = xdp_get_shared_info_from_buff(xdp);
1982                 shinfo->nr_frags = 0;
1983                 shinfo->xdp_frags_size = 0;
1984         }
1985
1986         if (*num_buf > MAX_SKB_FRAGS + 1)
1987                 return -EINVAL;
1988
1989         while (--*num_buf > 0) {
1990                 buf = virtnet_rq_get_buf(rq, &len, &ctx);
1991                 if (unlikely(!buf)) {
1992                         pr_debug("%s: rx error: %d buffers out of %d missing\n",
1993                                  dev->name, *num_buf,
1994                                  virtio16_to_cpu(vi->vdev, hdr->num_buffers));
1995                         DEV_STATS_INC(dev, rx_length_errors);
1996                         goto err;
1997                 }
1998
1999                 u64_stats_add(&stats->bytes, len);
2000                 page = virt_to_head_page(buf);
2001                 offset = buf - page_address(page);
2002
2003                 truesize = mergeable_ctx_to_truesize(ctx);
2004                 headroom = mergeable_ctx_to_headroom(ctx);
2005                 tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2006                 room = SKB_DATA_ALIGN(headroom + tailroom);
2007
2008                 cur_frag_size = truesize;
2009                 xdp_frags_truesz += cur_frag_size;
2010                 if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) {
2011                         put_page(page);
2012                         pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
2013                                  dev->name, len, (unsigned long)(truesize - room));
2014                         DEV_STATS_INC(dev, rx_length_errors);
2015                         goto err;
2016                 }
2017
2018                 frag = &shinfo->frags[shinfo->nr_frags++];
2019                 skb_frag_fill_page_desc(frag, page, offset, len);
2020                 if (page_is_pfmemalloc(page))
2021                         xdp_buff_set_frag_pfmemalloc(xdp);
2022
2023                 shinfo->xdp_frags_size += len;
2024         }
2025
2026         *xdp_frags_truesize = xdp_frags_truesz;
2027         return 0;
2028
2029 err:
2030         put_xdp_frags(xdp);
2031         return -EINVAL;
2032 }
2033
2034 static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
2035                                    struct receive_queue *rq,
2036                                    struct bpf_prog *xdp_prog,
2037                                    void *ctx,
2038                                    unsigned int *frame_sz,
2039                                    int *num_buf,
2040                                    struct page **page,
2041                                    int offset,
2042                                    unsigned int *len,
2043                                    struct virtio_net_hdr_mrg_rxbuf *hdr)
2044 {
2045         unsigned int truesize = mergeable_ctx_to_truesize(ctx);
2046         unsigned int headroom = mergeable_ctx_to_headroom(ctx);
2047         struct page *xdp_page;
2048         unsigned int xdp_room;
2049
2050         /* Transient failure which in theory could occur if
2051          * in-flight packets from before XDP was enabled reach
2052          * the receive path after XDP is loaded.
2053          */
2054         if (unlikely(hdr->hdr.gso_type))
2055                 return NULL;
2056
2057         /* Partially checksummed packets must be dropped. */
2058         if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM))
2059                 return NULL;
2060
2061         /* Now XDP core assumes frag size is PAGE_SIZE, but buffers
2062          * with headroom may add hole in truesize, which
2063          * make their length exceed PAGE_SIZE. So we disabled the
2064          * hole mechanism for xdp. See add_recvbuf_mergeable().
2065          */
2066         *frame_sz = truesize;
2067
2068         if (likely(headroom >= virtnet_get_headroom(vi) &&
2069                    (*num_buf == 1 || xdp_prog->aux->xdp_has_frags))) {
2070                 return page_address(*page) + offset;
2071         }
2072
2073         /* This happens when headroom is not enough because
2074          * of the buffer was prefilled before XDP is set.
2075          * This should only happen for the first several packets.
2076          * In fact, vq reset can be used here to help us clean up
2077          * the prefilled buffers, but many existing devices do not
2078          * support it, and we don't want to bother users who are
2079          * using xdp normally.
2080          */
2081         if (!xdp_prog->aux->xdp_has_frags) {
2082                 /* linearize data for XDP */
2083                 xdp_page = xdp_linearize_page(rq, num_buf,
2084                                               *page, offset,
2085                                               XDP_PACKET_HEADROOM,
2086                                               len);
2087                 if (!xdp_page)
2088                         return NULL;
2089         } else {
2090                 xdp_room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM +
2091                                           sizeof(struct skb_shared_info));
2092                 if (*len + xdp_room > PAGE_SIZE)
2093                         return NULL;
2094
2095                 xdp_page = alloc_page(GFP_ATOMIC);
2096                 if (!xdp_page)
2097                         return NULL;
2098
2099                 memcpy(page_address(xdp_page) + XDP_PACKET_HEADROOM,
2100                        page_address(*page) + offset, *len);
2101         }
2102
2103         *frame_sz = PAGE_SIZE;
2104
2105         put_page(*page);
2106
2107         *page = xdp_page;
2108
2109         return page_address(*page) + XDP_PACKET_HEADROOM;
2110 }
2111
2112 static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
2113                                              struct virtnet_info *vi,
2114                                              struct receive_queue *rq,
2115                                              struct bpf_prog *xdp_prog,
2116                                              void *buf,
2117                                              void *ctx,
2118                                              unsigned int len,
2119                                              unsigned int *xdp_xmit,
2120                                              struct virtnet_rq_stats *stats)
2121 {
2122         struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
2123         int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
2124         struct page *page = virt_to_head_page(buf);
2125         int offset = buf - page_address(page);
2126         unsigned int xdp_frags_truesz = 0;
2127         struct sk_buff *head_skb;
2128         unsigned int frame_sz;
2129         struct xdp_buff xdp;
2130         void *data;
2131         u32 act;
2132         int err;
2133
2134         data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz, &num_buf, &page,
2135                                      offset, &len, hdr);
2136         if (unlikely(!data))
2137                 goto err_xdp;
2138
2139         err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,
2140                                          &num_buf, &xdp_frags_truesz, stats);
2141         if (unlikely(err))
2142                 goto err_xdp;
2143
2144         act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
2145
2146         switch (act) {
2147         case XDP_PASS:
2148                 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
2149                 if (unlikely(!head_skb))
2150                         break;
2151                 return head_skb;
2152
2153         case XDP_TX:
2154         case XDP_REDIRECT:
2155                 return NULL;
2156
2157         default:
2158                 break;
2159         }
2160
2161         put_xdp_frags(&xdp);
2162
2163 err_xdp:
2164         put_page(page);
2165         mergeable_buf_free(rq, num_buf, dev, stats);
2166
2167         u64_stats_inc(&stats->xdp_drops);
2168         u64_stats_inc(&stats->drops);
2169         return NULL;
2170 }
2171
2172 static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
2173                                                struct sk_buff *curr_skb,
2174                                                struct page *page, void *buf,
2175                                                int len, int truesize)
2176 {
2177         int num_skb_frags;
2178         int offset;
2179
2180         num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
2181         if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
2182                 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
2183
2184                 if (unlikely(!nskb))
2185                         return NULL;
2186
2187                 if (curr_skb == head_skb)
2188                         skb_shinfo(curr_skb)->frag_list = nskb;
2189                 else
2190                         curr_skb->next = nskb;
2191                 curr_skb = nskb;
2192                 head_skb->truesize += nskb->truesize;
2193                 num_skb_frags = 0;
2194         }
2195
2196         if (curr_skb != head_skb) {
2197                 head_skb->data_len += len;
2198                 head_skb->len += len;
2199                 head_skb->truesize += truesize;
2200         }
2201
2202         offset = buf - page_address(page);
2203         if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
2204                 put_page(page);
2205                 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
2206                                      len, truesize);
2207         } else {
2208                 skb_add_rx_frag(curr_skb, num_skb_frags, page,
2209                                 offset, len, truesize);
2210         }
2211
2212         return curr_skb;
2213 }
2214
2215 static struct sk_buff *receive_mergeable(struct net_device *dev,
2216                                          struct virtnet_info *vi,
2217                                          struct receive_queue *rq,
2218                                          void *buf,
2219                                          void *ctx,
2220                                          unsigned int len,
2221                                          unsigned int *xdp_xmit,
2222                                          struct virtnet_rq_stats *stats)
2223 {
2224         struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
2225         int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
2226         struct page *page = virt_to_head_page(buf);
2227         int offset = buf - page_address(page);
2228         struct sk_buff *head_skb, *curr_skb;
2229         unsigned int truesize = mergeable_ctx_to_truesize(ctx);
2230         unsigned int headroom = mergeable_ctx_to_headroom(ctx);
2231         unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2232         unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
2233
2234         head_skb = NULL;
2235         u64_stats_add(&stats->bytes, len - vi->hdr_len);
2236
2237         if (unlikely(len > truesize - room)) {
2238                 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
2239                          dev->name, len, (unsigned long)(truesize - room));
2240                 DEV_STATS_INC(dev, rx_length_errors);
2241                 goto err_skb;
2242         }
2243
2244         if (unlikely(vi->xdp_enabled)) {
2245                 struct bpf_prog *xdp_prog;
2246
2247                 rcu_read_lock();
2248                 xdp_prog = rcu_dereference(rq->xdp_prog);
2249                 if (xdp_prog) {
2250                         head_skb = receive_mergeable_xdp(dev, vi, rq, xdp_prog, buf, ctx,
2251                                                          len, xdp_xmit, stats);
2252                         rcu_read_unlock();
2253                         return head_skb;
2254                 }
2255                 rcu_read_unlock();
2256         }
2257
2258         head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom);
2259         curr_skb = head_skb;
2260
2261         if (unlikely(!curr_skb))
2262                 goto err_skb;
2263         while (--num_buf) {
2264                 buf = virtnet_rq_get_buf(rq, &len, &ctx);
2265                 if (unlikely(!buf)) {
2266                         pr_debug("%s: rx error: %d buffers out of %d missing\n",
2267                                  dev->name, num_buf,
2268                                  virtio16_to_cpu(vi->vdev,
2269                                                  hdr->num_buffers));
2270                         DEV_STATS_INC(dev, rx_length_errors);
2271                         goto err_buf;
2272                 }
2273
2274                 u64_stats_add(&stats->bytes, len);
2275                 page = virt_to_head_page(buf);
2276
2277                 truesize = mergeable_ctx_to_truesize(ctx);
2278                 headroom = mergeable_ctx_to_headroom(ctx);
2279                 tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2280                 room = SKB_DATA_ALIGN(headroom + tailroom);
2281                 if (unlikely(len > truesize - room)) {
2282                         pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
2283                                  dev->name, len, (unsigned long)(truesize - room));
2284                         DEV_STATS_INC(dev, rx_length_errors);
2285                         goto err_skb;
2286                 }
2287
2288                 curr_skb  = virtnet_skb_append_frag(head_skb, curr_skb, page,
2289                                                     buf, len, truesize);
2290                 if (!curr_skb)
2291                         goto err_skb;
2292         }
2293
2294         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
2295         return head_skb;
2296
2297 err_skb:
2298         put_page(page);
2299         mergeable_buf_free(rq, num_buf, dev, stats);
2300
2301 err_buf:
2302         u64_stats_inc(&stats->drops);
2303         dev_kfree_skb(head_skb);
2304         return NULL;
2305 }
2306
2307 static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
2308                                 struct sk_buff *skb)
2309 {
2310         enum pkt_hash_types rss_hash_type;
2311
2312         if (!hdr_hash || !skb)
2313                 return;
2314
2315         switch (__le16_to_cpu(hdr_hash->hash_report)) {
2316         case VIRTIO_NET_HASH_REPORT_TCPv4:
2317         case VIRTIO_NET_HASH_REPORT_UDPv4:
2318         case VIRTIO_NET_HASH_REPORT_TCPv6:
2319         case VIRTIO_NET_HASH_REPORT_UDPv6:
2320         case VIRTIO_NET_HASH_REPORT_TCPv6_EX:
2321         case VIRTIO_NET_HASH_REPORT_UDPv6_EX:
2322                 rss_hash_type = PKT_HASH_TYPE_L4;
2323                 break;
2324         case VIRTIO_NET_HASH_REPORT_IPv4:
2325         case VIRTIO_NET_HASH_REPORT_IPv6:
2326         case VIRTIO_NET_HASH_REPORT_IPv6_EX:
2327                 rss_hash_type = PKT_HASH_TYPE_L3;
2328                 break;
2329         case VIRTIO_NET_HASH_REPORT_NONE:
2330         default:
2331                 rss_hash_type = PKT_HASH_TYPE_NONE;
2332         }
2333         skb_set_hash(skb, __le32_to_cpu(hdr_hash->hash_value), rss_hash_type);
2334 }
2335
2336 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq,
2337                                  struct sk_buff *skb, u8 flags)
2338 {
2339         struct virtio_net_common_hdr *hdr;
2340         struct net_device *dev = vi->dev;
2341
2342         hdr = skb_vnet_common_hdr(skb);
2343         if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report)
2344                 virtio_skb_set_hash(&hdr->hash_v1_hdr, skb);
2345
2346         if (flags & VIRTIO_NET_HDR_F_DATA_VALID)
2347                 skb->ip_summed = CHECKSUM_UNNECESSARY;
2348
2349         if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
2350                                   virtio_is_little_endian(vi->vdev))) {
2351                 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
2352                                      dev->name, hdr->hdr.gso_type,
2353                                      hdr->hdr.gso_size);
2354                 goto frame_err;
2355         }
2356
2357         skb_record_rx_queue(skb, vq2rxq(rq->vq));
2358         skb->protocol = eth_type_trans(skb, dev);
2359         pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
2360                  ntohs(skb->protocol), skb->len, skb->pkt_type);
2361
2362         napi_gro_receive(&rq->napi, skb);
2363         return;
2364
2365 frame_err:
2366         DEV_STATS_INC(dev, rx_frame_errors);
2367         dev_kfree_skb(skb);
2368 }
2369
2370 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
2371                         void *buf, unsigned int len, void **ctx,
2372                         unsigned int *xdp_xmit,
2373                         struct virtnet_rq_stats *stats)
2374 {
2375         struct net_device *dev = vi->dev;
2376         struct sk_buff *skb;
2377         u8 flags;
2378
2379         if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
2380                 pr_debug("%s: short packet %i\n", dev->name, len);
2381                 DEV_STATS_INC(dev, rx_length_errors);
2382                 virtnet_rq_free_buf(vi, rq, buf);
2383                 return;
2384         }
2385
2386         /* 1. Save the flags early, as the XDP program might overwrite them.
2387          * These flags ensure packets marked as VIRTIO_NET_HDR_F_DATA_VALID
2388          * stay valid after XDP processing.
2389          * 2. XDP doesn't work with partially checksummed packets (refer to
2390          * virtnet_xdp_set()), so packets marked as
2391          * VIRTIO_NET_HDR_F_NEEDS_CSUM get dropped during XDP processing.
2392          */
2393         flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags;
2394
2395         if (vi->mergeable_rx_bufs)
2396                 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
2397                                         stats);
2398         else if (vi->big_packets)
2399                 skb = receive_big(dev, vi, rq, buf, len, stats);
2400         else
2401                 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);
2402
2403         if (unlikely(!skb))
2404                 return;
2405
2406         virtnet_receive_done(vi, rq, skb, flags);
2407 }
2408
2409 /* Unlike mergeable buffers, all buffers are allocated to the
2410  * same size, except for the headroom. For this reason we do
2411  * not need to use  mergeable_len_to_ctx here - it is enough
2412  * to store the headroom as the context ignoring the truesize.
2413  */
2414 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
2415                              gfp_t gfp)
2416 {
2417         char *buf;
2418         unsigned int xdp_headroom = virtnet_get_headroom(vi);
2419         void *ctx = (void *)(unsigned long)xdp_headroom;
2420         int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
2421         int err;
2422
2423         len = SKB_DATA_ALIGN(len) +
2424               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2425
2426         buf = virtnet_rq_alloc(rq, len, gfp);
2427         if (unlikely(!buf))
2428                 return -ENOMEM;
2429
2430         buf += VIRTNET_RX_PAD + xdp_headroom;
2431
2432         virtnet_rq_init_one_sg(rq, buf, vi->hdr_len + GOOD_PACKET_LEN);
2433
2434         err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
2435         if (err < 0) {
2436                 if (rq->do_dma)
2437                         virtnet_rq_unmap(rq, buf, 0);
2438                 put_page(virt_to_head_page(buf));
2439         }
2440
2441         return err;
2442 }
2443
2444 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
2445                            gfp_t gfp)
2446 {
2447         struct page *first, *list = NULL;
2448         char *p;
2449         int i, err, offset;
2450
2451         sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2);
2452
2453         /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */
2454         for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) {
2455                 first = get_a_page(rq, gfp);
2456                 if (!first) {
2457                         if (list)
2458                                 give_pages(rq, list);
2459                         return -ENOMEM;
2460                 }
2461                 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
2462
2463                 /* chain new page in list head to match sg */
2464                 first->private = (unsigned long)list;
2465                 list = first;
2466         }
2467
2468         first = get_a_page(rq, gfp);
2469         if (!first) {
2470                 give_pages(rq, list);
2471                 return -ENOMEM;
2472         }
2473         p = page_address(first);
2474
2475         /* rq->sg[0], rq->sg[1] share the same page */
2476         /* a separated rq->sg[0] for header - required in case !any_header_sg */
2477         sg_set_buf(&rq->sg[0], p, vi->hdr_len);
2478
2479         /* rq->sg[1] for data packet, from offset */
2480         offset = sizeof(struct padded_vnet_hdr);
2481         sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
2482
2483         /* chain first in list head */
2484         first->private = (unsigned long)list;
2485         err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2,
2486                                   first, gfp);
2487         if (err < 0)
2488                 give_pages(rq, first);
2489
2490         return err;
2491 }
2492
2493 static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
2494                                           struct ewma_pkt_len *avg_pkt_len,
2495                                           unsigned int room)
2496 {
2497         struct virtnet_info *vi = rq->vq->vdev->priv;
2498         const size_t hdr_len = vi->hdr_len;
2499         unsigned int len;
2500
2501         if (room)
2502                 return PAGE_SIZE - room;
2503
2504         len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
2505                                 rq->min_buf_len, PAGE_SIZE - hdr_len);
2506
2507         return ALIGN(len, L1_CACHE_BYTES);
2508 }
2509
2510 static int add_recvbuf_mergeable(struct virtnet_info *vi,
2511                                  struct receive_queue *rq, gfp_t gfp)
2512 {
2513         struct page_frag *alloc_frag = &rq->alloc_frag;
2514         unsigned int headroom = virtnet_get_headroom(vi);
2515         unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2516         unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
2517         unsigned int len, hole;
2518         void *ctx;
2519         char *buf;
2520         int err;
2521
2522         /* Extra tailroom is needed to satisfy XDP's assumption. This
2523          * means rx frags coalescing won't work, but consider we've
2524          * disabled GSO for XDP, it won't be a big issue.
2525          */
2526         len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
2527
2528         buf = virtnet_rq_alloc(rq, len + room, gfp);
2529         if (unlikely(!buf))
2530                 return -ENOMEM;
2531
2532         buf += headroom; /* advance address leaving hole at front of pkt */
2533         hole = alloc_frag->size - alloc_frag->offset;
2534         if (hole < len + room) {
2535                 /* To avoid internal fragmentation, if there is very likely not
2536                  * enough space for another buffer, add the remaining space to
2537                  * the current buffer.
2538                  * XDP core assumes that frame_size of xdp_buff and the length
2539                  * of the frag are PAGE_SIZE, so we disable the hole mechanism.
2540                  */
2541                 if (!headroom)
2542                         len += hole;
2543                 alloc_frag->offset += hole;
2544         }
2545
2546         virtnet_rq_init_one_sg(rq, buf, len);
2547
2548         ctx = mergeable_len_to_ctx(len + room, headroom);
2549         err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
2550         if (err < 0) {
2551                 if (rq->do_dma)
2552                         virtnet_rq_unmap(rq, buf, 0);
2553                 put_page(virt_to_head_page(buf));
2554         }
2555
2556         return err;
2557 }
2558
2559 /*
2560  * Returns false if we couldn't fill entirely (OOM).
2561  *
2562  * Normally run in the receive path, but can also be run from ndo_open
2563  * before we're receiving packets, or from refill_work which is
2564  * careful to disable receiving (using napi_disable).
2565  */
2566 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
2567                           gfp_t gfp)
2568 {
2569         int err;
2570
2571         if (rq->xsk_pool) {
2572                 err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk_pool, gfp);
2573                 goto kick;
2574         }
2575
2576         do {
2577                 if (vi->mergeable_rx_bufs)
2578                         err = add_recvbuf_mergeable(vi, rq, gfp);
2579                 else if (vi->big_packets)
2580                         err = add_recvbuf_big(vi, rq, gfp);
2581                 else
2582                         err = add_recvbuf_small(vi, rq, gfp);
2583
2584                 if (err)
2585                         break;
2586         } while (rq->vq->num_free);
2587
2588 kick:
2589         if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
2590                 unsigned long flags;
2591
2592                 flags = u64_stats_update_begin_irqsave(&rq->stats.syncp);
2593                 u64_stats_inc(&rq->stats.kicks);
2594                 u64_stats_update_end_irqrestore(&rq->stats.syncp, flags);
2595         }
2596
2597         return err != -ENOMEM;
2598 }
2599
2600 static void skb_recv_done(struct virtqueue *rvq)
2601 {
2602         struct virtnet_info *vi = rvq->vdev->priv;
2603         struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
2604
2605         rq->calls++;
2606         virtqueue_napi_schedule(&rq->napi, rvq);
2607 }
2608
2609 static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
2610 {
2611         napi_enable(napi);
2612
2613         /* If all buffers were filled by other side before we napi_enabled, we
2614          * won't get another interrupt, so process any outstanding packets now.
2615          * Call local_bh_enable after to trigger softIRQ processing.
2616          */
2617         local_bh_disable();
2618         virtqueue_napi_schedule(napi, vq);
2619         local_bh_enable();
2620 }
2621
2622 static void virtnet_napi_tx_enable(struct virtnet_info *vi,
2623                                    struct virtqueue *vq,
2624                                    struct napi_struct *napi)
2625 {
2626         if (!napi->weight)
2627                 return;
2628
2629         /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
2630          * enable the feature if this is likely affine with the transmit path.
2631          */
2632         if (!vi->affinity_hint_set) {
2633                 napi->weight = 0;
2634                 return;
2635         }
2636
2637         return virtnet_napi_enable(vq, napi);
2638 }
2639
2640 static void virtnet_napi_tx_disable(struct napi_struct *napi)
2641 {
2642         if (napi->weight)
2643                 napi_disable(napi);
2644 }
2645
2646 static void refill_work(struct work_struct *work)
2647 {
2648         struct virtnet_info *vi =
2649                 container_of(work, struct virtnet_info, refill.work);
2650         bool still_empty;
2651         int i;
2652
2653         for (i = 0; i < vi->curr_queue_pairs; i++) {
2654                 struct receive_queue *rq = &vi->rq[i];
2655
2656                 napi_disable(&rq->napi);
2657                 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
2658                 virtnet_napi_enable(rq->vq, &rq->napi);
2659
2660                 /* In theory, this can happen: if we don't get any buffers in
2661                  * we will *never* try to fill again.
2662                  */
2663                 if (still_empty)
2664                         schedule_delayed_work(&vi->refill, HZ/2);
2665         }
2666 }
2667
2668 static int virtnet_receive_xsk_bufs(struct virtnet_info *vi,
2669                                     struct receive_queue *rq,
2670                                     int budget,
2671                                     unsigned int *xdp_xmit,
2672                                     struct virtnet_rq_stats *stats)
2673 {
2674         unsigned int len;
2675         int packets = 0;
2676         void *buf;
2677
2678         while (packets < budget) {
2679                 buf = virtqueue_get_buf(rq->vq, &len);
2680                 if (!buf)
2681                         break;
2682
2683                 virtnet_receive_xsk_buf(vi, rq, buf, len, xdp_xmit, stats);
2684                 packets++;
2685         }
2686
2687         return packets;
2688 }
2689
2690 static int virtnet_receive_packets(struct virtnet_info *vi,
2691                                    struct receive_queue *rq,
2692                                    int budget,
2693                                    unsigned int *xdp_xmit,
2694                                    struct virtnet_rq_stats *stats)
2695 {
2696         unsigned int len;
2697         int packets = 0;
2698         void *buf;
2699
2700         if (!vi->big_packets || vi->mergeable_rx_bufs) {
2701                 void *ctx;
2702                 while (packets < budget &&
2703                        (buf = virtnet_rq_get_buf(rq, &len, &ctx))) {
2704                         receive_buf(vi, rq, buf, len, ctx, xdp_xmit, stats);
2705                         packets++;
2706                 }
2707         } else {
2708                 while (packets < budget &&
2709                        (buf = virtnet_rq_get_buf(rq, &len, NULL)) != NULL) {
2710                         receive_buf(vi, rq, buf, len, NULL, xdp_xmit, stats);
2711                         packets++;
2712                 }
2713         }
2714
2715         return packets;
2716 }
2717
2718 static int virtnet_receive(struct receive_queue *rq, int budget,
2719                            unsigned int *xdp_xmit)
2720 {
2721         struct virtnet_info *vi = rq->vq->vdev->priv;
2722         struct virtnet_rq_stats stats = {};
2723         int i, packets;
2724
2725         if (rq->xsk_pool)
2726                 packets = virtnet_receive_xsk_bufs(vi, rq, budget, xdp_xmit, &stats);
2727         else
2728                 packets = virtnet_receive_packets(vi, rq, budget, xdp_xmit, &stats);
2729
2730         if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
2731                 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) {
2732                         spin_lock(&vi->refill_lock);
2733                         if (vi->refill_enabled)
2734                                 schedule_delayed_work(&vi->refill, 0);
2735                         spin_unlock(&vi->refill_lock);
2736                 }
2737         }
2738
2739         u64_stats_set(&stats.packets, packets);
2740         u64_stats_update_begin(&rq->stats.syncp);
2741         for (i = 0; i < ARRAY_SIZE(virtnet_rq_stats_desc); i++) {
2742                 size_t offset = virtnet_rq_stats_desc[i].offset;
2743                 u64_stats_t *item, *src;
2744
2745                 item = (u64_stats_t *)((u8 *)&rq->stats + offset);
2746                 src = (u64_stats_t *)((u8 *)&stats + offset);
2747                 u64_stats_add(item, u64_stats_read(src));
2748         }
2749
2750         u64_stats_add(&rq->stats.packets, u64_stats_read(&stats.packets));
2751         u64_stats_add(&rq->stats.bytes, u64_stats_read(&stats.bytes));
2752
2753         u64_stats_update_end(&rq->stats.syncp);
2754
2755         return packets;
2756 }
2757
2758 static void virtnet_poll_cleantx(struct receive_queue *rq, int budget)
2759 {
2760         struct virtnet_info *vi = rq->vq->vdev->priv;
2761         unsigned int index = vq2rxq(rq->vq);
2762         struct send_queue *sq = &vi->sq[index];
2763         struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
2764
2765         if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
2766                 return;
2767
2768         if (__netif_tx_trylock(txq)) {
2769                 if (sq->reset) {
2770                         __netif_tx_unlock(txq);
2771                         return;
2772                 }
2773
2774                 do {
2775                         virtqueue_disable_cb(sq->vq);
2776                         free_old_xmit(sq, txq, !!budget);
2777                 } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
2778
2779                 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
2780                         if (netif_tx_queue_stopped(txq)) {
2781                                 u64_stats_update_begin(&sq->stats.syncp);
2782                                 u64_stats_inc(&sq->stats.wake);
2783                                 u64_stats_update_end(&sq->stats.syncp);
2784                         }
2785                         netif_tx_wake_queue(txq);
2786                 }
2787
2788                 __netif_tx_unlock(txq);
2789         }
2790 }
2791
2792 static void virtnet_rx_dim_update(struct virtnet_info *vi, struct receive_queue *rq)
2793 {
2794         struct dim_sample cur_sample = {};
2795
2796         if (!rq->packets_in_napi)
2797                 return;
2798
2799         /* Don't need protection when fetching stats, since fetcher and
2800          * updater of the stats are in same context
2801          */
2802         dim_update_sample(rq->calls,
2803                           u64_stats_read(&rq->stats.packets),
2804                           u64_stats_read(&rq->stats.bytes),
2805                           &cur_sample);
2806
2807         net_dim(&rq->dim, cur_sample);
2808         rq->packets_in_napi = 0;
2809 }
2810
2811 static int virtnet_poll(struct napi_struct *napi, int budget)
2812 {
2813         struct receive_queue *rq =
2814                 container_of(napi, struct receive_queue, napi);
2815         struct virtnet_info *vi = rq->vq->vdev->priv;
2816         struct send_queue *sq;
2817         unsigned int received;
2818         unsigned int xdp_xmit = 0;
2819         bool napi_complete;
2820
2821         virtnet_poll_cleantx(rq, budget);
2822
2823         received = virtnet_receive(rq, budget, &xdp_xmit);
2824         rq->packets_in_napi += received;
2825
2826         if (xdp_xmit & VIRTIO_XDP_REDIR)
2827                 xdp_do_flush();
2828
2829         /* Out of packets? */
2830         if (received < budget) {
2831                 napi_complete = virtqueue_napi_complete(napi, rq->vq, received);
2832                 /* Intentionally not taking dim_lock here. This may result in a
2833                  * spurious net_dim call. But if that happens virtnet_rx_dim_work
2834                  * will not act on the scheduled work.
2835                  */
2836                 if (napi_complete && rq->dim_enabled)
2837                         virtnet_rx_dim_update(vi, rq);
2838         }
2839
2840         if (xdp_xmit & VIRTIO_XDP_TX) {
2841                 sq = virtnet_xdp_get_sq(vi);
2842                 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
2843                         u64_stats_update_begin(&sq->stats.syncp);
2844                         u64_stats_inc(&sq->stats.kicks);
2845                         u64_stats_update_end(&sq->stats.syncp);
2846                 }
2847                 virtnet_xdp_put_sq(vi, sq);
2848         }
2849
2850         return received;
2851 }
2852
2853 static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index)
2854 {
2855         virtnet_napi_tx_disable(&vi->sq[qp_index].napi);
2856         napi_disable(&vi->rq[qp_index].napi);
2857         xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq);
2858 }
2859
2860 static int virtnet_enable_queue_pair(struct virtnet_info *vi, int qp_index)
2861 {
2862         struct net_device *dev = vi->dev;
2863         int err;
2864
2865         err = xdp_rxq_info_reg(&vi->rq[qp_index].xdp_rxq, dev, qp_index,
2866                                vi->rq[qp_index].napi.napi_id);
2867         if (err < 0)
2868                 return err;
2869
2870         err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq,
2871                                          MEM_TYPE_PAGE_SHARED, NULL);
2872         if (err < 0)
2873                 goto err_xdp_reg_mem_model;
2874
2875         netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, qp_index));
2876         virtnet_napi_enable(vi->rq[qp_index].vq, &vi->rq[qp_index].napi);
2877         virtnet_napi_tx_enable(vi, vi->sq[qp_index].vq, &vi->sq[qp_index].napi);
2878
2879         return 0;
2880
2881 err_xdp_reg_mem_model:
2882         xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq);
2883         return err;
2884 }
2885
2886 static void virtnet_cancel_dim(struct virtnet_info *vi, struct dim *dim)
2887 {
2888         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
2889                 return;
2890         net_dim_work_cancel(dim);
2891 }
2892
2893 static void virtnet_update_settings(struct virtnet_info *vi)
2894 {
2895         u32 speed;
2896         u8 duplex;
2897
2898         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
2899                 return;
2900
2901         virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed);
2902
2903         if (ethtool_validate_speed(speed))
2904                 vi->speed = speed;
2905
2906         virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex);
2907
2908         if (ethtool_validate_duplex(duplex))
2909                 vi->duplex = duplex;
2910 }
2911
2912 static int virtnet_open(struct net_device *dev)
2913 {
2914         struct virtnet_info *vi = netdev_priv(dev);
2915         int i, err;
2916
2917         enable_delayed_refill(vi);
2918
2919         for (i = 0; i < vi->max_queue_pairs; i++) {
2920                 if (i < vi->curr_queue_pairs)
2921                         /* Make sure we have some buffers: if oom use wq. */
2922                         if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
2923                                 schedule_delayed_work(&vi->refill, 0);
2924
2925                 err = virtnet_enable_queue_pair(vi, i);
2926                 if (err < 0)
2927                         goto err_enable_qp;
2928         }
2929
2930         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
2931                 if (vi->status & VIRTIO_NET_S_LINK_UP)
2932                         netif_carrier_on(vi->dev);
2933                 virtio_config_driver_enable(vi->vdev);
2934         } else {
2935                 vi->status = VIRTIO_NET_S_LINK_UP;
2936                 netif_carrier_on(dev);
2937         }
2938
2939         return 0;
2940
2941 err_enable_qp:
2942         disable_delayed_refill(vi);
2943         cancel_delayed_work_sync(&vi->refill);
2944
2945         for (i--; i >= 0; i--) {
2946                 virtnet_disable_queue_pair(vi, i);
2947                 virtnet_cancel_dim(vi, &vi->rq[i].dim);
2948         }
2949
2950         return err;
2951 }
2952
2953 static int virtnet_poll_tx(struct napi_struct *napi, int budget)
2954 {
2955         struct send_queue *sq = container_of(napi, struct send_queue, napi);
2956         struct virtnet_info *vi = sq->vq->vdev->priv;
2957         unsigned int index = vq2txq(sq->vq);
2958         struct netdev_queue *txq;
2959         int opaque;
2960         bool done;
2961
2962         if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
2963                 /* We don't need to enable cb for XDP */
2964                 napi_complete_done(napi, 0);
2965                 return 0;
2966         }
2967
2968         txq = netdev_get_tx_queue(vi->dev, index);
2969         __netif_tx_lock(txq, raw_smp_processor_id());
2970         virtqueue_disable_cb(sq->vq);
2971         free_old_xmit(sq, txq, !!budget);
2972
2973         if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
2974                 if (netif_tx_queue_stopped(txq)) {
2975                         u64_stats_update_begin(&sq->stats.syncp);
2976                         u64_stats_inc(&sq->stats.wake);
2977                         u64_stats_update_end(&sq->stats.syncp);
2978                 }
2979                 netif_tx_wake_queue(txq);
2980         }
2981
2982         opaque = virtqueue_enable_cb_prepare(sq->vq);
2983
2984         done = napi_complete_done(napi, 0);
2985
2986         if (!done)
2987                 virtqueue_disable_cb(sq->vq);
2988
2989         __netif_tx_unlock(txq);
2990
2991         if (done) {
2992                 if (unlikely(virtqueue_poll(sq->vq, opaque))) {
2993                         if (napi_schedule_prep(napi)) {
2994                                 __netif_tx_lock(txq, raw_smp_processor_id());
2995                                 virtqueue_disable_cb(sq->vq);
2996                                 __netif_tx_unlock(txq);
2997                                 __napi_schedule(napi);
2998                         }
2999                 }
3000         }
3001
3002         return 0;
3003 }
3004
3005 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan)
3006 {
3007         struct virtio_net_hdr_mrg_rxbuf *hdr;
3008         const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
3009         struct virtnet_info *vi = sq->vq->vdev->priv;
3010         int num_sg;
3011         unsigned hdr_len = vi->hdr_len;
3012         bool can_push;
3013
3014         pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
3015
3016         can_push = vi->any_header_sg &&
3017                 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
3018                 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
3019         /* Even if we can, don't push here yet as this would skew
3020          * csum_start offset below. */
3021         if (can_push)
3022                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
3023         else
3024                 hdr = &skb_vnet_common_hdr(skb)->mrg_hdr;
3025
3026         if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
3027                                     virtio_is_little_endian(vi->vdev), false,
3028                                     0))
3029                 return -EPROTO;
3030
3031         if (vi->mergeable_rx_bufs)
3032                 hdr->num_buffers = 0;
3033
3034         sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
3035         if (can_push) {
3036                 __skb_push(skb, hdr_len);
3037                 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
3038                 if (unlikely(num_sg < 0))
3039                         return num_sg;
3040                 /* Pull header back to avoid skew in tx bytes calculations. */
3041                 __skb_pull(skb, hdr_len);
3042         } else {
3043                 sg_set_buf(sq->sg, hdr, hdr_len);
3044                 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
3045                 if (unlikely(num_sg < 0))
3046                         return num_sg;
3047                 num_sg++;
3048         }
3049         return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg,
3050                                     skb_to_ptr(skb, orphan), GFP_ATOMIC);
3051 }
3052
3053 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
3054 {
3055         struct virtnet_info *vi = netdev_priv(dev);
3056         int qnum = skb_get_queue_mapping(skb);
3057         struct send_queue *sq = &vi->sq[qnum];
3058         int err;
3059         struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
3060         bool xmit_more = netdev_xmit_more();
3061         bool use_napi = sq->napi.weight;
3062         bool kick;
3063
3064         /* Free up any pending old buffers before queueing new ones. */
3065         do {
3066                 if (use_napi)
3067                         virtqueue_disable_cb(sq->vq);
3068
3069                 free_old_xmit(sq, txq, false);
3070
3071         } while (use_napi && !xmit_more &&
3072                unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
3073
3074         /* timestamp packet in software */
3075         skb_tx_timestamp(skb);
3076
3077         /* Try to transmit */
3078         err = xmit_skb(sq, skb, !use_napi);
3079
3080         /* This should not happen! */
3081         if (unlikely(err)) {
3082                 DEV_STATS_INC(dev, tx_fifo_errors);
3083                 if (net_ratelimit())
3084                         dev_warn(&dev->dev,
3085                                  "Unexpected TXQ (%d) queue failure: %d\n",
3086                                  qnum, err);
3087                 DEV_STATS_INC(dev, tx_dropped);
3088                 dev_kfree_skb_any(skb);
3089                 return NETDEV_TX_OK;
3090         }
3091
3092         /* Don't wait up for transmitted skbs to be freed. */
3093         if (!use_napi) {
3094                 skb_orphan(skb);
3095                 nf_reset_ct(skb);
3096         }
3097
3098         check_sq_full_and_disable(vi, dev, sq);
3099
3100         kick = use_napi ? __netdev_tx_sent_queue(txq, skb->len, xmit_more) :
3101                           !xmit_more || netif_xmit_stopped(txq);
3102         if (kick) {
3103                 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
3104                         u64_stats_update_begin(&sq->stats.syncp);
3105                         u64_stats_inc(&sq->stats.kicks);
3106                         u64_stats_update_end(&sq->stats.syncp);
3107                 }
3108         }
3109
3110         return NETDEV_TX_OK;
3111 }
3112
3113 static void virtnet_rx_pause(struct virtnet_info *vi, struct receive_queue *rq)
3114 {
3115         bool running = netif_running(vi->dev);
3116
3117         if (running) {
3118                 napi_disable(&rq->napi);
3119                 virtnet_cancel_dim(vi, &rq->dim);
3120         }
3121 }
3122
3123 static void virtnet_rx_resume(struct virtnet_info *vi, struct receive_queue *rq)
3124 {
3125         bool running = netif_running(vi->dev);
3126
3127         if (!try_fill_recv(vi, rq, GFP_KERNEL))
3128                 schedule_delayed_work(&vi->refill, 0);
3129
3130         if (running)
3131                 virtnet_napi_enable(rq->vq, &rq->napi);
3132 }
3133
3134 static int virtnet_rx_resize(struct virtnet_info *vi,
3135                              struct receive_queue *rq, u32 ring_num)
3136 {
3137         int err, qindex;
3138
3139         qindex = rq - vi->rq;
3140
3141         virtnet_rx_pause(vi, rq);
3142
3143         err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_unmap_free_buf);
3144         if (err)
3145                 netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
3146
3147         virtnet_rx_resume(vi, rq);
3148         return err;
3149 }
3150
3151 static void virtnet_tx_pause(struct virtnet_info *vi, struct send_queue *sq)
3152 {
3153         bool running = netif_running(vi->dev);
3154         struct netdev_queue *txq;
3155         int qindex;
3156
3157         qindex = sq - vi->sq;
3158
3159         if (running)
3160                 virtnet_napi_tx_disable(&sq->napi);
3161
3162         txq = netdev_get_tx_queue(vi->dev, qindex);
3163
3164         /* 1. wait all ximt complete
3165          * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue()
3166          */
3167         __netif_tx_lock_bh(txq);
3168
3169         /* Prevent rx poll from accessing sq. */
3170         sq->reset = true;
3171
3172         /* Prevent the upper layer from trying to send packets. */
3173         netif_stop_subqueue(vi->dev, qindex);
3174
3175         __netif_tx_unlock_bh(txq);
3176 }
3177
3178 static void virtnet_tx_resume(struct virtnet_info *vi, struct send_queue *sq)
3179 {
3180         bool running = netif_running(vi->dev);
3181         struct netdev_queue *txq;
3182         int qindex;
3183
3184         qindex = sq - vi->sq;
3185
3186         txq = netdev_get_tx_queue(vi->dev, qindex);
3187
3188         __netif_tx_lock_bh(txq);
3189         sq->reset = false;
3190         netif_tx_wake_queue(txq);
3191         __netif_tx_unlock_bh(txq);
3192
3193         if (running)
3194                 virtnet_napi_tx_enable(vi, sq->vq, &sq->napi);
3195 }
3196
3197 static int virtnet_tx_resize(struct virtnet_info *vi, struct send_queue *sq,
3198                              u32 ring_num)
3199 {
3200         int qindex, err;
3201
3202         qindex = sq - vi->sq;
3203
3204         virtnet_tx_pause(vi, sq);
3205
3206         err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
3207         if (err)
3208                 netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
3209
3210         virtnet_tx_resume(vi, sq);
3211
3212         return err;
3213 }
3214
3215 /*
3216  * Send command via the control virtqueue and check status.  Commands
3217  * supported by the hypervisor, as indicated by feature bits, should
3218  * never fail unless improperly formatted.
3219  */
3220 static bool virtnet_send_command_reply(struct virtnet_info *vi, u8 class, u8 cmd,
3221                                        struct scatterlist *out,
3222                                        struct scatterlist *in)
3223 {
3224         struct scatterlist *sgs[5], hdr, stat;
3225         u32 out_num = 0, tmp, in_num = 0;
3226         bool ok;
3227         int ret;
3228
3229         /* Caller should know better */
3230         BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
3231
3232         mutex_lock(&vi->cvq_lock);
3233         vi->ctrl->status = ~0;
3234         vi->ctrl->hdr.class = class;
3235         vi->ctrl->hdr.cmd = cmd;
3236         /* Add header */
3237         sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
3238         sgs[out_num++] = &hdr;
3239
3240         if (out)
3241                 sgs[out_num++] = out;
3242
3243         /* Add return status. */
3244         sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
3245         sgs[out_num + in_num++] = &stat;
3246
3247         if (in)
3248                 sgs[out_num + in_num++] = in;
3249
3250         BUG_ON(out_num + in_num > ARRAY_SIZE(sgs));
3251         ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, in_num, vi, GFP_ATOMIC);
3252         if (ret < 0) {
3253                 dev_warn(&vi->vdev->dev,
3254                          "Failed to add sgs for command vq: %d\n.", ret);
3255                 mutex_unlock(&vi->cvq_lock);
3256                 return false;
3257         }
3258
3259         if (unlikely(!virtqueue_kick(vi->cvq)))
3260                 goto unlock;
3261
3262         /* Spin for a response, the kick causes an ioport write, trapping
3263          * into the hypervisor, so the request should be handled immediately.
3264          */
3265         while (!virtqueue_get_buf(vi->cvq, &tmp) &&
3266                !virtqueue_is_broken(vi->cvq)) {
3267                 cond_resched();
3268                 cpu_relax();
3269         }
3270
3271 unlock:
3272         ok = vi->ctrl->status == VIRTIO_NET_OK;
3273         mutex_unlock(&vi->cvq_lock);
3274         return ok;
3275 }
3276
3277 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
3278                                  struct scatterlist *out)
3279 {
3280         return virtnet_send_command_reply(vi, class, cmd, out, NULL);
3281 }
3282
3283 static int virtnet_set_mac_address(struct net_device *dev, void *p)
3284 {
3285         struct virtnet_info *vi = netdev_priv(dev);
3286         struct virtio_device *vdev = vi->vdev;
3287         int ret;
3288         struct sockaddr *addr;
3289         struct scatterlist sg;
3290
3291         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
3292                 return -EOPNOTSUPP;
3293
3294         addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
3295         if (!addr)
3296                 return -ENOMEM;
3297
3298         ret = eth_prepare_mac_addr_change(dev, addr);
3299         if (ret)
3300                 goto out;
3301
3302         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
3303                 sg_init_one(&sg, addr->sa_data, dev->addr_len);
3304                 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
3305                                           VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
3306                         dev_warn(&vdev->dev,
3307                                  "Failed to set mac address by vq command.\n");
3308                         ret = -EINVAL;
3309                         goto out;
3310                 }
3311         } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
3312                    !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3313                 unsigned int i;
3314
3315                 /* Naturally, this has an atomicity problem. */
3316                 for (i = 0; i < dev->addr_len; i++)
3317                         virtio_cwrite8(vdev,
3318                                        offsetof(struct virtio_net_config, mac) +
3319                                        i, addr->sa_data[i]);
3320         }
3321
3322         eth_commit_mac_addr_change(dev, p);
3323         ret = 0;
3324
3325 out:
3326         kfree(addr);
3327         return ret;
3328 }
3329
3330 static void virtnet_stats(struct net_device *dev,
3331                           struct rtnl_link_stats64 *tot)
3332 {
3333         struct virtnet_info *vi = netdev_priv(dev);
3334         unsigned int start;
3335         int i;
3336
3337         for (i = 0; i < vi->max_queue_pairs; i++) {
3338                 u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops;
3339                 struct receive_queue *rq = &vi->rq[i];
3340                 struct send_queue *sq = &vi->sq[i];
3341
3342                 do {
3343                         start = u64_stats_fetch_begin(&sq->stats.syncp);
3344                         tpackets = u64_stats_read(&sq->stats.packets);
3345                         tbytes   = u64_stats_read(&sq->stats.bytes);
3346                         terrors  = u64_stats_read(&sq->stats.tx_timeouts);
3347                 } while (u64_stats_fetch_retry(&sq->stats.syncp, start));
3348
3349                 do {
3350                         start = u64_stats_fetch_begin(&rq->stats.syncp);
3351                         rpackets = u64_stats_read(&rq->stats.packets);
3352                         rbytes   = u64_stats_read(&rq->stats.bytes);
3353                         rdrops   = u64_stats_read(&rq->stats.drops);
3354                 } while (u64_stats_fetch_retry(&rq->stats.syncp, start));
3355
3356                 tot->rx_packets += rpackets;
3357                 tot->tx_packets += tpackets;
3358                 tot->rx_bytes   += rbytes;
3359                 tot->tx_bytes   += tbytes;
3360                 tot->rx_dropped += rdrops;
3361                 tot->tx_errors  += terrors;
3362         }
3363
3364         tot->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
3365         tot->tx_fifo_errors = DEV_STATS_READ(dev, tx_fifo_errors);
3366         tot->rx_length_errors = DEV_STATS_READ(dev, rx_length_errors);
3367         tot->rx_frame_errors = DEV_STATS_READ(dev, rx_frame_errors);
3368 }
3369
3370 static void virtnet_ack_link_announce(struct virtnet_info *vi)
3371 {
3372         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
3373                                   VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
3374                 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
3375 }
3376
3377 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
3378 {
3379         struct virtio_net_ctrl_mq *mq __free(kfree) = NULL;
3380         struct scatterlist sg;
3381         struct net_device *dev = vi->dev;
3382
3383         if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
3384                 return 0;
3385
3386         mq = kzalloc(sizeof(*mq), GFP_KERNEL);
3387         if (!mq)
3388                 return -ENOMEM;
3389
3390         mq->virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
3391         sg_init_one(&sg, mq, sizeof(*mq));
3392
3393         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
3394                                   VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
3395                 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
3396                          queue_pairs);
3397                 return -EINVAL;
3398         } else {
3399                 vi->curr_queue_pairs = queue_pairs;
3400                 /* virtnet_open() will refill when device is going to up. */
3401                 if (dev->flags & IFF_UP)
3402                         schedule_delayed_work(&vi->refill, 0);
3403         }
3404
3405         return 0;
3406 }
3407
3408 static int virtnet_close(struct net_device *dev)
3409 {
3410         struct virtnet_info *vi = netdev_priv(dev);
3411         int i;
3412
3413         /* Make sure NAPI doesn't schedule refill work */
3414         disable_delayed_refill(vi);
3415         /* Make sure refill_work doesn't re-enable napi! */
3416         cancel_delayed_work_sync(&vi->refill);
3417         /* Prevent the config change callback from changing carrier
3418          * after close
3419          */
3420         virtio_config_driver_disable(vi->vdev);
3421         /* Stop getting status/speed updates: we don't care until next
3422          * open
3423          */
3424         cancel_work_sync(&vi->config_work);
3425
3426         for (i = 0; i < vi->max_queue_pairs; i++) {
3427                 virtnet_disable_queue_pair(vi, i);
3428                 virtnet_cancel_dim(vi, &vi->rq[i].dim);
3429         }
3430
3431         netif_carrier_off(dev);
3432
3433         return 0;
3434 }
3435
3436 static void virtnet_rx_mode_work(struct work_struct *work)
3437 {
3438         struct virtnet_info *vi =
3439                 container_of(work, struct virtnet_info, rx_mode_work);
3440         u8 *promisc_allmulti  __free(kfree) = NULL;
3441         struct net_device *dev = vi->dev;
3442         struct scatterlist sg[2];
3443         struct virtio_net_ctrl_mac *mac_data;
3444         struct netdev_hw_addr *ha;
3445         int uc_count;
3446         int mc_count;
3447         void *buf;
3448         int i;
3449
3450         /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
3451         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
3452                 return;
3453
3454         promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_KERNEL);
3455         if (!promisc_allmulti) {
3456                 dev_warn(&dev->dev, "Failed to set RX mode, no memory.\n");
3457                 return;
3458         }
3459
3460         rtnl_lock();
3461
3462         *promisc_allmulti = !!(dev->flags & IFF_PROMISC);
3463         sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti));
3464
3465         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
3466                                   VIRTIO_NET_CTRL_RX_PROMISC, sg))
3467                 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
3468                          *promisc_allmulti ? "en" : "dis");
3469
3470         *promisc_allmulti = !!(dev->flags & IFF_ALLMULTI);
3471         sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti));
3472
3473         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
3474                                   VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
3475                 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
3476                          *promisc_allmulti ? "en" : "dis");
3477
3478         netif_addr_lock_bh(dev);
3479
3480         uc_count = netdev_uc_count(dev);
3481         mc_count = netdev_mc_count(dev);
3482         /* MAC filter - use one buffer for both lists */
3483         buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
3484                       (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
3485         mac_data = buf;
3486         if (!buf) {
3487                 netif_addr_unlock_bh(dev);
3488                 rtnl_unlock();
3489                 return;
3490         }
3491
3492         sg_init_table(sg, 2);
3493
3494         /* Store the unicast list and count in the front of the buffer */
3495         mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
3496         i = 0;
3497         netdev_for_each_uc_addr(ha, dev)
3498                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
3499
3500         sg_set_buf(&sg[0], mac_data,
3501                    sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
3502
3503         /* multicast list and count fill the end */
3504         mac_data = (void *)&mac_data->macs[uc_count][0];
3505
3506         mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
3507         i = 0;
3508         netdev_for_each_mc_addr(ha, dev)
3509                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
3510
3511         netif_addr_unlock_bh(dev);
3512
3513         sg_set_buf(&sg[1], mac_data,
3514                    sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
3515
3516         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
3517                                   VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
3518                 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
3519
3520         rtnl_unlock();
3521
3522         kfree(buf);
3523 }
3524
3525 static void virtnet_set_rx_mode(struct net_device *dev)
3526 {
3527         struct virtnet_info *vi = netdev_priv(dev);
3528
3529         if (vi->rx_mode_work_enabled)
3530                 schedule_work(&vi->rx_mode_work);
3531 }
3532
3533 static int virtnet_vlan_rx_add_vid(struct net_device *dev,
3534                                    __be16 proto, u16 vid)
3535 {
3536         struct virtnet_info *vi = netdev_priv(dev);
3537         __virtio16 *_vid __free(kfree) = NULL;
3538         struct scatterlist sg;
3539
3540         _vid = kzalloc(sizeof(*_vid), GFP_KERNEL);
3541         if (!_vid)
3542                 return -ENOMEM;
3543
3544         *_vid = cpu_to_virtio16(vi->vdev, vid);
3545         sg_init_one(&sg, _vid, sizeof(*_vid));
3546
3547         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
3548                                   VIRTIO_NET_CTRL_VLAN_ADD, &sg))
3549                 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
3550         return 0;
3551 }
3552
3553 static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
3554                                     __be16 proto, u16 vid)
3555 {
3556         struct virtnet_info *vi = netdev_priv(dev);
3557         __virtio16 *_vid __free(kfree) = NULL;
3558         struct scatterlist sg;
3559
3560         _vid = kzalloc(sizeof(*_vid), GFP_KERNEL);
3561         if (!_vid)
3562                 return -ENOMEM;
3563
3564         *_vid = cpu_to_virtio16(vi->vdev, vid);
3565         sg_init_one(&sg, _vid, sizeof(*_vid));
3566
3567         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
3568                                   VIRTIO_NET_CTRL_VLAN_DEL, &sg))
3569                 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
3570         return 0;
3571 }
3572
3573 static void virtnet_clean_affinity(struct virtnet_info *vi)
3574 {
3575         int i;
3576
3577         if (vi->affinity_hint_set) {
3578                 for (i = 0; i < vi->max_queue_pairs; i++) {
3579                         virtqueue_set_affinity(vi->rq[i].vq, NULL);
3580                         virtqueue_set_affinity(vi->sq[i].vq, NULL);
3581                 }
3582
3583                 vi->affinity_hint_set = false;
3584         }
3585 }
3586
3587 static void virtnet_set_affinity(struct virtnet_info *vi)
3588 {
3589         cpumask_var_t mask;
3590         int stragglers;
3591         int group_size;
3592         int i, j, cpu;
3593         int num_cpu;
3594         int stride;
3595
3596         if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
3597                 virtnet_clean_affinity(vi);
3598                 return;
3599         }
3600
3601         num_cpu = num_online_cpus();
3602         stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);
3603         stragglers = num_cpu >= vi->curr_queue_pairs ?
3604                         num_cpu % vi->curr_queue_pairs :
3605                         0;
3606         cpu = cpumask_first(cpu_online_mask);
3607
3608         for (i = 0; i < vi->curr_queue_pairs; i++) {
3609                 group_size = stride + (i < stragglers ? 1 : 0);
3610
3611                 for (j = 0; j < group_size; j++) {
3612                         cpumask_set_cpu(cpu, mask);
3613                         cpu = cpumask_next_wrap(cpu, cpu_online_mask,
3614                                                 nr_cpu_ids, false);
3615                 }
3616                 virtqueue_set_affinity(vi->rq[i].vq, mask);
3617                 virtqueue_set_affinity(vi->sq[i].vq, mask);
3618                 __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS);
3619                 cpumask_clear(mask);
3620         }
3621
3622         vi->affinity_hint_set = true;
3623         free_cpumask_var(mask);
3624 }
3625
3626 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
3627 {
3628         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
3629                                                    node);
3630         virtnet_set_affinity(vi);
3631         return 0;
3632 }
3633
3634 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
3635 {
3636         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
3637                                                    node_dead);
3638         virtnet_set_affinity(vi);
3639         return 0;
3640 }
3641
3642 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
3643 {
3644         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
3645                                                    node);
3646
3647         virtnet_clean_affinity(vi);
3648         return 0;
3649 }
3650
3651 static enum cpuhp_state virtionet_online;
3652
3653 static int virtnet_cpu_notif_add(struct virtnet_info *vi)
3654 {
3655         int ret;
3656
3657         ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
3658         if (ret)
3659                 return ret;
3660         ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
3661                                                &vi->node_dead);
3662         if (!ret)
3663                 return ret;
3664         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
3665         return ret;
3666 }
3667
3668 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
3669 {
3670         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
3671         cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
3672                                             &vi->node_dead);
3673 }
3674
3675 static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi,
3676                                          u16 vqn, u32 max_usecs, u32 max_packets)
3677 {
3678         struct virtio_net_ctrl_coal_vq *coal_vq __free(kfree) = NULL;
3679         struct scatterlist sgs;
3680
3681         coal_vq = kzalloc(sizeof(*coal_vq), GFP_KERNEL);
3682         if (!coal_vq)
3683                 return -ENOMEM;
3684
3685         coal_vq->vqn = cpu_to_le16(vqn);
3686         coal_vq->coal.max_usecs = cpu_to_le32(max_usecs);
3687         coal_vq->coal.max_packets = cpu_to_le32(max_packets);
3688         sg_init_one(&sgs, coal_vq, sizeof(*coal_vq));
3689
3690         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
3691                                   VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET,
3692                                   &sgs))
3693                 return -EINVAL;
3694
3695         return 0;
3696 }
3697
3698 static int virtnet_send_rx_ctrl_coal_vq_cmd(struct virtnet_info *vi,
3699                                             u16 queue, u32 max_usecs,
3700                                             u32 max_packets)
3701 {
3702         int err;
3703
3704         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
3705                 return -EOPNOTSUPP;
3706
3707         err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(queue),
3708                                             max_usecs, max_packets);
3709         if (err)
3710                 return err;
3711
3712         vi->rq[queue].intr_coal.max_usecs = max_usecs;
3713         vi->rq[queue].intr_coal.max_packets = max_packets;
3714
3715         return 0;
3716 }
3717
3718 static int virtnet_send_tx_ctrl_coal_vq_cmd(struct virtnet_info *vi,
3719                                             u16 queue, u32 max_usecs,
3720                                             u32 max_packets)
3721 {
3722         int err;
3723
3724         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
3725                 return -EOPNOTSUPP;
3726
3727         err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(queue),
3728                                             max_usecs, max_packets);
3729         if (err)
3730                 return err;
3731
3732         vi->sq[queue].intr_coal.max_usecs = max_usecs;
3733         vi->sq[queue].intr_coal.max_packets = max_packets;
3734
3735         return 0;
3736 }
3737
3738 static void virtnet_get_ringparam(struct net_device *dev,
3739                                   struct ethtool_ringparam *ring,
3740                                   struct kernel_ethtool_ringparam *kernel_ring,
3741                                   struct netlink_ext_ack *extack)
3742 {
3743         struct virtnet_info *vi = netdev_priv(dev);
3744
3745         ring->rx_max_pending = vi->rq[0].vq->num_max;
3746         ring->tx_max_pending = vi->sq[0].vq->num_max;
3747         ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
3748         ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
3749 }
3750
3751 static int virtnet_set_ringparam(struct net_device *dev,
3752                                  struct ethtool_ringparam *ring,
3753                                  struct kernel_ethtool_ringparam *kernel_ring,
3754                                  struct netlink_ext_ack *extack)
3755 {
3756         struct virtnet_info *vi = netdev_priv(dev);
3757         u32 rx_pending, tx_pending;
3758         struct receive_queue *rq;
3759         struct send_queue *sq;
3760         int i, err;
3761
3762         if (ring->rx_mini_pending || ring->rx_jumbo_pending)
3763                 return -EINVAL;
3764
3765         rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
3766         tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
3767
3768         if (ring->rx_pending == rx_pending &&
3769             ring->tx_pending == tx_pending)
3770                 return 0;
3771
3772         if (ring->rx_pending > vi->rq[0].vq->num_max)
3773                 return -EINVAL;
3774
3775         if (ring->tx_pending > vi->sq[0].vq->num_max)
3776                 return -EINVAL;
3777
3778         for (i = 0; i < vi->max_queue_pairs; i++) {
3779                 rq = vi->rq + i;
3780                 sq = vi->sq + i;
3781
3782                 if (ring->tx_pending != tx_pending) {
3783                         err = virtnet_tx_resize(vi, sq, ring->tx_pending);
3784                         if (err)
3785                                 return err;
3786
3787                         /* Upon disabling and re-enabling a transmit virtqueue, the device must
3788                          * set the coalescing parameters of the virtqueue to those configured
3789                          * through the VIRTIO_NET_CTRL_NOTF_COAL_TX_SET command, or, if the driver
3790                          * did not set any TX coalescing parameters, to 0.
3791                          */
3792                         err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, i,
3793                                                                vi->intr_coal_tx.max_usecs,
3794                                                                vi->intr_coal_tx.max_packets);
3795
3796                         /* Don't break the tx resize action if the vq coalescing is not
3797                          * supported. The same is true for rx resize below.
3798                          */
3799                         if (err && err != -EOPNOTSUPP)
3800                                 return err;
3801                 }
3802
3803                 if (ring->rx_pending != rx_pending) {
3804                         err = virtnet_rx_resize(vi, rq, ring->rx_pending);
3805                         if (err)
3806                                 return err;
3807
3808                         /* The reason is same as the transmit virtqueue reset */
3809                         mutex_lock(&vi->rq[i].dim_lock);
3810                         err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, i,
3811                                                                vi->intr_coal_rx.max_usecs,
3812                                                                vi->intr_coal_rx.max_packets);
3813                         mutex_unlock(&vi->rq[i].dim_lock);
3814                         if (err && err != -EOPNOTSUPP)
3815                                 return err;
3816                 }
3817         }
3818
3819         return 0;
3820 }
3821
3822 static bool virtnet_commit_rss_command(struct virtnet_info *vi)
3823 {
3824         struct net_device *dev = vi->dev;
3825         struct scatterlist sgs[4];
3826         unsigned int sg_buf_size;
3827
3828         /* prepare sgs */
3829         sg_init_table(sgs, 4);
3830
3831         sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table);
3832         sg_set_buf(&sgs[0], &vi->rss, sg_buf_size);
3833
3834         sg_buf_size = sizeof(uint16_t) * (vi->rss.indirection_table_mask + 1);
3835         sg_set_buf(&sgs[1], vi->rss.indirection_table, sg_buf_size);
3836
3837         sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key)
3838                         - offsetof(struct virtio_net_ctrl_rss, max_tx_vq);
3839         sg_set_buf(&sgs[2], &vi->rss.max_tx_vq, sg_buf_size);
3840
3841         sg_buf_size = vi->rss_key_size;
3842         sg_set_buf(&sgs[3], vi->rss.key, sg_buf_size);
3843
3844         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
3845                                   vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG
3846                                   : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs))
3847                 goto err;
3848
3849         return true;
3850
3851 err:
3852         dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n");
3853         return false;
3854
3855 }
3856
3857 static void virtnet_init_default_rss(struct virtnet_info *vi)
3858 {
3859         u32 indir_val = 0;
3860         int i = 0;
3861
3862         vi->rss.hash_types = vi->rss_hash_types_supported;
3863         vi->rss_hash_types_saved = vi->rss_hash_types_supported;
3864         vi->rss.indirection_table_mask = vi->rss_indir_table_size
3865                                                 ? vi->rss_indir_table_size - 1 : 0;
3866         vi->rss.unclassified_queue = 0;
3867
3868         for (; i < vi->rss_indir_table_size; ++i) {
3869                 indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
3870                 vi->rss.indirection_table[i] = indir_val;
3871         }
3872
3873         vi->rss.max_tx_vq = vi->has_rss ? vi->curr_queue_pairs : 0;
3874         vi->rss.hash_key_length = vi->rss_key_size;
3875
3876         netdev_rss_key_fill(vi->rss.key, vi->rss_key_size);
3877 }
3878
3879 static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info)
3880 {
3881         info->data = 0;
3882         switch (info->flow_type) {
3883         case TCP_V4_FLOW:
3884                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {
3885                         info->data = RXH_IP_SRC | RXH_IP_DST |
3886                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
3887                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
3888                         info->data = RXH_IP_SRC | RXH_IP_DST;
3889                 }
3890                 break;
3891         case TCP_V6_FLOW:
3892                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {
3893                         info->data = RXH_IP_SRC | RXH_IP_DST |
3894                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
3895                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
3896                         info->data = RXH_IP_SRC | RXH_IP_DST;
3897                 }
3898                 break;
3899         case UDP_V4_FLOW:
3900                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {
3901                         info->data = RXH_IP_SRC | RXH_IP_DST |
3902                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
3903                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
3904                         info->data = RXH_IP_SRC | RXH_IP_DST;
3905                 }
3906                 break;
3907         case UDP_V6_FLOW:
3908                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {
3909                         info->data = RXH_IP_SRC | RXH_IP_DST |
3910                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
3911                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
3912                         info->data = RXH_IP_SRC | RXH_IP_DST;
3913                 }
3914                 break;
3915         case IPV4_FLOW:
3916                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4)
3917                         info->data = RXH_IP_SRC | RXH_IP_DST;
3918
3919                 break;
3920         case IPV6_FLOW:
3921                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6)
3922                         info->data = RXH_IP_SRC | RXH_IP_DST;
3923
3924                 break;
3925         default:
3926                 info->data = 0;
3927                 break;
3928         }
3929 }
3930
3931 static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info)
3932 {
3933         u32 new_hashtypes = vi->rss_hash_types_saved;
3934         bool is_disable = info->data & RXH_DISCARD;
3935         bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3);
3936
3937         /* supports only 'sd', 'sdfn' and 'r' */
3938         if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable))
3939                 return false;
3940
3941         switch (info->flow_type) {
3942         case TCP_V4_FLOW:
3943                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4);
3944                 if (!is_disable)
3945                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
3946                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0);
3947                 break;
3948         case UDP_V4_FLOW:
3949                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4);
3950                 if (!is_disable)
3951                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
3952                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0);
3953                 break;
3954         case IPV4_FLOW:
3955                 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4;
3956                 if (!is_disable)
3957                         new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4;
3958                 break;
3959         case TCP_V6_FLOW:
3960                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6);
3961                 if (!is_disable)
3962                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
3963                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0);
3964                 break;
3965         case UDP_V6_FLOW:
3966                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6);
3967                 if (!is_disable)
3968                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
3969                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0);
3970                 break;
3971         case IPV6_FLOW:
3972                 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6;
3973                 if (!is_disable)
3974                         new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6;
3975                 break;
3976         default:
3977                 /* unsupported flow */
3978                 return false;
3979         }
3980
3981         /* if unsupported hashtype was set */
3982         if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported))
3983                 return false;
3984
3985         if (new_hashtypes != vi->rss_hash_types_saved) {
3986                 vi->rss_hash_types_saved = new_hashtypes;
3987                 vi->rss.hash_types = vi->rss_hash_types_saved;
3988                 if (vi->dev->features & NETIF_F_RXHASH)
3989                         return virtnet_commit_rss_command(vi);
3990         }
3991
3992         return true;
3993 }
3994
3995 static void virtnet_get_drvinfo(struct net_device *dev,
3996                                 struct ethtool_drvinfo *info)
3997 {
3998         struct virtnet_info *vi = netdev_priv(dev);
3999         struct virtio_device *vdev = vi->vdev;
4000
4001         strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
4002         strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
4003         strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
4004
4005 }
4006
4007 /* TODO: Eliminate OOO packets during switching */
4008 static int virtnet_set_channels(struct net_device *dev,
4009                                 struct ethtool_channels *channels)
4010 {
4011         struct virtnet_info *vi = netdev_priv(dev);
4012         u16 queue_pairs = channels->combined_count;
4013         int err;
4014
4015         /* We don't support separate rx/tx channels.
4016          * We don't allow setting 'other' channels.
4017          */
4018         if (channels->rx_count || channels->tx_count || channels->other_count)
4019                 return -EINVAL;
4020
4021         if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
4022                 return -EINVAL;
4023
4024         /* For now we don't support modifying channels while XDP is loaded
4025          * also when XDP is loaded all RX queues have XDP programs so we only
4026          * need to check a single RX queue.
4027          */
4028         if (vi->rq[0].xdp_prog)
4029                 return -EINVAL;
4030
4031         cpus_read_lock();
4032         err = virtnet_set_queues(vi, queue_pairs);
4033         if (err) {
4034                 cpus_read_unlock();
4035                 goto err;
4036         }
4037         virtnet_set_affinity(vi);
4038         cpus_read_unlock();
4039
4040         netif_set_real_num_tx_queues(dev, queue_pairs);
4041         netif_set_real_num_rx_queues(dev, queue_pairs);
4042  err:
4043         return err;
4044 }
4045
4046 static void virtnet_stats_sprintf(u8 **p, const char *fmt, const char *noq_fmt,
4047                                   int num, int qid, const struct virtnet_stat_desc *desc)
4048 {
4049         int i;
4050
4051         if (qid < 0) {
4052                 for (i = 0; i < num; ++i)
4053                         ethtool_sprintf(p, noq_fmt, desc[i].desc);
4054         } else {
4055                 for (i = 0; i < num; ++i)
4056                         ethtool_sprintf(p, fmt, qid, desc[i].desc);
4057         }
4058 }
4059
4060 /* qid == -1: for rx/tx queue total field */
4061 static void virtnet_get_stats_string(struct virtnet_info *vi, int type, int qid, u8 **data)
4062 {
4063         const struct virtnet_stat_desc *desc;
4064         const char *fmt, *noq_fmt;
4065         u8 *p = *data;
4066         u32 num;
4067
4068         if (type == VIRTNET_Q_TYPE_CQ && qid >= 0) {
4069                 noq_fmt = "cq_hw_%s";
4070
4071                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) {
4072                         desc = &virtnet_stats_cvq_desc[0];
4073                         num = ARRAY_SIZE(virtnet_stats_cvq_desc);
4074
4075                         virtnet_stats_sprintf(&p, NULL, noq_fmt, num, -1, desc);
4076                 }
4077         }
4078
4079         if (type == VIRTNET_Q_TYPE_RX) {
4080                 fmt = "rx%u_%s";
4081                 noq_fmt = "rx_%s";
4082
4083                 desc = &virtnet_rq_stats_desc[0];
4084                 num = ARRAY_SIZE(virtnet_rq_stats_desc);
4085
4086                 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4087
4088                 fmt = "rx%u_hw_%s";
4089                 noq_fmt = "rx_hw_%s";
4090
4091                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4092                         desc = &virtnet_stats_rx_basic_desc[0];
4093                         num = ARRAY_SIZE(virtnet_stats_rx_basic_desc);
4094
4095                         virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4096                 }
4097
4098                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4099                         desc = &virtnet_stats_rx_csum_desc[0];
4100                         num = ARRAY_SIZE(virtnet_stats_rx_csum_desc);
4101
4102                         virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4103                 }
4104
4105                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4106                         desc = &virtnet_stats_rx_speed_desc[0];
4107                         num = ARRAY_SIZE(virtnet_stats_rx_speed_desc);
4108
4109                         virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4110                 }
4111         }
4112
4113         if (type == VIRTNET_Q_TYPE_TX) {
4114                 fmt = "tx%u_%s";
4115                 noq_fmt = "tx_%s";
4116
4117                 desc = &virtnet_sq_stats_desc[0];
4118                 num = ARRAY_SIZE(virtnet_sq_stats_desc);
4119
4120                 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4121
4122                 fmt = "tx%u_hw_%s";
4123                 noq_fmt = "tx_hw_%s";
4124
4125                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4126                         desc = &virtnet_stats_tx_basic_desc[0];
4127                         num = ARRAY_SIZE(virtnet_stats_tx_basic_desc);
4128
4129                         virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4130                 }
4131
4132                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4133                         desc = &virtnet_stats_tx_gso_desc[0];
4134                         num = ARRAY_SIZE(virtnet_stats_tx_gso_desc);
4135
4136                         virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4137                 }
4138
4139                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4140                         desc = &virtnet_stats_tx_speed_desc[0];
4141                         num = ARRAY_SIZE(virtnet_stats_tx_speed_desc);
4142
4143                         virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc);
4144                 }
4145         }
4146
4147         *data = p;
4148 }
4149
4150 struct virtnet_stats_ctx {
4151         /* The stats are write to qstats or ethtool -S */
4152         bool to_qstat;
4153
4154         /* Used to calculate the offset inside the output buffer. */
4155         u32 desc_num[3];
4156
4157         /* The actual supported stat types. */
4158         u32 bitmap[3];
4159
4160         /* Used to calculate the reply buffer size. */
4161         u32 size[3];
4162
4163         /* Record the output buffer. */
4164         u64 *data;
4165 };
4166
4167 static void virtnet_stats_ctx_init(struct virtnet_info *vi,
4168                                    struct virtnet_stats_ctx *ctx,
4169                                    u64 *data, bool to_qstat)
4170 {
4171         u32 queue_type;
4172
4173         ctx->data = data;
4174         ctx->to_qstat = to_qstat;
4175
4176         if (to_qstat) {
4177                 ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc_qstat);
4178                 ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc_qstat);
4179
4180                 queue_type = VIRTNET_Q_TYPE_RX;
4181
4182                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4183                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_RX_BASIC;
4184                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat);
4185                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_rx_basic);
4186                 }
4187
4188                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4189                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_RX_CSUM;
4190                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat);
4191                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_rx_csum);
4192                 }
4193
4194                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) {
4195                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_RX_GSO;
4196                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat);
4197                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_rx_gso);
4198                 }
4199
4200                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4201                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_RX_SPEED;
4202                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat);
4203                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_rx_speed);
4204                 }
4205
4206                 queue_type = VIRTNET_Q_TYPE_TX;
4207
4208                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4209                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_TX_BASIC;
4210                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat);
4211                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_tx_basic);
4212                 }
4213
4214                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) {
4215                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_TX_CSUM;
4216                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat);
4217                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_tx_csum);
4218                 }
4219
4220                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4221                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_TX_GSO;
4222                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat);
4223                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_tx_gso);
4224                 }
4225
4226                 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4227                         ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_TX_SPEED;
4228                         ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat);
4229                         ctx->size[queue_type]     += sizeof(struct virtio_net_stats_tx_speed);
4230                 }
4231
4232                 return;
4233         }
4234
4235         ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc);
4236         ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc);
4237
4238         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) {
4239                 queue_type = VIRTNET_Q_TYPE_CQ;
4240
4241                 ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_CVQ;
4242                 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_cvq_desc);
4243                 ctx->size[queue_type]     += sizeof(struct virtio_net_stats_cvq);
4244         }
4245
4246         queue_type = VIRTNET_Q_TYPE_RX;
4247
4248         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4249                 ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_RX_BASIC;
4250                 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc);
4251                 ctx->size[queue_type]     += sizeof(struct virtio_net_stats_rx_basic);
4252         }
4253
4254         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4255                 ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_RX_CSUM;
4256                 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc);
4257                 ctx->size[queue_type]     += sizeof(struct virtio_net_stats_rx_csum);
4258         }
4259
4260         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4261                 ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_RX_SPEED;
4262                 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc);
4263                 ctx->size[queue_type]     += sizeof(struct virtio_net_stats_rx_speed);
4264         }
4265
4266         queue_type = VIRTNET_Q_TYPE_TX;
4267
4268         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4269                 ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_TX_BASIC;
4270                 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc);
4271                 ctx->size[queue_type]     += sizeof(struct virtio_net_stats_tx_basic);
4272         }
4273
4274         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4275                 ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_TX_GSO;
4276                 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc);
4277                 ctx->size[queue_type]     += sizeof(struct virtio_net_stats_tx_gso);
4278         }
4279
4280         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4281                 ctx->bitmap[queue_type]   |= VIRTIO_NET_STATS_TYPE_TX_SPEED;
4282                 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc);
4283                 ctx->size[queue_type]     += sizeof(struct virtio_net_stats_tx_speed);
4284         }
4285 }
4286
4287 /* stats_sum_queue - Calculate the sum of the same fields in sq or rq.
4288  * @sum: the position to store the sum values
4289  * @num: field num
4290  * @q_value: the first queue fields
4291  * @q_num: number of the queues
4292  */
4293 static void stats_sum_queue(u64 *sum, u32 num, u64 *q_value, u32 q_num)
4294 {
4295         u32 step = num;
4296         int i, j;
4297         u64 *p;
4298
4299         for (i = 0; i < num; ++i) {
4300                 p = sum + i;
4301                 *p = 0;
4302
4303                 for (j = 0; j < q_num; ++j)
4304                         *p += *(q_value + i + j * step);
4305         }
4306 }
4307
4308 static void virtnet_fill_total_fields(struct virtnet_info *vi,
4309                                       struct virtnet_stats_ctx *ctx)
4310 {
4311         u64 *data, *first_rx_q, *first_tx_q;
4312         u32 num_cq, num_rx, num_tx;
4313
4314         num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ];
4315         num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX];
4316         num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX];
4317
4318         first_rx_q = ctx->data + num_rx + num_tx + num_cq;
4319         first_tx_q = first_rx_q + vi->curr_queue_pairs * num_rx;
4320
4321         data = ctx->data;
4322
4323         stats_sum_queue(data, num_rx, first_rx_q, vi->curr_queue_pairs);
4324
4325         data = ctx->data + num_rx;
4326
4327         stats_sum_queue(data, num_tx, first_tx_q, vi->curr_queue_pairs);
4328 }
4329
4330 static void virtnet_fill_stats_qstat(struct virtnet_info *vi, u32 qid,
4331                                      struct virtnet_stats_ctx *ctx,
4332                                      const u8 *base, bool drv_stats, u8 reply_type)
4333 {
4334         const struct virtnet_stat_desc *desc;
4335         const u64_stats_t *v_stat;
4336         u64 offset, bitmap;
4337         const __le64 *v;
4338         u32 queue_type;
4339         int i, num;
4340
4341         queue_type = vq_type(vi, qid);
4342         bitmap = ctx->bitmap[queue_type];
4343
4344         if (drv_stats) {
4345                 if (queue_type == VIRTNET_Q_TYPE_RX) {
4346                         desc = &virtnet_rq_stats_desc_qstat[0];
4347                         num = ARRAY_SIZE(virtnet_rq_stats_desc_qstat);
4348                 } else {
4349                         desc = &virtnet_sq_stats_desc_qstat[0];
4350                         num = ARRAY_SIZE(virtnet_sq_stats_desc_qstat);
4351                 }
4352
4353                 for (i = 0; i < num; ++i) {
4354                         offset = desc[i].qstat_offset / sizeof(*ctx->data);
4355                         v_stat = (const u64_stats_t *)(base + desc[i].offset);
4356                         ctx->data[offset] = u64_stats_read(v_stat);
4357                 }
4358                 return;
4359         }
4360
4361         if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4362                 desc = &virtnet_stats_rx_basic_desc_qstat[0];
4363                 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat);
4364                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC)
4365                         goto found;
4366         }
4367
4368         if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4369                 desc = &virtnet_stats_rx_csum_desc_qstat[0];
4370                 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat);
4371                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM)
4372                         goto found;
4373         }
4374
4375         if (bitmap & VIRTIO_NET_STATS_TYPE_RX_GSO) {
4376                 desc = &virtnet_stats_rx_gso_desc_qstat[0];
4377                 num = ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat);
4378                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_GSO)
4379                         goto found;
4380         }
4381
4382         if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4383                 desc = &virtnet_stats_rx_speed_desc_qstat[0];
4384                 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat);
4385                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED)
4386                         goto found;
4387         }
4388
4389         if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4390                 desc = &virtnet_stats_tx_basic_desc_qstat[0];
4391                 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat);
4392                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC)
4393                         goto found;
4394         }
4395
4396         if (bitmap & VIRTIO_NET_STATS_TYPE_TX_CSUM) {
4397                 desc = &virtnet_stats_tx_csum_desc_qstat[0];
4398                 num = ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat);
4399                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_CSUM)
4400                         goto found;
4401         }
4402
4403         if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4404                 desc = &virtnet_stats_tx_gso_desc_qstat[0];
4405                 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat);
4406                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO)
4407                         goto found;
4408         }
4409
4410         if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4411                 desc = &virtnet_stats_tx_speed_desc_qstat[0];
4412                 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat);
4413                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED)
4414                         goto found;
4415         }
4416
4417         return;
4418
4419 found:
4420         for (i = 0; i < num; ++i) {
4421                 offset = desc[i].qstat_offset / sizeof(*ctx->data);
4422                 v = (const __le64 *)(base + desc[i].offset);
4423                 ctx->data[offset] = le64_to_cpu(*v);
4424         }
4425 }
4426
4427 /* virtnet_fill_stats - copy the stats to qstats or ethtool -S
4428  * The stats source is the device or the driver.
4429  *
4430  * @vi: virtio net info
4431  * @qid: the vq id
4432  * @ctx: stats ctx (initiated by virtnet_stats_ctx_init())
4433  * @base: pointer to the device reply or the driver stats structure.
4434  * @drv_stats: designate the base type (device reply, driver stats)
4435  * @type: the type of the device reply (if drv_stats is true, this must be zero)
4436  */
4437 static void virtnet_fill_stats(struct virtnet_info *vi, u32 qid,
4438                                struct virtnet_stats_ctx *ctx,
4439                                const u8 *base, bool drv_stats, u8 reply_type)
4440 {
4441         u32 queue_type, num_rx, num_tx, num_cq;
4442         const struct virtnet_stat_desc *desc;
4443         const u64_stats_t *v_stat;
4444         u64 offset, bitmap;
4445         const __le64 *v;
4446         int i, num;
4447
4448         if (ctx->to_qstat)
4449                 return virtnet_fill_stats_qstat(vi, qid, ctx, base, drv_stats, reply_type);
4450
4451         num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ];
4452         num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX];
4453         num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX];
4454
4455         queue_type = vq_type(vi, qid);
4456         bitmap = ctx->bitmap[queue_type];
4457
4458         /* skip the total fields of pairs */
4459         offset = num_rx + num_tx;
4460
4461         if (queue_type == VIRTNET_Q_TYPE_TX) {
4462                 offset += num_cq + num_rx * vi->curr_queue_pairs + num_tx * (qid / 2);
4463
4464                 num = ARRAY_SIZE(virtnet_sq_stats_desc);
4465                 if (drv_stats) {
4466                         desc = &virtnet_sq_stats_desc[0];
4467                         goto drv_stats;
4468                 }
4469
4470                 offset += num;
4471
4472         } else if (queue_type == VIRTNET_Q_TYPE_RX) {
4473                 offset += num_cq + num_rx * (qid / 2);
4474
4475                 num = ARRAY_SIZE(virtnet_rq_stats_desc);
4476                 if (drv_stats) {
4477                         desc = &virtnet_rq_stats_desc[0];
4478                         goto drv_stats;
4479                 }
4480
4481                 offset += num;
4482         }
4483
4484         if (bitmap & VIRTIO_NET_STATS_TYPE_CVQ) {
4485                 desc = &virtnet_stats_cvq_desc[0];
4486                 num = ARRAY_SIZE(virtnet_stats_cvq_desc);
4487                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_CVQ)
4488                         goto found;
4489
4490                 offset += num;
4491         }
4492
4493         if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
4494                 desc = &virtnet_stats_rx_basic_desc[0];
4495                 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc);
4496                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC)
4497                         goto found;
4498
4499                 offset += num;
4500         }
4501
4502         if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
4503                 desc = &virtnet_stats_rx_csum_desc[0];
4504                 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc);
4505                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM)
4506                         goto found;
4507
4508                 offset += num;
4509         }
4510
4511         if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) {
4512                 desc = &virtnet_stats_rx_speed_desc[0];
4513                 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc);
4514                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED)
4515                         goto found;
4516
4517                 offset += num;
4518         }
4519
4520         if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
4521                 desc = &virtnet_stats_tx_basic_desc[0];
4522                 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc);
4523                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC)
4524                         goto found;
4525
4526                 offset += num;
4527         }
4528
4529         if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
4530                 desc = &virtnet_stats_tx_gso_desc[0];
4531                 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc);
4532                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO)
4533                         goto found;
4534
4535                 offset += num;
4536         }
4537
4538         if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) {
4539                 desc = &virtnet_stats_tx_speed_desc[0];
4540                 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc);
4541                 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED)
4542                         goto found;
4543
4544                 offset += num;
4545         }
4546
4547         return;
4548
4549 found:
4550         for (i = 0; i < num; ++i) {
4551                 v = (const __le64 *)(base + desc[i].offset);
4552                 ctx->data[offset + i] = le64_to_cpu(*v);
4553         }
4554
4555         return;
4556
4557 drv_stats:
4558         for (i = 0; i < num; ++i) {
4559                 v_stat = (const u64_stats_t *)(base + desc[i].offset);
4560                 ctx->data[offset + i] = u64_stats_read(v_stat);
4561         }
4562 }
4563
4564 static int __virtnet_get_hw_stats(struct virtnet_info *vi,
4565                                   struct virtnet_stats_ctx *ctx,
4566                                   struct virtio_net_ctrl_queue_stats *req,
4567                                   int req_size, void *reply, int res_size)
4568 {
4569         struct virtio_net_stats_reply_hdr *hdr;
4570         struct scatterlist sgs_in, sgs_out;
4571         void *p;
4572         u32 qid;
4573         int ok;
4574
4575         sg_init_one(&sgs_out, req, req_size);
4576         sg_init_one(&sgs_in, reply, res_size);
4577
4578         ok = virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS,
4579                                         VIRTIO_NET_CTRL_STATS_GET,
4580                                         &sgs_out, &sgs_in);
4581
4582         if (!ok)
4583                 return ok;
4584
4585         for (p = reply; p - reply < res_size; p += le16_to_cpu(hdr->size)) {
4586                 hdr = p;
4587                 qid = le16_to_cpu(hdr->vq_index);
4588                 virtnet_fill_stats(vi, qid, ctx, p, false, hdr->type);
4589         }
4590
4591         return 0;
4592 }
4593
4594 static void virtnet_make_stat_req(struct virtnet_info *vi,
4595                                   struct virtnet_stats_ctx *ctx,
4596                                   struct virtio_net_ctrl_queue_stats *req,
4597                                   int qid, int *idx)
4598 {
4599         int qtype = vq_type(vi, qid);
4600         u64 bitmap = ctx->bitmap[qtype];
4601
4602         if (!bitmap)
4603                 return;
4604
4605         req->stats[*idx].vq_index = cpu_to_le16(qid);
4606         req->stats[*idx].types_bitmap[0] = cpu_to_le64(bitmap);
4607         *idx += 1;
4608 }
4609
4610 /* qid: -1: get stats of all vq.
4611  *     > 0: get the stats for the special vq. This must not be cvq.
4612  */
4613 static int virtnet_get_hw_stats(struct virtnet_info *vi,
4614                                 struct virtnet_stats_ctx *ctx, int qid)
4615 {
4616         int qnum, i, j, res_size, qtype, last_vq, first_vq;
4617         struct virtio_net_ctrl_queue_stats *req;
4618         bool enable_cvq;
4619         void *reply;
4620         int ok;
4621
4622         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS))
4623                 return 0;
4624
4625         if (qid == -1) {
4626                 last_vq = vi->curr_queue_pairs * 2 - 1;
4627                 first_vq = 0;
4628                 enable_cvq = true;
4629         } else {
4630                 last_vq = qid;
4631                 first_vq = qid;
4632                 enable_cvq = false;
4633         }
4634
4635         qnum = 0;
4636         res_size = 0;
4637         for (i = first_vq; i <= last_vq ; ++i) {
4638                 qtype = vq_type(vi, i);
4639                 if (ctx->bitmap[qtype]) {
4640                         ++qnum;
4641                         res_size += ctx->size[qtype];
4642                 }
4643         }
4644
4645         if (enable_cvq && ctx->bitmap[VIRTNET_Q_TYPE_CQ]) {
4646                 res_size += ctx->size[VIRTNET_Q_TYPE_CQ];
4647                 qnum += 1;
4648         }
4649
4650         req = kcalloc(qnum, sizeof(*req), GFP_KERNEL);
4651         if (!req)
4652                 return -ENOMEM;
4653
4654         reply = kmalloc(res_size, GFP_KERNEL);
4655         if (!reply) {
4656                 kfree(req);
4657                 return -ENOMEM;
4658         }
4659
4660         j = 0;
4661         for (i = first_vq; i <= last_vq ; ++i)
4662                 virtnet_make_stat_req(vi, ctx, req, i, &j);
4663
4664         if (enable_cvq)
4665                 virtnet_make_stat_req(vi, ctx, req, vi->max_queue_pairs * 2, &j);
4666
4667         ok = __virtnet_get_hw_stats(vi, ctx, req, sizeof(*req) * j, reply, res_size);
4668
4669         kfree(req);
4670         kfree(reply);
4671
4672         return ok;
4673 }
4674
4675 static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
4676 {
4677         struct virtnet_info *vi = netdev_priv(dev);
4678         unsigned int i;
4679         u8 *p = data;
4680
4681         switch (stringset) {
4682         case ETH_SS_STATS:
4683                 /* Generate the total field names. */
4684                 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, -1, &p);
4685                 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, -1, &p);
4686
4687                 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_CQ, 0, &p);
4688
4689                 for (i = 0; i < vi->curr_queue_pairs; ++i)
4690                         virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, i, &p);
4691
4692                 for (i = 0; i < vi->curr_queue_pairs; ++i)
4693                         virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, i, &p);
4694                 break;
4695         }
4696 }
4697
4698 static int virtnet_get_sset_count(struct net_device *dev, int sset)
4699 {
4700         struct virtnet_info *vi = netdev_priv(dev);
4701         struct virtnet_stats_ctx ctx = {0};
4702         u32 pair_count;
4703
4704         switch (sset) {
4705         case ETH_SS_STATS:
4706                 virtnet_stats_ctx_init(vi, &ctx, NULL, false);
4707
4708                 pair_count = ctx.desc_num[VIRTNET_Q_TYPE_RX] + ctx.desc_num[VIRTNET_Q_TYPE_TX];
4709
4710                 return pair_count + ctx.desc_num[VIRTNET_Q_TYPE_CQ] +
4711                         vi->curr_queue_pairs * pair_count;
4712         default:
4713                 return -EOPNOTSUPP;
4714         }
4715 }
4716
4717 static void virtnet_get_ethtool_stats(struct net_device *dev,
4718                                       struct ethtool_stats *stats, u64 *data)
4719 {
4720         struct virtnet_info *vi = netdev_priv(dev);
4721         struct virtnet_stats_ctx ctx = {0};
4722         unsigned int start, i;
4723         const u8 *stats_base;
4724
4725         virtnet_stats_ctx_init(vi, &ctx, data, false);
4726         if (virtnet_get_hw_stats(vi, &ctx, -1))
4727                 dev_warn(&vi->dev->dev, "Failed to get hw stats.\n");
4728
4729         for (i = 0; i < vi->curr_queue_pairs; i++) {
4730                 struct receive_queue *rq = &vi->rq[i];
4731                 struct send_queue *sq = &vi->sq[i];
4732
4733                 stats_base = (const u8 *)&rq->stats;
4734                 do {
4735                         start = u64_stats_fetch_begin(&rq->stats.syncp);
4736                         virtnet_fill_stats(vi, i * 2, &ctx, stats_base, true, 0);
4737                 } while (u64_stats_fetch_retry(&rq->stats.syncp, start));
4738
4739                 stats_base = (const u8 *)&sq->stats;
4740                 do {
4741                         start = u64_stats_fetch_begin(&sq->stats.syncp);
4742                         virtnet_fill_stats(vi, i * 2 + 1, &ctx, stats_base, true, 0);
4743                 } while (u64_stats_fetch_retry(&sq->stats.syncp, start));
4744         }
4745
4746         virtnet_fill_total_fields(vi, &ctx);
4747 }
4748
4749 static void virtnet_get_channels(struct net_device *dev,
4750                                  struct ethtool_channels *channels)
4751 {
4752         struct virtnet_info *vi = netdev_priv(dev);
4753
4754         channels->combined_count = vi->curr_queue_pairs;
4755         channels->max_combined = vi->max_queue_pairs;
4756         channels->max_other = 0;
4757         channels->rx_count = 0;
4758         channels->tx_count = 0;
4759         channels->other_count = 0;
4760 }
4761
4762 static int virtnet_set_link_ksettings(struct net_device *dev,
4763                                       const struct ethtool_link_ksettings *cmd)
4764 {
4765         struct virtnet_info *vi = netdev_priv(dev);
4766
4767         return ethtool_virtdev_set_link_ksettings(dev, cmd,
4768                                                   &vi->speed, &vi->duplex);
4769 }
4770
4771 static int virtnet_get_link_ksettings(struct net_device *dev,
4772                                       struct ethtool_link_ksettings *cmd)
4773 {
4774         struct virtnet_info *vi = netdev_priv(dev);
4775
4776         cmd->base.speed = vi->speed;
4777         cmd->base.duplex = vi->duplex;
4778         cmd->base.port = PORT_OTHER;
4779
4780         return 0;
4781 }
4782
4783 static int virtnet_send_tx_notf_coal_cmds(struct virtnet_info *vi,
4784                                           struct ethtool_coalesce *ec)
4785 {
4786         struct virtio_net_ctrl_coal_tx *coal_tx __free(kfree) = NULL;
4787         struct scatterlist sgs_tx;
4788         int i;
4789
4790         coal_tx = kzalloc(sizeof(*coal_tx), GFP_KERNEL);
4791         if (!coal_tx)
4792                 return -ENOMEM;
4793
4794         coal_tx->tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs);
4795         coal_tx->tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames);
4796         sg_init_one(&sgs_tx, coal_tx, sizeof(*coal_tx));
4797
4798         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
4799                                   VIRTIO_NET_CTRL_NOTF_COAL_TX_SET,
4800                                   &sgs_tx))
4801                 return -EINVAL;
4802
4803         vi->intr_coal_tx.max_usecs = ec->tx_coalesce_usecs;
4804         vi->intr_coal_tx.max_packets = ec->tx_max_coalesced_frames;
4805         for (i = 0; i < vi->max_queue_pairs; i++) {
4806                 vi->sq[i].intr_coal.max_usecs = ec->tx_coalesce_usecs;
4807                 vi->sq[i].intr_coal.max_packets = ec->tx_max_coalesced_frames;
4808         }
4809
4810         return 0;
4811 }
4812
4813 static int virtnet_send_rx_notf_coal_cmds(struct virtnet_info *vi,
4814                                           struct ethtool_coalesce *ec)
4815 {
4816         struct virtio_net_ctrl_coal_rx *coal_rx __free(kfree) = NULL;
4817         bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce;
4818         struct scatterlist sgs_rx;
4819         int i;
4820
4821         if (rx_ctrl_dim_on && !virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
4822                 return -EOPNOTSUPP;
4823
4824         if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != vi->intr_coal_rx.max_usecs ||
4825                                ec->rx_max_coalesced_frames != vi->intr_coal_rx.max_packets))
4826                 return -EINVAL;
4827
4828         if (rx_ctrl_dim_on && !vi->rx_dim_enabled) {
4829                 vi->rx_dim_enabled = true;
4830                 for (i = 0; i < vi->max_queue_pairs; i++) {
4831                         mutex_lock(&vi->rq[i].dim_lock);
4832                         vi->rq[i].dim_enabled = true;
4833                         mutex_unlock(&vi->rq[i].dim_lock);
4834                 }
4835                 return 0;
4836         }
4837
4838         coal_rx = kzalloc(sizeof(*coal_rx), GFP_KERNEL);
4839         if (!coal_rx)
4840                 return -ENOMEM;
4841
4842         if (!rx_ctrl_dim_on && vi->rx_dim_enabled) {
4843                 vi->rx_dim_enabled = false;
4844                 for (i = 0; i < vi->max_queue_pairs; i++) {
4845                         mutex_lock(&vi->rq[i].dim_lock);
4846                         vi->rq[i].dim_enabled = false;
4847                         mutex_unlock(&vi->rq[i].dim_lock);
4848                 }
4849         }
4850
4851         /* Since the per-queue coalescing params can be set,
4852          * we need apply the global new params even if they
4853          * are not updated.
4854          */
4855         coal_rx->rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs);
4856         coal_rx->rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames);
4857         sg_init_one(&sgs_rx, coal_rx, sizeof(*coal_rx));
4858
4859         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
4860                                   VIRTIO_NET_CTRL_NOTF_COAL_RX_SET,
4861                                   &sgs_rx))
4862                 return -EINVAL;
4863
4864         vi->intr_coal_rx.max_usecs = ec->rx_coalesce_usecs;
4865         vi->intr_coal_rx.max_packets = ec->rx_max_coalesced_frames;
4866         for (i = 0; i < vi->max_queue_pairs; i++) {
4867                 mutex_lock(&vi->rq[i].dim_lock);
4868                 vi->rq[i].intr_coal.max_usecs = ec->rx_coalesce_usecs;
4869                 vi->rq[i].intr_coal.max_packets = ec->rx_max_coalesced_frames;
4870                 mutex_unlock(&vi->rq[i].dim_lock);
4871         }
4872
4873         return 0;
4874 }
4875
4876 static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi,
4877                                        struct ethtool_coalesce *ec)
4878 {
4879         int err;
4880
4881         err = virtnet_send_tx_notf_coal_cmds(vi, ec);
4882         if (err)
4883                 return err;
4884
4885         err = virtnet_send_rx_notf_coal_cmds(vi, ec);
4886         if (err)
4887                 return err;
4888
4889         return 0;
4890 }
4891
4892 static int virtnet_send_rx_notf_coal_vq_cmds(struct virtnet_info *vi,
4893                                              struct ethtool_coalesce *ec,
4894                                              u16 queue)
4895 {
4896         bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce;
4897         u32 max_usecs, max_packets;
4898         bool cur_rx_dim;
4899         int err;
4900
4901         mutex_lock(&vi->rq[queue].dim_lock);
4902         cur_rx_dim = vi->rq[queue].dim_enabled;
4903         max_usecs = vi->rq[queue].intr_coal.max_usecs;
4904         max_packets = vi->rq[queue].intr_coal.max_packets;
4905
4906         if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != max_usecs ||
4907                                ec->rx_max_coalesced_frames != max_packets)) {
4908                 mutex_unlock(&vi->rq[queue].dim_lock);
4909                 return -EINVAL;
4910         }
4911
4912         if (rx_ctrl_dim_on && !cur_rx_dim) {
4913                 vi->rq[queue].dim_enabled = true;
4914                 mutex_unlock(&vi->rq[queue].dim_lock);
4915                 return 0;
4916         }
4917
4918         if (!rx_ctrl_dim_on && cur_rx_dim)
4919                 vi->rq[queue].dim_enabled = false;
4920
4921         /* If no params are updated, userspace ethtool will
4922          * reject the modification.
4923          */
4924         err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, queue,
4925                                                ec->rx_coalesce_usecs,
4926                                                ec->rx_max_coalesced_frames);
4927         mutex_unlock(&vi->rq[queue].dim_lock);
4928         return err;
4929 }
4930
4931 static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi,
4932                                           struct ethtool_coalesce *ec,
4933                                           u16 queue)
4934 {
4935         int err;
4936
4937         err = virtnet_send_rx_notf_coal_vq_cmds(vi, ec, queue);
4938         if (err)
4939                 return err;
4940
4941         err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, queue,
4942                                                ec->tx_coalesce_usecs,
4943                                                ec->tx_max_coalesced_frames);
4944         if (err)
4945                 return err;
4946
4947         return 0;
4948 }
4949
4950 static void virtnet_rx_dim_work(struct work_struct *work)
4951 {
4952         struct dim *dim = container_of(work, struct dim, work);
4953         struct receive_queue *rq = container_of(dim,
4954                         struct receive_queue, dim);
4955         struct virtnet_info *vi = rq->vq->vdev->priv;
4956         struct net_device *dev = vi->dev;
4957         struct dim_cq_moder update_moder;
4958         int qnum, err;
4959
4960         qnum = rq - vi->rq;
4961
4962         mutex_lock(&rq->dim_lock);
4963         if (!rq->dim_enabled)
4964                 goto out;
4965
4966         update_moder = net_dim_get_rx_irq_moder(dev, dim);
4967         if (update_moder.usec != rq->intr_coal.max_usecs ||
4968             update_moder.pkts != rq->intr_coal.max_packets) {
4969                 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, qnum,
4970                                                        update_moder.usec,
4971                                                        update_moder.pkts);
4972                 if (err)
4973                         pr_debug("%s: Failed to send dim parameters on rxq%d\n",
4974                                  dev->name, qnum);
4975         }
4976 out:
4977         dim->state = DIM_START_MEASURE;
4978         mutex_unlock(&rq->dim_lock);
4979 }
4980
4981 static int virtnet_coal_params_supported(struct ethtool_coalesce *ec)
4982 {
4983         /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL
4984          * or VIRTIO_NET_F_VQ_NOTF_COAL feature is negotiated.
4985          */
4986         if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs)
4987                 return -EOPNOTSUPP;
4988
4989         if (ec->tx_max_coalesced_frames > 1 ||
4990             ec->rx_max_coalesced_frames != 1)
4991                 return -EINVAL;
4992
4993         return 0;
4994 }
4995
4996 static int virtnet_should_update_vq_weight(int dev_flags, int weight,
4997                                            int vq_weight, bool *should_update)
4998 {
4999         if (weight ^ vq_weight) {
5000                 if (dev_flags & IFF_UP)
5001                         return -EBUSY;
5002                 *should_update = true;
5003         }
5004
5005         return 0;
5006 }
5007
5008 static int virtnet_set_coalesce(struct net_device *dev,
5009                                 struct ethtool_coalesce *ec,
5010                                 struct kernel_ethtool_coalesce *kernel_coal,
5011                                 struct netlink_ext_ack *extack)
5012 {
5013         struct virtnet_info *vi = netdev_priv(dev);
5014         int ret, queue_number, napi_weight;
5015         bool update_napi = false;
5016
5017         /* Can't change NAPI weight if the link is up */
5018         napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
5019         for (queue_number = 0; queue_number < vi->max_queue_pairs; queue_number++) {
5020                 ret = virtnet_should_update_vq_weight(dev->flags, napi_weight,
5021                                                       vi->sq[queue_number].napi.weight,
5022                                                       &update_napi);
5023                 if (ret)
5024                         return ret;
5025
5026                 if (update_napi) {
5027                         /* All queues that belong to [queue_number, vi->max_queue_pairs] will be
5028                          * updated for the sake of simplicity, which might not be necessary
5029                          */
5030                         break;
5031                 }
5032         }
5033
5034         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL))
5035                 ret = virtnet_send_notf_coal_cmds(vi, ec);
5036         else
5037                 ret = virtnet_coal_params_supported(ec);
5038
5039         if (ret)
5040                 return ret;
5041
5042         if (update_napi) {
5043                 for (; queue_number < vi->max_queue_pairs; queue_number++)
5044                         vi->sq[queue_number].napi.weight = napi_weight;
5045         }
5046
5047         return ret;
5048 }
5049
5050 static int virtnet_get_coalesce(struct net_device *dev,
5051                                 struct ethtool_coalesce *ec,
5052                                 struct kernel_ethtool_coalesce *kernel_coal,
5053                                 struct netlink_ext_ack *extack)
5054 {
5055         struct virtnet_info *vi = netdev_priv(dev);
5056
5057         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
5058                 ec->rx_coalesce_usecs = vi->intr_coal_rx.max_usecs;
5059                 ec->tx_coalesce_usecs = vi->intr_coal_tx.max_usecs;
5060                 ec->tx_max_coalesced_frames = vi->intr_coal_tx.max_packets;
5061                 ec->rx_max_coalesced_frames = vi->intr_coal_rx.max_packets;
5062                 ec->use_adaptive_rx_coalesce = vi->rx_dim_enabled;
5063         } else {
5064                 ec->rx_max_coalesced_frames = 1;
5065
5066                 if (vi->sq[0].napi.weight)
5067                         ec->tx_max_coalesced_frames = 1;
5068         }
5069
5070         return 0;
5071 }
5072
5073 static int virtnet_set_per_queue_coalesce(struct net_device *dev,
5074                                           u32 queue,
5075                                           struct ethtool_coalesce *ec)
5076 {
5077         struct virtnet_info *vi = netdev_priv(dev);
5078         int ret, napi_weight;
5079         bool update_napi = false;
5080
5081         if (queue >= vi->max_queue_pairs)
5082                 return -EINVAL;
5083
5084         /* Can't change NAPI weight if the link is up */
5085         napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
5086         ret = virtnet_should_update_vq_weight(dev->flags, napi_weight,
5087                                               vi->sq[queue].napi.weight,
5088                                               &update_napi);
5089         if (ret)
5090                 return ret;
5091
5092         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
5093                 ret = virtnet_send_notf_coal_vq_cmds(vi, ec, queue);
5094         else
5095                 ret = virtnet_coal_params_supported(ec);
5096
5097         if (ret)
5098                 return ret;
5099
5100         if (update_napi)
5101                 vi->sq[queue].napi.weight = napi_weight;
5102
5103         return 0;
5104 }
5105
5106 static int virtnet_get_per_queue_coalesce(struct net_device *dev,
5107                                           u32 queue,
5108                                           struct ethtool_coalesce *ec)
5109 {
5110         struct virtnet_info *vi = netdev_priv(dev);
5111
5112         if (queue >= vi->max_queue_pairs)
5113                 return -EINVAL;
5114
5115         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) {
5116                 mutex_lock(&vi->rq[queue].dim_lock);
5117                 ec->rx_coalesce_usecs = vi->rq[queue].intr_coal.max_usecs;
5118                 ec->tx_coalesce_usecs = vi->sq[queue].intr_coal.max_usecs;
5119                 ec->tx_max_coalesced_frames = vi->sq[queue].intr_coal.max_packets;
5120                 ec->rx_max_coalesced_frames = vi->rq[queue].intr_coal.max_packets;
5121                 ec->use_adaptive_rx_coalesce = vi->rq[queue].dim_enabled;
5122                 mutex_unlock(&vi->rq[queue].dim_lock);
5123         } else {
5124                 ec->rx_max_coalesced_frames = 1;
5125
5126                 if (vi->sq[queue].napi.weight)
5127                         ec->tx_max_coalesced_frames = 1;
5128         }
5129
5130         return 0;
5131 }
5132
5133 static void virtnet_init_settings(struct net_device *dev)
5134 {
5135         struct virtnet_info *vi = netdev_priv(dev);
5136
5137         vi->speed = SPEED_UNKNOWN;
5138         vi->duplex = DUPLEX_UNKNOWN;
5139 }
5140
5141 static u32 virtnet_get_rxfh_key_size(struct net_device *dev)
5142 {
5143         return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size;
5144 }
5145
5146 static u32 virtnet_get_rxfh_indir_size(struct net_device *dev)
5147 {
5148         return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size;
5149 }
5150
5151 static int virtnet_get_rxfh(struct net_device *dev,
5152                             struct ethtool_rxfh_param *rxfh)
5153 {
5154         struct virtnet_info *vi = netdev_priv(dev);
5155         int i;
5156
5157         if (rxfh->indir) {
5158                 for (i = 0; i < vi->rss_indir_table_size; ++i)
5159                         rxfh->indir[i] = vi->rss.indirection_table[i];
5160         }
5161
5162         if (rxfh->key)
5163                 memcpy(rxfh->key, vi->rss.key, vi->rss_key_size);
5164
5165         rxfh->hfunc = ETH_RSS_HASH_TOP;
5166
5167         return 0;
5168 }
5169
5170 static int virtnet_set_rxfh(struct net_device *dev,
5171                             struct ethtool_rxfh_param *rxfh,
5172                             struct netlink_ext_ack *extack)
5173 {
5174         struct virtnet_info *vi = netdev_priv(dev);
5175         bool update = false;
5176         int i;
5177
5178         if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
5179             rxfh->hfunc != ETH_RSS_HASH_TOP)
5180                 return -EOPNOTSUPP;
5181
5182         if (rxfh->indir) {
5183                 if (!vi->has_rss)
5184                         return -EOPNOTSUPP;
5185
5186                 for (i = 0; i < vi->rss_indir_table_size; ++i)
5187                         vi->rss.indirection_table[i] = rxfh->indir[i];
5188                 update = true;
5189         }
5190
5191         if (rxfh->key) {
5192                 /* If either _F_HASH_REPORT or _F_RSS are negotiated, the
5193                  * device provides hash calculation capabilities, that is,
5194                  * hash_key is configured.
5195                  */
5196                 if (!vi->has_rss && !vi->has_rss_hash_report)
5197                         return -EOPNOTSUPP;
5198
5199                 memcpy(vi->rss.key, rxfh->key, vi->rss_key_size);
5200                 update = true;
5201         }
5202
5203         if (update)
5204                 virtnet_commit_rss_command(vi);
5205
5206         return 0;
5207 }
5208
5209 static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs)
5210 {
5211         struct virtnet_info *vi = netdev_priv(dev);
5212         int rc = 0;
5213
5214         switch (info->cmd) {
5215         case ETHTOOL_GRXRINGS:
5216                 info->data = vi->curr_queue_pairs;
5217                 break;
5218         case ETHTOOL_GRXFH:
5219                 virtnet_get_hashflow(vi, info);
5220                 break;
5221         default:
5222                 rc = -EOPNOTSUPP;
5223         }
5224
5225         return rc;
5226 }
5227
5228 static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info)
5229 {
5230         struct virtnet_info *vi = netdev_priv(dev);
5231         int rc = 0;
5232
5233         switch (info->cmd) {
5234         case ETHTOOL_SRXFH:
5235                 if (!virtnet_set_hashflow(vi, info))
5236                         rc = -EINVAL;
5237
5238                 break;
5239         default:
5240                 rc = -EOPNOTSUPP;
5241         }
5242
5243         return rc;
5244 }
5245
5246 static const struct ethtool_ops virtnet_ethtool_ops = {
5247         .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES |
5248                 ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
5249         .get_drvinfo = virtnet_get_drvinfo,
5250         .get_link = ethtool_op_get_link,
5251         .get_ringparam = virtnet_get_ringparam,
5252         .set_ringparam = virtnet_set_ringparam,
5253         .get_strings = virtnet_get_strings,
5254         .get_sset_count = virtnet_get_sset_count,
5255         .get_ethtool_stats = virtnet_get_ethtool_stats,
5256         .set_channels = virtnet_set_channels,
5257         .get_channels = virtnet_get_channels,
5258         .get_ts_info = ethtool_op_get_ts_info,
5259         .get_link_ksettings = virtnet_get_link_ksettings,
5260         .set_link_ksettings = virtnet_set_link_ksettings,
5261         .set_coalesce = virtnet_set_coalesce,
5262         .get_coalesce = virtnet_get_coalesce,
5263         .set_per_queue_coalesce = virtnet_set_per_queue_coalesce,
5264         .get_per_queue_coalesce = virtnet_get_per_queue_coalesce,
5265         .get_rxfh_key_size = virtnet_get_rxfh_key_size,
5266         .get_rxfh_indir_size = virtnet_get_rxfh_indir_size,
5267         .get_rxfh = virtnet_get_rxfh,
5268         .set_rxfh = virtnet_set_rxfh,
5269         .get_rxnfc = virtnet_get_rxnfc,
5270         .set_rxnfc = virtnet_set_rxnfc,
5271 };
5272
5273 static void virtnet_get_queue_stats_rx(struct net_device *dev, int i,
5274                                        struct netdev_queue_stats_rx *stats)
5275 {
5276         struct virtnet_info *vi = netdev_priv(dev);
5277         struct receive_queue *rq = &vi->rq[i];
5278         struct virtnet_stats_ctx ctx = {0};
5279
5280         virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true);
5281
5282         virtnet_get_hw_stats(vi, &ctx, i * 2);
5283         virtnet_fill_stats(vi, i * 2, &ctx, (void *)&rq->stats, true, 0);
5284 }
5285
5286 static void virtnet_get_queue_stats_tx(struct net_device *dev, int i,
5287                                        struct netdev_queue_stats_tx *stats)
5288 {
5289         struct virtnet_info *vi = netdev_priv(dev);
5290         struct send_queue *sq = &vi->sq[i];
5291         struct virtnet_stats_ctx ctx = {0};
5292
5293         virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true);
5294
5295         virtnet_get_hw_stats(vi, &ctx, i * 2 + 1);
5296         virtnet_fill_stats(vi, i * 2 + 1, &ctx, (void *)&sq->stats, true, 0);
5297 }
5298
5299 static void virtnet_get_base_stats(struct net_device *dev,
5300                                    struct netdev_queue_stats_rx *rx,
5301                                    struct netdev_queue_stats_tx *tx)
5302 {
5303         struct virtnet_info *vi = netdev_priv(dev);
5304
5305         /* The queue stats of the virtio-net will not be reset. So here we
5306          * return 0.
5307          */
5308         rx->bytes = 0;
5309         rx->packets = 0;
5310
5311         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
5312                 rx->hw_drops = 0;
5313                 rx->hw_drop_overruns = 0;
5314         }
5315
5316         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) {
5317                 rx->csum_unnecessary = 0;
5318                 rx->csum_none = 0;
5319                 rx->csum_bad = 0;
5320         }
5321
5322         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) {
5323                 rx->hw_gro_packets = 0;
5324                 rx->hw_gro_bytes = 0;
5325                 rx->hw_gro_wire_packets = 0;
5326                 rx->hw_gro_wire_bytes = 0;
5327         }
5328
5329         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED)
5330                 rx->hw_drop_ratelimits = 0;
5331
5332         tx->bytes = 0;
5333         tx->packets = 0;
5334         tx->stop = 0;
5335         tx->wake = 0;
5336
5337         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) {
5338                 tx->hw_drops = 0;
5339                 tx->hw_drop_errors = 0;
5340         }
5341
5342         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) {
5343                 tx->csum_none = 0;
5344                 tx->needs_csum = 0;
5345         }
5346
5347         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) {
5348                 tx->hw_gso_packets = 0;
5349                 tx->hw_gso_bytes = 0;
5350                 tx->hw_gso_wire_packets = 0;
5351                 tx->hw_gso_wire_bytes = 0;
5352         }
5353
5354         if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED)
5355                 tx->hw_drop_ratelimits = 0;
5356 }
5357
5358 static const struct netdev_stat_ops virtnet_stat_ops = {
5359         .get_queue_stats_rx     = virtnet_get_queue_stats_rx,
5360         .get_queue_stats_tx     = virtnet_get_queue_stats_tx,
5361         .get_base_stats         = virtnet_get_base_stats,
5362 };
5363
5364 static void virtnet_freeze_down(struct virtio_device *vdev)
5365 {
5366         struct virtnet_info *vi = vdev->priv;
5367
5368         /* Make sure no work handler is accessing the device */
5369         flush_work(&vi->config_work);
5370         disable_rx_mode_work(vi);
5371         flush_work(&vi->rx_mode_work);
5372
5373         netif_tx_lock_bh(vi->dev);
5374         netif_device_detach(vi->dev);
5375         netif_tx_unlock_bh(vi->dev);
5376         if (netif_running(vi->dev))
5377                 virtnet_close(vi->dev);
5378 }
5379
5380 static int init_vqs(struct virtnet_info *vi);
5381
5382 static int virtnet_restore_up(struct virtio_device *vdev)
5383 {
5384         struct virtnet_info *vi = vdev->priv;
5385         int err;
5386
5387         err = init_vqs(vi);
5388         if (err)
5389                 return err;
5390
5391         virtio_device_ready(vdev);
5392
5393         enable_delayed_refill(vi);
5394         enable_rx_mode_work(vi);
5395
5396         if (netif_running(vi->dev)) {
5397                 err = virtnet_open(vi->dev);
5398                 if (err)
5399                         return err;
5400         }
5401
5402         netif_tx_lock_bh(vi->dev);
5403         netif_device_attach(vi->dev);
5404         netif_tx_unlock_bh(vi->dev);
5405         return err;
5406 }
5407
5408 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
5409 {
5410         __virtio64 *_offloads __free(kfree) = NULL;
5411         struct scatterlist sg;
5412
5413         _offloads = kzalloc(sizeof(*_offloads), GFP_KERNEL);
5414         if (!_offloads)
5415                 return -ENOMEM;
5416
5417         *_offloads = cpu_to_virtio64(vi->vdev, offloads);
5418
5419         sg_init_one(&sg, _offloads, sizeof(*_offloads));
5420
5421         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
5422                                   VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
5423                 dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");
5424                 return -EINVAL;
5425         }
5426
5427         return 0;
5428 }
5429
5430 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
5431 {
5432         u64 offloads = 0;
5433
5434         if (!vi->guest_offloads)
5435                 return 0;
5436
5437         return virtnet_set_guest_offloads(vi, offloads);
5438 }
5439
5440 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
5441 {
5442         u64 offloads = vi->guest_offloads;
5443
5444         if (!vi->guest_offloads)
5445                 return 0;
5446
5447         return virtnet_set_guest_offloads(vi, offloads);
5448 }
5449
5450 static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq,
5451                                     struct xsk_buff_pool *pool)
5452 {
5453         int err, qindex;
5454
5455         qindex = rq - vi->rq;
5456
5457         if (pool) {
5458                 err = xdp_rxq_info_reg(&rq->xsk_rxq_info, vi->dev, qindex, rq->napi.napi_id);
5459                 if (err < 0)
5460                         return err;
5461
5462                 err = xdp_rxq_info_reg_mem_model(&rq->xsk_rxq_info,
5463                                                  MEM_TYPE_XSK_BUFF_POOL, NULL);
5464                 if (err < 0)
5465                         goto unreg;
5466
5467                 xsk_pool_set_rxq_info(pool, &rq->xsk_rxq_info);
5468         }
5469
5470         virtnet_rx_pause(vi, rq);
5471
5472         err = virtqueue_reset(rq->vq, virtnet_rq_unmap_free_buf);
5473         if (err) {
5474                 netdev_err(vi->dev, "reset rx fail: rx queue index: %d err: %d\n", qindex, err);
5475
5476                 pool = NULL;
5477         }
5478
5479         rq->xsk_pool = pool;
5480
5481         virtnet_rx_resume(vi, rq);
5482
5483         if (pool)
5484                 return 0;
5485
5486 unreg:
5487         xdp_rxq_info_unreg(&rq->xsk_rxq_info);
5488         return err;
5489 }
5490
5491 static int virtnet_xsk_pool_enable(struct net_device *dev,
5492                                    struct xsk_buff_pool *pool,
5493                                    u16 qid)
5494 {
5495         struct virtnet_info *vi = netdev_priv(dev);
5496         struct receive_queue *rq;
5497         struct device *dma_dev;
5498         struct send_queue *sq;
5499         int err, size;
5500
5501         if (vi->hdr_len > xsk_pool_get_headroom(pool))
5502                 return -EINVAL;
5503
5504         /* In big_packets mode, xdp cannot work, so there is no need to
5505          * initialize xsk of rq.
5506          */
5507         if (vi->big_packets && !vi->mergeable_rx_bufs)
5508                 return -ENOENT;
5509
5510         if (qid >= vi->curr_queue_pairs)
5511                 return -EINVAL;
5512
5513         sq = &vi->sq[qid];
5514         rq = &vi->rq[qid];
5515
5516         /* xsk assumes that tx and rx must have the same dma device. The af-xdp
5517          * may use one buffer to receive from the rx and reuse this buffer to
5518          * send by the tx. So the dma dev of sq and rq must be the same one.
5519          *
5520          * But vq->dma_dev allows every vq has the respective dma dev. So I
5521          * check the dma dev of vq and sq is the same dev.
5522          */
5523         if (virtqueue_dma_dev(rq->vq) != virtqueue_dma_dev(sq->vq))
5524                 return -EINVAL;
5525
5526         dma_dev = virtqueue_dma_dev(rq->vq);
5527         if (!dma_dev)
5528                 return -EINVAL;
5529
5530         size = virtqueue_get_vring_size(rq->vq);
5531
5532         rq->xsk_buffs = kvcalloc(size, sizeof(*rq->xsk_buffs), GFP_KERNEL);
5533         if (!rq->xsk_buffs)
5534                 return -ENOMEM;
5535
5536         err = xsk_pool_dma_map(pool, dma_dev, 0);
5537         if (err)
5538                 goto err_xsk_map;
5539
5540         err = virtnet_rq_bind_xsk_pool(vi, rq, pool);
5541         if (err)
5542                 goto err_rq;
5543
5544         return 0;
5545
5546 err_rq:
5547         xsk_pool_dma_unmap(pool, 0);
5548 err_xsk_map:
5549         return err;
5550 }
5551
5552 static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
5553 {
5554         struct virtnet_info *vi = netdev_priv(dev);
5555         struct xsk_buff_pool *pool;
5556         struct receive_queue *rq;
5557         int err;
5558
5559         if (qid >= vi->curr_queue_pairs)
5560                 return -EINVAL;
5561
5562         rq = &vi->rq[qid];
5563
5564         pool = rq->xsk_pool;
5565
5566         err = virtnet_rq_bind_xsk_pool(vi, rq, NULL);
5567
5568         xsk_pool_dma_unmap(pool, 0);
5569
5570         kvfree(rq->xsk_buffs);
5571
5572         return err;
5573 }
5574
5575 static int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp)
5576 {
5577         if (xdp->xsk.pool)
5578                 return virtnet_xsk_pool_enable(dev, xdp->xsk.pool,
5579                                                xdp->xsk.queue_id);
5580         else
5581                 return virtnet_xsk_pool_disable(dev, xdp->xsk.queue_id);
5582 }
5583
5584 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
5585                            struct netlink_ext_ack *extack)
5586 {
5587         unsigned int room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM +
5588                                            sizeof(struct skb_shared_info));
5589         unsigned int max_sz = PAGE_SIZE - room - ETH_HLEN;
5590         struct virtnet_info *vi = netdev_priv(dev);
5591         struct bpf_prog *old_prog;
5592         u16 xdp_qp = 0, curr_qp;
5593         int i, err;
5594
5595         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
5596             && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
5597                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
5598                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
5599                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
5600                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) ||
5601                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) ||
5602                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) {
5603                 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first");
5604                 return -EOPNOTSUPP;
5605         }
5606
5607         if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
5608                 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
5609                 return -EINVAL;
5610         }
5611
5612         if (prog && !prog->aux->xdp_has_frags && dev->mtu > max_sz) {
5613                 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP without frags");
5614                 netdev_warn(dev, "single-buffer XDP requires MTU less than %u\n", max_sz);
5615                 return -EINVAL;
5616         }
5617
5618         curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
5619         if (prog)
5620                 xdp_qp = nr_cpu_ids;
5621
5622         /* XDP requires extra queues for XDP_TX */
5623         if (curr_qp + xdp_qp > vi->max_queue_pairs) {
5624                 netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n",
5625                                  curr_qp + xdp_qp, vi->max_queue_pairs);
5626                 xdp_qp = 0;
5627         }
5628
5629         old_prog = rtnl_dereference(vi->rq[0].xdp_prog);
5630         if (!prog && !old_prog)
5631                 return 0;
5632
5633         if (prog)
5634                 bpf_prog_add(prog, vi->max_queue_pairs - 1);
5635
5636         /* Make sure NAPI is not using any XDP TX queues for RX. */
5637         if (netif_running(dev)) {
5638                 for (i = 0; i < vi->max_queue_pairs; i++) {
5639                         napi_disable(&vi->rq[i].napi);
5640                         virtnet_napi_tx_disable(&vi->sq[i].napi);
5641                 }
5642         }
5643
5644         if (!prog) {
5645                 for (i = 0; i < vi->max_queue_pairs; i++) {
5646                         rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
5647                         if (i == 0)
5648                                 virtnet_restore_guest_offloads(vi);
5649                 }
5650                 synchronize_net();
5651         }
5652
5653         err = virtnet_set_queues(vi, curr_qp + xdp_qp);
5654         if (err)
5655                 goto err;
5656         netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
5657         vi->xdp_queue_pairs = xdp_qp;
5658
5659         if (prog) {
5660                 vi->xdp_enabled = true;
5661                 for (i = 0; i < vi->max_queue_pairs; i++) {
5662                         rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
5663                         if (i == 0 && !old_prog)
5664                                 virtnet_clear_guest_offloads(vi);
5665                 }
5666                 if (!old_prog)
5667                         xdp_features_set_redirect_target(dev, true);
5668         } else {
5669                 xdp_features_clear_redirect_target(dev);
5670                 vi->xdp_enabled = false;
5671         }
5672
5673         for (i = 0; i < vi->max_queue_pairs; i++) {
5674                 if (old_prog)
5675                         bpf_prog_put(old_prog);
5676                 if (netif_running(dev)) {
5677                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
5678                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
5679                                                &vi->sq[i].napi);
5680                 }
5681         }
5682
5683         return 0;
5684
5685 err:
5686         if (!prog) {
5687                 virtnet_clear_guest_offloads(vi);
5688                 for (i = 0; i < vi->max_queue_pairs; i++)
5689                         rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);
5690         }
5691
5692         if (netif_running(dev)) {
5693                 for (i = 0; i < vi->max_queue_pairs; i++) {
5694                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
5695                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
5696                                                &vi->sq[i].napi);
5697                 }
5698         }
5699         if (prog)
5700                 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
5701         return err;
5702 }
5703
5704 static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
5705 {
5706         switch (xdp->command) {
5707         case XDP_SETUP_PROG:
5708                 return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
5709         case XDP_SETUP_XSK_POOL:
5710                 return virtnet_xsk_pool_setup(dev, xdp);
5711         default:
5712                 return -EINVAL;
5713         }
5714 }
5715
5716 static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
5717                                       size_t len)
5718 {
5719         struct virtnet_info *vi = netdev_priv(dev);
5720         int ret;
5721
5722         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
5723                 return -EOPNOTSUPP;
5724
5725         ret = snprintf(buf, len, "sby");
5726         if (ret >= len)
5727                 return -EOPNOTSUPP;
5728
5729         return 0;
5730 }
5731
5732 static int virtnet_set_features(struct net_device *dev,
5733                                 netdev_features_t features)
5734 {
5735         struct virtnet_info *vi = netdev_priv(dev);
5736         u64 offloads;
5737         int err;
5738
5739         if ((dev->features ^ features) & NETIF_F_GRO_HW) {
5740                 if (vi->xdp_enabled)
5741                         return -EBUSY;
5742
5743                 if (features & NETIF_F_GRO_HW)
5744                         offloads = vi->guest_offloads_capable;
5745                 else
5746                         offloads = vi->guest_offloads_capable &
5747                                    ~GUEST_OFFLOAD_GRO_HW_MASK;
5748
5749                 err = virtnet_set_guest_offloads(vi, offloads);
5750                 if (err)
5751                         return err;
5752                 vi->guest_offloads = offloads;
5753         }
5754
5755         if ((dev->features ^ features) & NETIF_F_RXHASH) {
5756                 if (features & NETIF_F_RXHASH)
5757                         vi->rss.hash_types = vi->rss_hash_types_saved;
5758                 else
5759                         vi->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE;
5760
5761                 if (!virtnet_commit_rss_command(vi))
5762                         return -EINVAL;
5763         }
5764
5765         return 0;
5766 }
5767
5768 static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue)
5769 {
5770         struct virtnet_info *priv = netdev_priv(dev);
5771         struct send_queue *sq = &priv->sq[txqueue];
5772         struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
5773
5774         u64_stats_update_begin(&sq->stats.syncp);
5775         u64_stats_inc(&sq->stats.tx_timeouts);
5776         u64_stats_update_end(&sq->stats.syncp);
5777
5778         netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n",
5779                    txqueue, sq->name, sq->vq->index, sq->vq->name,
5780                    jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start)));
5781 }
5782
5783 static int virtnet_init_irq_moder(struct virtnet_info *vi)
5784 {
5785         u8 profile_flags = 0, coal_flags = 0;
5786         int ret, i;
5787
5788         profile_flags |= DIM_PROFILE_RX;
5789         coal_flags |= DIM_COALESCE_USEC | DIM_COALESCE_PKTS;
5790         ret = net_dim_init_irq_moder(vi->dev, profile_flags, coal_flags,
5791                                      DIM_CQ_PERIOD_MODE_START_FROM_EQE,
5792                                      0, virtnet_rx_dim_work, NULL);
5793
5794         if (ret)
5795                 return ret;
5796
5797         for (i = 0; i < vi->max_queue_pairs; i++)
5798                 net_dim_setting(vi->dev, &vi->rq[i].dim, false);
5799
5800         return 0;
5801 }
5802
5803 static void virtnet_free_irq_moder(struct virtnet_info *vi)
5804 {
5805         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL))
5806                 return;
5807
5808         rtnl_lock();
5809         net_dim_free_irq_moder(vi->dev);
5810         rtnl_unlock();
5811 }
5812
5813 static const struct net_device_ops virtnet_netdev = {
5814         .ndo_open            = virtnet_open,
5815         .ndo_stop            = virtnet_close,
5816         .ndo_start_xmit      = start_xmit,
5817         .ndo_validate_addr   = eth_validate_addr,
5818         .ndo_set_mac_address = virtnet_set_mac_address,
5819         .ndo_set_rx_mode     = virtnet_set_rx_mode,
5820         .ndo_get_stats64     = virtnet_stats,
5821         .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
5822         .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
5823         .ndo_bpf                = virtnet_xdp,
5824         .ndo_xdp_xmit           = virtnet_xdp_xmit,
5825         .ndo_xsk_wakeup         = virtnet_xsk_wakeup,
5826         .ndo_features_check     = passthru_features_check,
5827         .ndo_get_phys_port_name = virtnet_get_phys_port_name,
5828         .ndo_set_features       = virtnet_set_features,
5829         .ndo_tx_timeout         = virtnet_tx_timeout,
5830 };
5831
5832 static void virtnet_config_changed_work(struct work_struct *work)
5833 {
5834         struct virtnet_info *vi =
5835                 container_of(work, struct virtnet_info, config_work);
5836         u16 v;
5837
5838         if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
5839                                  struct virtio_net_config, status, &v) < 0)
5840                 return;
5841
5842         if (v & VIRTIO_NET_S_ANNOUNCE) {
5843                 netdev_notify_peers(vi->dev);
5844                 virtnet_ack_link_announce(vi);
5845         }
5846
5847         /* Ignore unknown (future) status bits */
5848         v &= VIRTIO_NET_S_LINK_UP;
5849
5850         if (vi->status == v)
5851                 return;
5852
5853         vi->status = v;
5854
5855         if (vi->status & VIRTIO_NET_S_LINK_UP) {
5856                 virtnet_update_settings(vi);
5857                 netif_carrier_on(vi->dev);
5858                 netif_tx_wake_all_queues(vi->dev);
5859         } else {
5860                 netif_carrier_off(vi->dev);
5861                 netif_tx_stop_all_queues(vi->dev);
5862         }
5863 }
5864
5865 static void virtnet_config_changed(struct virtio_device *vdev)
5866 {
5867         struct virtnet_info *vi = vdev->priv;
5868
5869         schedule_work(&vi->config_work);
5870 }
5871
5872 static void virtnet_free_queues(struct virtnet_info *vi)
5873 {
5874         int i;
5875
5876         for (i = 0; i < vi->max_queue_pairs; i++) {
5877                 __netif_napi_del(&vi->rq[i].napi);
5878                 __netif_napi_del(&vi->sq[i].napi);
5879         }
5880
5881         /* We called __netif_napi_del(),
5882          * we need to respect an RCU grace period before freeing vi->rq
5883          */
5884         synchronize_net();
5885
5886         kfree(vi->rq);
5887         kfree(vi->sq);
5888         kfree(vi->ctrl);
5889 }
5890
5891 static void _free_receive_bufs(struct virtnet_info *vi)
5892 {
5893         struct bpf_prog *old_prog;
5894         int i;
5895
5896         for (i = 0; i < vi->max_queue_pairs; i++) {
5897                 while (vi->rq[i].pages)
5898                         __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
5899
5900                 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
5901                 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
5902                 if (old_prog)
5903                         bpf_prog_put(old_prog);
5904         }
5905 }
5906
5907 static void free_receive_bufs(struct virtnet_info *vi)
5908 {
5909         rtnl_lock();
5910         _free_receive_bufs(vi);
5911         rtnl_unlock();
5912 }
5913
5914 static void free_receive_page_frags(struct virtnet_info *vi)
5915 {
5916         int i;
5917         for (i = 0; i < vi->max_queue_pairs; i++)
5918                 if (vi->rq[i].alloc_frag.page) {
5919                         if (vi->rq[i].do_dma && vi->rq[i].last_dma)
5920                                 virtnet_rq_unmap(&vi->rq[i], vi->rq[i].last_dma, 0);
5921                         put_page(vi->rq[i].alloc_frag.page);
5922                 }
5923 }
5924
5925 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
5926 {
5927         if (!is_xdp_frame(buf))
5928                 dev_kfree_skb(buf);
5929         else
5930                 xdp_return_frame(ptr_to_xdp(buf));
5931 }
5932
5933 static void free_unused_bufs(struct virtnet_info *vi)
5934 {
5935         void *buf;
5936         int i;
5937
5938         for (i = 0; i < vi->max_queue_pairs; i++) {
5939                 struct virtqueue *vq = vi->sq[i].vq;
5940                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
5941                         virtnet_sq_free_unused_buf(vq, buf);
5942                 cond_resched();
5943         }
5944
5945         for (i = 0; i < vi->max_queue_pairs; i++) {
5946                 struct virtqueue *vq = vi->rq[i].vq;
5947
5948                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
5949                         virtnet_rq_unmap_free_buf(vq, buf);
5950                 cond_resched();
5951         }
5952 }
5953
5954 static void virtnet_del_vqs(struct virtnet_info *vi)
5955 {
5956         struct virtio_device *vdev = vi->vdev;
5957
5958         virtnet_clean_affinity(vi);
5959
5960         vdev->config->del_vqs(vdev);
5961
5962         virtnet_free_queues(vi);
5963 }
5964
5965 /* How large should a single buffer be so a queue full of these can fit at
5966  * least one full packet?
5967  * Logic below assumes the mergeable buffer header is used.
5968  */
5969 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
5970 {
5971         const unsigned int hdr_len = vi->hdr_len;
5972         unsigned int rq_size = virtqueue_get_vring_size(vq);
5973         unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
5974         unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
5975         unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
5976
5977         return max(max(min_buf_len, hdr_len) - hdr_len,
5978                    (unsigned int)GOOD_PACKET_LEN);
5979 }
5980
5981 static int virtnet_find_vqs(struct virtnet_info *vi)
5982 {
5983         struct virtqueue_info *vqs_info;
5984         struct virtqueue **vqs;
5985         int ret = -ENOMEM;
5986         int total_vqs;
5987         bool *ctx;
5988         u16 i;
5989
5990         /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
5991          * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
5992          * possible control vq.
5993          */
5994         total_vqs = vi->max_queue_pairs * 2 +
5995                     virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
5996
5997         /* Allocate space for find_vqs parameters */
5998         vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
5999         if (!vqs)
6000                 goto err_vq;
6001         vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL);
6002         if (!vqs_info)
6003                 goto err_vqs_info;
6004         if (!vi->big_packets || vi->mergeable_rx_bufs) {
6005                 ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
6006                 if (!ctx)
6007                         goto err_ctx;
6008         } else {
6009                 ctx = NULL;
6010         }
6011
6012         /* Parameters for control virtqueue, if any */
6013         if (vi->has_cvq) {
6014                 vqs_info[total_vqs - 1].name = "control";
6015         }
6016
6017         /* Allocate/initialize parameters for send/receive virtqueues */
6018         for (i = 0; i < vi->max_queue_pairs; i++) {
6019                 vqs_info[rxq2vq(i)].callback = skb_recv_done;
6020                 vqs_info[txq2vq(i)].callback = skb_xmit_done;
6021                 sprintf(vi->rq[i].name, "input.%u", i);
6022                 sprintf(vi->sq[i].name, "output.%u", i);
6023                 vqs_info[rxq2vq(i)].name = vi->rq[i].name;
6024                 vqs_info[txq2vq(i)].name = vi->sq[i].name;
6025                 if (ctx)
6026                         vqs_info[rxq2vq(i)].ctx = true;
6027         }
6028
6029         ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL);
6030         if (ret)
6031                 goto err_find;
6032
6033         if (vi->has_cvq) {
6034                 vi->cvq = vqs[total_vqs - 1];
6035                 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
6036                         vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
6037         }
6038
6039         for (i = 0; i < vi->max_queue_pairs; i++) {
6040                 vi->rq[i].vq = vqs[rxq2vq(i)];
6041                 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
6042                 vi->sq[i].vq = vqs[txq2vq(i)];
6043         }
6044
6045         /* run here: ret == 0. */
6046
6047
6048 err_find:
6049         kfree(ctx);
6050 err_ctx:
6051         kfree(vqs_info);
6052 err_vqs_info:
6053         kfree(vqs);
6054 err_vq:
6055         return ret;
6056 }
6057
6058 static int virtnet_alloc_queues(struct virtnet_info *vi)
6059 {
6060         int i;
6061
6062         if (vi->has_cvq) {
6063                 vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
6064                 if (!vi->ctrl)
6065                         goto err_ctrl;
6066         } else {
6067                 vi->ctrl = NULL;
6068         }
6069         vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);
6070         if (!vi->sq)
6071                 goto err_sq;
6072         vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);
6073         if (!vi->rq)
6074                 goto err_rq;
6075
6076         INIT_DELAYED_WORK(&vi->refill, refill_work);
6077         for (i = 0; i < vi->max_queue_pairs; i++) {
6078                 vi->rq[i].pages = NULL;
6079                 netif_napi_add_weight(vi->dev, &vi->rq[i].napi, virtnet_poll,
6080                                       napi_weight);
6081                 netif_napi_add_tx_weight(vi->dev, &vi->sq[i].napi,
6082                                          virtnet_poll_tx,
6083                                          napi_tx ? napi_weight : 0);
6084
6085                 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
6086                 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
6087                 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
6088
6089                 u64_stats_init(&vi->rq[i].stats.syncp);
6090                 u64_stats_init(&vi->sq[i].stats.syncp);
6091                 mutex_init(&vi->rq[i].dim_lock);
6092         }
6093
6094         return 0;
6095
6096 err_rq:
6097         kfree(vi->sq);
6098 err_sq:
6099         kfree(vi->ctrl);
6100 err_ctrl:
6101         return -ENOMEM;
6102 }
6103
6104 static int init_vqs(struct virtnet_info *vi)
6105 {
6106         int ret;
6107
6108         /* Allocate send & receive queues */
6109         ret = virtnet_alloc_queues(vi);
6110         if (ret)
6111                 goto err;
6112
6113         ret = virtnet_find_vqs(vi);
6114         if (ret)
6115                 goto err_free;
6116
6117         cpus_read_lock();
6118         virtnet_set_affinity(vi);
6119         cpus_read_unlock();
6120
6121         return 0;
6122
6123 err_free:
6124         virtnet_free_queues(vi);
6125 err:
6126         return ret;
6127 }
6128
6129 #ifdef CONFIG_SYSFS
6130 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
6131                 char *buf)
6132 {
6133         struct virtnet_info *vi = netdev_priv(queue->dev);
6134         unsigned int queue_index = get_netdev_rx_queue_index(queue);
6135         unsigned int headroom = virtnet_get_headroom(vi);
6136         unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
6137         struct ewma_pkt_len *avg;
6138
6139         BUG_ON(queue_index >= vi->max_queue_pairs);
6140         avg = &vi->rq[queue_index].mrg_avg_pkt_len;
6141         return sprintf(buf, "%u\n",
6142                        get_mergeable_buf_len(&vi->rq[queue_index], avg,
6143                                        SKB_DATA_ALIGN(headroom + tailroom)));
6144 }
6145
6146 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
6147         __ATTR_RO(mergeable_rx_buffer_size);
6148
6149 static struct attribute *virtio_net_mrg_rx_attrs[] = {
6150         &mergeable_rx_buffer_size_attribute.attr,
6151         NULL
6152 };
6153
6154 static const struct attribute_group virtio_net_mrg_rx_group = {
6155         .name = "virtio_net",
6156         .attrs = virtio_net_mrg_rx_attrs
6157 };
6158 #endif
6159
6160 static bool virtnet_fail_on_feature(struct virtio_device *vdev,
6161                                     unsigned int fbit,
6162                                     const char *fname, const char *dname)
6163 {
6164         if (!virtio_has_feature(vdev, fbit))
6165                 return false;
6166
6167         dev_err(&vdev->dev, "device advertises feature %s but not %s",
6168                 fname, dname);
6169
6170         return true;
6171 }
6172
6173 #define VIRTNET_FAIL_ON(vdev, fbit, dbit)                       \
6174         virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
6175
6176 static bool virtnet_validate_features(struct virtio_device *vdev)
6177 {
6178         if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
6179             (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
6180                              "VIRTIO_NET_F_CTRL_VQ") ||
6181              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
6182                              "VIRTIO_NET_F_CTRL_VQ") ||
6183              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
6184                              "VIRTIO_NET_F_CTRL_VQ") ||
6185              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
6186              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
6187                              "VIRTIO_NET_F_CTRL_VQ") ||
6188              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
6189                              "VIRTIO_NET_F_CTRL_VQ") ||
6190              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
6191                              "VIRTIO_NET_F_CTRL_VQ") ||
6192              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL,
6193                              "VIRTIO_NET_F_CTRL_VQ") ||
6194              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_VQ_NOTF_COAL,
6195                              "VIRTIO_NET_F_CTRL_VQ"))) {
6196                 return false;
6197         }
6198
6199         return true;
6200 }
6201
6202 #define MIN_MTU ETH_MIN_MTU
6203 #define MAX_MTU ETH_MAX_MTU
6204
6205 static int virtnet_validate(struct virtio_device *vdev)
6206 {
6207         if (!vdev->config->get) {
6208                 dev_err(&vdev->dev, "%s failure: config access disabled\n",
6209                         __func__);
6210                 return -EINVAL;
6211         }
6212
6213         if (!virtnet_validate_features(vdev))
6214                 return -EINVAL;
6215
6216         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
6217                 int mtu = virtio_cread16(vdev,
6218                                          offsetof(struct virtio_net_config,
6219                                                   mtu));
6220                 if (mtu < MIN_MTU)
6221                         __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
6222         }
6223
6224         if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY) &&
6225             !virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
6226                 dev_warn(&vdev->dev, "device advertises feature VIRTIO_NET_F_STANDBY but not VIRTIO_NET_F_MAC, disabling standby");
6227                 __virtio_clear_bit(vdev, VIRTIO_NET_F_STANDBY);
6228         }
6229
6230         return 0;
6231 }
6232
6233 static bool virtnet_check_guest_gso(const struct virtnet_info *vi)
6234 {
6235         return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
6236                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
6237                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
6238                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
6239                 (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) &&
6240                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6));
6241 }
6242
6243 static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu)
6244 {
6245         bool guest_gso = virtnet_check_guest_gso(vi);
6246
6247         /* If device can receive ANY guest GSO packets, regardless of mtu,
6248          * allocate packets of maximum size, otherwise limit it to only
6249          * mtu size worth only.
6250          */
6251         if (mtu > ETH_DATA_LEN || guest_gso) {
6252                 vi->big_packets = true;
6253                 vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE);
6254         }
6255 }
6256
6257 #define VIRTIO_NET_HASH_REPORT_MAX_TABLE      10
6258 static enum xdp_rss_hash_type
6259 virtnet_xdp_rss_type[VIRTIO_NET_HASH_REPORT_MAX_TABLE] = {
6260         [VIRTIO_NET_HASH_REPORT_NONE] = XDP_RSS_TYPE_NONE,
6261         [VIRTIO_NET_HASH_REPORT_IPv4] = XDP_RSS_TYPE_L3_IPV4,
6262         [VIRTIO_NET_HASH_REPORT_TCPv4] = XDP_RSS_TYPE_L4_IPV4_TCP,
6263         [VIRTIO_NET_HASH_REPORT_UDPv4] = XDP_RSS_TYPE_L4_IPV4_UDP,
6264         [VIRTIO_NET_HASH_REPORT_IPv6] = XDP_RSS_TYPE_L3_IPV6,
6265         [VIRTIO_NET_HASH_REPORT_TCPv6] = XDP_RSS_TYPE_L4_IPV6_TCP,
6266         [VIRTIO_NET_HASH_REPORT_UDPv6] = XDP_RSS_TYPE_L4_IPV6_UDP,
6267         [VIRTIO_NET_HASH_REPORT_IPv6_EX] = XDP_RSS_TYPE_L3_IPV6_EX,
6268         [VIRTIO_NET_HASH_REPORT_TCPv6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX,
6269         [VIRTIO_NET_HASH_REPORT_UDPv6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX
6270 };
6271
6272 static int virtnet_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
6273                                enum xdp_rss_hash_type *rss_type)
6274 {
6275         const struct xdp_buff *xdp = (void *)_ctx;
6276         struct virtio_net_hdr_v1_hash *hdr_hash;
6277         struct virtnet_info *vi;
6278         u16 hash_report;
6279
6280         if (!(xdp->rxq->dev->features & NETIF_F_RXHASH))
6281                 return -ENODATA;
6282
6283         vi = netdev_priv(xdp->rxq->dev);
6284         hdr_hash = (struct virtio_net_hdr_v1_hash *)(xdp->data - vi->hdr_len);
6285         hash_report = __le16_to_cpu(hdr_hash->hash_report);
6286
6287         if (hash_report >= VIRTIO_NET_HASH_REPORT_MAX_TABLE)
6288                 hash_report = VIRTIO_NET_HASH_REPORT_NONE;
6289
6290         *rss_type = virtnet_xdp_rss_type[hash_report];
6291         *hash = __le32_to_cpu(hdr_hash->hash_value);
6292         return 0;
6293 }
6294
6295 static const struct xdp_metadata_ops virtnet_xdp_metadata_ops = {
6296         .xmo_rx_hash                    = virtnet_xdp_rx_hash,
6297 };
6298
6299 static int virtnet_probe(struct virtio_device *vdev)
6300 {
6301         int i, err = -ENOMEM;
6302         struct net_device *dev;
6303         struct virtnet_info *vi;
6304         u16 max_queue_pairs;
6305         int mtu = 0;
6306
6307         /* Find if host supports multiqueue/rss virtio_net device */
6308         max_queue_pairs = 1;
6309         if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
6310                 max_queue_pairs =
6311                      virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs));
6312
6313         /* We need at least 2 queue's */
6314         if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
6315             max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
6316             !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
6317                 max_queue_pairs = 1;
6318
6319         /* Allocate ourselves a network device with room for our info */
6320         dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
6321         if (!dev)
6322                 return -ENOMEM;
6323
6324         /* Set up network device as normal. */
6325         dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE |
6326                            IFF_TX_SKB_NO_LINEAR;
6327         dev->netdev_ops = &virtnet_netdev;
6328         dev->stat_ops = &virtnet_stat_ops;
6329         dev->features = NETIF_F_HIGHDMA;
6330
6331         dev->ethtool_ops = &virtnet_ethtool_ops;
6332         SET_NETDEV_DEV(dev, &vdev->dev);
6333
6334         /* Do we support "hardware" checksums? */
6335         if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
6336                 /* This opens up the world of extra features. */
6337                 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
6338                 if (csum)
6339                         dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
6340
6341                 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
6342                         dev->hw_features |= NETIF_F_TSO
6343                                 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
6344                 }
6345                 /* Individual feature bits: what can host handle? */
6346                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
6347                         dev->hw_features |= NETIF_F_TSO;
6348                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
6349                         dev->hw_features |= NETIF_F_TSO6;
6350                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
6351                         dev->hw_features |= NETIF_F_TSO_ECN;
6352                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_USO))
6353                         dev->hw_features |= NETIF_F_GSO_UDP_L4;
6354
6355                 dev->features |= NETIF_F_GSO_ROBUST;
6356
6357                 if (gso)
6358                         dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
6359                 /* (!csum && gso) case will be fixed by register_netdev() */
6360         }
6361
6362         /* 1. With VIRTIO_NET_F_GUEST_CSUM negotiation, the driver doesn't
6363          * need to calculate checksums for partially checksummed packets,
6364          * as they're considered valid by the upper layer.
6365          * 2. Without VIRTIO_NET_F_GUEST_CSUM negotiation, the driver only
6366          * receives fully checksummed packets. The device may assist in
6367          * validating these packets' checksums, so the driver won't have to.
6368          */
6369         dev->features |= NETIF_F_RXCSUM;
6370
6371         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
6372             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6))
6373                 dev->features |= NETIF_F_GRO_HW;
6374         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))
6375                 dev->hw_features |= NETIF_F_GRO_HW;
6376
6377         dev->vlan_features = dev->features;
6378         dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
6379
6380         /* MTU range: 68 - 65535 */
6381         dev->min_mtu = MIN_MTU;
6382         dev->max_mtu = MAX_MTU;
6383
6384         /* Configuration may specify what MAC to use.  Otherwise random. */
6385         if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
6386                 u8 addr[ETH_ALEN];
6387
6388                 virtio_cread_bytes(vdev,
6389                                    offsetof(struct virtio_net_config, mac),
6390                                    addr, ETH_ALEN);
6391                 eth_hw_addr_set(dev, addr);
6392         } else {
6393                 eth_hw_addr_random(dev);
6394                 dev_info(&vdev->dev, "Assigned random MAC address %pM\n",
6395                          dev->dev_addr);
6396         }
6397
6398         /* Set up our device-specific information */
6399         vi = netdev_priv(dev);
6400         vi->dev = dev;
6401         vi->vdev = vdev;
6402         vdev->priv = vi;
6403
6404         INIT_WORK(&vi->config_work, virtnet_config_changed_work);
6405         INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work);
6406         spin_lock_init(&vi->refill_lock);
6407
6408         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) {
6409                 vi->mergeable_rx_bufs = true;
6410                 dev->xdp_features |= NETDEV_XDP_ACT_RX_SG;
6411         }
6412
6413         if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
6414                 vi->has_rss_hash_report = true;
6415
6416         if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) {
6417                 vi->has_rss = true;
6418
6419                 vi->rss_indir_table_size =
6420                         virtio_cread16(vdev, offsetof(struct virtio_net_config,
6421                                 rss_max_indirection_table_length));
6422         }
6423
6424         if (vi->has_rss || vi->has_rss_hash_report) {
6425                 vi->rss_key_size =
6426                         virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size));
6427
6428                 vi->rss_hash_types_supported =
6429                     virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types));
6430                 vi->rss_hash_types_supported &=
6431                                 ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX |
6432                                   VIRTIO_NET_RSS_HASH_TYPE_TCP_EX |
6433                                   VIRTIO_NET_RSS_HASH_TYPE_UDP_EX);
6434
6435                 dev->hw_features |= NETIF_F_RXHASH;
6436                 dev->xdp_metadata_ops = &virtnet_xdp_metadata_ops;
6437         }
6438
6439         if (vi->has_rss_hash_report)
6440                 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash);
6441         else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
6442                  virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
6443                 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
6444         else
6445                 vi->hdr_len = sizeof(struct virtio_net_hdr);
6446
6447         if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
6448             virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
6449                 vi->any_header_sg = true;
6450
6451         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
6452                 vi->has_cvq = true;
6453
6454         mutex_init(&vi->cvq_lock);
6455
6456         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
6457                 mtu = virtio_cread16(vdev,
6458                                      offsetof(struct virtio_net_config,
6459                                               mtu));
6460                 if (mtu < dev->min_mtu) {
6461                         /* Should never trigger: MTU was previously validated
6462                          * in virtnet_validate.
6463                          */
6464                         dev_err(&vdev->dev,
6465                                 "device MTU appears to have changed it is now %d < %d",
6466                                 mtu, dev->min_mtu);
6467                         err = -EINVAL;
6468                         goto free;
6469                 }
6470
6471                 dev->mtu = mtu;
6472                 dev->max_mtu = mtu;
6473         }
6474
6475         virtnet_set_big_packets(vi, mtu);
6476
6477         if (vi->any_header_sg)
6478                 dev->needed_headroom = vi->hdr_len;
6479
6480         /* Enable multiqueue by default */
6481         if (num_online_cpus() >= max_queue_pairs)
6482                 vi->curr_queue_pairs = max_queue_pairs;
6483         else
6484                 vi->curr_queue_pairs = num_online_cpus();
6485         vi->max_queue_pairs = max_queue_pairs;
6486
6487         /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
6488         err = init_vqs(vi);
6489         if (err)
6490                 goto free;
6491
6492         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
6493                 vi->intr_coal_rx.max_usecs = 0;
6494                 vi->intr_coal_tx.max_usecs = 0;
6495                 vi->intr_coal_rx.max_packets = 0;
6496
6497                 /* Keep the default values of the coalescing parameters
6498                  * aligned with the default napi_tx state.
6499                  */
6500                 if (vi->sq[0].napi.weight)
6501                         vi->intr_coal_tx.max_packets = 1;
6502                 else
6503                         vi->intr_coal_tx.max_packets = 0;
6504         }
6505
6506         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) {
6507                 /* The reason is the same as VIRTIO_NET_F_NOTF_COAL. */
6508                 for (i = 0; i < vi->max_queue_pairs; i++)
6509                         if (vi->sq[i].napi.weight)
6510                                 vi->sq[i].intr_coal.max_packets = 1;
6511
6512                 err = virtnet_init_irq_moder(vi);
6513                 if (err)
6514                         goto free;
6515         }
6516
6517 #ifdef CONFIG_SYSFS
6518         if (vi->mergeable_rx_bufs)
6519                 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
6520 #endif
6521         netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
6522         netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
6523
6524         virtnet_init_settings(dev);
6525
6526         if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
6527                 vi->failover = net_failover_create(vi->dev);
6528                 if (IS_ERR(vi->failover)) {
6529                         err = PTR_ERR(vi->failover);
6530                         goto free_vqs;
6531                 }
6532         }
6533
6534         if (vi->has_rss || vi->has_rss_hash_report)
6535                 virtnet_init_default_rss(vi);
6536
6537         enable_rx_mode_work(vi);
6538
6539         /* serialize netdev register + virtio_device_ready() with ndo_open() */
6540         rtnl_lock();
6541
6542         err = register_netdevice(dev);
6543         if (err) {
6544                 pr_debug("virtio_net: registering device failed\n");
6545                 rtnl_unlock();
6546                 goto free_failover;
6547         }
6548
6549         /* Disable config change notification until ndo_open. */
6550         virtio_config_driver_disable(vi->vdev);
6551
6552         virtio_device_ready(vdev);
6553
6554         virtnet_set_queues(vi, vi->curr_queue_pairs);
6555
6556         /* a random MAC address has been assigned, notify the device.
6557          * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there
6558          * because many devices work fine without getting MAC explicitly
6559          */
6560         if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
6561             virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
6562                 struct scatterlist sg;
6563
6564                 sg_init_one(&sg, dev->dev_addr, dev->addr_len);
6565                 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
6566                                           VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
6567                         pr_debug("virtio_net: setting MAC address failed\n");
6568                         rtnl_unlock();
6569                         err = -EINVAL;
6570                         goto free_unregister_netdev;
6571                 }
6572         }
6573
6574         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) {
6575                 struct virtio_net_stats_capabilities *stats_cap  __free(kfree) = NULL;
6576                 struct scatterlist sg;
6577                 __le64 v;
6578
6579                 stats_cap = kzalloc(sizeof(*stats_cap), GFP_KERNEL);
6580                 if (!stats_cap) {
6581                         rtnl_unlock();
6582                         err = -ENOMEM;
6583                         goto free_unregister_netdev;
6584                 }
6585
6586                 sg_init_one(&sg, stats_cap, sizeof(*stats_cap));
6587
6588                 if (!virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS,
6589                                                 VIRTIO_NET_CTRL_STATS_QUERY,
6590                                                 NULL, &sg)) {
6591                         pr_debug("virtio_net: fail to get stats capability\n");
6592                         rtnl_unlock();
6593                         err = -EINVAL;
6594                         goto free_unregister_netdev;
6595                 }
6596
6597                 v = stats_cap->supported_stats_types[0];
6598                 vi->device_stats_cap = le64_to_cpu(v);
6599         }
6600
6601         /* Assume link up if device can't report link status,
6602            otherwise get link status from config. */
6603         netif_carrier_off(dev);
6604         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
6605                 virtnet_config_changed_work(&vi->config_work);
6606         } else {
6607                 vi->status = VIRTIO_NET_S_LINK_UP;
6608                 virtnet_update_settings(vi);
6609                 netif_carrier_on(dev);
6610         }
6611
6612         for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
6613                 if (virtio_has_feature(vi->vdev, guest_offloads[i]))
6614                         set_bit(guest_offloads[i], &vi->guest_offloads);
6615         vi->guest_offloads_capable = vi->guest_offloads;
6616
6617         rtnl_unlock();
6618
6619         err = virtnet_cpu_notif_add(vi);
6620         if (err) {
6621                 pr_debug("virtio_net: registering cpu notifier failed\n");
6622                 goto free_unregister_netdev;
6623         }
6624
6625         pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
6626                  dev->name, max_queue_pairs);
6627
6628         return 0;
6629
6630 free_unregister_netdev:
6631         unregister_netdev(dev);
6632 free_failover:
6633         net_failover_destroy(vi->failover);
6634 free_vqs:
6635         virtio_reset_device(vdev);
6636         cancel_delayed_work_sync(&vi->refill);
6637         free_receive_page_frags(vi);
6638         virtnet_del_vqs(vi);
6639 free:
6640         free_netdev(dev);
6641         return err;
6642 }
6643
6644 static void remove_vq_common(struct virtnet_info *vi)
6645 {
6646         virtio_reset_device(vi->vdev);
6647
6648         /* Free unused buffers in both send and recv, if any. */
6649         free_unused_bufs(vi);
6650
6651         free_receive_bufs(vi);
6652
6653         free_receive_page_frags(vi);
6654
6655         virtnet_del_vqs(vi);
6656 }
6657
6658 static void virtnet_remove(struct virtio_device *vdev)
6659 {
6660         struct virtnet_info *vi = vdev->priv;
6661
6662         virtnet_cpu_notif_remove(vi);
6663
6664         /* Make sure no work handler is accessing the device. */
6665         flush_work(&vi->config_work);
6666         disable_rx_mode_work(vi);
6667         flush_work(&vi->rx_mode_work);
6668
6669         virtnet_free_irq_moder(vi);
6670
6671         unregister_netdev(vi->dev);
6672
6673         net_failover_destroy(vi->failover);
6674
6675         remove_vq_common(vi);
6676
6677         free_netdev(vi->dev);
6678 }
6679
6680 static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
6681 {
6682         struct virtnet_info *vi = vdev->priv;
6683
6684         virtnet_cpu_notif_remove(vi);
6685         virtnet_freeze_down(vdev);
6686         remove_vq_common(vi);
6687
6688         return 0;
6689 }
6690
6691 static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
6692 {
6693         struct virtnet_info *vi = vdev->priv;
6694         int err;
6695
6696         err = virtnet_restore_up(vdev);
6697         if (err)
6698                 return err;
6699         virtnet_set_queues(vi, vi->curr_queue_pairs);
6700
6701         err = virtnet_cpu_notif_add(vi);
6702         if (err) {
6703                 virtnet_freeze_down(vdev);
6704                 remove_vq_common(vi);
6705                 return err;
6706         }
6707
6708         return 0;
6709 }
6710
6711 static struct virtio_device_id id_table[] = {
6712         { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
6713         { 0 },
6714 };
6715
6716 #define VIRTNET_FEATURES \
6717         VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
6718         VIRTIO_NET_F_MAC, \
6719         VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
6720         VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
6721         VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
6722         VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_GUEST_USO4, VIRTIO_NET_F_GUEST_USO6, \
6723         VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
6724         VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
6725         VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
6726         VIRTIO_NET_F_CTRL_MAC_ADDR, \
6727         VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
6728         VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
6729         VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL, \
6730         VIRTIO_NET_F_VQ_NOTF_COAL, \
6731         VIRTIO_NET_F_GUEST_HDRLEN, VIRTIO_NET_F_DEVICE_STATS
6732
6733 static unsigned int features[] = {
6734         VIRTNET_FEATURES,
6735 };
6736
6737 static unsigned int features_legacy[] = {
6738         VIRTNET_FEATURES,
6739         VIRTIO_NET_F_GSO,
6740         VIRTIO_F_ANY_LAYOUT,
6741 };
6742
6743 static struct virtio_driver virtio_net_driver = {
6744         .feature_table = features,
6745         .feature_table_size = ARRAY_SIZE(features),
6746         .feature_table_legacy = features_legacy,
6747         .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
6748         .driver.name =  KBUILD_MODNAME,
6749         .id_table =     id_table,
6750         .validate =     virtnet_validate,
6751         .probe =        virtnet_probe,
6752         .remove =       virtnet_remove,
6753         .config_changed = virtnet_config_changed,
6754 #ifdef CONFIG_PM_SLEEP
6755         .freeze =       virtnet_freeze,
6756         .restore =      virtnet_restore,
6757 #endif
6758 };
6759
6760 static __init int virtio_net_driver_init(void)
6761 {
6762         int ret;
6763
6764         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
6765                                       virtnet_cpu_online,
6766                                       virtnet_cpu_down_prep);
6767         if (ret < 0)
6768                 goto out;
6769         virtionet_online = ret;
6770         ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
6771                                       NULL, virtnet_cpu_dead);
6772         if (ret)
6773                 goto err_dead;
6774         ret = register_virtio_driver(&virtio_net_driver);
6775         if (ret)
6776                 goto err_virtio;
6777         return 0;
6778 err_virtio:
6779         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
6780 err_dead:
6781         cpuhp_remove_multi_state(virtionet_online);
6782 out:
6783         return ret;
6784 }
6785 module_init(virtio_net_driver_init);
6786
6787 static __exit void virtio_net_driver_exit(void)
6788 {
6789         unregister_virtio_driver(&virtio_net_driver);
6790         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
6791         cpuhp_remove_multi_state(virtionet_online);
6792 }
6793 module_exit(virtio_net_driver_exit);
6794
6795 MODULE_DEVICE_TABLE(virtio, id_table);
6796 MODULE_DESCRIPTION("Virtio network driver");
6797 MODULE_LICENSE("GPL");