drivers/net/ethernet/pensando/ionic/ionic_txrx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip6_checksum.h>
   8 #include <net/netdev_queues.h>
   9 #include <net/page_pool/helpers.h>
  10
  11 #include "ionic.h"
  12 #include "ionic_lif.h"
  13 #include "ionic_txrx.h"
  14
  15 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
  16                                       void *data, size_t len);
  17
  18 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
  19                                     const skb_frag_t *frag,
  20                                     size_t offset, size_t len);
  21
  22 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
  23                                      struct ionic_tx_desc_info *desc_info);
  24
  25 static void ionic_tx_clean(struct ionic_queue *q,
  26                            struct ionic_tx_desc_info *desc_info,
  27                            struct ionic_txq_comp *comp,
  28                            bool in_napi);
  29
  30 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell)
  31 {
  32         ionic_q_post(q, ring_dbell);
  33 }
  34
  35 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell)
  36 {
  37         ionic_q_post(q, ring_dbell);
  38 }
  39
  40 bool ionic_txq_poke_doorbell(struct ionic_queue *q)
  41 {
  42         struct netdev_queue *netdev_txq;
  43         unsigned long now, then, dif;
  44         struct net_device *netdev;
  45
  46         netdev = q->lif->netdev;
  47         netdev_txq = netdev_get_tx_queue(netdev, q->index);
  48
  49         HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id());
  50
  51         if (q->tail_idx == q->head_idx) {
  52                 HARD_TX_UNLOCK(netdev, netdev_txq);
  53                 return false;
  54         }
  55
  56         now = READ_ONCE(jiffies);
  57         then = q->dbell_jiffies;
  58         dif = now - then;
  59
  60         if (dif > q->dbell_deadline) {
  61                 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
  62                                  q->dbval | q->head_idx);
  63
  64                 q->dbell_jiffies = now;
  65         }
  66
  67         HARD_TX_UNLOCK(netdev, netdev_txq);
  68
  69         return true;
  70 }
  71
  72 bool ionic_rxq_poke_doorbell(struct ionic_queue *q)
  73 {
  74         unsigned long now, then, dif;
  75
  76         /* no lock, called from rx napi or txrx napi, nothing else can fill */
  77
  78         if (q->tail_idx == q->head_idx)
  79                 return false;
  80
  81         now = READ_ONCE(jiffies);
  82         then = q->dbell_jiffies;
  83         dif = now - then;
  84
  85         if (dif > q->dbell_deadline) {
  86                 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
  87                                  q->dbval | q->head_idx);
  88
  89                 q->dbell_jiffies = now;
  90
  91                 dif = 2 * q->dbell_deadline;
  92                 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE)
  93                         dif = IONIC_RX_MAX_DOORBELL_DEADLINE;
  94
  95                 q->dbell_deadline = dif;
  96         }
  97
  98         return true;
  99 }
 100
 101 static inline struct ionic_txq_sg_elem *ionic_tx_sg_elems(struct ionic_queue *q)
 102 {
 103         if (likely(q->sg_desc_size == sizeof(struct ionic_txq_sg_desc_v1)))
 104                 return q->txq_sgl_v1[q->head_idx].elems;
 105         else
 106                 return q->txq_sgl[q->head_idx].elems;
 107 }
 108
 109 static inline struct netdev_queue *q_to_ndq(struct net_device *netdev,
 110                                             struct ionic_queue *q)
 111 {
 112         return netdev_get_tx_queue(netdev, q->index);
 113 }
 114
 115 static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info)
 116 {
 117         return page_address(buf_info->page) + buf_info->page_offset;
 118 }
 119
 120 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info)
 121 {
 122         return page_pool_get_dma_addr(buf_info->page) + buf_info->page_offset;
 123 }
 124
 125 static void __ionic_rx_put_buf(struct ionic_queue *q,
 126                                struct ionic_buf_info *buf_info,
 127                                bool recycle_direct)
 128 {
 129         if (!buf_info->page)
 130                 return;
 131
 132         page_pool_put_full_page(q->page_pool, buf_info->page, recycle_direct);
 133         buf_info->page = NULL;
 134         buf_info->len = 0;
 135         buf_info->page_offset = 0;
 136 }
 137
 138
 139 static void ionic_rx_put_buf(struct ionic_queue *q,
 140                              struct ionic_buf_info *buf_info)
 141 {
 142         __ionic_rx_put_buf(q, buf_info, false);
 143 }
 144
 145 static void ionic_rx_put_buf_direct(struct ionic_queue *q,
 146                                     struct ionic_buf_info *buf_info)
 147 {
 148         __ionic_rx_put_buf(q, buf_info, true);
 149 }
 150
 151 static void ionic_rx_add_skb_frag(struct ionic_queue *q,
 152                                   struct sk_buff *skb,
 153                                   struct ionic_buf_info *buf_info,
 154                                   u32 headroom, u32 len,
 155                                   bool synced)
 156 {
 157         if (!synced)
 158                 page_pool_dma_sync_for_cpu(q->page_pool,
 159                                            buf_info->page,
 160                                            buf_info->page_offset + headroom,
 161                                            len);
 162
 163         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
 164                         buf_info->page, buf_info->page_offset + headroom,
 165                         len, buf_info->len);
 166
 167         /* napi_gro_frags() will release/recycle the
 168          * page_pool buffers from the frags list
 169          */
 170         buf_info->page = NULL;
 171         buf_info->len = 0;
 172         buf_info->page_offset = 0;
 173 }
 174
 175 static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q,
 176                                           struct ionic_rx_desc_info *desc_info,
 177                                           unsigned int headroom,
 178                                           unsigned int len,
 179                                           unsigned int num_sg_elems,
 180                                           bool synced)
 181 {
 182         struct ionic_buf_info *buf_info;
 183         struct sk_buff *skb;
 184         unsigned int i;
 185         u16 frag_len;
 186
 187         buf_info = &desc_info->bufs[0];
 188         prefetchw(buf_info->page);
 189
 190         skb = napi_get_frags(&q_to_qcq(q)->napi);
 191         if (unlikely(!skb)) {
 192                 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
 193                                      dev_name(q->dev), q->name);
 194                 q_to_rx_stats(q)->alloc_err++;
 195                 return NULL;
 196         }
 197         skb_mark_for_recycle(skb);
 198
 199         if (headroom)
 200                 frag_len = min_t(u16, len,
 201                                  IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
 202         else
 203                 frag_len = min_t(u16, len, IONIC_PAGE_SIZE);
 204
 205         if (unlikely(!buf_info->page))
 206                 goto err_bad_buf_page;
 207         ionic_rx_add_skb_frag(q, skb, buf_info, headroom, frag_len, synced);
 208         len -= frag_len;
 209         buf_info++;
 210
 211         for (i = 0; i < num_sg_elems; i++, buf_info++) {
 212                 if (unlikely(!buf_info->page))
 213                         goto err_bad_buf_page;
 214                 frag_len = min_t(u16, len, buf_info->len);
 215                 ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced);
 216                 len -= frag_len;
 217         }
 218
 219         return skb;
 220
 221 err_bad_buf_page:
 222         dev_kfree_skb(skb);
 223         return NULL;
 224 }
 225
 226 static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev,
 227                                           struct ionic_queue *q,
 228                                           struct ionic_rx_desc_info *desc_info,
 229                                           unsigned int headroom,
 230                                           unsigned int len,
 231                                           unsigned int num_sg_elems,
 232                                           bool synced)
 233 {
 234         struct ionic_buf_info *buf_info;
 235         struct device *dev = q->dev;
 236         struct sk_buff *skb;
 237         int i;
 238
 239         buf_info = &desc_info->bufs[0];
 240
 241         skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
 242         if (unlikely(!skb)) {
 243                 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
 244                                      dev_name(dev), q->name);
 245                 q_to_rx_stats(q)->alloc_err++;
 246                 return NULL;
 247         }
 248         skb_mark_for_recycle(skb);
 249
 250         if (!synced)
 251                 page_pool_dma_sync_for_cpu(q->page_pool,
 252                                            buf_info->page,
 253                                            buf_info->page_offset + headroom,
 254                                            len);
 255
 256         skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len);
 257
 258         skb_put(skb, len);
 259         skb->protocol = eth_type_trans(skb, netdev);
 260
 261         /* recycle the Rx buffer now that we're done with it */
 262         ionic_rx_put_buf_direct(q, buf_info);
 263         buf_info++;
 264         for (i = 0; i < num_sg_elems; i++, buf_info++)
 265                 ionic_rx_put_buf_direct(q, buf_info);
 266
 267         return skb;
 268 }
 269
 270 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
 271                                     struct ionic_tx_desc_info *desc_info,
 272                                     bool in_napi)
 273 {
 274         struct xdp_frame_bulk bq;
 275
 276         if (!desc_info->nbufs)
 277                 return;
 278
 279         xdp_frame_bulk_init(&bq);
 280         rcu_read_lock(); /* need for xdp_return_frame_bulk */
 281
 282         if (desc_info->act == XDP_TX) {
 283                 if (likely(in_napi))
 284                         xdp_return_frame_rx_napi(desc_info->xdpf);
 285                 else
 286                         xdp_return_frame(desc_info->xdpf);
 287         } else if (desc_info->act == XDP_REDIRECT) {
 288                 ionic_tx_desc_unmap_bufs(q, desc_info);
 289                 xdp_return_frame_bulk(desc_info->xdpf, &bq);
 290         }
 291
 292         xdp_flush_frame_bulk(&bq);
 293         rcu_read_unlock();
 294
 295         desc_info->nbufs = 0;
 296         desc_info->xdpf = NULL;
 297         desc_info->act = 0;
 298 }
 299
 300 static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame,
 301                                 enum xdp_action act, struct page *page, int off,
 302                                 bool ring_doorbell)
 303 {
 304         struct ionic_tx_desc_info *desc_info;
 305         struct ionic_buf_info *buf_info;
 306         struct ionic_tx_stats *stats;
 307         struct ionic_txq_desc *desc;
 308         size_t len = frame->len;
 309         dma_addr_t dma_addr;
 310         u64 cmd;
 311
 312         desc_info = &q->tx_info[q->head_idx];
 313         desc = &q->txq[q->head_idx];
 314         buf_info = desc_info->bufs;
 315         stats = q_to_tx_stats(q);
 316
 317         if (act == XDP_TX) {
 318                 dma_addr = page_pool_get_dma_addr(page) +
 319                            off + XDP_PACKET_HEADROOM;
 320                 dma_sync_single_for_device(q->dev, dma_addr,
 321                                            len, DMA_TO_DEVICE);
 322         } else /* XDP_REDIRECT */ {
 323                 dma_addr = ionic_tx_map_single(q, frame->data, len);
 324                 if (!dma_addr)
 325                         return -EIO;
 326         }
 327
 328         buf_info->dma_addr = dma_addr;
 329         buf_info->len = len;
 330         buf_info->page = page;
 331         buf_info->page_offset = off;
 332
 333         desc_info->nbufs = 1;
 334         desc_info->xdpf = frame;
 335         desc_info->act = act;
 336
 337         if (xdp_frame_has_frags(frame)) {
 338                 struct ionic_txq_sg_elem *elem;
 339                 struct skb_shared_info *sinfo;
 340                 struct ionic_buf_info *bi;
 341                 skb_frag_t *frag;
 342                 int i;
 343
 344                 bi = &buf_info[1];
 345                 sinfo = xdp_get_shared_info_from_frame(frame);
 346                 frag = sinfo->frags;
 347                 elem = ionic_tx_sg_elems(q);
 348                 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) {
 349                         if (act == XDP_TX) {
 350                                 struct page *pg = skb_frag_page(frag);
 351
 352                                 dma_addr = page_pool_get_dma_addr(pg) +
 353                                            skb_frag_off(frag);
 354                                 dma_sync_single_for_device(q->dev, dma_addr,
 355                                                            skb_frag_size(frag),
 356                                                            DMA_TO_DEVICE);
 357                         } else {
 358                                 dma_addr = ionic_tx_map_frag(q, frag, 0,
 359                                                              skb_frag_size(frag));
 360                                 if (dma_mapping_error(q->dev, dma_addr)) {
 361                                         ionic_tx_desc_unmap_bufs(q, desc_info);
 362                                         return -EIO;
 363                                 }
 364                         }
 365                         bi->dma_addr = dma_addr;
 366                         bi->len = skb_frag_size(frag);
 367                         bi->page = skb_frag_page(frag);
 368
 369                         elem->addr = cpu_to_le64(bi->dma_addr);
 370                         elem->len = cpu_to_le16(bi->len);
 371                         elem++;
 372
 373                         desc_info->nbufs++;
 374                 }
 375         }
 376
 377         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
 378                                   0, (desc_info->nbufs - 1), buf_info->dma_addr);
 379         desc->cmd = cpu_to_le64(cmd);
 380         desc->len = cpu_to_le16(len);
 381         desc->csum_start = 0;
 382         desc->csum_offset = 0;
 383
 384         stats->xdp_frames++;
 385         stats->pkts++;
 386         stats->bytes += len;
 387
 388         ionic_txq_post(q, ring_doorbell);
 389
 390         return 0;
 391 }
 392
 393 int ionic_xdp_xmit(struct net_device *netdev, int n,
 394                    struct xdp_frame **xdp_frames, u32 flags)
 395 {
 396         struct ionic_lif *lif = netdev_priv(netdev);
 397         struct ionic_queue *txq;
 398         struct netdev_queue *nq;
 399         int nxmit;
 400         int space;
 401         int cpu;
 402         int qi;
 403
 404         if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state)))
 405                 return -ENETDOWN;
 406
 407         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 408                 return -EINVAL;
 409
 410         /* AdminQ is assumed on cpu 0, while we attempt to affinitize the
 411          * TxRx queue pairs 0..n-1 on cpus 1..n.  We try to keep with that
 412          * affinitization here, but of course irqbalance and friends might
 413          * have juggled things anyway, so we have to check for the 0 case.
 414          */
 415         cpu = smp_processor_id();
 416         qi = cpu ? (cpu - 1) % lif->nxqs : cpu;
 417
 418         txq = &lif->txqcqs[qi]->q;
 419         nq = netdev_get_tx_queue(netdev, txq->index);
 420         __netif_tx_lock(nq, cpu);
 421         txq_trans_cond_update(nq);
 422
 423         if (netif_tx_queue_stopped(nq) ||
 424             !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 425                                   ionic_q_space_avail(txq),
 426                                   1, 1)) {
 427                 __netif_tx_unlock(nq);
 428                 return -EIO;
 429         }
 430
 431         space = min_t(int, n, ionic_q_space_avail(txq));
 432         for (nxmit = 0; nxmit < space ; nxmit++) {
 433                 if (ionic_xdp_post_frame(txq, xdp_frames[nxmit],
 434                                          XDP_REDIRECT,
 435                                          virt_to_page(xdp_frames[nxmit]->data),
 436                                          0, false)) {
 437                         nxmit--;
 438                         break;
 439                 }
 440         }
 441
 442         if (flags & XDP_XMIT_FLUSH)
 443                 ionic_dbell_ring(lif->kern_dbpage, txq->hw_type,
 444                                  txq->dbval | txq->head_idx);
 445
 446         netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 447                              ionic_q_space_avail(txq),
 448                              4, 4);
 449         __netif_tx_unlock(nq);
 450
 451         return nxmit;
 452 }
 453
 454 static void ionic_xdp_rx_unlink_bufs(struct ionic_queue *q,
 455                                      struct ionic_buf_info *buf_info,
 456                                      int nbufs)
 457 {
 458         int i;
 459
 460         for (i = 0; i < nbufs; i++) {
 461                 buf_info->page = NULL;
 462                 buf_info++;
 463         }
 464 }
 465
 466 static bool ionic_run_xdp(struct ionic_rx_stats *stats,
 467                           struct net_device *netdev,
 468                           struct bpf_prog *xdp_prog,
 469                           struct ionic_queue *rxq,
 470                           struct ionic_buf_info *buf_info,
 471                           int len)
 472 {
 473         u32 xdp_action = XDP_ABORTED;
 474         struct xdp_buff xdp_buf;
 475         struct ionic_queue *txq;
 476         struct netdev_queue *nq;
 477         struct xdp_frame *xdpf;
 478         int remain_len;
 479         int nbufs = 1;
 480         int frag_len;
 481         int err = 0;
 482
 483         xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
 484         frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
 485         xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
 486                          XDP_PACKET_HEADROOM, frag_len, false);
 487         page_pool_dma_sync_for_cpu(rxq->page_pool, buf_info->page,
 488                                    buf_info->page_offset + XDP_PACKET_HEADROOM,
 489                                    frag_len);
 490         prefetchw(&xdp_buf.data_hard_start);
 491
 492         /*  We limit MTU size to one buffer if !xdp_has_frags, so
 493          *  if the recv len is bigger than one buffer
 494          *     then we know we have frag info to gather
 495          */
 496         remain_len = len - frag_len;
 497         if (remain_len) {
 498                 struct skb_shared_info *sinfo;
 499                 struct ionic_buf_info *bi;
 500                 skb_frag_t *frag;
 501
 502                 bi = buf_info;
 503                 sinfo = xdp_get_shared_info_from_buff(&xdp_buf);
 504                 sinfo->nr_frags = 0;
 505                 sinfo->xdp_frags_size = 0;
 506                 xdp_buff_set_frags_flag(&xdp_buf);
 507
 508                 do {
 509                         if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
 510                                 err = -ENOSPC;
 511                                 break;
 512                         }
 513
 514                         frag = &sinfo->frags[sinfo->nr_frags];
 515                         sinfo->nr_frags++;
 516                         bi++;
 517                         frag_len = min_t(u16, remain_len, bi->len);
 518                         page_pool_dma_sync_for_cpu(rxq->page_pool, bi->page,
 519                                                    buf_info->page_offset,
 520                                                    frag_len);
 521                         skb_frag_fill_page_desc(frag, bi->page, 0, frag_len);
 522                         sinfo->xdp_frags_size += frag_len;
 523                         remain_len -= frag_len;
 524
 525                         if (page_is_pfmemalloc(bi->page))
 526                                 xdp_buff_set_frag_pfmemalloc(&xdp_buf);
 527                 } while (remain_len > 0);
 528                 nbufs += sinfo->nr_frags;
 529         }
 530
 531         xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf);
 532
 533         switch (xdp_action) {
 534         case XDP_PASS:
 535                 stats->xdp_pass++;
 536                 return false;  /* false = we didn't consume the packet */
 537
 538         case XDP_DROP:
 539                 ionic_rx_put_buf_direct(rxq, buf_info);
 540                 stats->xdp_drop++;
 541                 break;
 542
 543         case XDP_TX:
 544                 xdpf = xdp_convert_buff_to_frame(&xdp_buf);
 545                 if (!xdpf) {
 546                         err = -ENOSPC;
 547                         break;
 548                 }
 549
 550                 txq = rxq->partner;
 551                 nq = netdev_get_tx_queue(netdev, txq->index);
 552                 __netif_tx_lock(nq, smp_processor_id());
 553                 txq_trans_cond_update(nq);
 554
 555                 if (netif_tx_queue_stopped(nq) ||
 556                     !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 557                                           ionic_q_space_avail(txq),
 558                                           1, 1)) {
 559                         __netif_tx_unlock(nq);
 560                         err = -EIO;
 561                         break;
 562                 }
 563
 564                 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX,
 565                                            buf_info->page,
 566                                            buf_info->page_offset,
 567                                            true);
 568                 __netif_tx_unlock(nq);
 569                 if (unlikely(err)) {
 570                         netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
 571                         break;
 572                 }
 573                 ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs);
 574                 stats->xdp_tx++;
 575                 break;
 576
 577         case XDP_REDIRECT:
 578                 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog);
 579                 if (unlikely(err)) {
 580                         netdev_dbg(netdev, "xdp_do_redirect err %d\n", err);
 581                         break;
 582                 }
 583                 ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs);
 584                 rxq->xdp_flush = true;
 585                 stats->xdp_redirect++;
 586                 break;
 587
 588         case XDP_ABORTED:
 589         default:
 590                 err = -EIO;
 591                 break;
 592         }
 593
 594         if (err) {
 595                 ionic_rx_put_buf_direct(rxq, buf_info);
 596                 trace_xdp_exception(netdev, xdp_prog, xdp_action);
 597                 stats->xdp_aborted++;
 598         }
 599
 600         return true;
 601 }
 602
 603 static void ionic_rx_clean(struct ionic_queue *q,
 604                            struct ionic_rx_desc_info *desc_info,
 605                            struct ionic_rxq_comp *comp,
 606                            struct bpf_prog *xdp_prog)
 607 {
 608         struct net_device *netdev = q->lif->netdev;
 609         struct ionic_qcq *qcq = q_to_qcq(q);
 610         struct ionic_rx_stats *stats;
 611         unsigned int headroom = 0;
 612         struct sk_buff *skb;
 613         bool synced = false;
 614         bool use_copybreak;
 615         u16 len;
 616
 617         stats = q_to_rx_stats(q);
 618
 619         if (unlikely(comp->status)) {
 620                 /* Most likely status==2 and the pkt received was bigger
 621                  * than the buffer available: comp->len will show the
 622                  * pkt size received that didn't fit the advertised desc.len
 623                  */
 624                 dev_dbg(q->dev, "q%d drop comp->status %d comp->len %d desc->len %d\n",
 625                         q->index, comp->status, comp->len, q->rxq[q->head_idx].len);
 626
 627                 stats->dropped++;
 628                 return;
 629         }
 630
 631         len = le16_to_cpu(comp->len);
 632         stats->pkts++;
 633         stats->bytes += len;
 634
 635         if (xdp_prog) {
 636                 if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len))
 637                         return;
 638                 synced = true;
 639                 headroom = XDP_PACKET_HEADROOM;
 640         }
 641
 642         use_copybreak = len <= q->lif->rx_copybreak;
 643         if (use_copybreak)
 644                 skb = ionic_rx_copybreak(netdev, q, desc_info,
 645                                          headroom, len,
 646                                          comp->num_sg_elems, synced);
 647         else
 648                 skb = ionic_rx_build_skb(q, desc_info, headroom, len,
 649                                          comp->num_sg_elems, synced);
 650
 651         if (unlikely(!skb)) {
 652                 stats->dropped++;
 653                 return;
 654         }
 655
 656         skb_record_rx_queue(skb, q->index);
 657
 658         if (likely(netdev->features & NETIF_F_RXHASH)) {
 659                 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 660                 case IONIC_PKT_TYPE_IPV4:
 661                 case IONIC_PKT_TYPE_IPV6:
 662                         skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
 663                                      PKT_HASH_TYPE_L3);
 664                         break;
 665                 case IONIC_PKT_TYPE_IPV4_TCP:
 666                 case IONIC_PKT_TYPE_IPV6_TCP:
 667                 case IONIC_PKT_TYPE_IPV4_UDP:
 668                 case IONIC_PKT_TYPE_IPV6_UDP:
 669                         skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
 670                                      PKT_HASH_TYPE_L4);
 671                         break;
 672                 }
 673         }
 674
 675         if (likely(netdev->features & NETIF_F_RXCSUM) &&
 676             (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) {
 677                 skb->ip_summed = CHECKSUM_COMPLETE;
 678                 skb->csum = (__force __wsum)le16_to_cpu(comp->csum);
 679                 stats->csum_complete++;
 680         } else {
 681                 stats->csum_none++;
 682         }
 683
 684         if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 685                      (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
 686                      (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
 687                 stats->csum_error++;
 688
 689         if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
 690             (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
 691                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 692                                        le16_to_cpu(comp->vlan_tci));
 693                 stats->vlan_stripped++;
 694         }
 695
 696         if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {
 697                 __le64 *cq_desc_hwstamp;
 698                 u64 hwstamp;
 699
 700                 cq_desc_hwstamp =
 701                         (void *)comp +
 702                         qcq->cq.desc_size -
 703                         sizeof(struct ionic_rxq_comp) -
 704                         IONIC_HWSTAMP_CQ_NEGOFFSET;
 705
 706                 hwstamp = le64_to_cpu(*cq_desc_hwstamp);
 707
 708                 if (hwstamp != IONIC_HWSTAMP_INVALID) {
 709                         skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
 710                         stats->hwstamp_valid++;
 711                 } else {
 712                         stats->hwstamp_invalid++;
 713                 }
 714         }
 715
 716         if (use_copybreak)
 717                 napi_gro_receive(&qcq->napi, skb);
 718         else
 719                 napi_gro_frags(&qcq->napi);
 720 }
 721
 722 static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog)
 723 {
 724         struct ionic_rx_desc_info *desc_info;
 725         struct ionic_queue *q = cq->bound_q;
 726         struct ionic_rxq_comp *comp;
 727
 728         comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx];
 729
 730         if (!color_match(comp->pkt_type_color, cq->done_color))
 731                 return false;
 732
 733         /* check for empty queue */
 734         if (q->tail_idx == q->head_idx)
 735                 return false;
 736
 737         if (q->tail_idx != le16_to_cpu(comp->comp_index))
 738                 return false;
 739
 740         desc_info = &q->rx_info[q->tail_idx];
 741         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 742
 743         /* clean the related q entry, only one per qc completion */
 744         ionic_rx_clean(q, desc_info, comp, xdp_prog);
 745
 746         return true;
 747 }
 748
 749 bool ionic_rx_service(struct ionic_cq *cq)
 750 {
 751         return __ionic_rx_service(cq, NULL);
 752 }
 753
 754 static inline void ionic_write_cmb_desc(struct ionic_queue *q,
 755                                         void *desc)
 756 {
 757         /* Since Rx and Tx descriptors are the same size, we can
 758          * save an instruction or two and skip the qtype check.
 759          */
 760         if (unlikely(q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS))
 761                 memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0]));
 762 }
 763
 764 void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog)
 765 {
 766         struct net_device *netdev = q->lif->netdev;
 767         struct ionic_rx_desc_info *desc_info;
 768         struct ionic_rxq_sg_elem *sg_elem;
 769         struct ionic_buf_info *buf_info;
 770         unsigned int fill_threshold;
 771         struct ionic_rxq_desc *desc;
 772         unsigned int first_frag_len;
 773         unsigned int first_buf_len;
 774         unsigned int headroom = 0;
 775         unsigned int remain_len;
 776         unsigned int frag_len;
 777         unsigned int nfrags;
 778         unsigned int n_fill;
 779         unsigned int len;
 780         unsigned int i;
 781         unsigned int j;
 782
 783         n_fill = ionic_q_space_avail(q);
 784
 785         fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD,
 786                                q->num_descs / IONIC_RX_FILL_DIV);
 787         if (n_fill < fill_threshold)
 788                 return;
 789
 790         len = netdev->mtu + VLAN_ETH_HLEN;
 791
 792         if (xdp_prog) {
 793                 /* Always alloc the full size buffer, but only need
 794                  * the actual frag_len in the descriptor
 795                  * XDP uses space in the first buffer, so account for
 796                  * head room, tail room, and ip header in the first frag size.
 797                  */
 798                 headroom = XDP_PACKET_HEADROOM;
 799                 first_buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN + headroom;
 800                 first_frag_len = min_t(u16, len + headroom, first_buf_len);
 801         } else {
 802                 /* Use MTU size if smaller than max buffer size */
 803                 first_frag_len = min_t(u16, len, IONIC_PAGE_SIZE);
 804                 first_buf_len = first_frag_len;
 805         }
 806
 807         for (i = n_fill; i; i--) {
 808                 /* fill main descriptor - buf[0] */
 809                 nfrags = 0;
 810                 remain_len = len;
 811                 desc = &q->rxq[q->head_idx];
 812                 desc_info = &q->rx_info[q->head_idx];
 813                 buf_info = &desc_info->bufs[0];
 814
 815                 buf_info->len = first_buf_len;
 816                 frag_len = first_frag_len - headroom;
 817
 818                 /* get a new buffer if we can't reuse one */
 819                 if (!buf_info->page)
 820                         buf_info->page = page_pool_alloc(q->page_pool,
 821                                                          &buf_info->page_offset,
 822                                                          &buf_info->len,
 823                                                          GFP_ATOMIC);
 824                 if (unlikely(!buf_info->page)) {
 825                         buf_info->len = 0;
 826                         return;
 827                 }
 828
 829                 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom);
 830                 desc->len = cpu_to_le16(frag_len);
 831                 remain_len -= frag_len;
 832                 buf_info++;
 833                 nfrags++;
 834
 835                 /* fill sg descriptors - buf[1..n] */
 836                 sg_elem = q->rxq_sgl[q->head_idx].elems;
 837                 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) {
 838                         frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE);
 839
 840                         /* Recycle any leftover buffers that are too small to reuse */
 841                         if (unlikely(buf_info->page && buf_info->len < frag_len))
 842                                 ionic_rx_put_buf_direct(q, buf_info);
 843
 844                         /* Get new buffer if needed */
 845                         if (!buf_info->page) {
 846                                 buf_info->len = frag_len;
 847                                 buf_info->page = page_pool_alloc(q->page_pool,
 848                                                                  &buf_info->page_offset,
 849                                                                  &buf_info->len,
 850                                                                  GFP_ATOMIC);
 851                                 if (unlikely(!buf_info->page)) {
 852                                         buf_info->len = 0;
 853                                         return;
 854                                 }
 855                         }
 856
 857                         sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
 858                         sg_elem->len = cpu_to_le16(frag_len);
 859                         remain_len -= frag_len;
 860                         buf_info++;
 861                         nfrags++;
 862                 }
 863
 864                 /* clear end sg element as a sentinel */
 865                 if (j < q->max_sg_elems)
 866                         memset(sg_elem, 0, sizeof(*sg_elem));
 867
 868                 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 869                                               IONIC_RXQ_DESC_OPCODE_SIMPLE;
 870                 desc_info->nbufs = nfrags;
 871
 872                 ionic_write_cmb_desc(q, desc);
 873
 874                 ionic_rxq_post(q, false);
 875         }
 876
 877         ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
 878                          q->dbval | q->head_idx);
 879
 880         q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
 881         q->dbell_jiffies = jiffies;
 882 }
 883
 884 void ionic_rx_empty(struct ionic_queue *q)
 885 {
 886         struct ionic_rx_desc_info *desc_info;
 887         unsigned int i, j;
 888
 889         for (i = 0; i < q->num_descs; i++) {
 890                 desc_info = &q->rx_info[i];
 891                 for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++)
 892                         ionic_rx_put_buf(q, &desc_info->bufs[j]);
 893                 desc_info->nbufs = 0;
 894         }
 895
 896         q->head_idx = 0;
 897         q->tail_idx = 0;
 898 }
 899
 900 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
 901 {
 902         struct dim_sample dim_sample;
 903         struct ionic_lif *lif;
 904         unsigned int qi;
 905         u64 pkts, bytes;
 906
 907         if (!qcq->intr.dim_coal_hw)
 908                 return;
 909
 910         lif = qcq->q.lif;
 911         qi = qcq->cq.bound_q->index;
 912
 913         switch (napi_mode) {
 914         case IONIC_LIF_F_TX_DIM_INTR:
 915                 pkts = lif->txqstats[qi].pkts;
 916                 bytes = lif->txqstats[qi].bytes;
 917                 break;
 918         case IONIC_LIF_F_RX_DIM_INTR:
 919                 pkts = lif->rxqstats[qi].pkts;
 920                 bytes = lif->rxqstats[qi].bytes;
 921                 break;
 922         default:
 923                 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts;
 924                 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes;
 925                 break;
 926         }
 927
 928         dim_update_sample(qcq->cq.bound_intr->rearm_count,
 929                           pkts, bytes, &dim_sample);
 930
 931         net_dim(&qcq->dim, &dim_sample);
 932 }
 933
 934 int ionic_tx_napi(struct napi_struct *napi, int budget)
 935 {
 936         struct ionic_qcq *qcq = napi_to_qcq(napi);
 937         struct ionic_cq *cq = napi_to_cq(napi);
 938         u32 work_done = 0;
 939         u32 flags = 0;
 940
 941         work_done = ionic_tx_cq_service(cq, budget, !!budget);
 942
 943         if (unlikely(!budget))
 944                 return budget;
 945
 946         if (work_done < budget && napi_complete_done(napi, work_done)) {
 947                 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR);
 948                 flags |= IONIC_INTR_CRED_UNMASK;
 949                 cq->bound_intr->rearm_count++;
 950         }
 951
 952         if (work_done || flags) {
 953                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
 954                 ionic_intr_credits(cq->idev->intr_ctrl,
 955                                    cq->bound_intr->index,
 956                                    work_done, flags);
 957         }
 958
 959         if (!work_done && cq->bound_q->lif->doorbell_wa)
 960                 ionic_txq_poke_doorbell(&qcq->q);
 961
 962         return work_done;
 963 }
 964
 965 static void ionic_xdp_do_flush(struct ionic_cq *cq)
 966 {
 967         if (cq->bound_q->xdp_flush) {
 968                 xdp_do_flush();
 969                 cq->bound_q->xdp_flush = false;
 970         }
 971 }
 972
 973 static unsigned int ionic_rx_cq_service(struct ionic_cq *cq,
 974                                         unsigned int work_to_do)
 975 {
 976         struct ionic_queue *q = cq->bound_q;
 977         unsigned int work_done = 0;
 978         struct bpf_prog *xdp_prog;
 979
 980         if (work_to_do == 0)
 981                 return 0;
 982
 983         xdp_prog = READ_ONCE(q->xdp_prog);
 984         while (__ionic_rx_service(cq, xdp_prog)) {
 985                 if (cq->tail_idx == cq->num_descs - 1)
 986                         cq->done_color = !cq->done_color;
 987
 988                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
 989
 990                 if (++work_done >= work_to_do)
 991                         break;
 992         }
 993         ionic_rx_fill(q, xdp_prog);
 994         ionic_xdp_do_flush(cq);
 995
 996         return work_done;
 997 }
 998
 999 int ionic_rx_napi(struct napi_struct *napi, int budget)
1000 {
1001         struct ionic_qcq *qcq = napi_to_qcq(napi);
1002         struct ionic_cq *cq = napi_to_cq(napi);
1003         u32 work_done = 0;
1004         u32 flags = 0;
1005
1006         if (unlikely(!budget))
1007                 return budget;
1008
1009         work_done = ionic_rx_cq_service(cq, budget);
1010
1011         if (work_done < budget && napi_complete_done(napi, work_done)) {
1012                 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
1013                 flags |= IONIC_INTR_CRED_UNMASK;
1014                 cq->bound_intr->rearm_count++;
1015         }
1016
1017         if (work_done || flags) {
1018                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1019                 ionic_intr_credits(cq->idev->intr_ctrl,
1020                                    cq->bound_intr->index,
1021                                    work_done, flags);
1022         }
1023
1024         if (!work_done && cq->bound_q->lif->doorbell_wa)
1025                 ionic_rxq_poke_doorbell(&qcq->q);
1026
1027         return work_done;
1028 }
1029
1030 int ionic_txrx_napi(struct napi_struct *napi, int budget)
1031 {
1032         struct ionic_qcq *rxqcq = napi_to_qcq(napi);
1033         struct ionic_cq *rxcq = napi_to_cq(napi);
1034         unsigned int qi = rxcq->bound_q->index;
1035         struct ionic_qcq *txqcq;
1036         struct ionic_lif *lif;
1037         struct ionic_cq *txcq;
1038         u32 rx_work_done = 0;
1039         u32 tx_work_done = 0;
1040         u32 flags = 0;
1041
1042         lif = rxcq->bound_q->lif;
1043         txqcq = lif->txqcqs[qi];
1044         txcq = &lif->txqcqs[qi]->cq;
1045
1046         tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, !!budget);
1047
1048         if (unlikely(!budget))
1049                 return budget;
1050
1051         rx_work_done = ionic_rx_cq_service(rxcq, budget);
1052
1053         if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
1054                 ionic_dim_update(rxqcq, 0);
1055                 flags |= IONIC_INTR_CRED_UNMASK;
1056                 rxcq->bound_intr->rearm_count++;
1057         }
1058
1059         if (rx_work_done || flags) {
1060                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1061                 ionic_intr_credits(rxcq->idev->intr_ctrl, rxcq->bound_intr->index,
1062                                    tx_work_done + rx_work_done, flags);
1063         }
1064
1065         if (lif->doorbell_wa) {
1066                 if (!rx_work_done)
1067                         ionic_rxq_poke_doorbell(&rxqcq->q);
1068                 if (!tx_work_done)
1069                         ionic_txq_poke_doorbell(&txqcq->q);
1070         }
1071
1072         return rx_work_done;
1073 }
1074
1075 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
1076                                       void *data, size_t len)
1077 {
1078         struct device *dev = q->dev;
1079         dma_addr_t dma_addr;
1080
1081         dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
1082         if (unlikely(dma_mapping_error(dev, dma_addr))) {
1083                 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
1084                                      dev_name(dev), q->name);
1085                 q_to_tx_stats(q)->dma_map_err++;
1086                 return 0;
1087         }
1088         return dma_addr;
1089 }
1090
1091 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
1092                                     const skb_frag_t *frag,
1093                                     size_t offset, size_t len)
1094 {
1095         struct device *dev = q->dev;
1096         dma_addr_t dma_addr;
1097
1098         dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
1099         if (unlikely(dma_mapping_error(dev, dma_addr))) {
1100                 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
1101                                      dev_name(dev), q->name);
1102                 q_to_tx_stats(q)->dma_map_err++;
1103                 return 0;
1104         }
1105         return dma_addr;
1106 }
1107
1108 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
1109                             struct ionic_tx_desc_info *desc_info)
1110 {
1111         struct ionic_buf_info *buf_info = desc_info->bufs;
1112         struct device *dev = q->dev;
1113         dma_addr_t dma_addr;
1114         unsigned int nfrags;
1115         skb_frag_t *frag;
1116         int frag_idx;
1117
1118         dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
1119         if (!dma_addr)
1120                 return -EIO;
1121         buf_info->dma_addr = dma_addr;
1122         buf_info->len = skb_headlen(skb);
1123         buf_info++;
1124
1125         frag = skb_shinfo(skb)->frags;
1126         nfrags = skb_shinfo(skb)->nr_frags;
1127         for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
1128                 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
1129                 if (!dma_addr)
1130                         goto dma_fail;
1131                 buf_info->dma_addr = dma_addr;
1132                 buf_info->len = skb_frag_size(frag);
1133                 buf_info++;
1134         }
1135
1136         desc_info->nbufs = 1 + nfrags;
1137
1138         return 0;
1139
1140 dma_fail:
1141         /* unwind the frag mappings and the head mapping */
1142         while (frag_idx > 0) {
1143                 frag_idx--;
1144                 buf_info--;
1145                 dma_unmap_page(dev, buf_info->dma_addr,
1146                                buf_info->len, DMA_TO_DEVICE);
1147         }
1148         dma_unmap_single(dev, desc_info->bufs[0].dma_addr,
1149                          desc_info->bufs[0].len, DMA_TO_DEVICE);
1150         return -EIO;
1151 }
1152
1153 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
1154                                      struct ionic_tx_desc_info *desc_info)
1155 {
1156         struct ionic_buf_info *buf_info = desc_info->bufs;
1157         struct device *dev = q->dev;
1158         unsigned int i;
1159
1160         if (!desc_info->nbufs)
1161                 return;
1162
1163         dma_unmap_single(dev, buf_info->dma_addr,
1164                          buf_info->len, DMA_TO_DEVICE);
1165         buf_info++;
1166         for (i = 1; i < desc_info->nbufs; i++, buf_info++)
1167                 dma_unmap_page(dev, buf_info->dma_addr,
1168                                buf_info->len, DMA_TO_DEVICE);
1169
1170         desc_info->nbufs = 0;
1171 }
1172
1173 static void ionic_tx_clean(struct ionic_queue *q,
1174                            struct ionic_tx_desc_info *desc_info,
1175                            struct ionic_txq_comp *comp,
1176                            bool in_napi)
1177 {
1178         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1179         struct ionic_qcq *qcq = q_to_qcq(q);
1180         struct sk_buff *skb;
1181
1182         if (desc_info->xdpf) {
1183                 ionic_xdp_tx_desc_clean(q->partner, desc_info, in_napi);
1184                 stats->clean++;
1185
1186                 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
1187                         netif_wake_subqueue(q->lif->netdev, q->index);
1188
1189                 return;
1190         }
1191
1192         ionic_tx_desc_unmap_bufs(q, desc_info);
1193
1194         skb = desc_info->skb;
1195         if (!skb)
1196                 return;
1197
1198         if (unlikely(ionic_txq_hwstamp_enabled(q))) {
1199                 if (comp) {
1200                         struct skb_shared_hwtstamps hwts = {};
1201                         __le64 *cq_desc_hwstamp;
1202                         u64 hwstamp;
1203
1204                         cq_desc_hwstamp =
1205                                 (void *)comp +
1206                                 qcq->cq.desc_size -
1207                                 sizeof(struct ionic_txq_comp) -
1208                                 IONIC_HWSTAMP_CQ_NEGOFFSET;
1209
1210                         hwstamp = le64_to_cpu(*cq_desc_hwstamp);
1211
1212                         if (hwstamp != IONIC_HWSTAMP_INVALID) {
1213                                 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
1214
1215                                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1216                                 skb_tstamp_tx(skb, &hwts);
1217
1218                                 stats->hwstamp_valid++;
1219                         } else {
1220                                 stats->hwstamp_invalid++;
1221                         }
1222                 }
1223         }
1224
1225         desc_info->bytes = skb->len;
1226         stats->clean++;
1227
1228         napi_consume_skb(skb, likely(in_napi) ? 1 : 0);
1229 }
1230
1231 static bool ionic_tx_service(struct ionic_cq *cq,
1232                              unsigned int *total_pkts,
1233                              unsigned int *total_bytes,
1234                              bool in_napi)
1235 {
1236         struct ionic_tx_desc_info *desc_info;
1237         struct ionic_queue *q = cq->bound_q;
1238         struct ionic_txq_comp *comp;
1239         unsigned int bytes = 0;
1240         unsigned int pkts = 0;
1241         u16 index;
1242
1243         comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx];
1244
1245         if (!color_match(comp->color, cq->done_color))
1246                 return false;
1247
1248         /* clean the related q entries, there could be
1249          * several q entries completed for each cq completion
1250          */
1251         do {
1252                 desc_info = &q->tx_info[q->tail_idx];
1253                 desc_info->bytes = 0;
1254                 index = q->tail_idx;
1255                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1256                 ionic_tx_clean(q, desc_info, comp, in_napi);
1257                 if (desc_info->skb) {
1258                         pkts++;
1259                         bytes += desc_info->bytes;
1260                         desc_info->skb = NULL;
1261                 }
1262         } while (index != le16_to_cpu(comp->comp_index));
1263
1264         (*total_pkts) += pkts;
1265         (*total_bytes) += bytes;
1266
1267         return true;
1268 }
1269
1270 unsigned int ionic_tx_cq_service(struct ionic_cq *cq,
1271                                  unsigned int work_to_do,
1272                                  bool in_napi)
1273 {
1274         unsigned int work_done = 0;
1275         unsigned int bytes = 0;
1276         unsigned int pkts = 0;
1277
1278         if (work_to_do == 0)
1279                 return 0;
1280
1281         while (ionic_tx_service(cq, &pkts, &bytes, in_napi)) {
1282                 if (cq->tail_idx == cq->num_descs - 1)
1283                         cq->done_color = !cq->done_color;
1284                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
1285
1286                 if (++work_done >= work_to_do)
1287                         break;
1288         }
1289
1290         if (work_done) {
1291                 struct ionic_queue *q = cq->bound_q;
1292
1293                 if (likely(!ionic_txq_hwstamp_enabled(q)))
1294                         netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q),
1295                                                  pkts, bytes,
1296                                                  ionic_q_space_avail(q),
1297                                                  IONIC_TSO_DESCS_NEEDED);
1298         }
1299
1300         return work_done;
1301 }
1302
1303 void ionic_tx_flush(struct ionic_cq *cq)
1304 {
1305         u32 work_done;
1306
1307         work_done = ionic_tx_cq_service(cq, cq->num_descs, false);
1308         if (work_done)
1309                 ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index,
1310                                    work_done, IONIC_INTR_CRED_RESET_COALESCE);
1311 }
1312
1313 void ionic_tx_empty(struct ionic_queue *q)
1314 {
1315         struct ionic_tx_desc_info *desc_info;
1316         int bytes = 0;
1317         int pkts = 0;
1318
1319         /* walk the not completed tx entries, if any */
1320         while (q->head_idx != q->tail_idx) {
1321                 desc_info = &q->tx_info[q->tail_idx];
1322                 desc_info->bytes = 0;
1323                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1324                 ionic_tx_clean(q, desc_info, NULL, false);
1325                 if (desc_info->skb) {
1326                         pkts++;
1327                         bytes += desc_info->bytes;
1328                         desc_info->skb = NULL;
1329                 }
1330         }
1331
1332         if (likely(!ionic_txq_hwstamp_enabled(q))) {
1333                 struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q);
1334
1335                 netdev_tx_completed_queue(ndq, pkts, bytes);
1336                 netdev_tx_reset_queue(ndq);
1337         }
1338 }
1339
1340 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
1341 {
1342         int err;
1343
1344         err = skb_cow_head(skb, 0);
1345         if (unlikely(err))
1346                 return err;
1347
1348         if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
1349                 inner_ip_hdr(skb)->check = 0;
1350                 inner_tcp_hdr(skb)->check =
1351                         ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
1352                                            inner_ip_hdr(skb)->daddr,
1353                                            0, IPPROTO_TCP, 0);
1354         } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
1355                 inner_tcp_hdr(skb)->check =
1356                         ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
1357                                          &inner_ipv6_hdr(skb)->daddr,
1358                                          0, IPPROTO_TCP, 0);
1359         }
1360
1361         return 0;
1362 }
1363
1364 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
1365 {
1366         int err;
1367
1368         err = skb_cow_head(skb, 0);
1369         if (unlikely(err))
1370                 return err;
1371
1372         if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
1373                 ip_hdr(skb)->check = 0;
1374                 tcp_hdr(skb)->check =
1375                         ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1376                                            ip_hdr(skb)->daddr,
1377                                            0, IPPROTO_TCP, 0);
1378         } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
1379                 tcp_v6_gso_csum_prep(skb);
1380         }
1381
1382         return 0;
1383 }
1384
1385 static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q,
1386                               struct ionic_txq_desc *desc,
1387                               struct sk_buff *skb,
1388                               dma_addr_t addr, u8 nsge, u16 len,
1389                               unsigned int hdrlen, unsigned int mss,
1390                               bool outer_csum,
1391                               u16 vlan_tci, bool has_vlan,
1392                               bool start, bool done)
1393 {
1394         u8 flags = 0;
1395         u64 cmd;
1396
1397         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1398         flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1399         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
1400         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
1401
1402         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
1403         desc->cmd = cpu_to_le64(cmd);
1404         desc->len = cpu_to_le16(len);
1405         desc->vlan_tci = cpu_to_le16(vlan_tci);
1406         desc->hdr_len = cpu_to_le16(hdrlen);
1407         desc->mss = cpu_to_le16(mss);
1408
1409         ionic_write_cmb_desc(q, desc);
1410
1411         if (start) {
1412                 skb_tx_timestamp(skb);
1413                 if (likely(!ionic_txq_hwstamp_enabled(q)))
1414                         netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len);
1415                 ionic_txq_post(q, false);
1416         } else {
1417                 ionic_txq_post(q, done);
1418         }
1419 }
1420
1421 static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q,
1422                         struct sk_buff *skb)
1423 {
1424         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1425         struct ionic_tx_desc_info *desc_info;
1426         struct ionic_buf_info *buf_info;
1427         struct ionic_txq_sg_elem *elem;
1428         struct ionic_txq_desc *desc;
1429         unsigned int chunk_len;
1430         unsigned int frag_rem;
1431         unsigned int tso_rem;
1432         unsigned int seg_rem;
1433         dma_addr_t desc_addr;
1434         dma_addr_t frag_addr;
1435         unsigned int hdrlen;
1436         unsigned int len;
1437         unsigned int mss;
1438         bool start, done;
1439         bool outer_csum;
1440         bool has_vlan;
1441         u16 desc_len;
1442         u8 desc_nsge;
1443         u16 vlan_tci;
1444         bool encap;
1445         int err;
1446
1447         desc_info = &q->tx_info[q->head_idx];
1448
1449         if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1450                 return -EIO;
1451
1452         len = skb->len;
1453         mss = skb_shinfo(skb)->gso_size;
1454         outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1455                                                    SKB_GSO_GRE_CSUM |
1456                                                    SKB_GSO_IPXIP4 |
1457                                                    SKB_GSO_IPXIP6 |
1458                                                    SKB_GSO_UDP_TUNNEL |
1459                                                    SKB_GSO_UDP_TUNNEL_CSUM));
1460         has_vlan = !!skb_vlan_tag_present(skb);
1461         vlan_tci = skb_vlan_tag_get(skb);
1462         encap = skb->encapsulation;
1463
1464         /* Preload inner-most TCP csum field with IP pseudo hdr
1465          * calculated with IP length set to zero.  HW will later
1466          * add in length to each TCP segment resulting from the TSO.
1467          */
1468
1469         if (encap)
1470                 err = ionic_tx_tcp_inner_pseudo_csum(skb);
1471         else
1472                 err = ionic_tx_tcp_pseudo_csum(skb);
1473         if (unlikely(err)) {
1474                 /* clean up mapping from ionic_tx_map_skb */
1475                 ionic_tx_desc_unmap_bufs(q, desc_info);
1476                 return err;
1477         }
1478
1479         if (encap)
1480                 hdrlen = skb_inner_tcp_all_headers(skb);
1481         else
1482                 hdrlen = skb_tcp_all_headers(skb);
1483
1484         desc_info->skb = skb;
1485         buf_info = desc_info->bufs;
1486         tso_rem = len;
1487         seg_rem = min(tso_rem, hdrlen + mss);
1488
1489         frag_addr = 0;
1490         frag_rem = 0;
1491
1492         start = true;
1493
1494         while (tso_rem > 0) {
1495                 desc = NULL;
1496                 elem = NULL;
1497                 desc_addr = 0;
1498                 desc_len = 0;
1499                 desc_nsge = 0;
1500                 /* use fragments until we have enough to post a single descriptor */
1501                 while (seg_rem > 0) {
1502                         /* if the fragment is exhausted then move to the next one */
1503                         if (frag_rem == 0) {
1504                                 /* grab the next fragment */
1505                                 frag_addr = buf_info->dma_addr;
1506                                 frag_rem = buf_info->len;
1507                                 buf_info++;
1508                         }
1509                         chunk_len = min(frag_rem, seg_rem);
1510                         if (!desc) {
1511                                 /* fill main descriptor */
1512                                 desc = &q->txq[q->head_idx];
1513                                 elem = ionic_tx_sg_elems(q);
1514                                 desc_addr = frag_addr;
1515                                 desc_len = chunk_len;
1516                         } else {
1517                                 /* fill sg descriptor */
1518                                 elem->addr = cpu_to_le64(frag_addr);
1519                                 elem->len = cpu_to_le16(chunk_len);
1520                                 elem++;
1521                                 desc_nsge++;
1522                         }
1523                         frag_addr += chunk_len;
1524                         frag_rem -= chunk_len;
1525                         tso_rem -= chunk_len;
1526                         seg_rem -= chunk_len;
1527                 }
1528                 seg_rem = min(tso_rem, mss);
1529                 done = (tso_rem == 0);
1530                 /* post descriptor */
1531                 ionic_tx_tso_post(netdev, q, desc, skb, desc_addr, desc_nsge,
1532                                   desc_len, hdrlen, mss, outer_csum, vlan_tci,
1533                                   has_vlan, start, done);
1534                 start = false;
1535                 /* Buffer information is stored with the first tso descriptor */
1536                 desc_info = &q->tx_info[q->head_idx];
1537                 desc_info->nbufs = 0;
1538         }
1539
1540         stats->pkts += DIV_ROUND_UP(len - hdrlen, mss);
1541         stats->bytes += len;
1542         stats->tso++;
1543         stats->tso_bytes = len;
1544
1545         return 0;
1546 }
1547
1548 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
1549                                struct ionic_tx_desc_info *desc_info)
1550 {
1551         struct ionic_txq_desc *desc = &q->txq[q->head_idx];
1552         struct ionic_buf_info *buf_info = desc_info->bufs;
1553         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1554         bool has_vlan;
1555         u8 flags = 0;
1556         bool encap;
1557         u64 cmd;
1558
1559         has_vlan = !!skb_vlan_tag_present(skb);
1560         encap = skb->encapsulation;
1561
1562         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1563         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1564
1565         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
1566                                   flags, skb_shinfo(skb)->nr_frags,
1567                                   buf_info->dma_addr);
1568         desc->cmd = cpu_to_le64(cmd);
1569         desc->len = cpu_to_le16(buf_info->len);
1570         if (has_vlan) {
1571                 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1572                 stats->vlan_inserted++;
1573         } else {
1574                 desc->vlan_tci = 0;
1575         }
1576         desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
1577         desc->csum_offset = cpu_to_le16(skb->csum_offset);
1578
1579         ionic_write_cmb_desc(q, desc);
1580
1581         if (skb_csum_is_sctp(skb))
1582                 stats->crc32_csum++;
1583         else
1584                 stats->csum++;
1585 }
1586
1587 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
1588                                   struct ionic_tx_desc_info *desc_info)
1589 {
1590         struct ionic_txq_desc *desc = &q->txq[q->head_idx];
1591         struct ionic_buf_info *buf_info = desc_info->bufs;
1592         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1593         bool has_vlan;
1594         u8 flags = 0;
1595         bool encap;
1596         u64 cmd;
1597
1598         has_vlan = !!skb_vlan_tag_present(skb);
1599         encap = skb->encapsulation;
1600
1601         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1602         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1603
1604         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
1605                                   flags, skb_shinfo(skb)->nr_frags,
1606                                   buf_info->dma_addr);
1607         desc->cmd = cpu_to_le64(cmd);
1608         desc->len = cpu_to_le16(buf_info->len);
1609         if (has_vlan) {
1610                 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1611                 stats->vlan_inserted++;
1612         } else {
1613                 desc->vlan_tci = 0;
1614         }
1615         desc->csum_start = 0;
1616         desc->csum_offset = 0;
1617
1618         ionic_write_cmb_desc(q, desc);
1619
1620         stats->csum_none++;
1621 }
1622
1623 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
1624                                struct ionic_tx_desc_info *desc_info)
1625 {
1626         struct ionic_buf_info *buf_info = &desc_info->bufs[1];
1627         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1628         struct ionic_txq_sg_elem *elem;
1629         unsigned int i;
1630
1631         elem = ionic_tx_sg_elems(q);
1632         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
1633                 elem->addr = cpu_to_le64(buf_info->dma_addr);
1634                 elem->len = cpu_to_le16(buf_info->len);
1635         }
1636
1637         stats->frags += skb_shinfo(skb)->nr_frags;
1638 }
1639
1640 static int ionic_tx(struct net_device *netdev, struct ionic_queue *q,
1641                     struct sk_buff *skb)
1642 {
1643         struct ionic_tx_desc_info *desc_info = &q->tx_info[q->head_idx];
1644         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1645         bool ring_dbell = true;
1646
1647         if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1648                 return -EIO;
1649
1650         desc_info->skb = skb;
1651
1652         /* set up the initial descriptor */
1653         if (skb->ip_summed == CHECKSUM_PARTIAL)
1654                 ionic_tx_calc_csum(q, skb, desc_info);
1655         else
1656                 ionic_tx_calc_no_csum(q, skb, desc_info);
1657
1658         /* add frags */
1659         ionic_tx_skb_frags(q, skb, desc_info);
1660
1661         skb_tx_timestamp(skb);
1662         stats->pkts++;
1663         stats->bytes += skb->len;
1664
1665         if (likely(!ionic_txq_hwstamp_enabled(q))) {
1666                 struct netdev_queue *ndq = q_to_ndq(netdev, q);
1667
1668                 if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1)))
1669                         netif_tx_stop_queue(ndq);
1670                 ring_dbell = __netdev_tx_sent_queue(ndq, skb->len,
1671                                                     netdev_xmit_more());
1672         }
1673         ionic_txq_post(q, ring_dbell);
1674
1675         return 0;
1676 }
1677
1678 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
1679 {
1680         int nr_frags = skb_shinfo(skb)->nr_frags;
1681         bool too_many_frags = false;
1682         skb_frag_t *frag;
1683         int desc_bufs;
1684         int chunk_len;
1685         int frag_rem;
1686         int tso_rem;
1687         int seg_rem;
1688         bool encap;
1689         int hdrlen;
1690         int ndescs;
1691         int err;
1692
1693         /* Each desc is mss long max, so a descriptor for each gso_seg */
1694         if (skb_is_gso(skb)) {
1695                 ndescs = skb_shinfo(skb)->gso_segs;
1696                 if (!nr_frags)
1697                         return ndescs;
1698         } else {
1699                 ndescs = 1;
1700                 if (!nr_frags)
1701                         return ndescs;
1702
1703                 if (unlikely(nr_frags > q->max_sg_elems)) {
1704                         too_many_frags = true;
1705                         goto linearize;
1706                 }
1707
1708                 return ndescs;
1709         }
1710
1711         /* We need to scan the skb to be sure that none of the MTU sized
1712          * packets in the TSO will require more sgs per descriptor than we
1713          * can support.  We loop through the frags, add up the lengths for
1714          * a packet, and count the number of sgs used per packet.
1715          */
1716         tso_rem = skb->len;
1717         frag = skb_shinfo(skb)->frags;
1718         encap = skb->encapsulation;
1719
1720         /* start with just hdr in first part of first descriptor */
1721         if (encap)
1722                 hdrlen = skb_inner_tcp_all_headers(skb);
1723         else
1724                 hdrlen = skb_tcp_all_headers(skb);
1725         seg_rem = min_t(int, tso_rem, hdrlen + skb_shinfo(skb)->gso_size);
1726         frag_rem = hdrlen;
1727
1728         while (tso_rem > 0) {
1729                 desc_bufs = 0;
1730                 while (seg_rem > 0) {
1731                         desc_bufs++;
1732
1733                         /* We add the +1 because we can take buffers for one
1734                          * more than we have SGs: one for the initial desc data
1735                          * in addition to the SG segments that might follow.
1736                          */
1737                         if (desc_bufs > q->max_sg_elems + 1) {
1738                                 too_many_frags = true;
1739                                 goto linearize;
1740                         }
1741
1742                         if (frag_rem == 0) {
1743                                 frag_rem = skb_frag_size(frag);
1744                                 frag++;
1745                         }
1746                         chunk_len = min(frag_rem, seg_rem);
1747                         frag_rem -= chunk_len;
1748                         tso_rem -= chunk_len;
1749                         seg_rem -= chunk_len;
1750                 }
1751
1752                 seg_rem = min_t(int, tso_rem, skb_shinfo(skb)->gso_size);
1753         }
1754
1755 linearize:
1756         if (too_many_frags) {
1757                 err = skb_linearize(skb);
1758                 if (unlikely(err))
1759                         return err;
1760                 q_to_tx_stats(q)->linearize++;
1761         }
1762
1763         return ndescs;
1764 }
1765
1766 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
1767                                             struct net_device *netdev)
1768 {
1769         struct ionic_lif *lif = netdev_priv(netdev);
1770         struct ionic_queue *q;
1771         int err, ndescs;
1772
1773         /* Does not stop/start txq, because we post to a separate tx queue
1774          * for timestamping, and if a packet can't be posted immediately to
1775          * the timestamping queue, it is dropped.
1776          */
1777
1778         q = &lif->hwstamp_txq->q;
1779         ndescs = ionic_tx_descs_needed(q, skb);
1780         if (unlikely(ndescs < 0))
1781                 goto err_out_drop;
1782
1783         if (unlikely(!ionic_q_has_space(q, ndescs)))
1784                 goto err_out_drop;
1785
1786         skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
1787         if (skb_is_gso(skb))
1788                 err = ionic_tx_tso(netdev, q, skb);
1789         else
1790                 err = ionic_tx(netdev, q, skb);
1791
1792         if (unlikely(err))
1793                 goto err_out_drop;
1794
1795         return NETDEV_TX_OK;
1796
1797 err_out_drop:
1798         q->drop++;
1799         dev_kfree_skb(skb);
1800         return NETDEV_TX_OK;
1801 }
1802
1803 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1804 {
1805         u16 queue_index = skb_get_queue_mapping(skb);
1806         struct ionic_lif *lif = netdev_priv(netdev);
1807         struct ionic_queue *q;
1808         int ndescs;
1809         int err;
1810
1811         if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
1812                 dev_kfree_skb(skb);
1813                 return NETDEV_TX_OK;
1814         }
1815
1816         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
1817                 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode)
1818                         return ionic_start_hwstamp_xmit(skb, netdev);
1819
1820         if (unlikely(queue_index >= lif->nxqs))
1821                 queue_index = 0;
1822         q = &lif->txqcqs[queue_index]->q;
1823
1824         ndescs = ionic_tx_descs_needed(q, skb);
1825         if (ndescs < 0)
1826                 goto err_out_drop;
1827
1828         if (!netif_txq_maybe_stop(q_to_ndq(netdev, q),
1829                                   ionic_q_space_avail(q),
1830                                   ndescs, ndescs))
1831                 return NETDEV_TX_BUSY;
1832
1833         if (skb_is_gso(skb))
1834                 err = ionic_tx_tso(netdev, q, skb);
1835         else
1836                 err = ionic_tx(netdev, q, skb);
1837
1838         if (unlikely(err))
1839                 goto err_out_drop;
1840
1841         return NETDEV_TX_OK;
1842
1843 err_out_drop:
1844         q->drop++;
1845         dev_kfree_skb(skb);
1846         return NETDEV_TX_OK;
1847 }