drivers/net/ethernet/intel/ice/ice_xsk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019, Intel Corporation. */
   3
   4 #include <linux/bpf_trace.h>
   5 #include <net/xdp_sock_drv.h>
   6 #include <net/xdp.h>
   7 #include "ice.h"
   8 #include "ice_base.h"
   9 #include "ice_type.h"
  10 #include "ice_xsk.h"
  11 #include "ice_txrx.h"
  12 #include "ice_txrx_lib.h"
  13 #include "ice_lib.h"
  14
  15 static struct xdp_buff **ice_xdp_buf(struct ice_rx_ring *rx_ring, u32 idx)
  16 {
  17         return &rx_ring->xdp_buf[idx];
  18 }
  19
  20 /**
  21  * ice_qp_reset_stats - Resets all stats for rings of given index
  22  * @vsi: VSI that contains rings of interest
  23  * @q_idx: ring index in array
  24  */
  25 static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx)
  26 {
  27         struct ice_vsi_stats *vsi_stat;
  28         struct ice_pf *pf;
  29
  30         pf = vsi->back;
  31         if (!pf->vsi_stats)
  32                 return;
  33
  34         vsi_stat = pf->vsi_stats[vsi->idx];
  35         if (!vsi_stat)
  36                 return;
  37
  38         memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0,
  39                sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats));
  40         memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0,
  41                sizeof(vsi_stat->tx_ring_stats[q_idx]->stats));
  42         if (vsi->xdp_rings)
  43                 memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0,
  44                        sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats));
  45 }
  46
  47 /**
  48  * ice_qp_clean_rings - Cleans all the rings of a given index
  49  * @vsi: VSI that contains rings of interest
  50  * @q_idx: ring index in array
  51  */
  52 static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx)
  53 {
  54         ice_clean_tx_ring(vsi->tx_rings[q_idx]);
  55         if (vsi->xdp_rings)
  56                 ice_clean_tx_ring(vsi->xdp_rings[q_idx]);
  57         ice_clean_rx_ring(vsi->rx_rings[q_idx]);
  58 }
  59
  60 /**
  61  * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector
  62  * @vsi: VSI that has netdev
  63  * @q_vector: q_vector that has NAPI context
  64  * @enable: true for enable, false for disable
  65  */
  66 static void
  67 ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector,
  68                      bool enable)
  69 {
  70         if (!vsi->netdev || !q_vector)
  71                 return;
  72
  73         if (enable)
  74                 napi_enable(&q_vector->napi);
  75         else
  76                 napi_disable(&q_vector->napi);
  77 }
  78
  79 /**
  80  * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring
  81  * @vsi: the VSI that contains queue vector being un-configured
  82  * @rx_ring: Rx ring that will have its IRQ disabled
  83  * @q_vector: queue vector
  84  */
  85 static void
  86 ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring,
  87                  struct ice_q_vector *q_vector)
  88 {
  89         struct ice_pf *pf = vsi->back;
  90         struct ice_hw *hw = &pf->hw;
  91         u16 reg;
  92         u32 val;
  93
  94         /* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle
  95          * here only QINT_RQCTL
  96          */
  97         reg = rx_ring->reg_idx;
  98         val = rd32(hw, QINT_RQCTL(reg));
  99         val &= ~QINT_RQCTL_CAUSE_ENA_M;
 100         wr32(hw, QINT_RQCTL(reg), val);
 101
 102         if (q_vector) {
 103                 wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0);
 104                 ice_flush(hw);
 105                 synchronize_irq(q_vector->irq.virq);
 106         }
 107 }
 108
 109 /**
 110  * ice_qvec_cfg_msix - Enable IRQ for given queue vector
 111  * @vsi: the VSI that contains queue vector
 112  * @q_vector: queue vector
 113  * @qid: queue index
 114  */
 115 static void
 116 ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector, u16 qid)
 117 {
 118         u16 reg_idx = q_vector->reg_idx;
 119         struct ice_pf *pf = vsi->back;
 120         struct ice_hw *hw = &pf->hw;
 121         int q, _qid = qid;
 122
 123         ice_cfg_itr(hw, q_vector);
 124
 125         for (q = 0; q < q_vector->num_ring_tx; q++) {
 126                 ice_cfg_txq_interrupt(vsi, _qid, reg_idx, q_vector->tx.itr_idx);
 127                 _qid++;
 128         }
 129
 130         _qid = qid;
 131
 132         for (q = 0; q < q_vector->num_ring_rx; q++) {
 133                 ice_cfg_rxq_interrupt(vsi, _qid, reg_idx, q_vector->rx.itr_idx);
 134                 _qid++;
 135         }
 136
 137         ice_flush(hw);
 138 }
 139
 140 /**
 141  * ice_qvec_ena_irq - Enable IRQ for given queue vector
 142  * @vsi: the VSI that contains queue vector
 143  * @q_vector: queue vector
 144  */
 145 static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
 146 {
 147         struct ice_pf *pf = vsi->back;
 148         struct ice_hw *hw = &pf->hw;
 149
 150         ice_irq_dynamic_ena(hw, vsi, q_vector);
 151
 152         ice_flush(hw);
 153 }
 154
 155 /**
 156  * ice_qp_dis - Disables a queue pair
 157  * @vsi: VSI of interest
 158  * @q_idx: ring index in array
 159  *
 160  * Returns 0 on success, negative on failure.
 161  */
 162 static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx)
 163 {
 164         struct ice_txq_meta txq_meta = { };
 165         struct ice_q_vector *q_vector;
 166         struct ice_tx_ring *tx_ring;
 167         struct ice_rx_ring *rx_ring;
 168         int fail = 0;
 169         int err;
 170
 171         if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
 172                 return -EINVAL;
 173
 174         tx_ring = vsi->tx_rings[q_idx];
 175         rx_ring = vsi->rx_rings[q_idx];
 176         q_vector = rx_ring->q_vector;
 177
 178         synchronize_net();
 179         netif_carrier_off(vsi->netdev);
 180         netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
 181
 182         ice_qvec_dis_irq(vsi, rx_ring, q_vector);
 183         ice_qvec_toggle_napi(vsi, q_vector, false);
 184
 185         ice_fill_txq_meta(vsi, tx_ring, &txq_meta);
 186         err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta);
 187         if (!fail)
 188                 fail = err;
 189         if (vsi->xdp_rings) {
 190                 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx];
 191
 192                 memset(&txq_meta, 0, sizeof(txq_meta));
 193                 ice_fill_txq_meta(vsi, xdp_ring, &txq_meta);
 194                 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring,
 195                                            &txq_meta);
 196                 if (!fail)
 197                         fail = err;
 198         }
 199
 200         ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, false);
 201         ice_qp_clean_rings(vsi, q_idx);
 202         ice_qp_reset_stats(vsi, q_idx);
 203
 204         return fail;
 205 }
 206
 207 /**
 208  * ice_qp_ena - Enables a queue pair
 209  * @vsi: VSI of interest
 210  * @q_idx: ring index in array
 211  *
 212  * Returns 0 on success, negative on failure.
 213  */
 214 static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx)
 215 {
 216         struct ice_q_vector *q_vector;
 217         int fail = 0;
 218         bool link_up;
 219         int err;
 220
 221         err = ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx);
 222         if (!fail)
 223                 fail = err;
 224
 225         if (ice_is_xdp_ena_vsi(vsi)) {
 226                 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx];
 227
 228                 err = ice_vsi_cfg_single_txq(vsi, vsi->xdp_rings, q_idx);
 229                 if (!fail)
 230                         fail = err;
 231                 ice_set_ring_xdp(xdp_ring);
 232                 ice_tx_xsk_pool(vsi, q_idx);
 233         }
 234
 235         err = ice_vsi_cfg_single_rxq(vsi, q_idx);
 236         if (!fail)
 237                 fail = err;
 238
 239         q_vector = vsi->rx_rings[q_idx]->q_vector;
 240         ice_qvec_cfg_msix(vsi, q_vector, q_idx);
 241
 242         err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true);
 243         if (!fail)
 244                 fail = err;
 245
 246         ice_qvec_toggle_napi(vsi, q_vector, true);
 247         ice_qvec_ena_irq(vsi, q_vector);
 248
 249         /* make sure NAPI sees updated ice_{t,x}_ring::xsk_pool */
 250         synchronize_net();
 251         ice_get_link_status(vsi->port_info, &link_up);
 252         if (link_up) {
 253                 netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
 254                 netif_carrier_on(vsi->netdev);
 255         }
 256
 257         return fail;
 258 }
 259
 260 /**
 261  * ice_xsk_pool_disable - disable a buffer pool region
 262  * @vsi: Current VSI
 263  * @qid: queue ID
 264  *
 265  * Returns 0 on success, negative on failure
 266  */
 267 static int ice_xsk_pool_disable(struct ice_vsi *vsi, u16 qid)
 268 {
 269         struct xsk_buff_pool *pool = xsk_get_pool_from_qid(vsi->netdev, qid);
 270
 271         if (!pool)
 272                 return -EINVAL;
 273
 274         xsk_pool_dma_unmap(pool, ICE_RX_DMA_ATTR);
 275
 276         return 0;
 277 }
 278
 279 /**
 280  * ice_xsk_pool_enable - enable a buffer pool region
 281  * @vsi: Current VSI
 282  * @pool: pointer to a requested buffer pool region
 283  * @qid: queue ID
 284  *
 285  * Returns 0 on success, negative on failure
 286  */
 287 static int
 288 ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid)
 289 {
 290         int err;
 291
 292         if (vsi->type != ICE_VSI_PF && vsi->type != ICE_VSI_SF)
 293                 return -EINVAL;
 294
 295         if (qid >= vsi->netdev->real_num_rx_queues ||
 296             qid >= vsi->netdev->real_num_tx_queues)
 297                 return -EINVAL;
 298
 299         err = xsk_pool_dma_map(pool, ice_pf_to_dev(vsi->back),
 300                                ICE_RX_DMA_ATTR);
 301         if (err)
 302                 return err;
 303
 304         return 0;
 305 }
 306
 307 /**
 308  * ice_realloc_rx_xdp_bufs - reallocate for either XSK or normal buffer
 309  * @rx_ring: Rx ring
 310  * @pool_present: is pool for XSK present
 311  *
 312  * Try allocating memory and return ENOMEM, if failed to allocate.
 313  * If allocation was successful, substitute buffer with allocated one.
 314  * Returns 0 on success, negative on failure
 315  */
 316 static int
 317 ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present)
 318 {
 319         size_t elem_size = pool_present ? sizeof(*rx_ring->xdp_buf) :
 320                                           sizeof(*rx_ring->rx_buf);
 321         void *sw_ring = kcalloc(rx_ring->count, elem_size, GFP_KERNEL);
 322
 323         if (!sw_ring)
 324                 return -ENOMEM;
 325
 326         if (pool_present) {
 327                 kfree(rx_ring->rx_buf);
 328                 rx_ring->rx_buf = NULL;
 329                 rx_ring->xdp_buf = sw_ring;
 330         } else {
 331                 kfree(rx_ring->xdp_buf);
 332                 rx_ring->xdp_buf = NULL;
 333                 rx_ring->rx_buf = sw_ring;
 334         }
 335
 336         return 0;
 337 }
 338
 339 /**
 340  * ice_realloc_zc_buf - reallocate XDP ZC queue pairs
 341  * @vsi: Current VSI
 342  * @zc: is zero copy set
 343  *
 344  * Reallocate buffer for rx_rings that might be used by XSK.
 345  * XDP requires more memory, than rx_buf provides.
 346  * Returns 0 on success, negative on failure
 347  */
 348 int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc)
 349 {
 350         struct ice_rx_ring *rx_ring;
 351         uint i;
 352
 353         ice_for_each_rxq(vsi, i) {
 354                 rx_ring = vsi->rx_rings[i];
 355                 if (!rx_ring->xsk_pool)
 356                         continue;
 357
 358                 if (ice_realloc_rx_xdp_bufs(rx_ring, zc))
 359                         return -ENOMEM;
 360         }
 361
 362         return 0;
 363 }
 364
 365 /**
 366  * ice_xsk_pool_setup - enable/disable a buffer pool region depending on its state
 367  * @vsi: Current VSI
 368  * @pool: buffer pool to enable/associate to a ring, NULL to disable
 369  * @qid: queue ID
 370  *
 371  * Returns 0 on success, negative on failure
 372  */
 373 int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid)
 374 {
 375         bool if_running, pool_present = !!pool;
 376         int ret = 0, pool_failure = 0;
 377
 378         if (qid >= vsi->num_rxq || qid >= vsi->num_txq) {
 379                 netdev_err(vsi->netdev, "Please use queue id in scope of combined queues count\n");
 380                 pool_failure = -EINVAL;
 381                 goto failure;
 382         }
 383
 384         if_running = !test_bit(ICE_VSI_DOWN, vsi->state) &&
 385                      ice_is_xdp_ena_vsi(vsi);
 386
 387         if (if_running) {
 388                 struct ice_rx_ring *rx_ring = vsi->rx_rings[qid];
 389
 390                 ret = ice_qp_dis(vsi, qid);
 391                 if (ret) {
 392                         netdev_err(vsi->netdev, "ice_qp_dis error = %d\n", ret);
 393                         goto xsk_pool_if_up;
 394                 }
 395
 396                 ret = ice_realloc_rx_xdp_bufs(rx_ring, pool_present);
 397                 if (ret)
 398                         goto xsk_pool_if_up;
 399         }
 400
 401         pool_failure = pool_present ? ice_xsk_pool_enable(vsi, pool, qid) :
 402                                       ice_xsk_pool_disable(vsi, qid);
 403
 404 xsk_pool_if_up:
 405         if (if_running) {
 406                 ret = ice_qp_ena(vsi, qid);
 407                 if (!ret && pool_present)
 408                         napi_schedule(&vsi->rx_rings[qid]->xdp_ring->q_vector->napi);
 409                 else if (ret)
 410                         netdev_err(vsi->netdev, "ice_qp_ena error = %d\n", ret);
 411         }
 412
 413 failure:
 414         if (pool_failure) {
 415                 netdev_err(vsi->netdev, "Could not %sable buffer pool, error = %d\n",
 416                            pool_present ? "en" : "dis", pool_failure);
 417                 return pool_failure;
 418         }
 419
 420         return ret;
 421 }
 422
 423 /**
 424  * ice_fill_rx_descs - pick buffers from XSK buffer pool and use it
 425  * @pool: XSK Buffer pool to pull the buffers from
 426  * @xdp: SW ring of xdp_buff that will hold the buffers
 427  * @rx_desc: Pointer to Rx descriptors that will be filled
 428  * @count: The number of buffers to allocate
 429  *
 430  * This function allocates a number of Rx buffers from the fill ring
 431  * or the internal recycle mechanism and places them on the Rx ring.
 432  *
 433  * Note that ring wrap should be handled by caller of this function.
 434  *
 435  * Returns the amount of allocated Rx descriptors
 436  */
 437 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp,
 438                              union ice_32b_rx_flex_desc *rx_desc, u16 count)
 439 {
 440         dma_addr_t dma;
 441         u16 buffs;
 442         int i;
 443
 444         buffs = xsk_buff_alloc_batch(pool, xdp, count);
 445         for (i = 0; i < buffs; i++) {
 446                 dma = xsk_buff_xdp_get_dma(*xdp);
 447                 rx_desc->read.pkt_addr = cpu_to_le64(dma);
 448                 rx_desc->wb.status_error0 = 0;
 449
 450                 /* Put private info that changes on a per-packet basis
 451                  * into xdp_buff_xsk->cb.
 452                  */
 453                 ice_xdp_meta_set_desc(*xdp, rx_desc);
 454
 455                 rx_desc++;
 456                 xdp++;
 457         }
 458
 459         return buffs;
 460 }
 461
 462 /**
 463  * __ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
 464  * @rx_ring: Rx ring
 465  * @xsk_pool: XSK buffer pool to pick buffers to be filled by HW
 466  * @count: The number of buffers to allocate
 467  *
 468  * Place the @count of descriptors onto Rx ring. Handle the ring wrap
 469  * for case where space from next_to_use up to the end of ring is less
 470  * than @count. Finally do a tail bump.
 471  *
 472  * Returns true if all allocations were successful, false if any fail.
 473  */
 474 static bool __ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring,
 475                                    struct xsk_buff_pool *xsk_pool, u16 count)
 476 {
 477         u32 nb_buffs_extra = 0, nb_buffs = 0;
 478         union ice_32b_rx_flex_desc *rx_desc;
 479         u16 ntu = rx_ring->next_to_use;
 480         u16 total_count = count;
 481         struct xdp_buff **xdp;
 482
 483         rx_desc = ICE_RX_DESC(rx_ring, ntu);
 484         xdp = ice_xdp_buf(rx_ring, ntu);
 485
 486         if (ntu + count >= rx_ring->count) {
 487                 nb_buffs_extra = ice_fill_rx_descs(xsk_pool, xdp, rx_desc,
 488                                                    rx_ring->count - ntu);
 489                 if (nb_buffs_extra != rx_ring->count - ntu) {
 490                         ntu += nb_buffs_extra;
 491                         goto exit;
 492                 }
 493                 rx_desc = ICE_RX_DESC(rx_ring, 0);
 494                 xdp = ice_xdp_buf(rx_ring, 0);
 495                 ntu = 0;
 496                 count -= nb_buffs_extra;
 497                 ice_release_rx_desc(rx_ring, 0);
 498         }
 499
 500         nb_buffs = ice_fill_rx_descs(xsk_pool, xdp, rx_desc, count);
 501
 502         ntu += nb_buffs;
 503         if (ntu == rx_ring->count)
 504                 ntu = 0;
 505
 506 exit:
 507         if (rx_ring->next_to_use != ntu)
 508                 ice_release_rx_desc(rx_ring, ntu);
 509
 510         return total_count == (nb_buffs_extra + nb_buffs);
 511 }
 512
 513 /**
 514  * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
 515  * @rx_ring: Rx ring
 516  * @xsk_pool: XSK buffer pool to pick buffers to be filled by HW
 517  * @count: The number of buffers to allocate
 518  *
 519  * Wrapper for internal allocation routine; figure out how many tail
 520  * bumps should take place based on the given threshold
 521  *
 522  * Returns true if all calls to internal alloc routine succeeded
 523  */
 524 bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring,
 525                           struct xsk_buff_pool *xsk_pool, u16 count)
 526 {
 527         u16 rx_thresh = ICE_RING_QUARTER(rx_ring);
 528         u16 leftover, i, tail_bumps;
 529
 530         tail_bumps = count / rx_thresh;
 531         leftover = count - (tail_bumps * rx_thresh);
 532
 533         for (i = 0; i < tail_bumps; i++)
 534                 if (!__ice_alloc_rx_bufs_zc(rx_ring, xsk_pool, rx_thresh))
 535                         return false;
 536         return __ice_alloc_rx_bufs_zc(rx_ring, xsk_pool, leftover);
 537 }
 538
 539 /**
 540  * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer
 541  * @rx_ring: Rx ring
 542  * @xdp: Pointer to XDP buffer
 543  *
 544  * This function allocates a new skb from a zero-copy Rx buffer.
 545  *
 546  * Returns the skb on success, NULL on failure.
 547  */
 548 static struct sk_buff *
 549 ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)
 550 {
 551         unsigned int totalsize = xdp->data_end - xdp->data_meta;
 552         unsigned int metasize = xdp->data - xdp->data_meta;
 553         struct skb_shared_info *sinfo = NULL;
 554         struct sk_buff *skb;
 555         u32 nr_frags = 0;
 556
 557         if (unlikely(xdp_buff_has_frags(xdp))) {
 558                 sinfo = xdp_get_shared_info_from_buff(xdp);
 559                 nr_frags = sinfo->nr_frags;
 560         }
 561         net_prefetch(xdp->data_meta);
 562
 563         skb = napi_alloc_skb(&rx_ring->q_vector->napi, totalsize);
 564         if (unlikely(!skb))
 565                 return NULL;
 566
 567         memcpy(__skb_put(skb, totalsize), xdp->data_meta,
 568                ALIGN(totalsize, sizeof(long)));
 569
 570         if (metasize) {
 571                 skb_metadata_set(skb, metasize);
 572                 __skb_pull(skb, metasize);
 573         }
 574
 575         if (likely(!xdp_buff_has_frags(xdp)))
 576                 goto out;
 577
 578         for (int i = 0; i < nr_frags; i++) {
 579                 struct skb_shared_info *skinfo = skb_shinfo(skb);
 580                 skb_frag_t *frag = &sinfo->frags[i];
 581                 struct page *page;
 582                 void *addr;
 583
 584                 page = dev_alloc_page();
 585                 if (!page) {
 586                         dev_kfree_skb(skb);
 587                         return NULL;
 588                 }
 589                 addr = page_to_virt(page);
 590
 591                 memcpy(addr, skb_frag_page(frag), skb_frag_size(frag));
 592
 593                 __skb_fill_page_desc_noacc(skinfo, skinfo->nr_frags++,
 594                                            addr, 0, skb_frag_size(frag));
 595         }
 596
 597 out:
 598         xsk_buff_free(xdp);
 599         return skb;
 600 }
 601
 602 /**
 603  * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ
 604  * @xdp_ring: XDP Tx ring
 605  * @xsk_pool: AF_XDP buffer pool pointer
 606  */
 607 static u32 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring,
 608                                 struct xsk_buff_pool *xsk_pool)
 609 {
 610         u16 ntc = xdp_ring->next_to_clean;
 611         struct ice_tx_desc *tx_desc;
 612         u16 cnt = xdp_ring->count;
 613         struct ice_tx_buf *tx_buf;
 614         u16 completed_frames = 0;
 615         u16 xsk_frames = 0;
 616         u16 last_rs;
 617         int i;
 618
 619         last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;
 620         tx_desc = ICE_TX_DESC(xdp_ring, last_rs);
 621         if (tx_desc->cmd_type_offset_bsz &
 622             cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) {
 623                 if (last_rs >= ntc)
 624                         completed_frames = last_rs - ntc + 1;
 625                 else
 626                         completed_frames = last_rs + cnt - ntc + 1;
 627         }
 628
 629         if (!completed_frames)
 630                 return 0;
 631
 632         if (likely(!xdp_ring->xdp_tx_active)) {
 633                 xsk_frames = completed_frames;
 634                 goto skip;
 635         }
 636
 637         ntc = xdp_ring->next_to_clean;
 638         for (i = 0; i < completed_frames; i++) {
 639                 tx_buf = &xdp_ring->tx_buf[ntc];
 640
 641                 if (tx_buf->type == ICE_TX_BUF_XSK_TX) {
 642                         tx_buf->type = ICE_TX_BUF_EMPTY;
 643                         xsk_buff_free(tx_buf->xdp);
 644                         xdp_ring->xdp_tx_active--;
 645                 } else {
 646                         xsk_frames++;
 647                 }
 648
 649                 ntc++;
 650                 if (ntc >= xdp_ring->count)
 651                         ntc = 0;
 652         }
 653 skip:
 654         tx_desc->cmd_type_offset_bsz = 0;
 655         xdp_ring->next_to_clean += completed_frames;
 656         if (xdp_ring->next_to_clean >= cnt)
 657                 xdp_ring->next_to_clean -= cnt;
 658         if (xsk_frames)
 659                 xsk_tx_completed(xsk_pool, xsk_frames);
 660
 661         return completed_frames;
 662 }
 663
 664 /**
 665  * ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX
 666  * @xdp: XDP buffer to xmit
 667  * @xdp_ring: XDP ring to produce descriptor onto
 668  * @xsk_pool: AF_XDP buffer pool pointer
 669  *
 670  * note that this function works directly on xdp_buff, no need to convert
 671  * it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning
 672  * side will be able to xsk_buff_free() it.
 673  *
 674  * Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there
 675  * was not enough space on XDP ring
 676  */
 677 static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,
 678                               struct ice_tx_ring *xdp_ring,
 679                               struct xsk_buff_pool *xsk_pool)
 680 {
 681         struct skb_shared_info *sinfo = NULL;
 682         u32 size = xdp->data_end - xdp->data;
 683         u32 ntu = xdp_ring->next_to_use;
 684         struct ice_tx_desc *tx_desc;
 685         struct ice_tx_buf *tx_buf;
 686         struct xdp_buff *head;
 687         u32 nr_frags = 0;
 688         u32 free_space;
 689         u32 frag = 0;
 690
 691         free_space = ICE_DESC_UNUSED(xdp_ring);
 692         if (free_space < ICE_RING_QUARTER(xdp_ring))
 693                 free_space += ice_clean_xdp_irq_zc(xdp_ring, xsk_pool);
 694
 695         if (unlikely(!free_space))
 696                 goto busy;
 697
 698         if (unlikely(xdp_buff_has_frags(xdp))) {
 699                 sinfo = xdp_get_shared_info_from_buff(xdp);
 700                 nr_frags = sinfo->nr_frags;
 701                 if (free_space < nr_frags + 1)
 702                         goto busy;
 703         }
 704
 705         tx_desc = ICE_TX_DESC(xdp_ring, ntu);
 706         tx_buf = &xdp_ring->tx_buf[ntu];
 707         head = xdp;
 708
 709         for (;;) {
 710                 dma_addr_t dma;
 711
 712                 dma = xsk_buff_xdp_get_dma(xdp);
 713                 xsk_buff_raw_dma_sync_for_device(xsk_pool, dma, size);
 714
 715                 tx_buf->xdp = xdp;
 716                 tx_buf->type = ICE_TX_BUF_XSK_TX;
 717                 tx_desc->buf_addr = cpu_to_le64(dma);
 718                 tx_desc->cmd_type_offset_bsz = ice_build_ctob(0, 0, size, 0);
 719                 /* account for each xdp_buff from xsk_buff_pool */
 720                 xdp_ring->xdp_tx_active++;
 721
 722                 if (++ntu == xdp_ring->count)
 723                         ntu = 0;
 724
 725                 if (frag == nr_frags)
 726                         break;
 727
 728                 tx_desc = ICE_TX_DESC(xdp_ring, ntu);
 729                 tx_buf = &xdp_ring->tx_buf[ntu];
 730
 731                 xdp = xsk_buff_get_frag(head);
 732                 size = skb_frag_size(&sinfo->frags[frag]);
 733                 frag++;
 734         }
 735
 736         xdp_ring->next_to_use = ntu;
 737         /* update last descriptor from a frame with EOP */
 738         tx_desc->cmd_type_offset_bsz |=
 739                 cpu_to_le64(ICE_TX_DESC_CMD_EOP << ICE_TXD_QW1_CMD_S);
 740
 741         return ICE_XDP_TX;
 742
 743 busy:
 744         xdp_ring->ring_stats->tx_stats.tx_busy++;
 745
 746         return ICE_XDP_CONSUMED;
 747 }
 748
 749 /**
 750  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
 751  * @rx_ring: Rx ring
 752  * @xdp: xdp_buff used as input to the XDP program
 753  * @xdp_prog: XDP program to run
 754  * @xdp_ring: ring to be used for XDP_TX action
 755  * @xsk_pool: AF_XDP buffer pool pointer
 756  *
 757  * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR}
 758  */
 759 static int
 760 ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
 761                struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring,
 762                struct xsk_buff_pool *xsk_pool)
 763 {
 764         int err, result = ICE_XDP_PASS;
 765         u32 act;
 766
 767         act = bpf_prog_run_xdp(xdp_prog, xdp);
 768
 769         if (likely(act == XDP_REDIRECT)) {
 770                 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
 771                 if (!err)
 772                         return ICE_XDP_REDIR;
 773                 if (xsk_uses_need_wakeup(xsk_pool) && err == -ENOBUFS)
 774                         result = ICE_XDP_EXIT;
 775                 else
 776                         result = ICE_XDP_CONSUMED;
 777                 goto out_failure;
 778         }
 779
 780         switch (act) {
 781         case XDP_PASS:
 782                 break;
 783         case XDP_TX:
 784                 result = ice_xmit_xdp_tx_zc(xdp, xdp_ring, xsk_pool);
 785                 if (result == ICE_XDP_CONSUMED)
 786                         goto out_failure;
 787                 break;
 788         case XDP_DROP:
 789                 result = ICE_XDP_CONSUMED;
 790                 break;
 791         default:
 792                 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
 793                 fallthrough;
 794         case XDP_ABORTED:
 795                 result = ICE_XDP_CONSUMED;
 796 out_failure:
 797                 trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
 798                 break;
 799         }
 800
 801         return result;
 802 }
 803
 804 static int
 805 ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first,
 806                  struct xdp_buff *xdp, const unsigned int size)
 807 {
 808         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first);
 809
 810         if (!size)
 811                 return 0;
 812
 813         if (!xdp_buff_has_frags(first)) {
 814                 sinfo->nr_frags = 0;
 815                 sinfo->xdp_frags_size = 0;
 816                 xdp_buff_set_frags_flag(first);
 817         }
 818
 819         if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) {
 820                 xsk_buff_free(first);
 821                 return -ENOMEM;
 822         }
 823
 824         __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++,
 825                                    virt_to_page(xdp->data_hard_start),
 826                                    XDP_PACKET_HEADROOM, size);
 827         sinfo->xdp_frags_size += size;
 828         xsk_buff_add_frag(xdp);
 829
 830         return 0;
 831 }
 832
 833 /**
 834  * ice_clean_rx_irq_zc - consumes packets from the hardware ring
 835  * @rx_ring: AF_XDP Rx ring
 836  * @xsk_pool: AF_XDP buffer pool pointer
 837  * @budget: NAPI budget
 838  *
 839  * Returns number of processed packets on success, remaining budget on failure.
 840  */
 841 int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring,
 842                         struct xsk_buff_pool *xsk_pool,
 843                         int budget)
 844 {
 845         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 846         u32 ntc = rx_ring->next_to_clean;
 847         u32 ntu = rx_ring->next_to_use;
 848         struct xdp_buff *first = NULL;
 849         struct ice_tx_ring *xdp_ring;
 850         unsigned int xdp_xmit = 0;
 851         struct bpf_prog *xdp_prog;
 852         u32 cnt = rx_ring->count;
 853         bool failure = false;
 854         int entries_to_alloc;
 855
 856         /* ZC patch is enabled only when XDP program is set,
 857          * so here it can not be NULL
 858          */
 859         xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 860         xdp_ring = rx_ring->xdp_ring;
 861
 862         if (ntc != rx_ring->first_desc)
 863                 first = *ice_xdp_buf(rx_ring, rx_ring->first_desc);
 864
 865         while (likely(total_rx_packets < (unsigned int)budget)) {
 866                 union ice_32b_rx_flex_desc *rx_desc;
 867                 unsigned int size, xdp_res = 0;
 868                 struct xdp_buff *xdp;
 869                 struct sk_buff *skb;
 870                 u16 stat_err_bits;
 871                 u16 vlan_tci;
 872
 873                 rx_desc = ICE_RX_DESC(rx_ring, ntc);
 874
 875                 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);
 876                 if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits))
 877                         break;
 878
 879                 /* This memory barrier is needed to keep us from reading
 880                  * any other fields out of the rx_desc until we have
 881                  * verified the descriptor has been written back.
 882                  */
 883                 dma_rmb();
 884
 885                 if (unlikely(ntc == ntu))
 886                         break;
 887
 888                 xdp = *ice_xdp_buf(rx_ring, ntc);
 889
 890                 size = le16_to_cpu(rx_desc->wb.pkt_len) &
 891                                    ICE_RX_FLX_DESC_PKT_LEN_M;
 892
 893                 xsk_buff_set_size(xdp, size);
 894                 xsk_buff_dma_sync_for_cpu(xdp);
 895
 896                 if (!first) {
 897                         first = xdp;
 898                 } else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) {
 899                         break;
 900                 }
 901
 902                 if (++ntc == cnt)
 903                         ntc = 0;
 904
 905                 if (ice_is_non_eop(rx_ring, rx_desc))
 906                         continue;
 907
 908                 xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring,
 909                                          xsk_pool);
 910                 if (likely(xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))) {
 911                         xdp_xmit |= xdp_res;
 912                 } else if (xdp_res == ICE_XDP_EXIT) {
 913                         failure = true;
 914                         first = NULL;
 915                         rx_ring->first_desc = ntc;
 916                         break;
 917                 } else if (xdp_res == ICE_XDP_CONSUMED) {
 918                         xsk_buff_free(first);
 919                 } else if (xdp_res == ICE_XDP_PASS) {
 920                         goto construct_skb;
 921                 }
 922
 923                 total_rx_bytes += xdp_get_buff_len(first);
 924                 total_rx_packets++;
 925
 926                 first = NULL;
 927                 rx_ring->first_desc = ntc;
 928                 continue;
 929
 930 construct_skb:
 931                 /* XDP_PASS path */
 932                 skb = ice_construct_skb_zc(rx_ring, first);
 933                 if (!skb) {
 934                         rx_ring->ring_stats->rx_stats.alloc_buf_failed++;
 935                         break;
 936                 }
 937
 938                 first = NULL;
 939                 rx_ring->first_desc = ntc;
 940
 941                 if (eth_skb_pad(skb)) {
 942                         skb = NULL;
 943                         continue;
 944                 }
 945
 946                 total_rx_bytes += skb->len;
 947                 total_rx_packets++;
 948
 949                 vlan_tci = ice_get_vlan_tci(rx_desc);
 950
 951                 ice_process_skb_fields(rx_ring, rx_desc, skb);
 952                 ice_receive_skb(rx_ring, skb, vlan_tci);
 953         }
 954
 955         rx_ring->next_to_clean = ntc;
 956         entries_to_alloc = ICE_RX_DESC_UNUSED(rx_ring);
 957         if (entries_to_alloc > ICE_RING_QUARTER(rx_ring))
 958                 failure |= !ice_alloc_rx_bufs_zc(rx_ring, xsk_pool,
 959                                                  entries_to_alloc);
 960
 961         ice_finalize_xdp_rx(xdp_ring, xdp_xmit, 0);
 962         ice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes);
 963
 964         if (xsk_uses_need_wakeup(xsk_pool)) {
 965                 /* ntu could have changed when allocating entries above, so
 966                  * use rx_ring value instead of stack based one
 967                  */
 968                 if (failure || ntc == rx_ring->next_to_use)
 969                         xsk_set_rx_need_wakeup(xsk_pool);
 970                 else
 971                         xsk_clear_rx_need_wakeup(xsk_pool);
 972
 973                 return (int)total_rx_packets;
 974         }
 975
 976         return failure ? budget : (int)total_rx_packets;
 977 }
 978
 979 /**
 980  * ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor
 981  * @xdp_ring: XDP ring to produce the HW Tx descriptor on
 982  * @xsk_pool: XSK buffer pool to pick buffers to be consumed by HW
 983  * @desc: AF_XDP descriptor to pull the DMA address and length from
 984  * @total_bytes: bytes accumulator that will be used for stats update
 985  */
 986 static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring,
 987                          struct xsk_buff_pool *xsk_pool, struct xdp_desc *desc,
 988                          unsigned int *total_bytes)
 989 {
 990         struct ice_tx_desc *tx_desc;
 991         dma_addr_t dma;
 992
 993         dma = xsk_buff_raw_get_dma(xsk_pool, desc->addr);
 994         xsk_buff_raw_dma_sync_for_device(xsk_pool, dma, desc->len);
 995
 996         tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use++);
 997         tx_desc->buf_addr = cpu_to_le64(dma);
 998         tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(desc),
 999                                                       0, desc->len, 0);
1000
1001         *total_bytes += desc->len;
1002 }
1003
1004 /**
1005  * ice_xmit_pkt_batch - produce a batch of HW Tx descriptors out of AF_XDP descriptors
1006  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1007  * @xsk_pool: XSK buffer pool to pick buffers to be consumed by HW
1008  * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from
1009  * @total_bytes: bytes accumulator that will be used for stats update
1010  */
1011 static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring,
1012                                struct xsk_buff_pool *xsk_pool,
1013                                struct xdp_desc *descs,
1014                                unsigned int *total_bytes)
1015 {
1016         u16 ntu = xdp_ring->next_to_use;
1017         struct ice_tx_desc *tx_desc;
1018         u32 i;
1019
1020         loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) {
1021                 dma_addr_t dma;
1022
1023                 dma = xsk_buff_raw_get_dma(xsk_pool, descs[i].addr);
1024                 xsk_buff_raw_dma_sync_for_device(xsk_pool, dma, descs[i].len);
1025
1026                 tx_desc = ICE_TX_DESC(xdp_ring, ntu++);
1027                 tx_desc->buf_addr = cpu_to_le64(dma);
1028                 tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(&descs[i]),
1029                                                               0, descs[i].len, 0);
1030
1031                 *total_bytes += descs[i].len;
1032         }
1033
1034         xdp_ring->next_to_use = ntu;
1035 }
1036
1037 /**
1038  * ice_fill_tx_hw_ring - produce the number of Tx descriptors onto ring
1039  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1040  * @xsk_pool: XSK buffer pool to pick buffers to be consumed by HW
1041  * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from
1042  * @nb_pkts: count of packets to be send
1043  * @total_bytes: bytes accumulator that will be used for stats update
1044  */
1045 static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring,
1046                                 struct xsk_buff_pool *xsk_pool,
1047                                 struct xdp_desc *descs, u32 nb_pkts,
1048                                 unsigned int *total_bytes)
1049 {
1050         u32 batched, leftover, i;
1051
1052         batched = ALIGN_DOWN(nb_pkts, PKTS_PER_BATCH);
1053         leftover = nb_pkts & (PKTS_PER_BATCH - 1);
1054         for (i = 0; i < batched; i += PKTS_PER_BATCH)
1055                 ice_xmit_pkt_batch(xdp_ring, xsk_pool, &descs[i], total_bytes);
1056         for (; i < batched + leftover; i++)
1057                 ice_xmit_pkt(xdp_ring, xsk_pool, &descs[i], total_bytes);
1058 }
1059
1060 /**
1061  * ice_xmit_zc - take entries from XSK Tx ring and place them onto HW Tx ring
1062  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1063  * @xsk_pool: AF_XDP buffer pool pointer
1064  *
1065  * Returns true if there is no more work that needs to be done, false otherwise
1066  */
1067 bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, struct xsk_buff_pool *xsk_pool)
1068 {
1069         struct xdp_desc *descs = xsk_pool->tx_descs;
1070         u32 nb_pkts, nb_processed = 0;
1071         unsigned int total_bytes = 0;
1072         int budget;
1073
1074         ice_clean_xdp_irq_zc(xdp_ring, xsk_pool);
1075
1076         if (!netif_carrier_ok(xdp_ring->vsi->netdev) ||
1077             !netif_running(xdp_ring->vsi->netdev))
1078                 return true;
1079
1080         budget = ICE_DESC_UNUSED(xdp_ring);
1081         budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring));
1082
1083         nb_pkts = xsk_tx_peek_release_desc_batch(xsk_pool, budget);
1084         if (!nb_pkts)
1085                 return true;
1086
1087         if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) {
1088                 nb_processed = xdp_ring->count - xdp_ring->next_to_use;
1089                 ice_fill_tx_hw_ring(xdp_ring, xsk_pool, descs, nb_processed,
1090                                     &total_bytes);
1091                 xdp_ring->next_to_use = 0;
1092         }
1093
1094         ice_fill_tx_hw_ring(xdp_ring, xsk_pool, &descs[nb_processed],
1095                             nb_pkts - nb_processed, &total_bytes);
1096
1097         ice_set_rs_bit(xdp_ring);
1098         ice_xdp_ring_update_tail(xdp_ring);
1099         ice_update_tx_ring_stats(xdp_ring, nb_pkts, total_bytes);
1100
1101         if (xsk_uses_need_wakeup(xsk_pool))
1102                 xsk_set_tx_need_wakeup(xsk_pool);
1103
1104         return nb_pkts < budget;
1105 }
1106
1107 /**
1108  * ice_xsk_wakeup - Implements ndo_xsk_wakeup
1109  * @netdev: net_device
1110  * @queue_id: queue to wake up
1111  * @flags: ignored in our case, since we have Rx and Tx in the same NAPI
1112  *
1113  * Returns negative on error, zero otherwise.
1114  */
1115 int
1116 ice_xsk_wakeup(struct net_device *netdev, u32 queue_id,
1117                u32 __always_unused flags)
1118 {
1119         struct ice_netdev_priv *np = netdev_priv(netdev);
1120         struct ice_q_vector *q_vector;
1121         struct ice_vsi *vsi = np->vsi;
1122         struct ice_tx_ring *ring;
1123
1124         if (test_bit(ICE_VSI_DOWN, vsi->state) || !netif_carrier_ok(netdev))
1125                 return -ENETDOWN;
1126
1127         if (!ice_is_xdp_ena_vsi(vsi))
1128                 return -EINVAL;
1129
1130         if (queue_id >= vsi->num_txq || queue_id >= vsi->num_rxq)
1131                 return -EINVAL;
1132
1133         ring = vsi->rx_rings[queue_id]->xdp_ring;
1134
1135         if (!READ_ONCE(ring->xsk_pool))
1136                 return -EINVAL;
1137
1138         /* The idea here is that if NAPI is running, mark a miss, so
1139          * it will run again. If not, trigger an interrupt and
1140          * schedule the NAPI from interrupt context. If NAPI would be
1141          * scheduled here, the interrupt affinity would not be
1142          * honored.
1143          */
1144         q_vector = ring->q_vector;
1145         if (!napi_if_scheduled_mark_missed(&q_vector->napi))
1146                 ice_trigger_sw_intr(&vsi->back->hw, q_vector);
1147
1148         return 0;
1149 }
1150
1151 /**
1152  * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP buff pool attached
1153  * @vsi: VSI to be checked
1154  *
1155  * Returns true if any of the Rx rings has an AF_XDP buff pool attached
1156  */
1157 bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi)
1158 {
1159         int i;
1160
1161         ice_for_each_rxq(vsi, i) {
1162                 if (xsk_get_pool_from_qid(vsi->netdev, i))
1163                         return true;
1164         }
1165
1166         return false;
1167 }
1168
1169 /**
1170  * ice_xsk_clean_rx_ring - clean buffer pool queues connected to a given Rx ring
1171  * @rx_ring: ring to be cleaned
1172  */
1173 void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring)
1174 {
1175         u16 ntc = rx_ring->next_to_clean;
1176         u16 ntu = rx_ring->next_to_use;
1177
1178         while (ntc != ntu) {
1179                 struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc);
1180
1181                 xsk_buff_free(xdp);
1182                 ntc++;
1183                 if (ntc >= rx_ring->count)
1184                         ntc = 0;
1185         }
1186 }
1187
1188 /**
1189  * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues
1190  * @xdp_ring: XDP_Tx ring
1191  */
1192 void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring)
1193 {
1194         u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use;
1195         u32 xsk_frames = 0;
1196
1197         while (ntc != ntu) {
1198                 struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];
1199
1200                 if (tx_buf->type == ICE_TX_BUF_XSK_TX) {
1201                         tx_buf->type = ICE_TX_BUF_EMPTY;
1202                         xsk_buff_free(tx_buf->xdp);
1203                 } else {
1204                         xsk_frames++;
1205                 }
1206
1207                 ntc++;
1208                 if (ntc >= xdp_ring->count)
1209                         ntc = 0;
1210         }
1211
1212         if (xsk_frames)
1213                 xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
1214 }