drivers/net/ethernet/amazon/ena/ena_netdev.c

   1 /*
   2  * Copyright 2015 Amazon.com, Inc. or its affiliates.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #ifdef CONFIG_RFS_ACCEL
  36 #include <linux/cpu_rmap.h>
  37 #endif /* CONFIG_RFS_ACCEL */
  38 #include <linux/ethtool.h>
  39 #include <linux/kernel.h>
  40 #include <linux/module.h>
  41 #include <linux/numa.h>
  42 #include <linux/pci.h>
  43 #include <linux/utsname.h>
  44 #include <linux/version.h>
  45 #include <linux/vmalloc.h>
  46 #include <net/ip.h>
  47
  48 #include "ena_netdev.h"
  49 #include <linux/bpf_trace.h>
  50 #include "ena_pci_id_tbl.h"
  51
  52 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
  53 MODULE_DESCRIPTION(DEVICE_NAME);
  54 MODULE_LICENSE("GPL");
  55
  56 /* Time in jiffies before concluding the transmitter is hung. */
  57 #define TX_TIMEOUT  (5 * HZ)
  58
  59 #define ENA_NAPI_BUDGET 64
  60
  61 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \
  62                 NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
  63 static int debug = -1;
  64 module_param(debug, int, 0);
  65 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  66
  67 static struct ena_aenq_handlers aenq_handlers;
  68
  69 static struct workqueue_struct *ena_wq;
  70
  71 MODULE_DEVICE_TABLE(pci, ena_pci_tbl);
  72
  73 static int ena_rss_init_default(struct ena_adapter *adapter);
  74 static void check_for_admin_com_state(struct ena_adapter *adapter);
  75 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful);
  76 static int ena_restore_device(struct ena_adapter *adapter);
  77
  78 static void ena_init_io_rings(struct ena_adapter *adapter,
  79                               int first_index, int count);
  80 static void ena_init_napi_in_range(struct ena_adapter *adapter, int first_index,
  81                                    int count);
  82 static void ena_del_napi_in_range(struct ena_adapter *adapter, int first_index,
  83                                   int count);
  84 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid);
  85 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
  86                                            int first_index,
  87                                            int count);
  88 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid);
  89 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid);
  90 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget);
  91 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter);
  92 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter);
  93 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
  94                                       int first_index, int count);
  95 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
  96                                      int first_index, int count);
  97 static int ena_up(struct ena_adapter *adapter);
  98 static void ena_down(struct ena_adapter *adapter);
  99 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
 100                                  struct ena_ring *rx_ring);
 101 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
 102                                       struct ena_ring *rx_ring);
 103 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
 104                               struct ena_tx_buffer *tx_info);
 105 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
 106                                             int first_index, int count);
 107
 108 static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue)
 109 {
 110         struct ena_adapter *adapter = netdev_priv(dev);
 111
 112         /* Change the state of the device to trigger reset
 113          * Check that we are not in the middle or a trigger already
 114          */
 115
 116         if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
 117                 return;
 118
 119         adapter->reset_reason = ENA_REGS_RESET_OS_NETDEV_WD;
 120         u64_stats_update_begin(&adapter->syncp);
 121         adapter->dev_stats.tx_timeout++;
 122         u64_stats_update_end(&adapter->syncp);
 123
 124         netif_err(adapter, tx_err, dev, "Transmit time out\n");
 125 }
 126
 127 static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
 128 {
 129         int i;
 130
 131         for (i = 0; i < adapter->num_io_queues; i++)
 132                 adapter->rx_ring[i].mtu = mtu;
 133 }
 134
 135 static int ena_change_mtu(struct net_device *dev, int new_mtu)
 136 {
 137         struct ena_adapter *adapter = netdev_priv(dev);
 138         int ret;
 139
 140         ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
 141         if (!ret) {
 142                 netif_dbg(adapter, drv, dev, "set MTU to %d\n", new_mtu);
 143                 update_rx_ring_mtu(adapter, new_mtu);
 144                 dev->mtu = new_mtu;
 145         } else {
 146                 netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",
 147                           new_mtu);
 148         }
 149
 150         return ret;
 151 }
 152
 153 static int ena_xmit_common(struct net_device *dev,
 154                            struct ena_ring *ring,
 155                            struct ena_tx_buffer *tx_info,
 156                            struct ena_com_tx_ctx *ena_tx_ctx,
 157                            u16 next_to_use,
 158                            u32 bytes)
 159 {
 160         struct ena_adapter *adapter = netdev_priv(dev);
 161         int rc, nb_hw_desc;
 162
 163         if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq,
 164                                                 ena_tx_ctx))) {
 165                 netif_dbg(adapter, tx_queued, dev,
 166                           "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
 167                           ring->qid);
 168                 ena_com_write_sq_doorbell(ring->ena_com_io_sq);
 169         }
 170
 171         /* prepare the packet's descriptors to dma engine */
 172         rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx,
 173                                 &nb_hw_desc);
 174
 175         /* In case there isn't enough space in the queue for the packet,
 176          * we simply drop it. All other failure reasons of
 177          * ena_com_prepare_tx() are fatal and therefore require a device reset.
 178          */
 179         if (unlikely(rc)) {
 180                 netif_err(adapter, tx_queued, dev,
 181                           "failed to prepare tx bufs\n");
 182                 u64_stats_update_begin(&ring->syncp);
 183                 ring->tx_stats.prepare_ctx_err++;
 184                 u64_stats_update_end(&ring->syncp);
 185                 if (rc != -ENOMEM) {
 186                         adapter->reset_reason =
 187                                 ENA_REGS_RESET_DRIVER_INVALID_STATE;
 188                         set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
 189                 }
 190                 return rc;
 191         }
 192
 193         u64_stats_update_begin(&ring->syncp);
 194         ring->tx_stats.cnt++;
 195         ring->tx_stats.bytes += bytes;
 196         u64_stats_update_end(&ring->syncp);
 197
 198         tx_info->tx_descs = nb_hw_desc;
 199         tx_info->last_jiffies = jiffies;
 200         tx_info->print_once = 0;
 201
 202         ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
 203                                                  ring->ring_size);
 204         return 0;
 205 }
 206
 207 /* This is the XDP napi callback. XDP queues use a separate napi callback
 208  * than Rx/Tx queues.
 209  */
 210 static int ena_xdp_io_poll(struct napi_struct *napi, int budget)
 211 {
 212         struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
 213         u32 xdp_work_done, xdp_budget;
 214         struct ena_ring *xdp_ring;
 215         int napi_comp_call = 0;
 216         int ret;
 217
 218         xdp_ring = ena_napi->xdp_ring;
 219         xdp_ring->first_interrupt = ena_napi->first_interrupt;
 220
 221         xdp_budget = budget;
 222
 223         if (!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags) ||
 224             test_bit(ENA_FLAG_TRIGGER_RESET, &xdp_ring->adapter->flags)) {
 225                 napi_complete_done(napi, 0);
 226                 return 0;
 227         }
 228
 229         xdp_work_done = ena_clean_xdp_irq(xdp_ring, xdp_budget);
 230
 231         /* If the device is about to reset or down, avoid unmask
 232          * the interrupt and return 0 so NAPI won't reschedule
 233          */
 234         if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags))) {
 235                 napi_complete_done(napi, 0);
 236                 ret = 0;
 237         } else if (xdp_budget > xdp_work_done) {
 238                 napi_comp_call = 1;
 239                 if (napi_complete_done(napi, xdp_work_done))
 240                         ena_unmask_interrupt(xdp_ring, NULL);
 241                 ena_update_ring_numa_node(xdp_ring, NULL);
 242                 ret = xdp_work_done;
 243         } else {
 244                 ret = xdp_budget;
 245         }
 246
 247         u64_stats_update_begin(&xdp_ring->syncp);
 248         xdp_ring->tx_stats.napi_comp += napi_comp_call;
 249         xdp_ring->tx_stats.tx_poll++;
 250         u64_stats_update_end(&xdp_ring->syncp);
 251
 252         return ret;
 253 }
 254
 255 static int ena_xdp_tx_map_buff(struct ena_ring *xdp_ring,
 256                                struct ena_tx_buffer *tx_info,
 257                                struct xdp_buff *xdp,
 258                                void **push_hdr,
 259                                u32 *push_len)
 260 {
 261         struct ena_adapter *adapter = xdp_ring->adapter;
 262         struct ena_com_buf *ena_buf;
 263         dma_addr_t dma = 0;
 264         u32 size;
 265
 266         tx_info->xdpf = xdp_convert_buff_to_frame(xdp);
 267         size = tx_info->xdpf->len;
 268         ena_buf = tx_info->bufs;
 269
 270         /* llq push buffer */
 271         *push_len = min_t(u32, size, xdp_ring->tx_max_header_size);
 272         *push_hdr = tx_info->xdpf->data;
 273
 274         if (size - *push_len > 0) {
 275                 dma = dma_map_single(xdp_ring->dev,
 276                                      *push_hdr + *push_len,
 277                                      size - *push_len,
 278                                      DMA_TO_DEVICE);
 279                 if (unlikely(dma_mapping_error(xdp_ring->dev, dma)))
 280                         goto error_report_dma_error;
 281
 282                 tx_info->map_linear_data = 1;
 283                 tx_info->num_of_bufs = 1;
 284         }
 285
 286         ena_buf->paddr = dma;
 287         ena_buf->len = size;
 288
 289         return 0;
 290
 291 error_report_dma_error:
 292         u64_stats_update_begin(&xdp_ring->syncp);
 293         xdp_ring->tx_stats.dma_mapping_err++;
 294         u64_stats_update_end(&xdp_ring->syncp);
 295         netdev_warn(adapter->netdev, "failed to map xdp buff\n");
 296
 297         xdp_return_frame_rx_napi(tx_info->xdpf);
 298         tx_info->xdpf = NULL;
 299         tx_info->num_of_bufs = 0;
 300
 301         return -EINVAL;
 302 }
 303
 304 static int ena_xdp_xmit_buff(struct net_device *dev,
 305                              struct xdp_buff *xdp,
 306                              int qid,
 307                              struct ena_rx_buffer *rx_info)
 308 {
 309         struct ena_adapter *adapter = netdev_priv(dev);
 310         struct ena_com_tx_ctx ena_tx_ctx = {};
 311         struct ena_tx_buffer *tx_info;
 312         struct ena_ring *xdp_ring;
 313         u16 next_to_use, req_id;
 314         int rc;
 315         void *push_hdr;
 316         u32 push_len;
 317
 318         xdp_ring = &adapter->tx_ring[qid];
 319         next_to_use = xdp_ring->next_to_use;
 320         req_id = xdp_ring->free_ids[next_to_use];
 321         tx_info = &xdp_ring->tx_buffer_info[req_id];
 322         tx_info->num_of_bufs = 0;
 323         page_ref_inc(rx_info->page);
 324         tx_info->xdp_rx_page = rx_info->page;
 325
 326         rc = ena_xdp_tx_map_buff(xdp_ring, tx_info, xdp, &push_hdr, &push_len);
 327         if (unlikely(rc))
 328                 goto error_drop_packet;
 329
 330         ena_tx_ctx.ena_bufs = tx_info->bufs;
 331         ena_tx_ctx.push_header = push_hdr;
 332         ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
 333         ena_tx_ctx.req_id = req_id;
 334         ena_tx_ctx.header_len = push_len;
 335
 336         rc = ena_xmit_common(dev,
 337                              xdp_ring,
 338                              tx_info,
 339                              &ena_tx_ctx,
 340                              next_to_use,
 341                              xdp->data_end - xdp->data);
 342         if (rc)
 343                 goto error_unmap_dma;
 344         /* trigger the dma engine. ena_com_write_sq_doorbell()
 345          * has a mb
 346          */
 347         ena_com_write_sq_doorbell(xdp_ring->ena_com_io_sq);
 348         u64_stats_update_begin(&xdp_ring->syncp);
 349         xdp_ring->tx_stats.doorbells++;
 350         u64_stats_update_end(&xdp_ring->syncp);
 351
 352         return NETDEV_TX_OK;
 353
 354 error_unmap_dma:
 355         ena_unmap_tx_buff(xdp_ring, tx_info);
 356         tx_info->xdpf = NULL;
 357 error_drop_packet:
 358         __free_page(tx_info->xdp_rx_page);
 359         return NETDEV_TX_OK;
 360 }
 361
 362 static int ena_xdp_execute(struct ena_ring *rx_ring,
 363                            struct xdp_buff *xdp,
 364                            struct ena_rx_buffer *rx_info)
 365 {
 366         struct bpf_prog *xdp_prog;
 367         u32 verdict = XDP_PASS;
 368
 369         rcu_read_lock();
 370         xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog);
 371
 372         if (!xdp_prog)
 373                 goto out;
 374
 375         verdict = bpf_prog_run_xdp(xdp_prog, xdp);
 376
 377         if (verdict == XDP_TX)
 378                 ena_xdp_xmit_buff(rx_ring->netdev,
 379                                   xdp,
 380                                   rx_ring->qid + rx_ring->adapter->num_io_queues,
 381                                   rx_info);
 382         else if (unlikely(verdict == XDP_ABORTED))
 383                 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
 384         else if (unlikely(verdict > XDP_TX))
 385                 bpf_warn_invalid_xdp_action(verdict);
 386 out:
 387         rcu_read_unlock();
 388         return verdict;
 389 }
 390
 391 static void ena_init_all_xdp_queues(struct ena_adapter *adapter)
 392 {
 393         adapter->xdp_first_ring = adapter->num_io_queues;
 394         adapter->xdp_num_queues = adapter->num_io_queues;
 395
 396         ena_init_io_rings(adapter,
 397                           adapter->xdp_first_ring,
 398                           adapter->xdp_num_queues);
 399 }
 400
 401 static int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)
 402 {
 403         int rc = 0;
 404
 405         rc = ena_setup_tx_resources_in_range(adapter, adapter->xdp_first_ring,
 406                                              adapter->xdp_num_queues);
 407         if (rc)
 408                 goto setup_err;
 409
 410         rc = ena_create_io_tx_queues_in_range(adapter,
 411                                               adapter->xdp_first_ring,
 412                                               adapter->xdp_num_queues);
 413         if (rc)
 414                 goto create_err;
 415
 416         return 0;
 417
 418 create_err:
 419         ena_free_all_io_tx_resources(adapter);
 420 setup_err:
 421         return rc;
 422 }
 423
 424 /* Provides a way for both kernel and bpf-prog to know
 425  * more about the RX-queue a given XDP frame arrived on.
 426  */
 427 static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
 428 {
 429         int rc;
 430
 431         rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid);
 432
 433         if (rc) {
 434                 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
 435                           "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",
 436                           rx_ring->qid, rc);
 437                 goto err;
 438         }
 439
 440         rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
 441                                         NULL);
 442
 443         if (rc) {
 444                 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
 445                           "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",
 446                           rx_ring->qid, rc);
 447                 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 448         }
 449
 450 err:
 451         return rc;
 452 }
 453
 454 static void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)
 455 {
 456         xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);
 457         xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 458 }
 459
 460 static void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
 461                                                  struct bpf_prog *prog,
 462                                                  int first, int count)
 463 {
 464         struct ena_ring *rx_ring;
 465         int i = 0;
 466
 467         for (i = first; i < count; i++) {
 468                 rx_ring = &adapter->rx_ring[i];
 469                 xchg(&rx_ring->xdp_bpf_prog, prog);
 470                 if (prog) {
 471                         ena_xdp_register_rxq_info(rx_ring);
 472                         rx_ring->rx_headroom = XDP_PACKET_HEADROOM;
 473                 } else {
 474                         ena_xdp_unregister_rxq_info(rx_ring);
 475                         rx_ring->rx_headroom = 0;
 476                 }
 477         }
 478 }
 479
 480 static void ena_xdp_exchange_program(struct ena_adapter *adapter,
 481                                      struct bpf_prog *prog)
 482 {
 483         struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);
 484
 485         ena_xdp_exchange_program_rx_in_range(adapter,
 486                                              prog,
 487                                              0,
 488                                              adapter->num_io_queues);
 489
 490         if (old_bpf_prog)
 491                 bpf_prog_put(old_bpf_prog);
 492 }
 493
 494 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)
 495 {
 496         bool was_up;
 497         int rc;
 498
 499         was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
 500
 501         if (was_up)
 502                 ena_down(adapter);
 503
 504         adapter->xdp_first_ring = 0;
 505         adapter->xdp_num_queues = 0;
 506         ena_xdp_exchange_program(adapter, NULL);
 507         if (was_up) {
 508                 rc = ena_up(adapter);
 509                 if (rc)
 510                         return rc;
 511         }
 512         return 0;
 513 }
 514
 515 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf)
 516 {
 517         struct ena_adapter *adapter = netdev_priv(netdev);
 518         struct bpf_prog *prog = bpf->prog;
 519         struct bpf_prog *old_bpf_prog;
 520         int rc, prev_mtu;
 521         bool is_up;
 522
 523         is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
 524         rc = ena_xdp_allowed(adapter);
 525         if (rc == ENA_XDP_ALLOWED) {
 526                 old_bpf_prog = adapter->xdp_bpf_prog;
 527                 if (prog) {
 528                         if (!is_up) {
 529                                 ena_init_all_xdp_queues(adapter);
 530                         } else if (!old_bpf_prog) {
 531                                 ena_down(adapter);
 532                                 ena_init_all_xdp_queues(adapter);
 533                         }
 534                         ena_xdp_exchange_program(adapter, prog);
 535
 536                         if (is_up && !old_bpf_prog) {
 537                                 rc = ena_up(adapter);
 538                                 if (rc)
 539                                         return rc;
 540                         }
 541                 } else if (old_bpf_prog) {
 542                         rc = ena_destroy_and_free_all_xdp_queues(adapter);
 543                         if (rc)
 544                                 return rc;
 545                 }
 546
 547                 prev_mtu = netdev->max_mtu;
 548                 netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;
 549
 550                 if (!old_bpf_prog)
 551                         netif_info(adapter, drv, adapter->netdev,
 552                                    "xdp program set, changing the max_mtu from %d to %d",
 553                                    prev_mtu, netdev->max_mtu);
 554
 555         } else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {
 556                 netif_err(adapter, drv, adapter->netdev,
 557                           "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",
 558                           netdev->mtu, ENA_XDP_MAX_MTU);
 559                 NL_SET_ERR_MSG_MOD(bpf->extack,
 560                                    "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");
 561                 return -EINVAL;
 562         } else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {
 563                 netif_err(adapter, drv, adapter->netdev,
 564                           "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",
 565                           adapter->num_io_queues, adapter->max_num_io_queues);
 566                 NL_SET_ERR_MSG_MOD(bpf->extack,
 567                                    "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");
 568                 return -EINVAL;
 569         }
 570
 571         return 0;
 572 }
 573
 574 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp
 575  * program as well as to query the current xdp program id.
 576  */
 577 static int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
 578 {
 579         switch (bpf->command) {
 580         case XDP_SETUP_PROG:
 581                 return ena_xdp_set(netdev, bpf);
 582         default:
 583                 return -EINVAL;
 584         }
 585         return 0;
 586 }
 587
 588 static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
 589 {
 590 #ifdef CONFIG_RFS_ACCEL
 591         u32 i;
 592         int rc;
 593
 594         adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_io_queues);
 595         if (!adapter->netdev->rx_cpu_rmap)
 596                 return -ENOMEM;
 597         for (i = 0; i < adapter->num_io_queues; i++) {
 598                 int irq_idx = ENA_IO_IRQ_IDX(i);
 599
 600                 rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
 601                                       pci_irq_vector(adapter->pdev, irq_idx));
 602                 if (rc) {
 603                         free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
 604                         adapter->netdev->rx_cpu_rmap = NULL;
 605                         return rc;
 606                 }
 607         }
 608 #endif /* CONFIG_RFS_ACCEL */
 609         return 0;
 610 }
 611
 612 static void ena_init_io_rings_common(struct ena_adapter *adapter,
 613                                      struct ena_ring *ring, u16 qid)
 614 {
 615         ring->qid = qid;
 616         ring->pdev = adapter->pdev;
 617         ring->dev = &adapter->pdev->dev;
 618         ring->netdev = adapter->netdev;
 619         ring->napi = &adapter->ena_napi[qid].napi;
 620         ring->adapter = adapter;
 621         ring->ena_dev = adapter->ena_dev;
 622         ring->per_napi_packets = 0;
 623         ring->cpu = 0;
 624         ring->first_interrupt = false;
 625         ring->no_interrupt_event_cnt = 0;
 626         u64_stats_init(&ring->syncp);
 627 }
 628
 629 static void ena_init_io_rings(struct ena_adapter *adapter,
 630                               int first_index, int count)
 631 {
 632         struct ena_com_dev *ena_dev;
 633         struct ena_ring *txr, *rxr;
 634         int i;
 635
 636         ena_dev = adapter->ena_dev;
 637
 638         for (i = first_index; i < first_index + count; i++) {
 639                 txr = &adapter->tx_ring[i];
 640                 rxr = &adapter->rx_ring[i];
 641
 642                 /* TX common ring state */
 643                 ena_init_io_rings_common(adapter, txr, i);
 644
 645                 /* TX specific ring state */
 646                 txr->ring_size = adapter->requested_tx_ring_size;
 647                 txr->tx_max_header_size = ena_dev->tx_max_header_size;
 648                 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
 649                 txr->sgl_size = adapter->max_tx_sgl_size;
 650                 txr->smoothed_interval =
 651                         ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
 652                 txr->disable_meta_caching = adapter->disable_meta_caching;
 653
 654                 /* Don't init RX queues for xdp queues */
 655                 if (!ENA_IS_XDP_INDEX(adapter, i)) {
 656                         /* RX common ring state */
 657                         ena_init_io_rings_common(adapter, rxr, i);
 658
 659                         /* RX specific ring state */
 660                         rxr->ring_size = adapter->requested_rx_ring_size;
 661                         rxr->rx_copybreak = adapter->rx_copybreak;
 662                         rxr->sgl_size = adapter->max_rx_sgl_size;
 663                         rxr->smoothed_interval =
 664                                 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
 665                         rxr->empty_rx_queue = 0;
 666                         adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 667                 }
 668         }
 669 }
 670
 671 /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)
 672  * @adapter: network interface device structure
 673  * @qid: queue index
 674  *
 675  * Return 0 on success, negative on failure
 676  */
 677 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
 678 {
 679         struct ena_ring *tx_ring = &adapter->tx_ring[qid];
 680         struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
 681         int size, i, node;
 682
 683         if (tx_ring->tx_buffer_info) {
 684                 netif_err(adapter, ifup,
 685                           adapter->netdev, "tx_buffer_info info is not NULL");
 686                 return -EEXIST;
 687         }
 688
 689         size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
 690         node = cpu_to_node(ena_irq->cpu);
 691
 692         tx_ring->tx_buffer_info = vzalloc_node(size, node);
 693         if (!tx_ring->tx_buffer_info) {
 694                 tx_ring->tx_buffer_info = vzalloc(size);
 695                 if (!tx_ring->tx_buffer_info)
 696                         goto err_tx_buffer_info;
 697         }
 698
 699         size = sizeof(u16) * tx_ring->ring_size;
 700         tx_ring->free_ids = vzalloc_node(size, node);
 701         if (!tx_ring->free_ids) {
 702                 tx_ring->free_ids = vzalloc(size);
 703                 if (!tx_ring->free_ids)
 704                         goto err_tx_free_ids;
 705         }
 706
 707         size = tx_ring->tx_max_header_size;
 708         tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
 709         if (!tx_ring->push_buf_intermediate_buf) {
 710                 tx_ring->push_buf_intermediate_buf = vzalloc(size);
 711                 if (!tx_ring->push_buf_intermediate_buf)
 712                         goto err_push_buf_intermediate_buf;
 713         }
 714
 715         /* Req id ring for TX out of order completions */
 716         for (i = 0; i < tx_ring->ring_size; i++)
 717                 tx_ring->free_ids[i] = i;
 718
 719         /* Reset tx statistics */
 720         memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
 721
 722         tx_ring->next_to_use = 0;
 723         tx_ring->next_to_clean = 0;
 724         tx_ring->cpu = ena_irq->cpu;
 725         return 0;
 726
 727 err_push_buf_intermediate_buf:
 728         vfree(tx_ring->free_ids);
 729         tx_ring->free_ids = NULL;
 730 err_tx_free_ids:
 731         vfree(tx_ring->tx_buffer_info);
 732         tx_ring->tx_buffer_info = NULL;
 733 err_tx_buffer_info:
 734         return -ENOMEM;
 735 }
 736
 737 /* ena_free_tx_resources - Free I/O Tx Resources per Queue
 738  * @adapter: network interface device structure
 739  * @qid: queue index
 740  *
 741  * Free all transmit software resources
 742  */
 743 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
 744 {
 745         struct ena_ring *tx_ring = &adapter->tx_ring[qid];
 746
 747         vfree(tx_ring->tx_buffer_info);
 748         tx_ring->tx_buffer_info = NULL;
 749
 750         vfree(tx_ring->free_ids);
 751         tx_ring->free_ids = NULL;
 752
 753         vfree(tx_ring->push_buf_intermediate_buf);
 754         tx_ring->push_buf_intermediate_buf = NULL;
 755 }
 756
 757 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
 758                                            int first_index,
 759                                            int count)
 760 {
 761         int i, rc = 0;
 762
 763         for (i = first_index; i < first_index + count; i++) {
 764                 rc = ena_setup_tx_resources(adapter, i);
 765                 if (rc)
 766                         goto err_setup_tx;
 767         }
 768
 769         return 0;
 770
 771 err_setup_tx:
 772
 773         netif_err(adapter, ifup, adapter->netdev,
 774                   "Tx queue %d: allocation failed\n", i);
 775
 776         /* rewind the index freeing the rings as we go */
 777         while (first_index < i--)
 778                 ena_free_tx_resources(adapter, i);
 779         return rc;
 780 }
 781
 782 static void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,
 783                                                   int first_index, int count)
 784 {
 785         int i;
 786
 787         for (i = first_index; i < first_index + count; i++)
 788                 ena_free_tx_resources(adapter, i);
 789 }
 790
 791 /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
 792  * @adapter: board private structure
 793  *
 794  * Free all transmit software resources
 795  */
 796 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
 797 {
 798         ena_free_all_io_tx_resources_in_range(adapter,
 799                                               0,
 800                                               adapter->xdp_num_queues +
 801                                               adapter->num_io_queues);
 802 }
 803
 804 static int validate_rx_req_id(struct ena_ring *rx_ring, u16 req_id)
 805 {
 806         if (likely(req_id < rx_ring->ring_size))
 807                 return 0;
 808
 809         netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
 810                   "Invalid rx req_id: %hu\n", req_id);
 811
 812         u64_stats_update_begin(&rx_ring->syncp);
 813         rx_ring->rx_stats.bad_req_id++;
 814         u64_stats_update_end(&rx_ring->syncp);
 815
 816         /* Trigger device reset */
 817         rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
 818         set_bit(ENA_FLAG_TRIGGER_RESET, &rx_ring->adapter->flags);
 819         return -EFAULT;
 820 }
 821
 822 /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
 823  * @adapter: network interface device structure
 824  * @qid: queue index
 825  *
 826  * Returns 0 on success, negative on failure
 827  */
 828 static int ena_setup_rx_resources(struct ena_adapter *adapter,
 829                                   u32 qid)
 830 {
 831         struct ena_ring *rx_ring = &adapter->rx_ring[qid];
 832         struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
 833         int size, node, i;
 834
 835         if (rx_ring->rx_buffer_info) {
 836                 netif_err(adapter, ifup, adapter->netdev,
 837                           "rx_buffer_info is not NULL");
 838                 return -EEXIST;
 839         }
 840
 841         /* alloc extra element so in rx path
 842          * we can always prefetch rx_info + 1
 843          */
 844         size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);
 845         node = cpu_to_node(ena_irq->cpu);
 846
 847         rx_ring->rx_buffer_info = vzalloc_node(size, node);
 848         if (!rx_ring->rx_buffer_info) {
 849                 rx_ring->rx_buffer_info = vzalloc(size);
 850                 if (!rx_ring->rx_buffer_info)
 851                         return -ENOMEM;
 852         }
 853
 854         size = sizeof(u16) * rx_ring->ring_size;
 855         rx_ring->free_ids = vzalloc_node(size, node);
 856         if (!rx_ring->free_ids) {
 857                 rx_ring->free_ids = vzalloc(size);
 858                 if (!rx_ring->free_ids) {
 859                         vfree(rx_ring->rx_buffer_info);
 860                         rx_ring->rx_buffer_info = NULL;
 861                         return -ENOMEM;
 862                 }
 863         }
 864
 865         /* Req id ring for receiving RX pkts out of order */
 866         for (i = 0; i < rx_ring->ring_size; i++)
 867                 rx_ring->free_ids[i] = i;
 868
 869         /* Reset rx statistics */
 870         memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
 871
 872         rx_ring->next_to_clean = 0;
 873         rx_ring->next_to_use = 0;
 874         rx_ring->cpu = ena_irq->cpu;
 875
 876         return 0;
 877 }
 878
 879 /* ena_free_rx_resources - Free I/O Rx Resources
 880  * @adapter: network interface device structure
 881  * @qid: queue index
 882  *
 883  * Free all receive software resources
 884  */
 885 static void ena_free_rx_resources(struct ena_adapter *adapter,
 886                                   u32 qid)
 887 {
 888         struct ena_ring *rx_ring = &adapter->rx_ring[qid];
 889
 890         vfree(rx_ring->rx_buffer_info);
 891         rx_ring->rx_buffer_info = NULL;
 892
 893         vfree(rx_ring->free_ids);
 894         rx_ring->free_ids = NULL;
 895 }
 896
 897 /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
 898  * @adapter: board private structure
 899  *
 900  * Return 0 on success, negative on failure
 901  */
 902 static int ena_setup_all_rx_resources(struct ena_adapter *adapter)
 903 {
 904         int i, rc = 0;
 905
 906         for (i = 0; i < adapter->num_io_queues; i++) {
 907                 rc = ena_setup_rx_resources(adapter, i);
 908                 if (rc)
 909                         goto err_setup_rx;
 910         }
 911
 912         return 0;
 913
 914 err_setup_rx:
 915
 916         netif_err(adapter, ifup, adapter->netdev,
 917                   "Rx queue %d: allocation failed\n", i);
 918
 919         /* rewind the index freeing the rings as we go */
 920         while (i--)
 921                 ena_free_rx_resources(adapter, i);
 922         return rc;
 923 }
 924
 925 /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues
 926  * @adapter: board private structure
 927  *
 928  * Free all receive software resources
 929  */
 930 static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)
 931 {
 932         int i;
 933
 934         for (i = 0; i < adapter->num_io_queues; i++)
 935                 ena_free_rx_resources(adapter, i);
 936 }
 937
 938 static int ena_alloc_rx_page(struct ena_ring *rx_ring,
 939                                     struct ena_rx_buffer *rx_info, gfp_t gfp)
 940 {
 941         struct ena_com_buf *ena_buf;
 942         struct page *page;
 943         dma_addr_t dma;
 944
 945         /* if previous allocated page is not used */
 946         if (unlikely(rx_info->page))
 947                 return 0;
 948
 949         page = alloc_page(gfp);
 950         if (unlikely(!page)) {
 951                 u64_stats_update_begin(&rx_ring->syncp);
 952                 rx_ring->rx_stats.page_alloc_fail++;
 953                 u64_stats_update_end(&rx_ring->syncp);
 954                 return -ENOMEM;
 955         }
 956
 957         /* To enable NIC-side port-mirroring, AKA SPAN port,
 958          * we make the buffer readable from the nic as well
 959          */
 960         dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE,
 961                            DMA_BIDIRECTIONAL);
 962         if (unlikely(dma_mapping_error(rx_ring->dev, dma))) {
 963                 u64_stats_update_begin(&rx_ring->syncp);
 964                 rx_ring->rx_stats.dma_mapping_err++;
 965                 u64_stats_update_end(&rx_ring->syncp);
 966
 967                 __free_page(page);
 968                 return -EIO;
 969         }
 970         netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
 971                   "alloc page %p, rx_info %p\n", page, rx_info);
 972
 973         rx_info->page = page;
 974         rx_info->page_offset = 0;
 975         ena_buf = &rx_info->ena_buf;
 976         ena_buf->paddr = dma + rx_ring->rx_headroom;
 977         ena_buf->len = ENA_PAGE_SIZE - rx_ring->rx_headroom;
 978
 979         return 0;
 980 }
 981
 982 static void ena_free_rx_page(struct ena_ring *rx_ring,
 983                              struct ena_rx_buffer *rx_info)
 984 {
 985         struct page *page = rx_info->page;
 986         struct ena_com_buf *ena_buf = &rx_info->ena_buf;
 987
 988         if (unlikely(!page)) {
 989                 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
 990                            "Trying to free unallocated buffer\n");
 991                 return;
 992         }
 993
 994         dma_unmap_page(rx_ring->dev, ena_buf->paddr - rx_ring->rx_headroom,
 995                        ENA_PAGE_SIZE,
 996                        DMA_BIDIRECTIONAL);
 997
 998         __free_page(page);
 999         rx_info->page = NULL;
1000 }
1001
1002 static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
1003 {
1004         u16 next_to_use, req_id;
1005         u32 i;
1006         int rc;
1007
1008         next_to_use = rx_ring->next_to_use;
1009
1010         for (i = 0; i < num; i++) {
1011                 struct ena_rx_buffer *rx_info;
1012
1013                 req_id = rx_ring->free_ids[next_to_use];
1014
1015                 rx_info = &rx_ring->rx_buffer_info[req_id];
1016
1017                 rc = ena_alloc_rx_page(rx_ring, rx_info,
1018                                        GFP_ATOMIC | __GFP_COMP);
1019                 if (unlikely(rc < 0)) {
1020                         netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
1021                                    "failed to alloc buffer for rx queue %d\n",
1022                                    rx_ring->qid);
1023                         break;
1024                 }
1025                 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1026                                                 &rx_info->ena_buf,
1027                                                 req_id);
1028                 if (unlikely(rc)) {
1029                         netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
1030                                    "failed to add buffer for rx queue %d\n",
1031                                    rx_ring->qid);
1032                         break;
1033                 }
1034                 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1035                                                    rx_ring->ring_size);
1036         }
1037
1038         if (unlikely(i < num)) {
1039                 u64_stats_update_begin(&rx_ring->syncp);
1040                 rx_ring->rx_stats.refil_partial++;
1041                 u64_stats_update_end(&rx_ring->syncp);
1042                 netdev_warn(rx_ring->netdev,
1043                             "refilled rx qid %d with only %d buffers (from %d)\n",
1044                             rx_ring->qid, i, num);
1045         }
1046
1047         /* ena_com_write_sq_doorbell issues a wmb() */
1048         if (likely(i))
1049                 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1050
1051         rx_ring->next_to_use = next_to_use;
1052
1053         return i;
1054 }
1055
1056 static void ena_free_rx_bufs(struct ena_adapter *adapter,
1057                              u32 qid)
1058 {
1059         struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1060         u32 i;
1061
1062         for (i = 0; i < rx_ring->ring_size; i++) {
1063                 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1064
1065                 if (rx_info->page)
1066                         ena_free_rx_page(rx_ring, rx_info);
1067         }
1068 }
1069
1070 /* ena_refill_all_rx_bufs - allocate all queues Rx buffers
1071  * @adapter: board private structure
1072  */
1073 static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1074 {
1075         struct ena_ring *rx_ring;
1076         int i, rc, bufs_num;
1077
1078         for (i = 0; i < adapter->num_io_queues; i++) {
1079                 rx_ring = &adapter->rx_ring[i];
1080                 bufs_num = rx_ring->ring_size - 1;
1081                 rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1082
1083                 if (unlikely(rc != bufs_num))
1084                         netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
1085                                    "refilling Queue %d failed. allocated %d buffers from: %d\n",
1086                                    i, rc, bufs_num);
1087         }
1088 }
1089
1090 static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
1091 {
1092         int i;
1093
1094         for (i = 0; i < adapter->num_io_queues; i++)
1095                 ena_free_rx_bufs(adapter, i);
1096 }
1097
1098 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
1099                               struct ena_tx_buffer *tx_info)
1100 {
1101         struct ena_com_buf *ena_buf;
1102         u32 cnt;
1103         int i;
1104
1105         ena_buf = tx_info->bufs;
1106         cnt = tx_info->num_of_bufs;
1107
1108         if (unlikely(!cnt))
1109                 return;
1110
1111         if (tx_info->map_linear_data) {
1112                 dma_unmap_single(tx_ring->dev,
1113                                  dma_unmap_addr(ena_buf, paddr),
1114                                  dma_unmap_len(ena_buf, len),
1115                                  DMA_TO_DEVICE);
1116                 ena_buf++;
1117                 cnt--;
1118         }
1119
1120         /* unmap remaining mapped pages */
1121         for (i = 0; i < cnt; i++) {
1122                 dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
1123                                dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
1124                 ena_buf++;
1125         }
1126 }
1127
1128 /* ena_free_tx_bufs - Free Tx Buffers per Queue
1129  * @tx_ring: TX ring for which buffers be freed
1130  */
1131 static void ena_free_tx_bufs(struct ena_ring *tx_ring)
1132 {
1133         bool print_once = true;
1134         u32 i;
1135
1136         for (i = 0; i < tx_ring->ring_size; i++) {
1137                 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1138
1139                 if (!tx_info->skb)
1140                         continue;
1141
1142                 if (print_once) {
1143                         netdev_notice(tx_ring->netdev,
1144                                       "free uncompleted tx skb qid %d idx 0x%x\n",
1145                                       tx_ring->qid, i);
1146                         print_once = false;
1147                 } else {
1148                         netdev_dbg(tx_ring->netdev,
1149                                    "free uncompleted tx skb qid %d idx 0x%x\n",
1150                                    tx_ring->qid, i);
1151                 }
1152
1153                 ena_unmap_tx_buff(tx_ring, tx_info);
1154
1155                 dev_kfree_skb_any(tx_info->skb);
1156         }
1157         netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
1158                                                   tx_ring->qid));
1159 }
1160
1161 static void ena_free_all_tx_bufs(struct ena_adapter *adapter)
1162 {
1163         struct ena_ring *tx_ring;
1164         int i;
1165
1166         for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
1167                 tx_ring = &adapter->tx_ring[i];
1168                 ena_free_tx_bufs(tx_ring);
1169         }
1170 }
1171
1172 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1173 {
1174         u16 ena_qid;
1175         int i;
1176
1177         for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
1178                 ena_qid = ENA_IO_TXQ_IDX(i);
1179                 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1180         }
1181 }
1182
1183 static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1184 {
1185         u16 ena_qid;
1186         int i;
1187
1188         for (i = 0; i < adapter->num_io_queues; i++) {
1189                 ena_qid = ENA_IO_RXQ_IDX(i);
1190                 cancel_work_sync(&adapter->ena_napi[i].dim.work);
1191                 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1192         }
1193 }
1194
1195 static void ena_destroy_all_io_queues(struct ena_adapter *adapter)
1196 {
1197         ena_destroy_all_tx_queues(adapter);
1198         ena_destroy_all_rx_queues(adapter);
1199 }
1200
1201 static int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,
1202                                  struct ena_tx_buffer *tx_info, bool is_xdp)
1203 {
1204         if (tx_info)
1205                 netif_err(ring->adapter,
1206                           tx_done,
1207                           ring->netdev,
1208                           "tx_info doesn't have valid %s",
1209                            is_xdp ? "xdp frame" : "skb");
1210         else
1211                 netif_err(ring->adapter,
1212                           tx_done,
1213                           ring->netdev,
1214                           "Invalid req_id: %hu\n",
1215                           req_id);
1216
1217         u64_stats_update_begin(&ring->syncp);
1218         ring->tx_stats.bad_req_id++;
1219         u64_stats_update_end(&ring->syncp);
1220
1221         /* Trigger device reset */
1222         ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
1223         set_bit(ENA_FLAG_TRIGGER_RESET, &ring->adapter->flags);
1224         return -EFAULT;
1225 }
1226
1227 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
1228 {
1229         struct ena_tx_buffer *tx_info = NULL;
1230
1231         if (likely(req_id < tx_ring->ring_size)) {
1232                 tx_info = &tx_ring->tx_buffer_info[req_id];
1233                 if (likely(tx_info->skb))
1234                         return 0;
1235         }
1236
1237         return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
1238 }
1239
1240 static int validate_xdp_req_id(struct ena_ring *xdp_ring, u16 req_id)
1241 {
1242         struct ena_tx_buffer *tx_info = NULL;
1243
1244         if (likely(req_id < xdp_ring->ring_size)) {
1245                 tx_info = &xdp_ring->tx_buffer_info[req_id];
1246                 if (likely(tx_info->xdpf))
1247                         return 0;
1248         }
1249
1250         return handle_invalid_req_id(xdp_ring, req_id, tx_info, true);
1251 }
1252
1253 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
1254 {
1255         struct netdev_queue *txq;
1256         bool above_thresh;
1257         u32 tx_bytes = 0;
1258         u32 total_done = 0;
1259         u16 next_to_clean;
1260         u16 req_id;
1261         int tx_pkts = 0;
1262         int rc;
1263
1264         next_to_clean = tx_ring->next_to_clean;
1265         txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
1266
1267         while (tx_pkts < budget) {
1268                 struct ena_tx_buffer *tx_info;
1269                 struct sk_buff *skb;
1270
1271                 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
1272                                                 &req_id);
1273                 if (rc)
1274                         break;
1275
1276                 rc = validate_tx_req_id(tx_ring, req_id);
1277                 if (rc)
1278                         break;
1279
1280                 tx_info = &tx_ring->tx_buffer_info[req_id];
1281                 skb = tx_info->skb;
1282
1283                 /* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */
1284                 prefetch(&skb->end);
1285
1286                 tx_info->skb = NULL;
1287                 tx_info->last_jiffies = 0;
1288
1289                 ena_unmap_tx_buff(tx_ring, tx_info);
1290
1291                 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
1292                           "tx_poll: q %d skb %p completed\n", tx_ring->qid,
1293                           skb);
1294
1295                 tx_bytes += skb->len;
1296                 dev_kfree_skb(skb);
1297                 tx_pkts++;
1298                 total_done += tx_info->tx_descs;
1299
1300                 tx_ring->free_ids[next_to_clean] = req_id;
1301                 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1302                                                      tx_ring->ring_size);
1303         }
1304
1305         tx_ring->next_to_clean = next_to_clean;
1306         ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
1307         ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
1308
1309         netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1310
1311         netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
1312                   "tx_poll: q %d done. total pkts: %d\n",
1313                   tx_ring->qid, tx_pkts);
1314
1315         /* need to make the rings circular update visible to
1316          * ena_start_xmit() before checking for netif_queue_stopped().
1317          */
1318         smp_mb();
1319
1320         above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1321                                                     ENA_TX_WAKEUP_THRESH);
1322         if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
1323                 __netif_tx_lock(txq, smp_processor_id());
1324                 above_thresh =
1325                         ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1326                                                      ENA_TX_WAKEUP_THRESH);
1327                 if (netif_tx_queue_stopped(txq) && above_thresh &&
1328                     test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
1329                         netif_tx_wake_queue(txq);
1330                         u64_stats_update_begin(&tx_ring->syncp);
1331                         tx_ring->tx_stats.queue_wakeup++;
1332                         u64_stats_update_end(&tx_ring->syncp);
1333                 }
1334                 __netif_tx_unlock(txq);
1335         }
1336
1337         return tx_pkts;
1338 }
1339
1340 static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, bool frags)
1341 {
1342         struct sk_buff *skb;
1343
1344         if (frags)
1345                 skb = napi_get_frags(rx_ring->napi);
1346         else
1347                 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1348                                                 rx_ring->rx_copybreak);
1349
1350         if (unlikely(!skb)) {
1351                 u64_stats_update_begin(&rx_ring->syncp);
1352                 rx_ring->rx_stats.skb_alloc_fail++;
1353                 u64_stats_update_end(&rx_ring->syncp);
1354                 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1355                           "Failed to allocate skb. frags: %d\n", frags);
1356                 return NULL;
1357         }
1358
1359         return skb;
1360 }
1361
1362 static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
1363                                   struct ena_com_rx_buf_info *ena_bufs,
1364                                   u32 descs,
1365                                   u16 *next_to_clean)
1366 {
1367         struct sk_buff *skb;
1368         struct ena_rx_buffer *rx_info;
1369         u16 len, req_id, buf = 0;
1370         void *va;
1371         int rc;
1372
1373         len = ena_bufs[buf].len;
1374         req_id = ena_bufs[buf].req_id;
1375
1376         rc = validate_rx_req_id(rx_ring, req_id);
1377         if (unlikely(rc < 0))
1378                 return NULL;
1379
1380         rx_info = &rx_ring->rx_buffer_info[req_id];
1381
1382         if (unlikely(!rx_info->page)) {
1383                 netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
1384                           "Page is NULL\n");
1385                 return NULL;
1386         }
1387
1388         netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1389                   "rx_info %p page %p\n",
1390                   rx_info, rx_info->page);
1391
1392         /* save virt address of first buffer */
1393         va = page_address(rx_info->page) + rx_info->page_offset;
1394         prefetch(va + NET_IP_ALIGN);
1395
1396         if (len <= rx_ring->rx_copybreak) {
1397                 skb = ena_alloc_skb(rx_ring, false);
1398                 if (unlikely(!skb))
1399                         return NULL;
1400
1401                 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1402                           "rx allocated small packet. len %d. data_len %d\n",
1403                           skb->len, skb->data_len);
1404
1405                 /* sync this buffer for CPU use */
1406                 dma_sync_single_for_cpu(rx_ring->dev,
1407                                         dma_unmap_addr(&rx_info->ena_buf, paddr),
1408                                         len,
1409                                         DMA_FROM_DEVICE);
1410                 skb_copy_to_linear_data(skb, va, len);
1411                 dma_sync_single_for_device(rx_ring->dev,
1412                                            dma_unmap_addr(&rx_info->ena_buf, paddr),
1413                                            len,
1414                                            DMA_FROM_DEVICE);
1415
1416                 skb_put(skb, len);
1417                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1418                 rx_ring->free_ids[*next_to_clean] = req_id;
1419                 *next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
1420                                                      rx_ring->ring_size);
1421                 return skb;
1422         }
1423
1424         skb = ena_alloc_skb(rx_ring, true);
1425         if (unlikely(!skb))
1426                 return NULL;
1427
1428         do {
1429                 dma_unmap_page(rx_ring->dev,
1430                                dma_unmap_addr(&rx_info->ena_buf, paddr),
1431                                ENA_PAGE_SIZE, DMA_BIDIRECTIONAL);
1432
1433                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
1434                                 rx_info->page_offset, len, ENA_PAGE_SIZE);
1435                 /* The offset is non zero only for the first buffer */
1436                 rx_info->page_offset = 0;
1437
1438                 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1439                           "rx skb updated. len %d. data_len %d\n",
1440                           skb->len, skb->data_len);
1441
1442                 rx_info->page = NULL;
1443
1444                 rx_ring->free_ids[*next_to_clean] = req_id;
1445                 *next_to_clean =
1446                         ENA_RX_RING_IDX_NEXT(*next_to_clean,
1447                                              rx_ring->ring_size);
1448                 if (likely(--descs == 0))
1449                         break;
1450
1451                 buf++;
1452                 len = ena_bufs[buf].len;
1453                 req_id = ena_bufs[buf].req_id;
1454
1455                 rc = validate_rx_req_id(rx_ring, req_id);
1456                 if (unlikely(rc < 0))
1457                         return NULL;
1458
1459                 rx_info = &rx_ring->rx_buffer_info[req_id];
1460         } while (1);
1461
1462         return skb;
1463 }
1464
1465 /* ena_rx_checksum - indicate in skb if hw indicated a good cksum
1466  * @adapter: structure containing adapter specific data
1467  * @ena_rx_ctx: received packet context/metadata
1468  * @skb: skb currently being received and modified
1469  */
1470 static void ena_rx_checksum(struct ena_ring *rx_ring,
1471                                    struct ena_com_rx_ctx *ena_rx_ctx,
1472                                    struct sk_buff *skb)
1473 {
1474         /* Rx csum disabled */
1475         if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {
1476                 skb->ip_summed = CHECKSUM_NONE;
1477                 return;
1478         }
1479
1480         /* For fragmented packets the checksum isn't valid */
1481         if (ena_rx_ctx->frag) {
1482                 skb->ip_summed = CHECKSUM_NONE;
1483                 return;
1484         }
1485
1486         /* if IP and error */
1487         if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
1488                      (ena_rx_ctx->l3_csum_err))) {
1489                 /* ipv4 checksum error */
1490                 skb->ip_summed = CHECKSUM_NONE;
1491                 u64_stats_update_begin(&rx_ring->syncp);
1492                 rx_ring->rx_stats.bad_csum++;
1493                 u64_stats_update_end(&rx_ring->syncp);
1494                 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1495                           "RX IPv4 header checksum error\n");
1496                 return;
1497         }
1498
1499         /* if TCP/UDP */
1500         if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1501                    (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {
1502                 if (unlikely(ena_rx_ctx->l4_csum_err)) {
1503                         /* TCP/UDP checksum error */
1504                         u64_stats_update_begin(&rx_ring->syncp);
1505                         rx_ring->rx_stats.bad_csum++;
1506                         u64_stats_update_end(&rx_ring->syncp);
1507                         netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1508                                   "RX L4 checksum error\n");
1509                         skb->ip_summed = CHECKSUM_NONE;
1510                         return;
1511                 }
1512
1513                 if (likely(ena_rx_ctx->l4_csum_checked)) {
1514                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1515                         u64_stats_update_begin(&rx_ring->syncp);
1516                         rx_ring->rx_stats.csum_good++;
1517                         u64_stats_update_end(&rx_ring->syncp);
1518                 } else {
1519                         u64_stats_update_begin(&rx_ring->syncp);
1520                         rx_ring->rx_stats.csum_unchecked++;
1521                         u64_stats_update_end(&rx_ring->syncp);
1522                         skb->ip_summed = CHECKSUM_NONE;
1523                 }
1524         } else {
1525                 skb->ip_summed = CHECKSUM_NONE;
1526                 return;
1527         }
1528
1529 }
1530
1531 static void ena_set_rx_hash(struct ena_ring *rx_ring,
1532                             struct ena_com_rx_ctx *ena_rx_ctx,
1533                             struct sk_buff *skb)
1534 {
1535         enum pkt_hash_types hash_type;
1536
1537         if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {
1538                 if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1539                            (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))
1540
1541                         hash_type = PKT_HASH_TYPE_L4;
1542                 else
1543                         hash_type = PKT_HASH_TYPE_NONE;
1544
1545                 /* Override hash type if the packet is fragmented */
1546                 if (ena_rx_ctx->frag)
1547                         hash_type = PKT_HASH_TYPE_NONE;
1548
1549                 skb_set_hash(skb, ena_rx_ctx->hash, hash_type);
1550         }
1551 }
1552
1553 static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp)
1554 {
1555         struct ena_rx_buffer *rx_info;
1556         int ret;
1557
1558         rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1559         xdp->data = page_address(rx_info->page) +
1560                 rx_info->page_offset + rx_ring->rx_headroom;
1561         xdp_set_data_meta_invalid(xdp);
1562         xdp->data_hard_start = page_address(rx_info->page);
1563         xdp->data_end = xdp->data + rx_ring->ena_bufs[0].len;
1564         /* If for some reason we received a bigger packet than
1565          * we expect, then we simply drop it
1566          */
1567         if (unlikely(rx_ring->ena_bufs[0].len > ENA_XDP_MAX_MTU))
1568                 return XDP_DROP;
1569
1570         ret = ena_xdp_execute(rx_ring, xdp, rx_info);
1571
1572         /* The xdp program might expand the headers */
1573         if (ret == XDP_PASS) {
1574                 rx_info->page_offset = xdp->data - xdp->data_hard_start;
1575                 rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data;
1576         }
1577
1578         return ret;
1579 }
1580 /* ena_clean_rx_irq - Cleanup RX irq
1581  * @rx_ring: RX ring to clean
1582  * @napi: napi handler
1583  * @budget: how many packets driver is allowed to clean
1584  *
1585  * Returns the number of cleaned buffers.
1586  */
1587 static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
1588                             u32 budget)
1589 {
1590         u16 next_to_clean = rx_ring->next_to_clean;
1591         struct ena_com_rx_ctx ena_rx_ctx;
1592         struct ena_rx_buffer *rx_info;
1593         struct ena_adapter *adapter;
1594         u32 res_budget, work_done;
1595         int rx_copybreak_pkt = 0;
1596         int refill_threshold;
1597         struct sk_buff *skb;
1598         int refill_required;
1599         struct xdp_buff xdp;
1600         int total_len = 0;
1601         int xdp_verdict;
1602         int rc = 0;
1603         int i;
1604
1605         netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1606                   "%s qid %d\n", __func__, rx_ring->qid);
1607         res_budget = budget;
1608         xdp.rxq = &rx_ring->xdp_rxq;
1609         xdp.frame_sz = ENA_PAGE_SIZE;
1610
1611         do {
1612                 xdp_verdict = XDP_PASS;
1613                 skb = NULL;
1614                 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1615                 ena_rx_ctx.max_bufs = rx_ring->sgl_size;
1616                 ena_rx_ctx.descs = 0;
1617                 ena_rx_ctx.pkt_offset = 0;
1618                 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
1619                                     rx_ring->ena_com_io_sq,
1620                                     &ena_rx_ctx);
1621                 if (unlikely(rc))
1622                         goto error;
1623
1624                 if (unlikely(ena_rx_ctx.descs == 0))
1625                         break;
1626
1627                 rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1628                 rx_info->page_offset = ena_rx_ctx.pkt_offset;
1629
1630                 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1631                           "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
1632                           rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
1633                           ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
1634
1635                 if (ena_xdp_present_ring(rx_ring))
1636                         xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp);
1637
1638                 /* allocate skb and fill it */
1639                 if (xdp_verdict == XDP_PASS)
1640                         skb = ena_rx_skb(rx_ring,
1641                                          rx_ring->ena_bufs,
1642                                          ena_rx_ctx.descs,
1643                                          &next_to_clean);
1644
1645                 if (unlikely(!skb)) {
1646                         if (xdp_verdict == XDP_TX)
1647                                 ena_free_rx_page(rx_ring,
1648                                                  &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id]);
1649                         for (i = 0; i < ena_rx_ctx.descs; i++) {
1650                                 rx_ring->free_ids[next_to_clean] =
1651                                         rx_ring->ena_bufs[i].req_id;
1652                                 next_to_clean =
1653                                         ENA_RX_RING_IDX_NEXT(next_to_clean,
1654                                                              rx_ring->ring_size);
1655                         }
1656                         if (xdp_verdict != XDP_PASS) {
1657                                 res_budget--;
1658                                 continue;
1659                         }
1660                         break;
1661                 }
1662
1663                 ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);
1664
1665                 ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);
1666
1667                 skb_record_rx_queue(skb, rx_ring->qid);
1668
1669                 if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak) {
1670                         total_len += rx_ring->ena_bufs[0].len;
1671                         rx_copybreak_pkt++;
1672                         napi_gro_receive(napi, skb);
1673                 } else {
1674                         total_len += skb->len;
1675                         napi_gro_frags(napi);
1676                 }
1677
1678                 res_budget--;
1679         } while (likely(res_budget));
1680
1681         work_done = budget - res_budget;
1682         rx_ring->per_napi_packets += work_done;
1683         u64_stats_update_begin(&rx_ring->syncp);
1684         rx_ring->rx_stats.bytes += total_len;
1685         rx_ring->rx_stats.cnt += work_done;
1686         rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;
1687         u64_stats_update_end(&rx_ring->syncp);
1688
1689         rx_ring->next_to_clean = next_to_clean;
1690
1691         refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
1692         refill_threshold =
1693                 min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
1694                       ENA_RX_REFILL_THRESH_PACKET);
1695
1696         /* Optimization, try to batch new rx buffers */
1697         if (refill_required > refill_threshold) {
1698                 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
1699                 ena_refill_rx_bufs(rx_ring, refill_required);
1700         }
1701
1702         return work_done;
1703
1704 error:
1705         adapter = netdev_priv(rx_ring->netdev);
1706
1707         u64_stats_update_begin(&rx_ring->syncp);
1708         rx_ring->rx_stats.bad_desc_num++;
1709         u64_stats_update_end(&rx_ring->syncp);
1710
1711         /* Too many desc from the device. Trigger reset */
1712         adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
1713         set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
1714
1715         return 0;
1716 }
1717
1718 static void ena_dim_work(struct work_struct *w)
1719 {
1720         struct dim *dim = container_of(w, struct dim, work);
1721         struct dim_cq_moder cur_moder =
1722                 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1723         struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);
1724
1725         ena_napi->rx_ring->smoothed_interval = cur_moder.usec;
1726         dim->state = DIM_START_MEASURE;
1727 }
1728
1729 static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)
1730 {
1731         struct dim_sample dim_sample;
1732         struct ena_ring *rx_ring = ena_napi->rx_ring;
1733
1734         if (!rx_ring->per_napi_packets)
1735                 return;
1736
1737         rx_ring->non_empty_napi_events++;
1738
1739         dim_update_sample(rx_ring->non_empty_napi_events,
1740                           rx_ring->rx_stats.cnt,
1741                           rx_ring->rx_stats.bytes,
1742                           &dim_sample);
1743
1744         net_dim(&ena_napi->dim, dim_sample);
1745
1746         rx_ring->per_napi_packets = 0;
1747 }
1748
1749 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
1750                                         struct ena_ring *rx_ring)
1751 {
1752         struct ena_eth_io_intr_reg intr_reg;
1753         u32 rx_interval = 0;
1754         /* Rx ring can be NULL when for XDP tx queues which don't have an
1755          * accompanying rx_ring pair.
1756          */
1757         if (rx_ring)
1758                 rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?
1759                         rx_ring->smoothed_interval :
1760                         ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);
1761
1762         /* Update intr register: rx intr delay,
1763          * tx intr delay and interrupt unmask
1764          */
1765         ena_com_update_intr_reg(&intr_reg,
1766                                 rx_interval,
1767                                 tx_ring->smoothed_interval,
1768                                 true);
1769
1770         u64_stats_update_begin(&tx_ring->syncp);
1771         tx_ring->tx_stats.unmask_interrupt++;
1772         u64_stats_update_end(&tx_ring->syncp);
1773         /* It is a shared MSI-X.
1774          * Tx and Rx CQ have pointer to it.
1775          * So we use one of them to reach the intr reg
1776          * The Tx ring is used because the rx_ring is NULL for XDP queues
1777          */
1778         ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);
1779 }
1780
1781 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
1782                                              struct ena_ring *rx_ring)
1783 {
1784         int cpu = get_cpu();
1785         int numa_node;
1786
1787         /* Check only one ring since the 2 rings are running on the same cpu */
1788         if (likely(tx_ring->cpu == cpu))
1789                 goto out;
1790
1791         numa_node = cpu_to_node(cpu);
1792         put_cpu();
1793
1794         if (numa_node != NUMA_NO_NODE) {
1795                 ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
1796                 if (rx_ring)
1797                         ena_com_update_numa_node(rx_ring->ena_com_io_cq,
1798                                                  numa_node);
1799         }
1800
1801         tx_ring->cpu = cpu;
1802         if (rx_ring)
1803                 rx_ring->cpu = cpu;
1804
1805         return;
1806 out:
1807         put_cpu();
1808 }
1809
1810 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)
1811 {
1812         u32 total_done = 0;
1813         u16 next_to_clean;
1814         u32 tx_bytes = 0;
1815         int tx_pkts = 0;
1816         u16 req_id;
1817         int rc;
1818
1819         if (unlikely(!xdp_ring))
1820                 return 0;
1821         next_to_clean = xdp_ring->next_to_clean;
1822
1823         while (tx_pkts < budget) {
1824                 struct ena_tx_buffer *tx_info;
1825                 struct xdp_frame *xdpf;
1826
1827                 rc = ena_com_tx_comp_req_id_get(xdp_ring->ena_com_io_cq,
1828                                                 &req_id);
1829                 if (rc)
1830                         break;
1831
1832                 rc = validate_xdp_req_id(xdp_ring, req_id);
1833                 if (rc)
1834                         break;
1835
1836                 tx_info = &xdp_ring->tx_buffer_info[req_id];
1837                 xdpf = tx_info->xdpf;
1838
1839                 tx_info->xdpf = NULL;
1840                 tx_info->last_jiffies = 0;
1841                 ena_unmap_tx_buff(xdp_ring, tx_info);
1842
1843                 netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
1844                           "tx_poll: q %d skb %p completed\n", xdp_ring->qid,
1845                           xdpf);
1846
1847                 tx_bytes += xdpf->len;
1848                 tx_pkts++;
1849                 total_done += tx_info->tx_descs;
1850
1851                 __free_page(tx_info->xdp_rx_page);
1852                 xdp_ring->free_ids[next_to_clean] = req_id;
1853                 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1854                                                      xdp_ring->ring_size);
1855         }
1856
1857         xdp_ring->next_to_clean = next_to_clean;
1858         ena_com_comp_ack(xdp_ring->ena_com_io_sq, total_done);
1859         ena_com_update_dev_comp_head(xdp_ring->ena_com_io_cq);
1860
1861         netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
1862                   "tx_poll: q %d done. total pkts: %d\n",
1863                   xdp_ring->qid, tx_pkts);
1864
1865         return tx_pkts;
1866 }
1867
1868 static int ena_io_poll(struct napi_struct *napi, int budget)
1869 {
1870         struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
1871         struct ena_ring *tx_ring, *rx_ring;
1872         int tx_work_done;
1873         int rx_work_done = 0;
1874         int tx_budget;
1875         int napi_comp_call = 0;
1876         int ret;
1877
1878         tx_ring = ena_napi->tx_ring;
1879         rx_ring = ena_napi->rx_ring;
1880
1881         tx_ring->first_interrupt = ena_napi->first_interrupt;
1882         rx_ring->first_interrupt = ena_napi->first_interrupt;
1883
1884         tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
1885
1886         if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1887             test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
1888                 napi_complete_done(napi, 0);
1889                 return 0;
1890         }
1891
1892         tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
1893         /* On netpoll the budget is zero and the handler should only clean the
1894          * tx completions.
1895          */
1896         if (likely(budget))
1897                 rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
1898
1899         /* If the device is about to reset or down, avoid unmask
1900          * the interrupt and return 0 so NAPI won't reschedule
1901          */
1902         if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1903                      test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
1904                 napi_complete_done(napi, 0);
1905                 ret = 0;
1906
1907         } else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
1908                 napi_comp_call = 1;
1909
1910                 /* Update numa and unmask the interrupt only when schedule
1911                  * from the interrupt context (vs from sk_busy_loop)
1912                  */
1913                 if (napi_complete_done(napi, rx_work_done) &&
1914                     READ_ONCE(ena_napi->interrupts_masked)) {
1915                         smp_rmb(); /* make sure interrupts_masked is read */
1916                         WRITE_ONCE(ena_napi->interrupts_masked, false);
1917                         /* We apply adaptive moderation on Rx path only.
1918                          * Tx uses static interrupt moderation.
1919                          */
1920                         if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
1921                                 ena_adjust_adaptive_rx_intr_moderation(ena_napi);
1922
1923                         ena_unmask_interrupt(tx_ring, rx_ring);
1924                 }
1925
1926                 ena_update_ring_numa_node(tx_ring, rx_ring);
1927
1928                 ret = rx_work_done;
1929         } else {
1930                 ret = budget;
1931         }
1932
1933         u64_stats_update_begin(&tx_ring->syncp);
1934         tx_ring->tx_stats.napi_comp += napi_comp_call;
1935         tx_ring->tx_stats.tx_poll++;
1936         u64_stats_update_end(&tx_ring->syncp);
1937
1938         return ret;
1939 }
1940
1941 static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)
1942 {
1943         struct ena_adapter *adapter = (struct ena_adapter *)data;
1944
1945         ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1946
1947         /* Don't call the aenq handler before probe is done */
1948         if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))
1949                 ena_com_aenq_intr_handler(adapter->ena_dev, data);
1950
1951         return IRQ_HANDLED;
1952 }
1953
1954 /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx
1955  * @irq: interrupt number
1956  * @data: pointer to a network interface private napi device structure
1957  */
1958 static irqreturn_t ena_intr_msix_io(int irq, void *data)
1959 {
1960         struct ena_napi *ena_napi = data;
1961
1962         ena_napi->first_interrupt = true;
1963
1964         WRITE_ONCE(ena_napi->interrupts_masked, true);
1965         smp_wmb(); /* write interrupts_masked before calling napi */
1966
1967         napi_schedule_irqoff(&ena_napi->napi);
1968
1969         return IRQ_HANDLED;
1970 }
1971
1972 /* Reserve a single MSI-X vector for management (admin + aenq).
1973  * plus reserve one vector for each potential io queue.
1974  * the number of potential io queues is the minimum of what the device
1975  * supports and the number of vCPUs.
1976  */
1977 static int ena_enable_msix(struct ena_adapter *adapter)
1978 {
1979         int msix_vecs, irq_cnt;
1980
1981         if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
1982                 netif_err(adapter, probe, adapter->netdev,
1983                           "Error, MSI-X is already enabled\n");
1984                 return -EPERM;
1985         }
1986
1987         /* Reserved the max msix vectors we might need */
1988         msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
1989         netif_dbg(adapter, probe, adapter->netdev,
1990                   "trying to enable MSI-X, vectors %d\n", msix_vecs);
1991
1992         irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
1993                                         msix_vecs, PCI_IRQ_MSIX);
1994
1995         if (irq_cnt < 0) {
1996                 netif_err(adapter, probe, adapter->netdev,
1997                           "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);
1998                 return -ENOSPC;
1999         }
2000
2001         if (irq_cnt != msix_vecs) {
2002                 netif_notice(adapter, probe, adapter->netdev,
2003                              "enable only %d MSI-X (out of %d), reduce the number of queues\n",
2004                              irq_cnt, msix_vecs);
2005                 adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
2006         }
2007
2008         if (ena_init_rx_cpu_rmap(adapter))
2009                 netif_warn(adapter, probe, adapter->netdev,
2010                            "Failed to map IRQs to CPUs\n");
2011
2012         adapter->msix_vecs = irq_cnt;
2013         set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
2014
2015         return 0;
2016 }
2017
2018 static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
2019 {
2020         u32 cpu;
2021
2022         snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
2023                  ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
2024                  pci_name(adapter->pdev));
2025         adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =
2026                 ena_intr_msix_mgmnt;
2027         adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
2028         adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
2029                 pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
2030         cpu = cpumask_first(cpu_online_mask);
2031         adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
2032         cpumask_set_cpu(cpu,
2033                         &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);
2034 }
2035
2036 static void ena_setup_io_intr(struct ena_adapter *adapter)
2037 {
2038         struct net_device *netdev;
2039         int irq_idx, i, cpu;
2040         int io_queue_count;
2041
2042         netdev = adapter->netdev;
2043         io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2044
2045         for (i = 0; i < io_queue_count; i++) {
2046                 irq_idx = ENA_IO_IRQ_IDX(i);
2047                 cpu = i % num_online_cpus();
2048
2049                 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
2050                          "%s-Tx-Rx-%d", netdev->name, i);
2051                 adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
2052                 adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
2053                 adapter->irq_tbl[irq_idx].vector =
2054                         pci_irq_vector(adapter->pdev, irq_idx);
2055                 adapter->irq_tbl[irq_idx].cpu = cpu;
2056
2057                 cpumask_set_cpu(cpu,
2058                                 &adapter->irq_tbl[irq_idx].affinity_hint_mask);
2059         }
2060 }
2061
2062 static int ena_request_mgmnt_irq(struct ena_adapter *adapter)
2063 {
2064         unsigned long flags = 0;
2065         struct ena_irq *irq;
2066         int rc;
2067
2068         irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2069         rc = request_irq(irq->vector, irq->handler, flags, irq->name,
2070                          irq->data);
2071         if (rc) {
2072                 netif_err(adapter, probe, adapter->netdev,
2073                           "failed to request admin irq\n");
2074                 return rc;
2075         }
2076
2077         netif_dbg(adapter, probe, adapter->netdev,
2078                   "set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
2079                   irq->affinity_hint_mask.bits[0], irq->vector);
2080
2081         irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
2082
2083         return rc;
2084 }
2085
2086 static int ena_request_io_irq(struct ena_adapter *adapter)
2087 {
2088         u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2089         unsigned long flags = 0;
2090         struct ena_irq *irq;
2091         int rc = 0, i, k;
2092
2093         if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
2094                 netif_err(adapter, ifup, adapter->netdev,
2095                           "Failed to request I/O IRQ: MSI-X is not enabled\n");
2096                 return -EINVAL;
2097         }
2098
2099         for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
2100                 irq = &adapter->irq_tbl[i];
2101                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
2102                                  irq->data);
2103                 if (rc) {
2104                         netif_err(adapter, ifup, adapter->netdev,
2105                                   "Failed to request I/O IRQ. index %d rc %d\n",
2106                                    i, rc);
2107                         goto err;
2108                 }
2109
2110                 netif_dbg(adapter, ifup, adapter->netdev,
2111                           "set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
2112                           i, irq->affinity_hint_mask.bits[0], irq->vector);
2113
2114                 irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
2115         }
2116
2117         return rc;
2118
2119 err:
2120         for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {
2121                 irq = &adapter->irq_tbl[k];
2122                 free_irq(irq->vector, irq->data);
2123         }
2124
2125         return rc;
2126 }
2127
2128 static void ena_free_mgmnt_irq(struct ena_adapter *adapter)
2129 {
2130         struct ena_irq *irq;
2131
2132         irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2133         synchronize_irq(irq->vector);
2134         irq_set_affinity_hint(irq->vector, NULL);
2135         free_irq(irq->vector, irq->data);
2136 }
2137
2138 static void ena_free_io_irq(struct ena_adapter *adapter)
2139 {
2140         u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2141         struct ena_irq *irq;
2142         int i;
2143
2144 #ifdef CONFIG_RFS_ACCEL
2145         if (adapter->msix_vecs >= 1) {
2146                 free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
2147                 adapter->netdev->rx_cpu_rmap = NULL;
2148         }
2149 #endif /* CONFIG_RFS_ACCEL */
2150
2151         for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
2152                 irq = &adapter->irq_tbl[i];
2153                 irq_set_affinity_hint(irq->vector, NULL);
2154                 free_irq(irq->vector, irq->data);
2155         }
2156 }
2157
2158 static void ena_disable_msix(struct ena_adapter *adapter)
2159 {
2160         if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
2161                 pci_free_irq_vectors(adapter->pdev);
2162 }
2163
2164 static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
2165 {
2166         u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2167         int i;
2168
2169         if (!netif_running(adapter->netdev))
2170                 return;
2171
2172         for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++)
2173                 synchronize_irq(adapter->irq_tbl[i].vector);
2174 }
2175
2176 static void ena_del_napi_in_range(struct ena_adapter *adapter,
2177                                   int first_index,
2178                                   int count)
2179 {
2180         int i;
2181
2182         for (i = first_index; i < first_index + count; i++) {
2183                 /* Check if napi was initialized before */
2184                 if (!ENA_IS_XDP_INDEX(adapter, i) ||
2185                     adapter->ena_napi[i].xdp_ring)
2186                         netif_napi_del(&adapter->ena_napi[i].napi);
2187                 else
2188                         WARN_ON(ENA_IS_XDP_INDEX(adapter, i) &&
2189                                 adapter->ena_napi[i].xdp_ring);
2190         }
2191 }
2192
2193 static void ena_init_napi_in_range(struct ena_adapter *adapter,
2194                                    int first_index, int count)
2195 {
2196         int i;
2197
2198         for (i = first_index; i < first_index + count; i++) {
2199                 struct ena_napi *napi = &adapter->ena_napi[i];
2200
2201                 netif_napi_add(adapter->netdev,
2202                                &napi->napi,
2203                                ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll,
2204                                ENA_NAPI_BUDGET);
2205
2206                 if (!ENA_IS_XDP_INDEX(adapter, i)) {
2207                         napi->rx_ring = &adapter->rx_ring[i];
2208                         napi->tx_ring = &adapter->tx_ring[i];
2209                 } else {
2210                         napi->xdp_ring = &adapter->tx_ring[i];
2211                 }
2212                 napi->qid = i;
2213         }
2214 }
2215
2216 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
2217                                       int first_index,
2218                                       int count)
2219 {
2220         int i;
2221
2222         for (i = first_index; i < first_index + count; i++)
2223                 napi_disable(&adapter->ena_napi[i].napi);
2224 }
2225
2226 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
2227                                      int first_index,
2228                                      int count)
2229 {
2230         int i;
2231
2232         for (i = first_index; i < first_index + count; i++)
2233                 napi_enable(&adapter->ena_napi[i].napi);
2234 }
2235
2236 /* Configure the Rx forwarding */
2237 static int ena_rss_configure(struct ena_adapter *adapter)
2238 {
2239         struct ena_com_dev *ena_dev = adapter->ena_dev;
2240         int rc;
2241
2242         /* In case the RSS table wasn't initialized by probe */
2243         if (!ena_dev->rss.tbl_log_size) {
2244                 rc = ena_rss_init_default(adapter);
2245                 if (rc && (rc != -EOPNOTSUPP)) {
2246                         netif_err(adapter, ifup, adapter->netdev,
2247                                   "Failed to init RSS rc: %d\n", rc);
2248                         return rc;
2249                 }
2250         }
2251
2252         /* Set indirect table */
2253         rc = ena_com_indirect_table_set(ena_dev);
2254         if (unlikely(rc && rc != -EOPNOTSUPP))
2255                 return rc;
2256
2257         /* Configure hash function (if supported) */
2258         rc = ena_com_set_hash_function(ena_dev);
2259         if (unlikely(rc && (rc != -EOPNOTSUPP)))
2260                 return rc;
2261
2262         /* Configure hash inputs (if supported) */
2263         rc = ena_com_set_hash_ctrl(ena_dev);
2264         if (unlikely(rc && (rc != -EOPNOTSUPP)))
2265                 return rc;
2266
2267         return 0;
2268 }
2269
2270 static int ena_up_complete(struct ena_adapter *adapter)
2271 {
2272         int rc;
2273
2274         rc = ena_rss_configure(adapter);
2275         if (rc)
2276                 return rc;
2277
2278         ena_change_mtu(adapter->netdev, adapter->netdev->mtu);
2279
2280         ena_refill_all_rx_bufs(adapter);
2281
2282         /* enable transmits */
2283         netif_tx_start_all_queues(adapter->netdev);
2284
2285         ena_napi_enable_in_range(adapter,
2286                                  0,
2287                                  adapter->xdp_num_queues + adapter->num_io_queues);
2288
2289         return 0;
2290 }
2291
2292 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
2293 {
2294         struct ena_com_create_io_ctx ctx;
2295         struct ena_com_dev *ena_dev;
2296         struct ena_ring *tx_ring;
2297         u32 msix_vector;
2298         u16 ena_qid;
2299         int rc;
2300
2301         ena_dev = adapter->ena_dev;
2302
2303         tx_ring = &adapter->tx_ring[qid];
2304         msix_vector = ENA_IO_IRQ_IDX(qid);
2305         ena_qid = ENA_IO_TXQ_IDX(qid);
2306
2307         memset(&ctx, 0x0, sizeof(ctx));
2308
2309         ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
2310         ctx.qid = ena_qid;
2311         ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
2312         ctx.msix_vector = msix_vector;
2313         ctx.queue_size = tx_ring->ring_size;
2314         ctx.numa_node = cpu_to_node(tx_ring->cpu);
2315
2316         rc = ena_com_create_io_queue(ena_dev, &ctx);
2317         if (rc) {
2318                 netif_err(adapter, ifup, adapter->netdev,
2319                           "Failed to create I/O TX queue num %d rc: %d\n",
2320                           qid, rc);
2321                 return rc;
2322         }
2323
2324         rc = ena_com_get_io_handlers(ena_dev, ena_qid,
2325                                      &tx_ring->ena_com_io_sq,
2326                                      &tx_ring->ena_com_io_cq);
2327         if (rc) {
2328                 netif_err(adapter, ifup, adapter->netdev,
2329                           "Failed to get TX queue handlers. TX queue num %d rc: %d\n",
2330                           qid, rc);
2331                 ena_com_destroy_io_queue(ena_dev, ena_qid);
2332                 return rc;
2333         }
2334
2335         ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
2336         return rc;
2337 }
2338
2339 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
2340                                             int first_index, int count)
2341 {
2342         struct ena_com_dev *ena_dev = adapter->ena_dev;
2343         int rc, i;
2344
2345         for (i = first_index; i < first_index + count; i++) {
2346                 rc = ena_create_io_tx_queue(adapter, i);
2347                 if (rc)
2348                         goto create_err;
2349         }
2350
2351         return 0;
2352
2353 create_err:
2354         while (i-- > first_index)
2355                 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
2356
2357         return rc;
2358 }
2359
2360 static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
2361 {
2362         struct ena_com_dev *ena_dev;
2363         struct ena_com_create_io_ctx ctx;
2364         struct ena_ring *rx_ring;
2365         u32 msix_vector;
2366         u16 ena_qid;
2367         int rc;
2368
2369         ena_dev = adapter->ena_dev;
2370
2371         rx_ring = &adapter->rx_ring[qid];
2372         msix_vector = ENA_IO_IRQ_IDX(qid);
2373         ena_qid = ENA_IO_RXQ_IDX(qid);
2374
2375         memset(&ctx, 0x0, sizeof(ctx));
2376
2377         ctx.qid = ena_qid;
2378         ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
2379         ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2380         ctx.msix_vector = msix_vector;
2381         ctx.queue_size = rx_ring->ring_size;
2382         ctx.numa_node = cpu_to_node(rx_ring->cpu);
2383
2384         rc = ena_com_create_io_queue(ena_dev, &ctx);
2385         if (rc) {
2386                 netif_err(adapter, ifup, adapter->netdev,
2387                           "Failed to create I/O RX queue num %d rc: %d\n",
2388                           qid, rc);
2389                 return rc;
2390         }
2391
2392         rc = ena_com_get_io_handlers(ena_dev, ena_qid,
2393                                      &rx_ring->ena_com_io_sq,
2394                                      &rx_ring->ena_com_io_cq);
2395         if (rc) {
2396                 netif_err(adapter, ifup, adapter->netdev,
2397                           "Failed to get RX queue handlers. RX queue num %d rc: %d\n",
2398                           qid, rc);
2399                 goto err;
2400         }
2401
2402         ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
2403
2404         return rc;
2405 err:
2406         ena_com_destroy_io_queue(ena_dev, ena_qid);
2407         return rc;
2408 }
2409
2410 static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
2411 {
2412         struct ena_com_dev *ena_dev = adapter->ena_dev;
2413         int rc, i;
2414
2415         for (i = 0; i < adapter->num_io_queues; i++) {
2416                 rc = ena_create_io_rx_queue(adapter, i);
2417                 if (rc)
2418                         goto create_err;
2419                 INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);
2420         }
2421
2422         return 0;
2423
2424 create_err:
2425         while (i--) {
2426                 cancel_work_sync(&adapter->ena_napi[i].dim.work);
2427                 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
2428         }
2429
2430         return rc;
2431 }
2432
2433 static void set_io_rings_size(struct ena_adapter *adapter,
2434                               int new_tx_size,
2435                               int new_rx_size)
2436 {
2437         int i;
2438
2439         for (i = 0; i < adapter->num_io_queues; i++) {
2440                 adapter->tx_ring[i].ring_size = new_tx_size;
2441                 adapter->rx_ring[i].ring_size = new_rx_size;
2442         }
2443 }
2444
2445 /* This function allows queue allocation to backoff when the system is
2446  * low on memory. If there is not enough memory to allocate io queues
2447  * the driver will try to allocate smaller queues.
2448  *
2449  * The backoff algorithm is as follows:
2450  *  1. Try to allocate TX and RX and if successful.
2451  *  1.1. return success
2452  *
2453  *  2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same).
2454  *
2455  *  3. If TX or RX is smaller than 256
2456  *  3.1. return failure.
2457  *  4. else
2458  *  4.1. go back to 1.
2459  */
2460 static int create_queues_with_size_backoff(struct ena_adapter *adapter)
2461 {
2462         int rc, cur_rx_ring_size, cur_tx_ring_size;
2463         int new_rx_ring_size, new_tx_ring_size;
2464
2465         /* current queue sizes might be set to smaller than the requested
2466          * ones due to past queue allocation failures.
2467          */
2468         set_io_rings_size(adapter, adapter->requested_tx_ring_size,
2469                           adapter->requested_rx_ring_size);
2470
2471         while (1) {
2472                 if (ena_xdp_present(adapter)) {
2473                         rc = ena_setup_and_create_all_xdp_queues(adapter);
2474
2475                         if (rc)
2476                                 goto err_setup_tx;
2477                 }
2478                 rc = ena_setup_tx_resources_in_range(adapter,
2479                                                      0,
2480                                                      adapter->num_io_queues);
2481                 if (rc)
2482                         goto err_setup_tx;
2483
2484                 rc = ena_create_io_tx_queues_in_range(adapter,
2485                                                       0,
2486                                                       adapter->num_io_queues);
2487                 if (rc)
2488                         goto err_create_tx_queues;
2489
2490                 rc = ena_setup_all_rx_resources(adapter);
2491                 if (rc)
2492                         goto err_setup_rx;
2493
2494                 rc = ena_create_all_io_rx_queues(adapter);
2495                 if (rc)
2496                         goto err_create_rx_queues;
2497
2498                 return 0;
2499
2500 err_create_rx_queues:
2501                 ena_free_all_io_rx_resources(adapter);
2502 err_setup_rx:
2503                 ena_destroy_all_tx_queues(adapter);
2504 err_create_tx_queues:
2505                 ena_free_all_io_tx_resources(adapter);
2506 err_setup_tx:
2507                 if (rc != -ENOMEM) {
2508                         netif_err(adapter, ifup, adapter->netdev,
2509                                   "Queue creation failed with error code %d\n",
2510                                   rc);
2511                         return rc;
2512                 }
2513
2514                 cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2515                 cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2516
2517                 netif_err(adapter, ifup, adapter->netdev,
2518                           "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2519                           cur_tx_ring_size, cur_rx_ring_size);
2520
2521                 new_tx_ring_size = cur_tx_ring_size;
2522                 new_rx_ring_size = cur_rx_ring_size;
2523
2524                 /* Decrease the size of the larger queue, or
2525                  * decrease both if they are the same size.
2526                  */
2527                 if (cur_rx_ring_size <= cur_tx_ring_size)
2528                         new_tx_ring_size = cur_tx_ring_size / 2;
2529                 if (cur_rx_ring_size >= cur_tx_ring_size)
2530                         new_rx_ring_size = cur_rx_ring_size / 2;
2531
2532                 if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2533                     new_rx_ring_size < ENA_MIN_RING_SIZE) {
2534                         netif_err(adapter, ifup, adapter->netdev,
2535                                   "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n",
2536                                   ENA_MIN_RING_SIZE);
2537                         return rc;
2538                 }
2539
2540                 netif_err(adapter, ifup, adapter->netdev,
2541                           "Retrying queue creation with sizes TX=%d, RX=%d\n",
2542                           new_tx_ring_size,
2543                           new_rx_ring_size);
2544
2545                 set_io_rings_size(adapter, new_tx_ring_size,
2546                                   new_rx_ring_size);
2547         }
2548 }
2549
2550 static int ena_up(struct ena_adapter *adapter)
2551 {
2552         int io_queue_count, rc, i;
2553
2554         netdev_dbg(adapter->netdev, "%s\n", __func__);
2555
2556         io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2557         ena_setup_io_intr(adapter);
2558
2559         /* napi poll functions should be initialized before running
2560          * request_irq(), to handle a rare condition where there is a pending
2561          * interrupt, causing the ISR to fire immediately while the poll
2562          * function wasn't set yet, causing a null dereference
2563          */
2564         ena_init_napi_in_range(adapter, 0, io_queue_count);
2565
2566         rc = ena_request_io_irq(adapter);
2567         if (rc)
2568                 goto err_req_irq;
2569
2570         rc = create_queues_with_size_backoff(adapter);
2571         if (rc)
2572                 goto err_create_queues_with_backoff;
2573
2574         rc = ena_up_complete(adapter);
2575         if (rc)
2576                 goto err_up;
2577
2578         if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
2579                 netif_carrier_on(adapter->netdev);
2580
2581         u64_stats_update_begin(&adapter->syncp);
2582         adapter->dev_stats.interface_up++;
2583         u64_stats_update_end(&adapter->syncp);
2584
2585         set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2586
2587         /* Enable completion queues interrupt */
2588         for (i = 0; i < adapter->num_io_queues; i++)
2589                 ena_unmask_interrupt(&adapter->tx_ring[i],
2590                                      &adapter->rx_ring[i]);
2591
2592         /* schedule napi in case we had pending packets
2593          * from the last time we disable napi
2594          */
2595         for (i = 0; i < io_queue_count; i++)
2596                 napi_schedule(&adapter->ena_napi[i].napi);
2597
2598         return rc;
2599
2600 err_up:
2601         ena_destroy_all_tx_queues(adapter);
2602         ena_free_all_io_tx_resources(adapter);
2603         ena_destroy_all_rx_queues(adapter);
2604         ena_free_all_io_rx_resources(adapter);
2605 err_create_queues_with_backoff:
2606         ena_free_io_irq(adapter);
2607 err_req_irq:
2608         ena_del_napi_in_range(adapter, 0, io_queue_count);
2609
2610         return rc;
2611 }
2612
2613 static void ena_down(struct ena_adapter *adapter)
2614 {
2615         int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2616
2617         netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__);
2618
2619         clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2620
2621         u64_stats_update_begin(&adapter->syncp);
2622         adapter->dev_stats.interface_down++;
2623         u64_stats_update_end(&adapter->syncp);
2624
2625         netif_carrier_off(adapter->netdev);
2626         netif_tx_disable(adapter->netdev);
2627
2628         /* After this point the napi handler won't enable the tx queue */
2629         ena_napi_disable_in_range(adapter, 0, io_queue_count);
2630
2631         /* After destroy the queue there won't be any new interrupts */
2632
2633         if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
2634                 int rc;
2635
2636                 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
2637                 if (rc)
2638                         dev_err(&adapter->pdev->dev, "Device reset failed\n");
2639                 /* stop submitting admin commands on a device that was reset */
2640                 ena_com_set_admin_running_state(adapter->ena_dev, false);
2641         }
2642
2643         ena_destroy_all_io_queues(adapter);
2644
2645         ena_disable_io_intr_sync(adapter);
2646         ena_free_io_irq(adapter);
2647         ena_del_napi_in_range(adapter, 0, io_queue_count);
2648
2649         ena_free_all_tx_bufs(adapter);
2650         ena_free_all_rx_bufs(adapter);
2651         ena_free_all_io_tx_resources(adapter);
2652         ena_free_all_io_rx_resources(adapter);
2653 }
2654
2655 /* ena_open - Called when a network interface is made active
2656  * @netdev: network interface device structure
2657  *
2658  * Returns 0 on success, negative value on failure
2659  *
2660  * The open entry point is called when a network interface is made
2661  * active by the system (IFF_UP).  At this point all resources needed
2662  * for transmit and receive operations are allocated, the interrupt
2663  * handler is registered with the OS, the watchdog timer is started,
2664  * and the stack is notified that the interface is ready.
2665  */
2666 static int ena_open(struct net_device *netdev)
2667 {
2668         struct ena_adapter *adapter = netdev_priv(netdev);
2669         int rc;
2670
2671         /* Notify the stack of the actual queue counts. */
2672         rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues);
2673         if (rc) {
2674                 netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
2675                 return rc;
2676         }
2677
2678         rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues);
2679         if (rc) {
2680                 netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
2681                 return rc;
2682         }
2683
2684         rc = ena_up(adapter);
2685         if (rc)
2686                 return rc;
2687
2688         return rc;
2689 }
2690
2691 /* ena_close - Disables a network interface
2692  * @netdev: network interface device structure
2693  *
2694  * Returns 0, this is not allowed to fail
2695  *
2696  * The close entry point is called when an interface is de-activated
2697  * by the OS.  The hardware is still under the drivers control, but
2698  * needs to be disabled.  A global MAC reset is issued to stop the
2699  * hardware, and all transmit and receive resources are freed.
2700  */
2701 static int ena_close(struct net_device *netdev)
2702 {
2703         struct ena_adapter *adapter = netdev_priv(netdev);
2704
2705         netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
2706
2707         if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
2708                 return 0;
2709
2710         if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2711                 ena_down(adapter);
2712
2713         /* Check for device status and issue reset if needed*/
2714         check_for_admin_com_state(adapter);
2715         if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
2716                 netif_err(adapter, ifdown, adapter->netdev,
2717                           "Destroy failure, restarting device\n");
2718                 ena_dump_stats_to_dmesg(adapter);
2719                 /* rtnl lock already obtained in dev_ioctl() layer */
2720                 ena_destroy_device(adapter, false);
2721                 ena_restore_device(adapter);
2722         }
2723
2724         return 0;
2725 }
2726
2727 int ena_update_queue_sizes(struct ena_adapter *adapter,
2728                            u32 new_tx_size,
2729                            u32 new_rx_size)
2730 {
2731         bool dev_was_up;
2732
2733         dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2734         ena_close(adapter->netdev);
2735         adapter->requested_tx_ring_size = new_tx_size;
2736         adapter->requested_rx_ring_size = new_rx_size;
2737         ena_init_io_rings(adapter,
2738                           0,
2739                           adapter->xdp_num_queues +
2740                           adapter->num_io_queues);
2741         return dev_was_up ? ena_up(adapter) : 0;
2742 }
2743
2744 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)
2745 {
2746         struct ena_com_dev *ena_dev = adapter->ena_dev;
2747         int prev_channel_count;
2748         bool dev_was_up;
2749
2750         dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2751         ena_close(adapter->netdev);
2752         prev_channel_count = adapter->num_io_queues;
2753         adapter->num_io_queues = new_channel_count;
2754         if (ena_xdp_present(adapter) &&
2755             ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) {
2756                 adapter->xdp_first_ring = new_channel_count;
2757                 adapter->xdp_num_queues = new_channel_count;
2758                 if (prev_channel_count > new_channel_count)
2759                         ena_xdp_exchange_program_rx_in_range(adapter,
2760                                                              NULL,
2761                                                              new_channel_count,
2762                                                              prev_channel_count);
2763                 else
2764                         ena_xdp_exchange_program_rx_in_range(adapter,
2765                                                              adapter->xdp_bpf_prog,
2766                                                              prev_channel_count,
2767                                                              new_channel_count);
2768         }
2769
2770         /* We need to destroy the rss table so that the indirection
2771          * table will be reinitialized by ena_up()
2772          */
2773         ena_com_rss_destroy(ena_dev);
2774         ena_init_io_rings(adapter,
2775                           0,
2776                           adapter->xdp_num_queues +
2777                           adapter->num_io_queues);
2778         return dev_was_up ? ena_open(adapter->netdev) : 0;
2779 }
2780
2781 static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,
2782                         struct sk_buff *skb,
2783                         bool disable_meta_caching)
2784 {
2785         u32 mss = skb_shinfo(skb)->gso_size;
2786         struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
2787         u8 l4_protocol = 0;
2788
2789         if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {
2790                 ena_tx_ctx->l4_csum_enable = 1;
2791                 if (mss) {
2792                         ena_tx_ctx->tso_enable = 1;
2793                         ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;
2794                         ena_tx_ctx->l4_csum_partial = 0;
2795                 } else {
2796                         ena_tx_ctx->tso_enable = 0;
2797                         ena_meta->l4_hdr_len = 0;
2798                         ena_tx_ctx->l4_csum_partial = 1;
2799                 }
2800
2801                 switch (ip_hdr(skb)->version) {
2802                 case IPVERSION:
2803                         ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
2804                         if (ip_hdr(skb)->frag_off & htons(IP_DF))
2805                                 ena_tx_ctx->df = 1;
2806                         if (mss)
2807                                 ena_tx_ctx->l3_csum_enable = 1;
2808                         l4_protocol = ip_hdr(skb)->protocol;
2809                         break;
2810                 case 6:
2811                         ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
2812                         l4_protocol = ipv6_hdr(skb)->nexthdr;
2813                         break;
2814                 default:
2815                         break;
2816                 }
2817
2818                 if (l4_protocol == IPPROTO_TCP)
2819                         ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
2820                 else
2821                         ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
2822
2823                 ena_meta->mss = mss;
2824                 ena_meta->l3_hdr_len = skb_network_header_len(skb);
2825                 ena_meta->l3_hdr_offset = skb_network_offset(skb);
2826                 ena_tx_ctx->meta_valid = 1;
2827         } else if (disable_meta_caching) {
2828                 memset(ena_meta, 0, sizeof(*ena_meta));
2829                 ena_tx_ctx->meta_valid = 1;
2830         } else {
2831                 ena_tx_ctx->meta_valid = 0;
2832         }
2833 }
2834
2835 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
2836                                        struct sk_buff *skb)
2837 {
2838         int num_frags, header_len, rc;
2839
2840         num_frags = skb_shinfo(skb)->nr_frags;
2841         header_len = skb_headlen(skb);
2842
2843         if (num_frags < tx_ring->sgl_size)
2844                 return 0;
2845
2846         if ((num_frags == tx_ring->sgl_size) &&
2847             (header_len < tx_ring->tx_max_header_size))
2848                 return 0;
2849
2850         u64_stats_update_begin(&tx_ring->syncp);
2851         tx_ring->tx_stats.linearize++;
2852         u64_stats_update_end(&tx_ring->syncp);
2853
2854         rc = skb_linearize(skb);
2855         if (unlikely(rc)) {
2856                 u64_stats_update_begin(&tx_ring->syncp);
2857                 tx_ring->tx_stats.linearize_failed++;
2858                 u64_stats_update_end(&tx_ring->syncp);
2859         }
2860
2861         return rc;
2862 }
2863
2864 static int ena_tx_map_skb(struct ena_ring *tx_ring,
2865                           struct ena_tx_buffer *tx_info,
2866                           struct sk_buff *skb,
2867                           void **push_hdr,
2868                           u16 *header_len)
2869 {
2870         struct ena_adapter *adapter = tx_ring->adapter;
2871         struct ena_com_buf *ena_buf;
2872         dma_addr_t dma;
2873         u32 skb_head_len, frag_len, last_frag;
2874         u16 push_len = 0;
2875         u16 delta = 0;
2876         int i = 0;
2877
2878         skb_head_len = skb_headlen(skb);
2879         tx_info->skb = skb;
2880         ena_buf = tx_info->bufs;
2881
2882         if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2883                 /* When the device is LLQ mode, the driver will copy
2884                  * the header into the device memory space.
2885                  * the ena_com layer assume the header is in a linear
2886                  * memory space.
2887                  * This assumption might be wrong since part of the header
2888                  * can be in the fragmented buffers.
2889                  * Use skb_header_pointer to make sure the header is in a
2890                  * linear memory space.
2891                  */
2892
2893                 push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
2894                 *push_hdr = skb_header_pointer(skb, 0, push_len,
2895                                                tx_ring->push_buf_intermediate_buf);
2896                 *header_len = push_len;
2897                 if (unlikely(skb->data != *push_hdr)) {
2898                         u64_stats_update_begin(&tx_ring->syncp);
2899                         tx_ring->tx_stats.llq_buffer_copy++;
2900                         u64_stats_update_end(&tx_ring->syncp);
2901
2902                         delta = push_len - skb_head_len;
2903                 }
2904         } else {
2905                 *push_hdr = NULL;
2906                 *header_len = min_t(u32, skb_head_len,
2907                                     tx_ring->tx_max_header_size);
2908         }
2909
2910         netif_dbg(adapter, tx_queued, adapter->netdev,
2911                   "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
2912                   *push_hdr, push_len);
2913
2914         if (skb_head_len > push_len) {
2915                 dma = dma_map_single(tx_ring->dev, skb->data + push_len,
2916                                      skb_head_len - push_len, DMA_TO_DEVICE);
2917                 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2918                         goto error_report_dma_error;
2919
2920                 ena_buf->paddr = dma;
2921                 ena_buf->len = skb_head_len - push_len;
2922
2923                 ena_buf++;
2924                 tx_info->num_of_bufs++;
2925                 tx_info->map_linear_data = 1;
2926         } else {
2927                 tx_info->map_linear_data = 0;
2928         }
2929
2930         last_frag = skb_shinfo(skb)->nr_frags;
2931
2932         for (i = 0; i < last_frag; i++) {
2933                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2934
2935                 frag_len = skb_frag_size(frag);
2936
2937                 if (unlikely(delta >= frag_len)) {
2938                         delta -= frag_len;
2939                         continue;
2940                 }
2941
2942                 dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
2943                                        frag_len - delta, DMA_TO_DEVICE);
2944                 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2945                         goto error_report_dma_error;
2946
2947                 ena_buf->paddr = dma;
2948                 ena_buf->len = frag_len - delta;
2949                 ena_buf++;
2950                 tx_info->num_of_bufs++;
2951                 delta = 0;
2952         }
2953
2954         return 0;
2955
2956 error_report_dma_error:
2957         u64_stats_update_begin(&tx_ring->syncp);
2958         tx_ring->tx_stats.dma_mapping_err++;
2959         u64_stats_update_end(&tx_ring->syncp);
2960         netdev_warn(adapter->netdev, "failed to map skb\n");
2961
2962         tx_info->skb = NULL;
2963
2964         tx_info->num_of_bufs += i;
2965         ena_unmap_tx_buff(tx_ring, tx_info);
2966
2967         return -EINVAL;
2968 }
2969
2970 /* Called with netif_tx_lock. */
2971 static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
2972 {
2973         struct ena_adapter *adapter = netdev_priv(dev);
2974         struct ena_tx_buffer *tx_info;
2975         struct ena_com_tx_ctx ena_tx_ctx;
2976         struct ena_ring *tx_ring;
2977         struct netdev_queue *txq;
2978         void *push_hdr;
2979         u16 next_to_use, req_id, header_len;
2980         int qid, rc;
2981
2982         netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
2983         /*  Determine which tx ring we will be placed on */
2984         qid = skb_get_queue_mapping(skb);
2985         tx_ring = &adapter->tx_ring[qid];
2986         txq = netdev_get_tx_queue(dev, qid);
2987
2988         rc = ena_check_and_linearize_skb(tx_ring, skb);
2989         if (unlikely(rc))
2990                 goto error_drop_packet;
2991
2992         skb_tx_timestamp(skb);
2993
2994         next_to_use = tx_ring->next_to_use;
2995         req_id = tx_ring->free_ids[next_to_use];
2996         tx_info = &tx_ring->tx_buffer_info[req_id];
2997         tx_info->num_of_bufs = 0;
2998
2999         WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
3000
3001         rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
3002         if (unlikely(rc))
3003                 goto error_drop_packet;
3004
3005         memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
3006         ena_tx_ctx.ena_bufs = tx_info->bufs;
3007         ena_tx_ctx.push_header = push_hdr;
3008         ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
3009         ena_tx_ctx.req_id = req_id;
3010         ena_tx_ctx.header_len = header_len;
3011
3012         /* set flags and meta data */
3013         ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
3014
3015         rc = ena_xmit_common(dev,
3016                              tx_ring,
3017                              tx_info,
3018                              &ena_tx_ctx,
3019                              next_to_use,
3020                              skb->len);
3021         if (rc)
3022                 goto error_unmap_dma;
3023
3024         netdev_tx_sent_queue(txq, skb->len);
3025
3026         /* stop the queue when no more space available, the packet can have up
3027          * to sgl_size + 2. one for the meta descriptor and one for header
3028          * (if the header is larger than tx_max_header_size).
3029          */
3030         if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
3031                                                    tx_ring->sgl_size + 2))) {
3032                 netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
3033                           __func__, qid);
3034
3035                 netif_tx_stop_queue(txq);
3036                 u64_stats_update_begin(&tx_ring->syncp);
3037                 tx_ring->tx_stats.queue_stop++;
3038                 u64_stats_update_end(&tx_ring->syncp);
3039
3040                 /* There is a rare condition where this function decide to
3041                  * stop the queue but meanwhile clean_tx_irq updates
3042                  * next_to_completion and terminates.
3043                  * The queue will remain stopped forever.
3044                  * To solve this issue add a mb() to make sure that
3045                  * netif_tx_stop_queue() write is vissible before checking if
3046                  * there is additional space in the queue.
3047                  */
3048                 smp_mb();
3049
3050                 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
3051                                                  ENA_TX_WAKEUP_THRESH)) {
3052                         netif_tx_wake_queue(txq);
3053                         u64_stats_update_begin(&tx_ring->syncp);
3054                         tx_ring->tx_stats.queue_wakeup++;
3055                         u64_stats_update_end(&tx_ring->syncp);
3056                 }
3057         }
3058
3059         if (netif_xmit_stopped(txq) || !netdev_xmit_more()) {
3060                 /* trigger the dma engine. ena_com_write_sq_doorbell()
3061                  * has a mb
3062                  */
3063                 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
3064                 u64_stats_update_begin(&tx_ring->syncp);
3065                 tx_ring->tx_stats.doorbells++;
3066                 u64_stats_update_end(&tx_ring->syncp);
3067         }
3068
3069         return NETDEV_TX_OK;
3070
3071 error_unmap_dma:
3072         ena_unmap_tx_buff(tx_ring, tx_info);
3073         tx_info->skb = NULL;
3074
3075 error_drop_packet:
3076         dev_kfree_skb(skb);
3077         return NETDEV_TX_OK;
3078 }
3079
3080 static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb,
3081                             struct net_device *sb_dev)
3082 {
3083         u16 qid;
3084         /* we suspect that this is good for in--kernel network services that
3085          * want to loop incoming skb rx to tx in normal user generated traffic,
3086          * most probably we will not get to this
3087          */
3088         if (skb_rx_queue_recorded(skb))
3089                 qid = skb_get_rx_queue(skb);
3090         else
3091                 qid = netdev_pick_tx(dev, skb, NULL);
3092
3093         return qid;
3094 }
3095
3096 static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
3097 {
3098         struct ena_admin_host_info *host_info;
3099         int rc;
3100
3101         /* Allocate only the host info */
3102         rc = ena_com_allocate_host_info(ena_dev);
3103         if (rc) {
3104                 pr_err("Cannot allocate host info\n");
3105                 return;
3106         }
3107
3108         host_info = ena_dev->host_attr.host_info;
3109
3110         host_info->bdf = (pdev->bus->number << 8) | pdev->devfn;
3111         host_info->os_type = ENA_ADMIN_OS_LINUX;
3112         host_info->kernel_ver = LINUX_VERSION_CODE;
3113         strlcpy(host_info->kernel_ver_str, utsname()->version,
3114                 sizeof(host_info->kernel_ver_str) - 1);
3115         host_info->os_dist = 0;
3116         strncpy(host_info->os_dist_str, utsname()->release,
3117                 sizeof(host_info->os_dist_str) - 1);
3118         host_info->driver_version =
3119                 (DRV_MODULE_GEN_MAJOR) |
3120                 (DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
3121                 (DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) |
3122                 ("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);
3123         host_info->num_cpus = num_online_cpus();
3124
3125         host_info->driver_supported_features =
3126                 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
3127                 ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK |
3128                 ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK |
3129                 ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
3130
3131         rc = ena_com_set_host_attributes(ena_dev);
3132         if (rc) {
3133                 if (rc == -EOPNOTSUPP)
3134                         pr_warn("Cannot set host attributes\n");
3135                 else
3136                         pr_err("Cannot set host attributes\n");
3137
3138                 goto err;
3139         }
3140
3141         return;
3142
3143 err:
3144         ena_com_delete_host_info(ena_dev);
3145 }
3146
3147 static void ena_config_debug_area(struct ena_adapter *adapter)
3148 {
3149         u32 debug_area_size;
3150         int rc, ss_count;
3151
3152         ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);
3153         if (ss_count <= 0) {
3154                 netif_err(adapter, drv, adapter->netdev,
3155                           "SS count is negative\n");
3156                 return;
3157         }
3158
3159         /* allocate 32 bytes for each string and 64bit for the value */
3160         debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
3161
3162         rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
3163         if (rc) {
3164                 pr_err("Cannot allocate debug area\n");
3165                 return;
3166         }
3167
3168         rc = ena_com_set_host_attributes(adapter->ena_dev);
3169         if (rc) {
3170                 if (rc == -EOPNOTSUPP)
3171                         netif_warn(adapter, drv, adapter->netdev,
3172                                    "Cannot set host attributes\n");
3173                 else
3174                         netif_err(adapter, drv, adapter->netdev,
3175                                   "Cannot set host attributes\n");
3176                 goto err;
3177         }
3178
3179         return;
3180 err:
3181         ena_com_delete_debug_area(adapter->ena_dev);
3182 }
3183
3184 static void ena_get_stats64(struct net_device *netdev,
3185                             struct rtnl_link_stats64 *stats)
3186 {
3187         struct ena_adapter *adapter = netdev_priv(netdev);
3188         struct ena_ring *rx_ring, *tx_ring;
3189         unsigned int start;
3190         u64 rx_drops;
3191         u64 tx_drops;
3192         int i;
3193
3194         if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3195                 return;
3196
3197         for (i = 0; i < adapter->num_io_queues; i++) {
3198                 u64 bytes, packets;
3199
3200                 tx_ring = &adapter->tx_ring[i];
3201
3202                 do {
3203                         start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
3204                         packets = tx_ring->tx_stats.cnt;
3205                         bytes = tx_ring->tx_stats.bytes;
3206                 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
3207
3208                 stats->tx_packets += packets;
3209                 stats->tx_bytes += bytes;
3210
3211                 rx_ring = &adapter->rx_ring[i];
3212
3213                 do {
3214                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
3215                         packets = rx_ring->rx_stats.cnt;
3216                         bytes = rx_ring->rx_stats.bytes;
3217                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
3218
3219                 stats->rx_packets += packets;
3220                 stats->rx_bytes += bytes;
3221         }
3222
3223         do {
3224                 start = u64_stats_fetch_begin_irq(&adapter->syncp);
3225                 rx_drops = adapter->dev_stats.rx_drops;
3226                 tx_drops = adapter->dev_stats.tx_drops;
3227         } while (u64_stats_fetch_retry_irq(&adapter->syncp, start));
3228
3229         stats->rx_dropped = rx_drops;
3230         stats->tx_dropped = tx_drops;
3231
3232         stats->multicast = 0;
3233         stats->collisions = 0;
3234
3235         stats->rx_length_errors = 0;
3236         stats->rx_crc_errors = 0;
3237         stats->rx_frame_errors = 0;
3238         stats->rx_fifo_errors = 0;
3239         stats->rx_missed_errors = 0;
3240         stats->tx_window_errors = 0;
3241
3242         stats->rx_errors = 0;
3243         stats->tx_errors = 0;
3244 }
3245
3246 static const struct net_device_ops ena_netdev_ops = {
3247         .ndo_open               = ena_open,
3248         .ndo_stop               = ena_close,
3249         .ndo_start_xmit         = ena_start_xmit,
3250         .ndo_select_queue       = ena_select_queue,
3251         .ndo_get_stats64        = ena_get_stats64,
3252         .ndo_tx_timeout         = ena_tx_timeout,
3253         .ndo_change_mtu         = ena_change_mtu,
3254         .ndo_set_mac_address    = NULL,
3255         .ndo_validate_addr      = eth_validate_addr,
3256         .ndo_bpf                = ena_xdp,
3257 };
3258
3259 static int ena_device_validate_params(struct ena_adapter *adapter,
3260                                       struct ena_com_dev_get_features_ctx *get_feat_ctx)
3261 {
3262         struct net_device *netdev = adapter->netdev;
3263         int rc;
3264
3265         rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,
3266                               adapter->mac_addr);
3267         if (!rc) {
3268                 netif_err(adapter, drv, netdev,
3269                           "Error, mac address are different\n");
3270                 return -EINVAL;
3271         }
3272
3273         if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
3274                 netif_err(adapter, drv, netdev,
3275                           "Error, device max mtu is smaller than netdev MTU\n");
3276                 return -EINVAL;
3277         }
3278
3279         return 0;
3280 }
3281
3282 static void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
3283 {
3284         llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
3285         llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
3286         llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
3287         llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
3288         llq_config->llq_ring_entry_size_value = 128;
3289 }
3290
3291 static int ena_set_queues_placement_policy(struct pci_dev *pdev,
3292                                            struct ena_com_dev *ena_dev,
3293                                            struct ena_admin_feature_llq_desc *llq,
3294                                            struct ena_llq_configurations *llq_default_configurations)
3295 {
3296         int rc;
3297         u32 llq_feature_mask;
3298
3299         llq_feature_mask = 1 << ENA_ADMIN_LLQ;
3300         if (!(ena_dev->supported_features & llq_feature_mask)) {
3301                 dev_err(&pdev->dev,
3302                         "LLQ is not supported Fallback to host mode policy.\n");
3303                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3304                 return 0;
3305         }
3306
3307         rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
3308         if (unlikely(rc)) {
3309                 dev_err(&pdev->dev,
3310                         "Failed to configure the device mode.  Fallback to host mode policy.\n");
3311                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3312         }
3313
3314         return 0;
3315 }
3316
3317 static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
3318                                int bars)
3319 {
3320         bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));
3321
3322         if (!has_mem_bar) {
3323                 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
3324                         dev_err(&pdev->dev,
3325                                 "ENA device does not expose LLQ bar. Fallback to host mode policy.\n");
3326                         ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3327                 }
3328
3329                 return 0;
3330         }
3331
3332         ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
3333                                            pci_resource_start(pdev, ENA_MEM_BAR),
3334                                            pci_resource_len(pdev, ENA_MEM_BAR));
3335
3336         if (!ena_dev->mem_bar)
3337                 return -EFAULT;
3338
3339         return 0;
3340 }
3341
3342 static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
3343                            struct ena_com_dev_get_features_ctx *get_feat_ctx,
3344                            bool *wd_state)
3345 {
3346         struct ena_llq_configurations llq_config;
3347         struct device *dev = &pdev->dev;
3348         bool readless_supported;
3349         u32 aenq_groups;
3350         int dma_width;
3351         int rc;
3352
3353         rc = ena_com_mmio_reg_read_request_init(ena_dev);
3354         if (rc) {
3355                 dev_err(dev, "failed to init mmio read less\n");
3356                 return rc;
3357         }
3358
3359         /* The PCIe configuration space revision id indicate if mmio reg
3360          * read is disabled
3361          */
3362         readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
3363         ena_com_set_mmio_read_mode(ena_dev, readless_supported);
3364
3365         rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
3366         if (rc) {
3367                 dev_err(dev, "Can not reset device\n");
3368                 goto err_mmio_read_less;
3369         }
3370
3371         rc = ena_com_validate_version(ena_dev);
3372         if (rc) {
3373                 dev_err(dev, "device version is too low\n");
3374                 goto err_mmio_read_less;
3375         }
3376
3377         dma_width = ena_com_get_dma_width(ena_dev);
3378         if (dma_width < 0) {
3379                 dev_err(dev, "Invalid dma width value %d", dma_width);
3380                 rc = dma_width;
3381                 goto err_mmio_read_less;
3382         }
3383
3384         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_width));
3385         if (rc) {
3386                 dev_err(dev, "pci_set_dma_mask failed 0x%x\n", rc);
3387                 goto err_mmio_read_less;
3388         }
3389
3390         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_width));
3391         if (rc) {
3392                 dev_err(dev, "err_pci_set_consistent_dma_mask failed 0x%x\n",
3393                         rc);
3394                 goto err_mmio_read_less;
3395         }
3396
3397         /* ENA admin level init */
3398         rc = ena_com_admin_init(ena_dev, &aenq_handlers);
3399         if (rc) {
3400                 dev_err(dev,
3401                         "Can not initialize ena admin queue with device\n");
3402                 goto err_mmio_read_less;
3403         }
3404
3405         /* To enable the msix interrupts the driver needs to know the number
3406          * of queues. So the driver uses polling mode to retrieve this
3407          * information
3408          */
3409         ena_com_set_admin_polling_mode(ena_dev, true);
3410
3411         ena_config_host_info(ena_dev, pdev);
3412
3413         /* Get Device Attributes*/
3414         rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3415         if (rc) {
3416                 dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);
3417                 goto err_admin_init;
3418         }
3419
3420         /* Try to turn all the available aenq groups */
3421         aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3422                 BIT(ENA_ADMIN_FATAL_ERROR) |
3423                 BIT(ENA_ADMIN_WARNING) |
3424                 BIT(ENA_ADMIN_NOTIFICATION) |
3425                 BIT(ENA_ADMIN_KEEP_ALIVE);
3426
3427         aenq_groups &= get_feat_ctx->aenq.supported_groups;
3428
3429         rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3430         if (rc) {
3431                 dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);
3432                 goto err_admin_init;
3433         }
3434
3435         *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3436
3437         set_default_llq_configurations(&llq_config);
3438
3439         rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
3440                                              &llq_config);
3441         if (rc) {
3442                 dev_err(&pdev->dev, "ena device init failed\n");
3443                 goto err_admin_init;
3444         }
3445
3446         return 0;
3447
3448 err_admin_init:
3449         ena_com_delete_host_info(ena_dev);
3450         ena_com_admin_destroy(ena_dev);
3451 err_mmio_read_less:
3452         ena_com_mmio_reg_read_request_destroy(ena_dev);
3453
3454         return rc;
3455 }
3456
3457 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
3458 {
3459         struct ena_com_dev *ena_dev = adapter->ena_dev;
3460         struct device *dev = &adapter->pdev->dev;
3461         int rc;
3462
3463         rc = ena_enable_msix(adapter);
3464         if (rc) {
3465                 dev_err(dev, "Can not reserve msix vectors\n");
3466                 return rc;
3467         }
3468
3469         ena_setup_mgmnt_intr(adapter);
3470
3471         rc = ena_request_mgmnt_irq(adapter);
3472         if (rc) {
3473                 dev_err(dev, "Can not setup management interrupts\n");
3474                 goto err_disable_msix;
3475         }
3476
3477         ena_com_set_admin_polling_mode(ena_dev, false);
3478
3479         ena_com_admin_aenq_enable(ena_dev);
3480
3481         return 0;
3482
3483 err_disable_msix:
3484         ena_disable_msix(adapter);
3485
3486         return rc;
3487 }
3488
3489 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3490 {
3491         struct net_device *netdev = adapter->netdev;
3492         struct ena_com_dev *ena_dev = adapter->ena_dev;
3493         bool dev_up;
3494
3495         if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
3496                 return;
3497
3498         netif_carrier_off(netdev);
3499
3500         del_timer_sync(&adapter->timer_service);
3501
3502         dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
3503         adapter->dev_up_before_reset = dev_up;
3504         if (!graceful)
3505                 ena_com_set_admin_running_state(ena_dev, false);
3506
3507         if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3508                 ena_down(adapter);
3509
3510         /* Stop the device from sending AENQ events (in case reset flag is set
3511          *  and device is up, ena_down() already reset the device.
3512          */
3513         if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up))
3514                 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3515
3516         ena_free_mgmnt_irq(adapter);
3517
3518         ena_disable_msix(adapter);
3519
3520         ena_com_abort_admin_commands(ena_dev);
3521
3522         ena_com_wait_for_abort_completion(ena_dev);
3523
3524         ena_com_admin_destroy(ena_dev);
3525
3526         ena_com_mmio_reg_read_request_destroy(ena_dev);
3527
3528         /* return reset reason to default value */
3529         adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3530
3531         clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3532         clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3533 }
3534
3535 static int ena_restore_device(struct ena_adapter *adapter)
3536 {
3537         struct ena_com_dev_get_features_ctx get_feat_ctx;
3538         struct ena_com_dev *ena_dev = adapter->ena_dev;
3539         struct pci_dev *pdev = adapter->pdev;
3540         bool wd_state;
3541         int rc;
3542
3543         set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3544         rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state);
3545         if (rc) {
3546                 dev_err(&pdev->dev, "Can not initialize device\n");
3547                 goto err;
3548         }
3549         adapter->wd_state = wd_state;
3550
3551         rc = ena_device_validate_params(adapter, &get_feat_ctx);
3552         if (rc) {
3553                 dev_err(&pdev->dev, "Validation of device parameters failed\n");
3554                 goto err_device_destroy;
3555         }
3556
3557         rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3558         if (rc) {
3559                 dev_err(&pdev->dev, "Enable MSI-X failed\n");
3560                 goto err_device_destroy;
3561         }
3562         /* If the interface was up before the reset bring it up */
3563         if (adapter->dev_up_before_reset) {
3564                 rc = ena_up(adapter);
3565                 if (rc) {
3566                         dev_err(&pdev->dev, "Failed to create I/O queues\n");
3567                         goto err_disable_msix;
3568                 }
3569         }
3570
3571         set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3572
3573         clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3574         if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
3575                 netif_carrier_on(adapter->netdev);
3576
3577         mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3578         dev_err(&pdev->dev, "Device reset completed successfully\n");
3579         adapter->last_keep_alive_jiffies = jiffies;
3580
3581         return rc;
3582 err_disable_msix:
3583         ena_free_mgmnt_irq(adapter);
3584         ena_disable_msix(adapter);
3585 err_device_destroy:
3586         ena_com_abort_admin_commands(ena_dev);
3587         ena_com_wait_for_abort_completion(ena_dev);
3588         ena_com_admin_destroy(ena_dev);
3589         ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3590         ena_com_mmio_reg_read_request_destroy(ena_dev);
3591 err:
3592         clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3593         clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3594         dev_err(&pdev->dev,
3595                 "Reset attempt failed. Can not reset the device\n");
3596
3597         return rc;
3598 }
3599
3600 static void ena_fw_reset_device(struct work_struct *work)
3601 {
3602         struct ena_adapter *adapter =
3603                 container_of(work, struct ena_adapter, reset_task);
3604         struct pci_dev *pdev = adapter->pdev;
3605
3606         if (unlikely(!test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3607                 dev_err(&pdev->dev,
3608                         "device reset schedule while reset bit is off\n");
3609                 return;
3610         }
3611         rtnl_lock();
3612         ena_destroy_device(adapter, false);
3613         ena_restore_device(adapter);
3614         rtnl_unlock();
3615 }
3616
3617 static int check_for_rx_interrupt_queue(struct ena_adapter *adapter,
3618                                         struct ena_ring *rx_ring)
3619 {
3620         if (likely(rx_ring->first_interrupt))
3621                 return 0;
3622
3623         if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
3624                 return 0;
3625
3626         rx_ring->no_interrupt_event_cnt++;
3627
3628         if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {
3629                 netif_err(adapter, rx_err, adapter->netdev,
3630                           "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
3631                           rx_ring->qid);
3632                 adapter->reset_reason = ENA_REGS_RESET_MISS_INTERRUPT;
3633                 smp_mb__before_atomic();
3634                 set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3635                 return -EIO;
3636         }
3637
3638         return 0;
3639 }
3640
3641 static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
3642                                           struct ena_ring *tx_ring)
3643 {
3644         struct ena_tx_buffer *tx_buf;
3645         unsigned long last_jiffies;
3646         u32 missed_tx = 0;
3647         int i, rc = 0;
3648
3649         for (i = 0; i < tx_ring->ring_size; i++) {
3650                 tx_buf = &tx_ring->tx_buffer_info[i];
3651                 last_jiffies = tx_buf->last_jiffies;
3652
3653                 if (last_jiffies == 0)
3654                         /* no pending Tx at this location */
3655                         continue;
3656
3657                 if (unlikely(!tx_ring->first_interrupt && time_is_before_jiffies(last_jiffies +
3658                              2 * adapter->missing_tx_completion_to))) {
3659                         /* If after graceful period interrupt is still not
3660                          * received, we schedule a reset
3661                          */
3662                         netif_err(adapter, tx_err, adapter->netdev,
3663                                   "Potential MSIX issue on Tx side Queue = %d. Reset the device\n",
3664                                   tx_ring->qid);
3665                         adapter->reset_reason = ENA_REGS_RESET_MISS_INTERRUPT;
3666                         smp_mb__before_atomic();
3667                         set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3668                         return -EIO;
3669                 }
3670
3671                 if (unlikely(time_is_before_jiffies(last_jiffies +
3672                                 adapter->missing_tx_completion_to))) {
3673                         if (!tx_buf->print_once)
3674                                 netif_notice(adapter, tx_err, adapter->netdev,
3675                                              "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
3676                                              tx_ring->qid, i);
3677
3678                         tx_buf->print_once = 1;
3679                         missed_tx++;
3680                 }
3681         }
3682
3683         if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
3684                 netif_err(adapter, tx_err, adapter->netdev,
3685                           "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n",
3686                           missed_tx,
3687                           adapter->missing_tx_completion_threshold);
3688                 adapter->reset_reason =
3689                         ENA_REGS_RESET_MISS_TX_CMPL;
3690                 set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3691                 rc = -EIO;
3692         }
3693
3694         u64_stats_update_begin(&tx_ring->syncp);
3695         tx_ring->tx_stats.missed_tx = missed_tx;
3696         u64_stats_update_end(&tx_ring->syncp);
3697
3698         return rc;
3699 }
3700
3701 static void check_for_missing_completions(struct ena_adapter *adapter)
3702 {
3703         struct ena_ring *tx_ring;
3704         struct ena_ring *rx_ring;
3705         int i, budget, rc;
3706         int io_queue_count;
3707
3708         io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues;
3709         /* Make sure the driver doesn't turn the device in other process */
3710         smp_rmb();
3711
3712         if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3713                 return;
3714
3715         if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3716                 return;
3717
3718         if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)
3719                 return;
3720
3721         budget = ENA_MONITORED_TX_QUEUES;
3722
3723         for (i = adapter->last_monitored_tx_qid; i < io_queue_count; i++) {
3724                 tx_ring = &adapter->tx_ring[i];
3725                 rx_ring = &adapter->rx_ring[i];
3726
3727                 rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
3728                 if (unlikely(rc))
3729                         return;
3730
3731                 rc =  !ENA_IS_XDP_INDEX(adapter, i) ?
3732                         check_for_rx_interrupt_queue(adapter, rx_ring) : 0;
3733                 if (unlikely(rc))
3734                         return;
3735
3736                 budget--;
3737                 if (!budget)
3738                         break;
3739         }
3740
3741         adapter->last_monitored_tx_qid = i % io_queue_count;
3742 }
3743
3744 /* trigger napi schedule after 2 consecutive detections */
3745 #define EMPTY_RX_REFILL 2
3746 /* For the rare case where the device runs out of Rx descriptors and the
3747  * napi handler failed to refill new Rx descriptors (due to a lack of memory
3748  * for example).
3749  * This case will lead to a deadlock:
3750  * The device won't send interrupts since all the new Rx packets will be dropped
3751  * The napi handler won't allocate new Rx descriptors so the device will be
3752  * able to send new packets.
3753  *
3754  * This scenario can happen when the kernel's vm.min_free_kbytes is too small.
3755  * It is recommended to have at least 512MB, with a minimum of 128MB for
3756  * constrained environment).
3757  *
3758  * When such a situation is detected - Reschedule napi
3759  */
3760 static void check_for_empty_rx_ring(struct ena_adapter *adapter)
3761 {
3762         struct ena_ring *rx_ring;
3763         int i, refill_required;
3764
3765         if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3766                 return;
3767
3768         if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3769                 return;
3770
3771         for (i = 0; i < adapter->num_io_queues; i++) {
3772                 rx_ring = &adapter->rx_ring[i];
3773
3774                 refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
3775                 if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3776                         rx_ring->empty_rx_queue++;
3777
3778                         if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
3779                                 u64_stats_update_begin(&rx_ring->syncp);
3780                                 rx_ring->rx_stats.empty_rx_ring++;
3781                                 u64_stats_update_end(&rx_ring->syncp);
3782
3783                                 netif_err(adapter, drv, adapter->netdev,
3784                                           "trigger refill for ring %d\n", i);
3785
3786                                 napi_schedule(rx_ring->napi);
3787                                 rx_ring->empty_rx_queue = 0;
3788                         }
3789                 } else {
3790                         rx_ring->empty_rx_queue = 0;
3791                 }
3792         }
3793 }
3794
3795 /* Check for keep alive expiration */
3796 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
3797 {
3798         unsigned long keep_alive_expired;
3799
3800         if (!adapter->wd_state)
3801                 return;
3802
3803         if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3804                 return;
3805
3806         keep_alive_expired = adapter->last_keep_alive_jiffies +
3807                              adapter->keep_alive_timeout;
3808         if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
3809                 netif_err(adapter, drv, adapter->netdev,
3810                           "Keep alive watchdog timeout.\n");
3811                 u64_stats_update_begin(&adapter->syncp);
3812                 adapter->dev_stats.wd_expired++;
3813                 u64_stats_update_end(&adapter->syncp);
3814                 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
3815                 set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3816         }
3817 }
3818
3819 static void check_for_admin_com_state(struct ena_adapter *adapter)
3820 {
3821         if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
3822                 netif_err(adapter, drv, adapter->netdev,
3823                           "ENA admin queue is not in running state!\n");
3824                 u64_stats_update_begin(&adapter->syncp);
3825                 adapter->dev_stats.admin_q_pause++;
3826                 u64_stats_update_end(&adapter->syncp);
3827                 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
3828                 set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3829         }
3830 }
3831
3832 static void ena_update_hints(struct ena_adapter *adapter,
3833                              struct ena_admin_ena_hw_hints *hints)
3834 {
3835         struct net_device *netdev = adapter->netdev;
3836
3837         if (hints->admin_completion_tx_timeout)
3838                 adapter->ena_dev->admin_queue.completion_timeout =
3839                         hints->admin_completion_tx_timeout * 1000;
3840
3841         if (hints->mmio_read_timeout)
3842                 /* convert to usec */
3843                 adapter->ena_dev->mmio_read.reg_read_to =
3844                         hints->mmio_read_timeout * 1000;
3845
3846         if (hints->missed_tx_completion_count_threshold_to_reset)
3847                 adapter->missing_tx_completion_threshold =
3848                         hints->missed_tx_completion_count_threshold_to_reset;
3849
3850         if (hints->missing_tx_completion_timeout) {
3851                 if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3852                         adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT;
3853                 else
3854                         adapter->missing_tx_completion_to =
3855                                 msecs_to_jiffies(hints->missing_tx_completion_timeout);
3856         }
3857
3858         if (hints->netdev_wd_timeout)
3859                 netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout);
3860
3861         if (hints->driver_watchdog_timeout) {
3862                 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3863                         adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3864                 else
3865                         adapter->keep_alive_timeout =
3866                                 msecs_to_jiffies(hints->driver_watchdog_timeout);
3867         }
3868 }
3869
3870 static void ena_update_host_info(struct ena_admin_host_info *host_info,
3871                                  struct net_device *netdev)
3872 {
3873         host_info->supported_network_features[0] =
3874                 netdev->features & GENMASK_ULL(31, 0);
3875         host_info->supported_network_features[1] =
3876                 (netdev->features & GENMASK_ULL(63, 32)) >> 32;
3877 }
3878
3879 static void ena_timer_service(struct timer_list *t)
3880 {
3881         struct ena_adapter *adapter = from_timer(adapter, t, timer_service);
3882         u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;
3883         struct ena_admin_host_info *host_info =
3884                 adapter->ena_dev->host_attr.host_info;
3885
3886         check_for_missing_keep_alive(adapter);
3887
3888         check_for_admin_com_state(adapter);
3889
3890         check_for_missing_completions(adapter);
3891
3892         check_for_empty_rx_ring(adapter);
3893
3894         if (debug_area)
3895                 ena_dump_stats_to_buf(adapter, debug_area);
3896
3897         if (host_info)
3898                 ena_update_host_info(host_info, adapter->netdev);
3899
3900         if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3901                 netif_err(adapter, drv, adapter->netdev,
3902                           "Trigger reset is on\n");
3903                 ena_dump_stats_to_dmesg(adapter);
3904                 queue_work(ena_wq, &adapter->reset_task);
3905                 return;
3906         }
3907
3908         /* Reset the timer */
3909         mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3910 }
3911
3912 static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,
3913                                      struct ena_com_dev *ena_dev,
3914                                      struct ena_com_dev_get_features_ctx *get_feat_ctx)
3915 {
3916         u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
3917
3918         if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
3919                 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
3920                         &get_feat_ctx->max_queue_ext.max_queue_ext;
3921                 io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num,
3922                                   max_queue_ext->max_rx_cq_num);
3923
3924                 io_tx_sq_num = max_queue_ext->max_tx_sq_num;
3925                 io_tx_cq_num = max_queue_ext->max_tx_cq_num;
3926         } else {
3927                 struct ena_admin_queue_feature_desc *max_queues =
3928                         &get_feat_ctx->max_queues;
3929                 io_tx_sq_num = max_queues->max_sq_num;
3930                 io_tx_cq_num = max_queues->max_cq_num;
3931                 io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num);
3932         }
3933
3934         /* In case of LLQ use the llq fields for the tx SQ/CQ */
3935         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3936                 io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
3937
3938         max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
3939         max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num);
3940         max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num);
3941         max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);
3942         /* 1 IRQ for for mgmnt and 1 IRQs for each IO direction */
3943         max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);
3944         if (unlikely(!max_num_io_queues)) {
3945                 dev_err(&pdev->dev, "The device doesn't have io queues\n");
3946                 return -EFAULT;
3947         }
3948
3949         return max_num_io_queues;
3950 }
3951
3952 static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
3953                                  struct net_device *netdev)
3954 {
3955         netdev_features_t dev_features = 0;
3956
3957         /* Set offload features */
3958         if (feat->offload.tx &
3959                 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
3960                 dev_features |= NETIF_F_IP_CSUM;
3961
3962         if (feat->offload.tx &
3963                 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
3964                 dev_features |= NETIF_F_IPV6_CSUM;
3965
3966         if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
3967                 dev_features |= NETIF_F_TSO;
3968
3969         if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
3970                 dev_features |= NETIF_F_TSO6;
3971
3972         if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
3973                 dev_features |= NETIF_F_TSO_ECN;
3974
3975         if (feat->offload.rx_supported &
3976                 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
3977                 dev_features |= NETIF_F_RXCSUM;
3978
3979         if (feat->offload.rx_supported &
3980                 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
3981                 dev_features |= NETIF_F_RXCSUM;
3982
3983         netdev->features =
3984                 dev_features |
3985                 NETIF_F_SG |
3986                 NETIF_F_RXHASH |
3987                 NETIF_F_HIGHDMA;
3988
3989         netdev->hw_features |= netdev->features;
3990         netdev->vlan_features |= netdev->features;
3991 }
3992
3993 static void ena_set_conf_feat_params(struct ena_adapter *adapter,
3994                                      struct ena_com_dev_get_features_ctx *feat)
3995 {
3996         struct net_device *netdev = adapter->netdev;
3997
3998         /* Copy mac address */
3999         if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {
4000                 eth_hw_addr_random(netdev);
4001                 ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
4002         } else {
4003                 ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);
4004                 ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
4005         }
4006
4007         /* Set offload features */
4008         ena_set_dev_offloads(feat, netdev);
4009
4010         adapter->max_mtu = feat->dev_attr.max_mtu;
4011         netdev->max_mtu = adapter->max_mtu;
4012         netdev->min_mtu = ENA_MIN_MTU;
4013 }
4014
4015 static int ena_rss_init_default(struct ena_adapter *adapter)
4016 {
4017         struct ena_com_dev *ena_dev = adapter->ena_dev;
4018         struct device *dev = &adapter->pdev->dev;
4019         int rc, i;
4020         u32 val;
4021
4022         rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
4023         if (unlikely(rc)) {
4024                 dev_err(dev, "Cannot init indirect table\n");
4025                 goto err_rss_init;
4026         }
4027
4028         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
4029                 val = ethtool_rxfh_indir_default(i, adapter->num_io_queues);
4030                 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
4031                                                        ENA_IO_RXQ_IDX(val));
4032                 if (unlikely(rc && (rc != -EOPNOTSUPP))) {
4033                         dev_err(dev, "Cannot fill indirect table\n");
4034                         goto err_fill_indir;
4035                 }
4036         }
4037
4038         rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL,
4039                                         ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
4040         if (unlikely(rc && (rc != -EOPNOTSUPP))) {
4041                 dev_err(dev, "Cannot fill hash function\n");
4042                 goto err_fill_indir;
4043         }
4044
4045         rc = ena_com_set_default_hash_ctrl(ena_dev);
4046         if (unlikely(rc && (rc != -EOPNOTSUPP))) {
4047                 dev_err(dev, "Cannot fill hash control\n");
4048                 goto err_fill_indir;
4049         }
4050
4051         return 0;
4052
4053 err_fill_indir:
4054         ena_com_rss_destroy(ena_dev);
4055 err_rss_init:
4056
4057         return rc;
4058 }
4059
4060 static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
4061 {
4062         int release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
4063
4064         pci_release_selected_regions(pdev, release_bars);
4065 }
4066
4067
4068 static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx)
4069 {
4070         struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq;
4071         struct ena_com_dev *ena_dev = ctx->ena_dev;
4072         u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
4073         u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
4074         u32 max_tx_queue_size;
4075         u32 max_rx_queue_size;
4076
4077         if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
4078                 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
4079                         &ctx->get_feat_ctx->max_queue_ext.max_queue_ext;
4080                 max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
4081                                           max_queue_ext->max_rx_sq_depth);
4082                 max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
4083
4084                 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4085                         max_tx_queue_size = min_t(u32, max_tx_queue_size,
4086                                                   llq->max_llq_depth);
4087                 else
4088                         max_tx_queue_size = min_t(u32, max_tx_queue_size,
4089                                                   max_queue_ext->max_tx_sq_depth);
4090
4091                 ctx->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4092                                              max_queue_ext->max_per_packet_tx_descs);
4093                 ctx->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4094                                              max_queue_ext->max_per_packet_rx_descs);
4095         } else {
4096                 struct ena_admin_queue_feature_desc *max_queues =
4097                         &ctx->get_feat_ctx->max_queues;
4098                 max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
4099                                           max_queues->max_sq_depth);
4100                 max_tx_queue_size = max_queues->max_cq_depth;
4101
4102                 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4103                         max_tx_queue_size = min_t(u32, max_tx_queue_size,
4104                                                   llq->max_llq_depth);
4105                 else
4106                         max_tx_queue_size = min_t(u32, max_tx_queue_size,
4107                                                   max_queues->max_sq_depth);
4108
4109                 ctx->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4110                                              max_queues->max_packet_tx_descs);
4111                 ctx->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4112                                              max_queues->max_packet_rx_descs);
4113         }
4114
4115         max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
4116         max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
4117
4118         tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
4119                                   max_tx_queue_size);
4120         rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
4121                                   max_rx_queue_size);
4122
4123         tx_queue_size = rounddown_pow_of_two(tx_queue_size);
4124         rx_queue_size = rounddown_pow_of_two(rx_queue_size);
4125
4126         ctx->max_tx_queue_size = max_tx_queue_size;
4127         ctx->max_rx_queue_size = max_rx_queue_size;
4128         ctx->tx_queue_size = tx_queue_size;
4129         ctx->rx_queue_size = rx_queue_size;
4130
4131         return 0;
4132 }
4133
4134 /* ena_probe - Device Initialization Routine
4135  * @pdev: PCI device information struct
4136  * @ent: entry in ena_pci_tbl
4137  *
4138  * Returns 0 on success, negative on failure
4139  *
4140  * ena_probe initializes an adapter identified by a pci_dev structure.
4141  * The OS initialization, configuring of the adapter private structure,
4142  * and a hardware reset occur.
4143  */
4144 static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4145 {
4146         struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 };
4147         struct ena_com_dev_get_features_ctx get_feat_ctx;
4148         struct ena_com_dev *ena_dev = NULL;
4149         struct ena_adapter *adapter;
4150         struct net_device *netdev;
4151         static int adapters_found;
4152         u32 max_num_io_queues;
4153         char *queue_type_str;
4154         bool wd_state;
4155         int bars, rc;
4156
4157         dev_dbg(&pdev->dev, "%s\n", __func__);
4158
4159         rc = pci_enable_device_mem(pdev);
4160         if (rc) {
4161                 dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
4162                 return rc;
4163         }
4164
4165         pci_set_master(pdev);
4166
4167         ena_dev = vzalloc(sizeof(*ena_dev));
4168         if (!ena_dev) {
4169                 rc = -ENOMEM;
4170                 goto err_disable_device;
4171         }
4172
4173         bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
4174         rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
4175         if (rc) {
4176                 dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
4177                         rc);
4178                 goto err_free_ena_dev;
4179         }
4180
4181         ena_dev->reg_bar = devm_ioremap(&pdev->dev,
4182                                         pci_resource_start(pdev, ENA_REG_BAR),
4183                                         pci_resource_len(pdev, ENA_REG_BAR));
4184         if (!ena_dev->reg_bar) {
4185                 dev_err(&pdev->dev, "failed to remap regs bar\n");
4186                 rc = -EFAULT;
4187                 goto err_free_region;
4188         }
4189
4190         ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
4191
4192         ena_dev->dmadev = &pdev->dev;
4193
4194         rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state);
4195         if (rc) {
4196                 dev_err(&pdev->dev, "ena device init failed\n");
4197                 if (rc == -ETIME)
4198                         rc = -EPROBE_DEFER;
4199                 goto err_free_region;
4200         }
4201
4202         rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
4203         if (rc) {
4204                 dev_err(&pdev->dev, "ena llq bar mapping failed\n");
4205                 goto err_free_ena_dev;
4206         }
4207
4208         calc_queue_ctx.ena_dev = ena_dev;
4209         calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
4210         calc_queue_ctx.pdev = pdev;
4211
4212         /* Initial TX and RX interrupt delay. Assumes 1 usec granularity.
4213          * Updated during device initialization with the real granularity
4214          */
4215         ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
4216         ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
4217         ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
4218         max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);
4219         rc = ena_calc_io_queue_size(&calc_queue_ctx);
4220         if (rc || !max_num_io_queues) {
4221                 rc = -EFAULT;
4222                 goto err_device_destroy;
4223         }
4224
4225         /* dev zeroed in init_etherdev */
4226         netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), max_num_io_queues);
4227         if (!netdev) {
4228                 dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
4229                 rc = -ENOMEM;
4230                 goto err_device_destroy;
4231         }
4232
4233         SET_NETDEV_DEV(netdev, &pdev->dev);
4234
4235         adapter = netdev_priv(netdev);
4236         pci_set_drvdata(pdev, adapter);
4237
4238         adapter->ena_dev = ena_dev;
4239         adapter->netdev = netdev;
4240         adapter->pdev = pdev;
4241
4242         ena_set_conf_feat_params(adapter, &get_feat_ctx);
4243
4244         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
4245         adapter->reset_reason = ENA_REGS_RESET_NORMAL;
4246
4247         adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size;
4248         adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size;
4249         adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;
4250         adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;
4251         adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
4252         adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
4253
4254         adapter->num_io_queues = max_num_io_queues;
4255         adapter->max_num_io_queues = max_num_io_queues;
4256         adapter->last_monitored_tx_qid = 0;
4257
4258         adapter->xdp_first_ring = 0;
4259         adapter->xdp_num_queues = 0;
4260
4261         adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
4262         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4263                 adapter->disable_meta_caching =
4264                         !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
4265                            BIT(ENA_ADMIN_DISABLE_META_CACHING));
4266
4267         adapter->wd_state = wd_state;
4268
4269         snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
4270
4271         rc = ena_com_init_interrupt_moderation(adapter->ena_dev);
4272         if (rc) {
4273                 dev_err(&pdev->dev,
4274                         "Failed to query interrupt moderation feature\n");
4275                 goto err_netdev_destroy;
4276         }
4277         ena_init_io_rings(adapter,
4278                           0,
4279                           adapter->xdp_num_queues +
4280                           adapter->num_io_queues);
4281
4282         netdev->netdev_ops = &ena_netdev_ops;
4283         netdev->watchdog_timeo = TX_TIMEOUT;
4284         ena_set_ethtool_ops(netdev);
4285
4286         netdev->priv_flags |= IFF_UNICAST_FLT;
4287
4288         u64_stats_init(&adapter->syncp);
4289
4290         rc = ena_enable_msix_and_set_admin_interrupts(adapter);
4291         if (rc) {
4292                 dev_err(&pdev->dev,
4293                         "Failed to enable and set the admin interrupts\n");
4294                 goto err_worker_destroy;
4295         }
4296         rc = ena_rss_init_default(adapter);
4297         if (rc && (rc != -EOPNOTSUPP)) {
4298                 dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
4299                 goto err_free_msix;
4300         }
4301
4302         ena_config_debug_area(adapter);
4303
4304         memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
4305
4306         netif_carrier_off(netdev);
4307
4308         rc = register_netdev(netdev);
4309         if (rc) {
4310                 dev_err(&pdev->dev, "Cannot register net device\n");
4311                 goto err_rss;
4312         }
4313
4314         INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
4315
4316         adapter->last_keep_alive_jiffies = jiffies;
4317         adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
4318         adapter->missing_tx_completion_to = TX_TIMEOUT;
4319         adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS;
4320
4321         ena_update_hints(adapter, &get_feat_ctx.hw_hints);
4322
4323         timer_setup(&adapter->timer_service, ena_timer_service, 0);
4324         mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
4325
4326         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
4327                 queue_type_str = "Regular";
4328         else
4329                 queue_type_str = "Low Latency";
4330
4331         dev_info(&pdev->dev,
4332                  "%s found at mem %lx, mac addr %pM, Placement policy: %s\n",
4333                  DEVICE_NAME, (long)pci_resource_start(pdev, 0),
4334                  netdev->dev_addr, queue_type_str);
4335
4336         set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
4337
4338         adapters_found++;
4339
4340         return 0;
4341
4342 err_rss:
4343         ena_com_delete_debug_area(ena_dev);
4344         ena_com_rss_destroy(ena_dev);
4345 err_free_msix:
4346         ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
4347         /* stop submitting admin commands on a device that was reset */
4348         ena_com_set_admin_running_state(ena_dev, false);
4349         ena_free_mgmnt_irq(adapter);
4350         ena_disable_msix(adapter);
4351 err_worker_destroy:
4352         del_timer(&adapter->timer_service);
4353 err_netdev_destroy:
4354         free_netdev(netdev);
4355 err_device_destroy:
4356         ena_com_delete_host_info(ena_dev);
4357         ena_com_admin_destroy(ena_dev);
4358 err_free_region:
4359         ena_release_bars(ena_dev, pdev);
4360 err_free_ena_dev:
4361         vfree(ena_dev);
4362 err_disable_device:
4363         pci_disable_device(pdev);
4364         return rc;
4365 }
4366
4367 /*****************************************************************************/
4368
4369 /* __ena_shutoff - Helper used in both PCI remove/shutdown routines
4370  * @pdev: PCI device information struct
4371  * @shutdown: Is it a shutdown operation? If false, means it is a removal
4372  *
4373  * __ena_shutoff is a helper routine that does the real work on shutdown and
4374  * removal paths; the difference between those paths is with regards to whether
4375  * dettach or unregister the netdevice.
4376  */
4377 static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)
4378 {
4379         struct ena_adapter *adapter = pci_get_drvdata(pdev);
4380         struct ena_com_dev *ena_dev;
4381         struct net_device *netdev;
4382
4383         ena_dev = adapter->ena_dev;
4384         netdev = adapter->netdev;
4385
4386 #ifdef CONFIG_RFS_ACCEL
4387         if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) {
4388                 free_irq_cpu_rmap(netdev->rx_cpu_rmap);
4389                 netdev->rx_cpu_rmap = NULL;
4390         }
4391 #endif /* CONFIG_RFS_ACCEL */
4392         del_timer_sync(&adapter->timer_service);
4393
4394         cancel_work_sync(&adapter->reset_task);
4395
4396         rtnl_lock(); /* lock released inside the below if-else block */
4397         adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;
4398         ena_destroy_device(adapter, true);
4399         if (shutdown) {
4400                 netif_device_detach(netdev);
4401                 dev_close(netdev);
4402                 rtnl_unlock();
4403         } else {
4404                 rtnl_unlock();
4405                 unregister_netdev(netdev);
4406                 free_netdev(netdev);
4407         }
4408
4409         ena_com_rss_destroy(ena_dev);
4410
4411         ena_com_delete_debug_area(ena_dev);
4412
4413         ena_com_delete_host_info(ena_dev);
4414
4415         ena_release_bars(ena_dev, pdev);
4416
4417         pci_disable_device(pdev);
4418
4419         vfree(ena_dev);
4420 }
4421
4422 /* ena_remove - Device Removal Routine
4423  * @pdev: PCI device information struct
4424  *
4425  * ena_remove is called by the PCI subsystem to alert the driver
4426  * that it should release a PCI device.
4427  */
4428
4429 static void ena_remove(struct pci_dev *pdev)
4430 {
4431         __ena_shutoff(pdev, false);
4432 }
4433
4434 /* ena_shutdown - Device Shutdown Routine
4435  * @pdev: PCI device information struct
4436  *
4437  * ena_shutdown is called by the PCI subsystem to alert the driver that
4438  * a shutdown/reboot (or kexec) is happening and device must be disabled.
4439  */
4440
4441 static void ena_shutdown(struct pci_dev *pdev)
4442 {
4443         __ena_shutoff(pdev, true);
4444 }
4445
4446 /* ena_suspend - PM suspend callback
4447  * @dev_d: Device information struct
4448  */
4449 static int __maybe_unused ena_suspend(struct device *dev_d)
4450 {
4451         struct pci_dev *pdev = to_pci_dev(dev_d);
4452         struct ena_adapter *adapter = pci_get_drvdata(pdev);
4453
4454         u64_stats_update_begin(&adapter->syncp);
4455         adapter->dev_stats.suspend++;
4456         u64_stats_update_end(&adapter->syncp);
4457
4458         rtnl_lock();
4459         if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
4460                 dev_err(&pdev->dev,
4461                         "ignoring device reset request as the device is being suspended\n");
4462                 clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
4463         }
4464         ena_destroy_device(adapter, true);
4465         rtnl_unlock();
4466         return 0;
4467 }
4468
4469 /* ena_resume - PM resume callback
4470  * @dev_d: Device information struct
4471  */
4472 static int __maybe_unused ena_resume(struct device *dev_d)
4473 {
4474         struct ena_adapter *adapter = dev_get_drvdata(dev_d);
4475         int rc;
4476
4477         u64_stats_update_begin(&adapter->syncp);
4478         adapter->dev_stats.resume++;
4479         u64_stats_update_end(&adapter->syncp);
4480
4481         rtnl_lock();
4482         rc = ena_restore_device(adapter);
4483         rtnl_unlock();
4484         return rc;
4485 }
4486
4487 static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume);
4488
4489 static struct pci_driver ena_pci_driver = {
4490         .name           = DRV_MODULE_NAME,
4491         .id_table       = ena_pci_tbl,
4492         .probe          = ena_probe,
4493         .remove         = ena_remove,
4494         .shutdown       = ena_shutdown,
4495         .driver.pm      = &ena_pm_ops,
4496         .sriov_configure = pci_sriov_configure_simple,
4497 };
4498
4499 static int __init ena_init(void)
4500 {
4501         ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
4502         if (!ena_wq) {
4503                 pr_err("Failed to create workqueue\n");
4504                 return -ENOMEM;
4505         }
4506
4507         return pci_register_driver(&ena_pci_driver);
4508 }
4509
4510 static void __exit ena_cleanup(void)
4511 {
4512         pci_unregister_driver(&ena_pci_driver);
4513
4514         if (ena_wq) {
4515                 destroy_workqueue(ena_wq);
4516                 ena_wq = NULL;
4517         }
4518 }
4519
4520 /******************************************************************************
4521  ******************************** AENQ Handlers *******************************
4522  *****************************************************************************/
4523 /* ena_update_on_link_change:
4524  * Notify the network interface about the change in link status
4525  */
4526 static void ena_update_on_link_change(void *adapter_data,
4527                                       struct ena_admin_aenq_entry *aenq_e)
4528 {
4529         struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4530         struct ena_admin_aenq_link_change_desc *aenq_desc =
4531                 (struct ena_admin_aenq_link_change_desc *)aenq_e;
4532         int status = aenq_desc->flags &
4533                 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
4534
4535         if (status) {
4536                 netdev_dbg(adapter->netdev, "%s\n", __func__);
4537                 set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4538                 if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags))
4539                         netif_carrier_on(adapter->netdev);
4540         } else {
4541                 clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4542                 netif_carrier_off(adapter->netdev);
4543         }
4544 }
4545
4546 static void ena_keep_alive_wd(void *adapter_data,
4547                               struct ena_admin_aenq_entry *aenq_e)
4548 {
4549         struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4550         struct ena_admin_aenq_keep_alive_desc *desc;
4551         u64 rx_drops;
4552         u64 tx_drops;
4553
4554         desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
4555         adapter->last_keep_alive_jiffies = jiffies;
4556
4557         rx_drops = ((u64)desc->rx_drops_high << 32) | desc->rx_drops_low;
4558         tx_drops = ((u64)desc->tx_drops_high << 32) | desc->tx_drops_low;
4559
4560         u64_stats_update_begin(&adapter->syncp);
4561         adapter->dev_stats.rx_drops = rx_drops;
4562         adapter->dev_stats.tx_drops = tx_drops;
4563         u64_stats_update_end(&adapter->syncp);
4564 }
4565
4566 static void ena_notification(void *adapter_data,
4567                              struct ena_admin_aenq_entry *aenq_e)
4568 {
4569         struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4570         struct ena_admin_ena_hw_hints *hints;
4571
4572         WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
4573              "Invalid group(%x) expected %x\n",
4574              aenq_e->aenq_common_desc.group,
4575              ENA_ADMIN_NOTIFICATION);
4576
4577         switch (aenq_e->aenq_common_desc.syndrom) {
4578         case ENA_ADMIN_UPDATE_HINTS:
4579                 hints = (struct ena_admin_ena_hw_hints *)
4580                         (&aenq_e->inline_data_w4);
4581                 ena_update_hints(adapter, hints);
4582                 break;
4583         default:
4584                 netif_err(adapter, drv, adapter->netdev,
4585                           "Invalid aenq notification link state %d\n",
4586                           aenq_e->aenq_common_desc.syndrom);
4587         }
4588 }
4589
4590 /* This handler will called for unknown event group or unimplemented handlers*/
4591 static void unimplemented_aenq_handler(void *data,
4592                                        struct ena_admin_aenq_entry *aenq_e)
4593 {
4594         struct ena_adapter *adapter = (struct ena_adapter *)data;
4595
4596         netif_err(adapter, drv, adapter->netdev,
4597                   "Unknown event was received or event with unimplemented handler\n");
4598 }
4599
4600 static struct ena_aenq_handlers aenq_handlers = {
4601         .handlers = {
4602                 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
4603                 [ENA_ADMIN_NOTIFICATION] = ena_notification,
4604                 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
4605         },
4606         .unimplemented_handler = unimplemented_aenq_handler
4607 };
4608
4609 module_init(ena_init);
4610 module_exit(ena_cleanup);