drivers/net/ethernet/amazon/ena/ena_ethtool.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
   4  */
   5
   6 #include <linux/ethtool.h>
   7 #include <linux/pci.h>
   8
   9 #include "ena_netdev.h"
  10 #include "ena_xdp.h"
  11
  12 struct ena_stats {
  13         char name[ETH_GSTRING_LEN];
  14         int stat_offset;
  15 };
  16
  17 #define ENA_STAT_ENA_COM_ENTRY(stat) { \
  18         .name = #stat, \
  19         .stat_offset = offsetof(struct ena_com_stats_admin, stat) / sizeof(u64) \
  20 }
  21
  22 #define ENA_STAT_ENTRY(stat, stat_type) { \
  23         .name = #stat, \
  24         .stat_offset = offsetof(struct ena_stats_##stat_type, stat) / sizeof(u64) \
  25 }
  26
  27 #define ENA_STAT_HW_ENTRY(stat, stat_type) { \
  28         .name = #stat, \
  29         .stat_offset = offsetof(struct ena_admin_##stat_type, stat) / sizeof(u64) \
  30 }
  31
  32 #define ENA_STAT_RX_ENTRY(stat) \
  33         ENA_STAT_ENTRY(stat, rx)
  34
  35 #define ENA_STAT_TX_ENTRY(stat) \
  36         ENA_STAT_ENTRY(stat, tx)
  37
  38 #define ENA_STAT_GLOBAL_ENTRY(stat) \
  39         ENA_STAT_ENTRY(stat, dev)
  40
  41 #define ENA_STAT_ENI_ENTRY(stat) \
  42         ENA_STAT_HW_ENTRY(stat, eni_stats)
  43
  44 static const struct ena_stats ena_stats_global_strings[] = {
  45         ENA_STAT_GLOBAL_ENTRY(tx_timeout),
  46         ENA_STAT_GLOBAL_ENTRY(suspend),
  47         ENA_STAT_GLOBAL_ENTRY(resume),
  48         ENA_STAT_GLOBAL_ENTRY(wd_expired),
  49         ENA_STAT_GLOBAL_ENTRY(interface_up),
  50         ENA_STAT_GLOBAL_ENTRY(interface_down),
  51         ENA_STAT_GLOBAL_ENTRY(admin_q_pause),
  52         ENA_STAT_GLOBAL_ENTRY(reset_fail),
  53 };
  54
  55 static const struct ena_stats ena_stats_eni_strings[] = {
  56         ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded),
  57         ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded),
  58         ENA_STAT_ENI_ENTRY(pps_allowance_exceeded),
  59         ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded),
  60         ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded),
  61 };
  62
  63 static const struct ena_stats ena_stats_tx_strings[] = {
  64         ENA_STAT_TX_ENTRY(cnt),
  65         ENA_STAT_TX_ENTRY(bytes),
  66         ENA_STAT_TX_ENTRY(queue_stop),
  67         ENA_STAT_TX_ENTRY(queue_wakeup),
  68         ENA_STAT_TX_ENTRY(dma_mapping_err),
  69         ENA_STAT_TX_ENTRY(linearize),
  70         ENA_STAT_TX_ENTRY(linearize_failed),
  71         ENA_STAT_TX_ENTRY(napi_comp),
  72         ENA_STAT_TX_ENTRY(tx_poll),
  73         ENA_STAT_TX_ENTRY(doorbells),
  74         ENA_STAT_TX_ENTRY(prepare_ctx_err),
  75         ENA_STAT_TX_ENTRY(bad_req_id),
  76         ENA_STAT_TX_ENTRY(llq_buffer_copy),
  77         ENA_STAT_TX_ENTRY(missed_tx),
  78         ENA_STAT_TX_ENTRY(unmask_interrupt),
  79 };
  80
  81 static const struct ena_stats ena_stats_rx_strings[] = {
  82         ENA_STAT_RX_ENTRY(cnt),
  83         ENA_STAT_RX_ENTRY(bytes),
  84         ENA_STAT_RX_ENTRY(rx_copybreak_pkt),
  85         ENA_STAT_RX_ENTRY(csum_good),
  86         ENA_STAT_RX_ENTRY(refil_partial),
  87         ENA_STAT_RX_ENTRY(csum_bad),
  88         ENA_STAT_RX_ENTRY(page_alloc_fail),
  89         ENA_STAT_RX_ENTRY(skb_alloc_fail),
  90         ENA_STAT_RX_ENTRY(dma_mapping_err),
  91         ENA_STAT_RX_ENTRY(bad_desc_num),
  92         ENA_STAT_RX_ENTRY(bad_req_id),
  93         ENA_STAT_RX_ENTRY(empty_rx_ring),
  94         ENA_STAT_RX_ENTRY(csum_unchecked),
  95         ENA_STAT_RX_ENTRY(xdp_aborted),
  96         ENA_STAT_RX_ENTRY(xdp_drop),
  97         ENA_STAT_RX_ENTRY(xdp_pass),
  98         ENA_STAT_RX_ENTRY(xdp_tx),
  99         ENA_STAT_RX_ENTRY(xdp_invalid),
 100         ENA_STAT_RX_ENTRY(xdp_redirect),
 101 };
 102
 103 static const struct ena_stats ena_stats_ena_com_strings[] = {
 104         ENA_STAT_ENA_COM_ENTRY(aborted_cmd),
 105         ENA_STAT_ENA_COM_ENTRY(submitted_cmd),
 106         ENA_STAT_ENA_COM_ENTRY(completed_cmd),
 107         ENA_STAT_ENA_COM_ENTRY(out_of_space),
 108         ENA_STAT_ENA_COM_ENTRY(no_completion),
 109 };
 110
 111 #define ENA_STATS_ARRAY_GLOBAL          ARRAY_SIZE(ena_stats_global_strings)
 112 #define ENA_STATS_ARRAY_TX              ARRAY_SIZE(ena_stats_tx_strings)
 113 #define ENA_STATS_ARRAY_RX              ARRAY_SIZE(ena_stats_rx_strings)
 114 #define ENA_STATS_ARRAY_ENA_COM         ARRAY_SIZE(ena_stats_ena_com_strings)
 115 #define ENA_STATS_ARRAY_ENI(adapter)    ARRAY_SIZE(ena_stats_eni_strings)
 116
 117 static void ena_safe_update_stat(u64 *src, u64 *dst,
 118                                  struct u64_stats_sync *syncp)
 119 {
 120         unsigned int start;
 121
 122         do {
 123                 start = u64_stats_fetch_begin(syncp);
 124                 *(dst) = *src;
 125         } while (u64_stats_fetch_retry(syncp, start));
 126 }
 127
 128 static void ena_queue_stats(struct ena_adapter *adapter, u64 **data)
 129 {
 130         const struct ena_stats *ena_stats;
 131         struct ena_ring *ring;
 132
 133         u64 *ptr;
 134         int i, j;
 135
 136         for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
 137                 /* Tx stats */
 138                 ring = &adapter->tx_ring[i];
 139
 140                 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
 141                         ena_stats = &ena_stats_tx_strings[j];
 142
 143                         ptr = (u64 *)&ring->tx_stats + ena_stats->stat_offset;
 144
 145                         ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
 146                 }
 147                 /* XDP TX queues don't have a RX queue counterpart */
 148                 if (!ENA_IS_XDP_INDEX(adapter, i)) {
 149                         /* Rx stats */
 150                         ring = &adapter->rx_ring[i];
 151
 152                         for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
 153                                 ena_stats = &ena_stats_rx_strings[j];
 154
 155                                 ptr = (u64 *)&ring->rx_stats +
 156                                         ena_stats->stat_offset;
 157
 158                                 ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
 159                         }
 160                 }
 161         }
 162 }
 163
 164 static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data)
 165 {
 166         const struct ena_stats *ena_stats;
 167         u64 *ptr;
 168         int i;
 169
 170         for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
 171                 ena_stats = &ena_stats_ena_com_strings[i];
 172
 173                 ptr = (u64 *)&adapter->ena_dev->admin_queue.stats +
 174                         ena_stats->stat_offset;
 175
 176                 *(*data)++ = *ptr;
 177         }
 178 }
 179
 180 static void ena_get_stats(struct ena_adapter *adapter,
 181                           u64 *data,
 182                           bool eni_stats_needed)
 183 {
 184         const struct ena_stats *ena_stats;
 185         u64 *ptr;
 186         int i;
 187
 188         for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
 189                 ena_stats = &ena_stats_global_strings[i];
 190
 191                 ptr = (u64 *)&adapter->dev_stats + ena_stats->stat_offset;
 192
 193                 ena_safe_update_stat(ptr, data++, &adapter->syncp);
 194         }
 195
 196         if (eni_stats_needed) {
 197                 ena_update_hw_stats(adapter);
 198                 for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) {
 199                         ena_stats = &ena_stats_eni_strings[i];
 200
 201                         ptr = (u64 *)&adapter->eni_stats +
 202                                 ena_stats->stat_offset;
 203
 204                         ena_safe_update_stat(ptr, data++, &adapter->syncp);
 205                 }
 206         }
 207
 208         ena_queue_stats(adapter, &data);
 209         ena_dev_admin_queue_stats(adapter, &data);
 210 }
 211
 212 static void ena_get_ethtool_stats(struct net_device *netdev,
 213                                   struct ethtool_stats *stats,
 214                                   u64 *data)
 215 {
 216         struct ena_adapter *adapter = netdev_priv(netdev);
 217         struct ena_com_dev *dev = adapter->ena_dev;
 218
 219         ena_get_stats(adapter, data, ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS));
 220 }
 221
 222 static int ena_get_sw_stats_count(struct ena_adapter *adapter)
 223 {
 224         return adapter->num_io_queues * (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX)
 225                 + adapter->xdp_num_queues * ENA_STATS_ARRAY_TX
 226                 + ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM;
 227 }
 228
 229 static int ena_get_hw_stats_count(struct ena_adapter *adapter)
 230 {
 231         bool supported = ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENI_STATS);
 232
 233         return ENA_STATS_ARRAY_ENI(adapter) * supported;
 234 }
 235
 236 int ena_get_sset_count(struct net_device *netdev, int sset)
 237 {
 238         struct ena_adapter *adapter = netdev_priv(netdev);
 239
 240         switch (sset) {
 241         case ETH_SS_STATS:
 242                 return ena_get_sw_stats_count(adapter) +
 243                        ena_get_hw_stats_count(adapter);
 244         }
 245
 246         return -EOPNOTSUPP;
 247 }
 248
 249 static void ena_queue_strings(struct ena_adapter *adapter, u8 **data)
 250 {
 251         const struct ena_stats *ena_stats;
 252         bool is_xdp;
 253         int i, j;
 254
 255         for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
 256                 is_xdp = ENA_IS_XDP_INDEX(adapter, i);
 257                 /* Tx stats */
 258                 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
 259                         ena_stats = &ena_stats_tx_strings[j];
 260
 261                         ethtool_sprintf(data,
 262                                         "queue_%u_%s_%s", i,
 263                                         is_xdp ? "xdp_tx" : "tx",
 264                                         ena_stats->name);
 265                 }
 266
 267                 /* In XDP there isn't an RX queue counterpart */
 268                 if (is_xdp)
 269                         continue;
 270
 271                 for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
 272                         ena_stats = &ena_stats_rx_strings[j];
 273
 274                         ethtool_sprintf(data, "queue_%u_rx_%s", i, ena_stats->name);
 275                 }
 276         }
 277 }
 278
 279 static void ena_com_dev_strings(u8 **data)
 280 {
 281         const struct ena_stats *ena_stats;
 282         int i;
 283
 284         for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
 285                 ena_stats = &ena_stats_ena_com_strings[i];
 286
 287                 ethtool_sprintf(data,
 288                                 "ena_admin_q_%s", ena_stats->name);
 289         }
 290 }
 291
 292 static void ena_get_strings(struct ena_adapter *adapter,
 293                             u8 *data,
 294                             bool eni_stats_needed)
 295 {
 296         const struct ena_stats *ena_stats;
 297         int i;
 298
 299         for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
 300                 ena_stats = &ena_stats_global_strings[i];
 301                 ethtool_puts(&data, ena_stats->name);
 302         }
 303
 304         if (eni_stats_needed) {
 305                 for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) {
 306                         ena_stats = &ena_stats_eni_strings[i];
 307                         ethtool_puts(&data, ena_stats->name);
 308                 }
 309         }
 310
 311         ena_queue_strings(adapter, &data);
 312         ena_com_dev_strings(&data);
 313 }
 314
 315 static void ena_get_ethtool_strings(struct net_device *netdev,
 316                                     u32 sset,
 317                                     u8 *data)
 318 {
 319         struct ena_adapter *adapter = netdev_priv(netdev);
 320         struct ena_com_dev *dev = adapter->ena_dev;
 321
 322         switch (sset) {
 323         case ETH_SS_STATS:
 324                 ena_get_strings(adapter, data, ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS));
 325                 break;
 326         }
 327 }
 328
 329 static int ena_get_link_ksettings(struct net_device *netdev,
 330                                   struct ethtool_link_ksettings *link_ksettings)
 331 {
 332         struct ena_adapter *adapter = netdev_priv(netdev);
 333         struct ena_com_dev *ena_dev = adapter->ena_dev;
 334         struct ena_admin_get_feature_link_desc *link;
 335         struct ena_admin_get_feat_resp feat_resp;
 336         int rc;
 337
 338         rc = ena_com_get_link_params(ena_dev, &feat_resp);
 339         if (rc)
 340                 return rc;
 341
 342         link = &feat_resp.u.link;
 343         link_ksettings->base.speed = link->speed;
 344
 345         if (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) {
 346                 ethtool_link_ksettings_add_link_mode(link_ksettings,
 347                                                      supported, Autoneg);
 348                 ethtool_link_ksettings_add_link_mode(link_ksettings,
 349                                                      supported, Autoneg);
 350         }
 351
 352         link_ksettings->base.autoneg =
 353                 (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) ?
 354                 AUTONEG_ENABLE : AUTONEG_DISABLE;
 355
 356         link_ksettings->base.duplex = DUPLEX_FULL;
 357
 358         return 0;
 359 }
 360
 361 static int ena_get_coalesce(struct net_device *net_dev,
 362                             struct ethtool_coalesce *coalesce,
 363                             struct kernel_ethtool_coalesce *kernel_coal,
 364                             struct netlink_ext_ack *extack)
 365 {
 366         struct ena_adapter *adapter = netdev_priv(net_dev);
 367         struct ena_com_dev *ena_dev = adapter->ena_dev;
 368
 369         if (!ena_com_interrupt_moderation_supported(ena_dev))
 370                 return -EOPNOTSUPP;
 371
 372         coalesce->tx_coalesce_usecs =
 373                 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) *
 374                         ena_dev->intr_delay_resolution;
 375
 376         coalesce->rx_coalesce_usecs =
 377                 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev)
 378                 * ena_dev->intr_delay_resolution;
 379
 380         coalesce->use_adaptive_rx_coalesce =
 381                 ena_com_get_adaptive_moderation_enabled(ena_dev);
 382
 383         return 0;
 384 }
 385
 386 static void ena_update_tx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter)
 387 {
 388         unsigned int val;
 389         int i;
 390
 391         val = ena_com_get_nonadaptive_moderation_interval_tx(adapter->ena_dev);
 392
 393         for (i = 0; i < adapter->num_io_queues; i++)
 394                 adapter->tx_ring[i].smoothed_interval = val;
 395 }
 396
 397 static void ena_update_rx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter)
 398 {
 399         unsigned int val;
 400         int i;
 401
 402         val = ena_com_get_nonadaptive_moderation_interval_rx(adapter->ena_dev);
 403
 404         for (i = 0; i < adapter->num_io_queues; i++)
 405                 adapter->rx_ring[i].smoothed_interval = val;
 406 }
 407
 408 static int ena_set_coalesce(struct net_device *net_dev,
 409                             struct ethtool_coalesce *coalesce,
 410                             struct kernel_ethtool_coalesce *kernel_coal,
 411                             struct netlink_ext_ack *extack)
 412 {
 413         struct ena_adapter *adapter = netdev_priv(net_dev);
 414         struct ena_com_dev *ena_dev = adapter->ena_dev;
 415         int rc;
 416
 417         if (!ena_com_interrupt_moderation_supported(ena_dev))
 418                 return -EOPNOTSUPP;
 419
 420         rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev,
 421                                                                coalesce->tx_coalesce_usecs);
 422         if (rc)
 423                 return rc;
 424
 425         ena_update_tx_rings_nonadaptive_intr_moderation(adapter);
 426
 427         rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
 428                                                                coalesce->rx_coalesce_usecs);
 429         if (rc)
 430                 return rc;
 431
 432         ena_update_rx_rings_nonadaptive_intr_moderation(adapter);
 433
 434         if (coalesce->use_adaptive_rx_coalesce &&
 435             !ena_com_get_adaptive_moderation_enabled(ena_dev))
 436                 ena_com_enable_adaptive_moderation(ena_dev);
 437
 438         if (!coalesce->use_adaptive_rx_coalesce &&
 439             ena_com_get_adaptive_moderation_enabled(ena_dev))
 440                 ena_com_disable_adaptive_moderation(ena_dev);
 441
 442         return 0;
 443 }
 444
 445 static u32 ena_get_msglevel(struct net_device *netdev)
 446 {
 447         struct ena_adapter *adapter = netdev_priv(netdev);
 448
 449         return adapter->msg_enable;
 450 }
 451
 452 static void ena_set_msglevel(struct net_device *netdev, u32 value)
 453 {
 454         struct ena_adapter *adapter = netdev_priv(netdev);
 455
 456         adapter->msg_enable = value;
 457 }
 458
 459 static void ena_get_drvinfo(struct net_device *dev,
 460                             struct ethtool_drvinfo *info)
 461 {
 462         struct ena_adapter *adapter = netdev_priv(dev);
 463         ssize_t ret = 0;
 464
 465         ret = strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 466         if (ret < 0)
 467                 netif_dbg(adapter, drv, dev,
 468                           "module name will be truncated, status = %zd\n", ret);
 469
 470         ret = strscpy(info->bus_info, pci_name(adapter->pdev),
 471                       sizeof(info->bus_info));
 472         if (ret < 0)
 473                 netif_dbg(adapter, drv, dev,
 474                           "bus info will be truncated, status = %zd\n", ret);
 475 }
 476
 477 static void ena_get_ringparam(struct net_device *netdev,
 478                               struct ethtool_ringparam *ring,
 479                               struct kernel_ethtool_ringparam *kernel_ring,
 480                               struct netlink_ext_ack *extack)
 481 {
 482         struct ena_adapter *adapter = netdev_priv(netdev);
 483
 484         ring->tx_max_pending = adapter->max_tx_ring_size;
 485         ring->rx_max_pending = adapter->max_rx_ring_size;
 486         if (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
 487                 bool large_llq_supported = adapter->large_llq_header_supported;
 488
 489                 kernel_ring->tx_push = true;
 490                 kernel_ring->tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
 491                 if (large_llq_supported)
 492                         kernel_ring->tx_push_buf_max_len = ENA_LLQ_LARGE_HEADER;
 493                 else
 494                         kernel_ring->tx_push_buf_max_len = ENA_LLQ_HEADER;
 495         } else {
 496                 kernel_ring->tx_push = false;
 497                 kernel_ring->tx_push_buf_max_len = 0;
 498                 kernel_ring->tx_push_buf_len = 0;
 499         }
 500
 501         ring->tx_pending = adapter->tx_ring[0].ring_size;
 502         ring->rx_pending = adapter->rx_ring[0].ring_size;
 503 }
 504
 505 static int ena_set_ringparam(struct net_device *netdev,
 506                              struct ethtool_ringparam *ring,
 507                              struct kernel_ethtool_ringparam *kernel_ring,
 508                              struct netlink_ext_ack *extack)
 509 {
 510         struct ena_adapter *adapter = netdev_priv(netdev);
 511         u32 new_tx_size, new_rx_size, new_tx_push_buf_len;
 512         bool changed = false;
 513
 514         new_tx_size = ring->tx_pending < ENA_MIN_RING_SIZE ?
 515                         ENA_MIN_RING_SIZE : ring->tx_pending;
 516         new_tx_size = rounddown_pow_of_two(new_tx_size);
 517
 518         new_rx_size = ring->rx_pending < ENA_MIN_RING_SIZE ?
 519                         ENA_MIN_RING_SIZE : ring->rx_pending;
 520         new_rx_size = rounddown_pow_of_two(new_rx_size);
 521
 522         changed |= new_tx_size != adapter->requested_tx_ring_size ||
 523                    new_rx_size != adapter->requested_rx_ring_size;
 524
 525         /* This value is ignored if LLQ is not supported */
 526         new_tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
 527
 528         if ((adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) !=
 529             kernel_ring->tx_push) {
 530                 NL_SET_ERR_MSG_MOD(extack, "Push mode state cannot be modified");
 531                 return -EINVAL;
 532         }
 533
 534         /* Validate that the push buffer is supported on the underlying device */
 535         if (kernel_ring->tx_push_buf_len) {
 536                 enum ena_admin_placement_policy_type placement;
 537
 538                 new_tx_push_buf_len = kernel_ring->tx_push_buf_len;
 539
 540                 placement = adapter->ena_dev->tx_mem_queue_type;
 541                 if (placement == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 542                         return -EOPNOTSUPP;
 543
 544                 if (new_tx_push_buf_len != ENA_LLQ_HEADER &&
 545                     new_tx_push_buf_len != ENA_LLQ_LARGE_HEADER) {
 546                         bool large_llq_sup = adapter->large_llq_header_supported;
 547                         char large_llq_size_str[40];
 548
 549                         snprintf(large_llq_size_str, 40, ", %lu", ENA_LLQ_LARGE_HEADER);
 550
 551                         NL_SET_ERR_MSG_FMT_MOD(extack,
 552                                                "Supported tx push buff values: [%lu%s]",
 553                                                ENA_LLQ_HEADER,
 554                                                large_llq_sup ? large_llq_size_str : "");
 555
 556                         return -EINVAL;
 557                 }
 558
 559                 changed |= new_tx_push_buf_len != adapter->ena_dev->tx_max_header_size;
 560         }
 561
 562         if (!changed)
 563                 return 0;
 564
 565         return ena_update_queue_params(adapter, new_tx_size, new_rx_size,
 566                                        new_tx_push_buf_len);
 567 }
 568
 569 static u32 ena_flow_hash_to_flow_type(u16 hash_fields)
 570 {
 571         u32 data = 0;
 572
 573         if (hash_fields & ENA_ADMIN_RSS_L2_DA)
 574                 data |= RXH_L2DA;
 575
 576         if (hash_fields & ENA_ADMIN_RSS_L3_DA)
 577                 data |= RXH_IP_DST;
 578
 579         if (hash_fields & ENA_ADMIN_RSS_L3_SA)
 580                 data |= RXH_IP_SRC;
 581
 582         if (hash_fields & ENA_ADMIN_RSS_L4_DP)
 583                 data |= RXH_L4_B_2_3;
 584
 585         if (hash_fields & ENA_ADMIN_RSS_L4_SP)
 586                 data |= RXH_L4_B_0_1;
 587
 588         return data;
 589 }
 590
 591 static u16 ena_flow_data_to_flow_hash(u32 hash_fields)
 592 {
 593         u16 data = 0;
 594
 595         if (hash_fields & RXH_L2DA)
 596                 data |= ENA_ADMIN_RSS_L2_DA;
 597
 598         if (hash_fields & RXH_IP_DST)
 599                 data |= ENA_ADMIN_RSS_L3_DA;
 600
 601         if (hash_fields & RXH_IP_SRC)
 602                 data |= ENA_ADMIN_RSS_L3_SA;
 603
 604         if (hash_fields & RXH_L4_B_2_3)
 605                 data |= ENA_ADMIN_RSS_L4_DP;
 606
 607         if (hash_fields & RXH_L4_B_0_1)
 608                 data |= ENA_ADMIN_RSS_L4_SP;
 609
 610         return data;
 611 }
 612
 613 static int ena_get_rss_hash(struct ena_com_dev *ena_dev,
 614                             struct ethtool_rxnfc *cmd)
 615 {
 616         enum ena_admin_flow_hash_proto proto;
 617         u16 hash_fields;
 618         int rc;
 619
 620         cmd->data = 0;
 621
 622         switch (cmd->flow_type) {
 623         case TCP_V4_FLOW:
 624                 proto = ENA_ADMIN_RSS_TCP4;
 625                 break;
 626         case UDP_V4_FLOW:
 627                 proto = ENA_ADMIN_RSS_UDP4;
 628                 break;
 629         case TCP_V6_FLOW:
 630                 proto = ENA_ADMIN_RSS_TCP6;
 631                 break;
 632         case UDP_V6_FLOW:
 633                 proto = ENA_ADMIN_RSS_UDP6;
 634                 break;
 635         case IPV4_FLOW:
 636                 proto = ENA_ADMIN_RSS_IP4;
 637                 break;
 638         case IPV6_FLOW:
 639                 proto = ENA_ADMIN_RSS_IP6;
 640                 break;
 641         case ETHER_FLOW:
 642                 proto = ENA_ADMIN_RSS_NOT_IP;
 643                 break;
 644         case AH_V4_FLOW:
 645         case ESP_V4_FLOW:
 646         case AH_V6_FLOW:
 647         case ESP_V6_FLOW:
 648         case SCTP_V4_FLOW:
 649         case AH_ESP_V4_FLOW:
 650                 return -EOPNOTSUPP;
 651         default:
 652                 return -EINVAL;
 653         }
 654
 655         rc = ena_com_get_hash_ctrl(ena_dev, proto, &hash_fields);
 656         if (rc)
 657                 return rc;
 658
 659         cmd->data = ena_flow_hash_to_flow_type(hash_fields);
 660
 661         return 0;
 662 }
 663
 664 static int ena_set_rss_hash(struct ena_com_dev *ena_dev,
 665                             struct ethtool_rxnfc *cmd)
 666 {
 667         enum ena_admin_flow_hash_proto proto;
 668         u16 hash_fields;
 669
 670         switch (cmd->flow_type) {
 671         case TCP_V4_FLOW:
 672                 proto = ENA_ADMIN_RSS_TCP4;
 673                 break;
 674         case UDP_V4_FLOW:
 675                 proto = ENA_ADMIN_RSS_UDP4;
 676                 break;
 677         case TCP_V6_FLOW:
 678                 proto = ENA_ADMIN_RSS_TCP6;
 679                 break;
 680         case UDP_V6_FLOW:
 681                 proto = ENA_ADMIN_RSS_UDP6;
 682                 break;
 683         case IPV4_FLOW:
 684                 proto = ENA_ADMIN_RSS_IP4;
 685                 break;
 686         case IPV6_FLOW:
 687                 proto = ENA_ADMIN_RSS_IP6;
 688                 break;
 689         case ETHER_FLOW:
 690                 proto = ENA_ADMIN_RSS_NOT_IP;
 691                 break;
 692         case AH_V4_FLOW:
 693         case ESP_V4_FLOW:
 694         case AH_V6_FLOW:
 695         case ESP_V6_FLOW:
 696         case SCTP_V4_FLOW:
 697         case AH_ESP_V4_FLOW:
 698                 return -EOPNOTSUPP;
 699         default:
 700                 return -EINVAL;
 701         }
 702
 703         hash_fields = ena_flow_data_to_flow_hash(cmd->data);
 704
 705         return ena_com_fill_hash_ctrl(ena_dev, proto, hash_fields);
 706 }
 707
 708 static int ena_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info)
 709 {
 710         struct ena_adapter *adapter = netdev_priv(netdev);
 711         int rc = 0;
 712
 713         switch (info->cmd) {
 714         case ETHTOOL_SRXFH:
 715                 rc = ena_set_rss_hash(adapter->ena_dev, info);
 716                 break;
 717         case ETHTOOL_SRXCLSRLDEL:
 718         case ETHTOOL_SRXCLSRLINS:
 719         default:
 720                 netif_err(adapter, drv, netdev,
 721                           "Command parameter %d is not supported\n", info->cmd);
 722                 rc = -EOPNOTSUPP;
 723         }
 724
 725         return rc;
 726 }
 727
 728 static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
 729                          u32 *rules)
 730 {
 731         struct ena_adapter *adapter = netdev_priv(netdev);
 732         int rc = 0;
 733
 734         switch (info->cmd) {
 735         case ETHTOOL_GRXRINGS:
 736                 info->data = adapter->num_io_queues;
 737                 rc = 0;
 738                 break;
 739         case ETHTOOL_GRXFH:
 740                 rc = ena_get_rss_hash(adapter->ena_dev, info);
 741                 break;
 742         case ETHTOOL_GRXCLSRLCNT:
 743         case ETHTOOL_GRXCLSRULE:
 744         case ETHTOOL_GRXCLSRLALL:
 745         default:
 746                 netif_err(adapter, drv, netdev,
 747                           "Command parameter %d is not supported\n", info->cmd);
 748                 rc = -EOPNOTSUPP;
 749         }
 750
 751         return rc;
 752 }
 753
 754 static u32 ena_get_rxfh_indir_size(struct net_device *netdev)
 755 {
 756         return ENA_RX_RSS_TABLE_SIZE;
 757 }
 758
 759 static u32 ena_get_rxfh_key_size(struct net_device *netdev)
 760 {
 761         return ENA_HASH_KEY_SIZE;
 762 }
 763
 764 static int ena_indirection_table_set(struct ena_adapter *adapter,
 765                                      const u32 *indir)
 766 {
 767         struct ena_com_dev *ena_dev = adapter->ena_dev;
 768         int i, rc;
 769
 770         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
 771                 rc = ena_com_indirect_table_fill_entry(ena_dev,
 772                                                        i,
 773                                                        ENA_IO_RXQ_IDX(indir[i]));
 774                 if (unlikely(rc)) {
 775                         netif_err(adapter, drv, adapter->netdev,
 776                                   "Cannot fill indirect table (index is too large)\n");
 777                         return rc;
 778                 }
 779         }
 780
 781         rc = ena_com_indirect_table_set(ena_dev);
 782         if (rc) {
 783                 netif_err(adapter, drv, adapter->netdev,
 784                           "Cannot set indirect table\n");
 785                 return rc == -EPERM ? -EOPNOTSUPP : rc;
 786         }
 787         return rc;
 788 }
 789
 790 static int ena_indirection_table_get(struct ena_adapter *adapter, u32 *indir)
 791 {
 792         struct ena_com_dev *ena_dev = adapter->ena_dev;
 793         int i, rc;
 794
 795         if (!indir)
 796                 return 0;
 797
 798         rc = ena_com_indirect_table_get(ena_dev, indir);
 799         if (rc)
 800                 return rc;
 801
 802         /* Our internal representation of the indices is: even indices
 803          * for Tx and uneven indices for Rx. We need to convert the Rx
 804          * indices to be consecutive
 805          */
 806         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++)
 807                 indir[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(indir[i]);
 808
 809         return rc;
 810 }
 811
 812 static int ena_get_rxfh(struct net_device *netdev,
 813                         struct ethtool_rxfh_param *rxfh)
 814 {
 815         struct ena_adapter *adapter = netdev_priv(netdev);
 816         enum ena_admin_hash_functions ena_func;
 817         u8 func;
 818         int rc;
 819
 820         rc = ena_indirection_table_get(adapter, rxfh->indir);
 821         if (rc)
 822                 return rc;
 823
 824         /* We call this function in order to check if the device
 825          * supports getting/setting the hash function.
 826          */
 827         rc = ena_com_get_hash_function(adapter->ena_dev, &ena_func);
 828         if (rc) {
 829                 if (rc == -EOPNOTSUPP)
 830                         rc = 0;
 831
 832                 return rc;
 833         }
 834
 835         rc = ena_com_get_hash_key(adapter->ena_dev, rxfh->key);
 836         if (rc)
 837                 return rc;
 838
 839         switch (ena_func) {
 840         case ENA_ADMIN_TOEPLITZ:
 841                 func = ETH_RSS_HASH_TOP;
 842                 break;
 843         case ENA_ADMIN_CRC32:
 844                 func = ETH_RSS_HASH_CRC32;
 845                 break;
 846         default:
 847                 netif_err(adapter, drv, netdev,
 848                           "Command parameter is not supported\n");
 849                 return -EOPNOTSUPP;
 850         }
 851
 852         rxfh->hfunc = func;
 853
 854         return 0;
 855 }
 856
 857 static int ena_set_rxfh(struct net_device *netdev,
 858                         struct ethtool_rxfh_param *rxfh,
 859                         struct netlink_ext_ack *extack)
 860 {
 861         struct ena_adapter *adapter = netdev_priv(netdev);
 862         struct ena_com_dev *ena_dev = adapter->ena_dev;
 863         enum ena_admin_hash_functions func = 0;
 864         int rc;
 865
 866         if (rxfh->indir) {
 867                 rc = ena_indirection_table_set(adapter, rxfh->indir);
 868                 if (rc)
 869                         return rc;
 870         }
 871
 872         switch (rxfh->hfunc) {
 873         case ETH_RSS_HASH_NO_CHANGE:
 874                 func = ena_com_get_current_hash_function(ena_dev);
 875                 break;
 876         case ETH_RSS_HASH_TOP:
 877                 func = ENA_ADMIN_TOEPLITZ;
 878                 break;
 879         case ETH_RSS_HASH_CRC32:
 880                 func = ENA_ADMIN_CRC32;
 881                 break;
 882         default:
 883                 netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n",
 884                           rxfh->hfunc);
 885                 return -EOPNOTSUPP;
 886         }
 887
 888         if (rxfh->key || func) {
 889                 rc = ena_com_fill_hash_function(ena_dev, func, rxfh->key,
 890                                                 ENA_HASH_KEY_SIZE,
 891                                                 0xFFFFFFFF);
 892                 if (unlikely(rc)) {
 893                         netif_err(adapter, drv, netdev, "Cannot fill key\n");
 894                         return rc == -EPERM ? -EOPNOTSUPP : rc;
 895                 }
 896         }
 897
 898         return 0;
 899 }
 900
 901 static void ena_get_channels(struct net_device *netdev,
 902                              struct ethtool_channels *channels)
 903 {
 904         struct ena_adapter *adapter = netdev_priv(netdev);
 905
 906         channels->max_combined = adapter->max_num_io_queues;
 907         channels->combined_count = adapter->num_io_queues;
 908 }
 909
 910 static int ena_set_channels(struct net_device *netdev,
 911                             struct ethtool_channels *channels)
 912 {
 913         struct ena_adapter *adapter = netdev_priv(netdev);
 914         u32 count = channels->combined_count;
 915         /* The check for max value is already done in ethtool */
 916         if (count < ENA_MIN_NUM_IO_QUEUES)
 917                 return -EINVAL;
 918
 919         if (!ena_xdp_legal_queue_count(adapter, count)) {
 920                 if (ena_xdp_present(adapter))
 921                         return -EINVAL;
 922
 923                 xdp_clear_features_flag(netdev);
 924         } else {
 925                 xdp_set_features_flag(netdev,
 926                                       NETDEV_XDP_ACT_BASIC |
 927                                       NETDEV_XDP_ACT_REDIRECT);
 928         }
 929
 930         return ena_update_queue_count(adapter, count);
 931 }
 932
 933 static int ena_get_tunable(struct net_device *netdev,
 934                            const struct ethtool_tunable *tuna, void *data)
 935 {
 936         struct ena_adapter *adapter = netdev_priv(netdev);
 937         int ret = 0;
 938
 939         switch (tuna->id) {
 940         case ETHTOOL_RX_COPYBREAK:
 941                 *(u32 *)data = adapter->rx_copybreak;
 942                 break;
 943         default:
 944                 ret = -EINVAL;
 945                 break;
 946         }
 947
 948         return ret;
 949 }
 950
 951 static int ena_set_tunable(struct net_device *netdev,
 952                            const struct ethtool_tunable *tuna,
 953                            const void *data)
 954 {
 955         struct ena_adapter *adapter = netdev_priv(netdev);
 956         int ret = 0;
 957         u32 len;
 958
 959         switch (tuna->id) {
 960         case ETHTOOL_RX_COPYBREAK:
 961                 len = *(u32 *)data;
 962                 ret = ena_set_rx_copybreak(adapter, len);
 963                 break;
 964         default:
 965                 ret = -EINVAL;
 966                 break;
 967         }
 968
 969         return ret;
 970 }
 971
 972 static const struct ethtool_ops ena_ethtool_ops = {
 973         .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
 974                                      ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
 975         .supported_ring_params  = ETHTOOL_RING_USE_TX_PUSH_BUF_LEN |
 976                                   ETHTOOL_RING_USE_TX_PUSH,
 977         .get_link_ksettings     = ena_get_link_ksettings,
 978         .get_drvinfo            = ena_get_drvinfo,
 979         .get_msglevel           = ena_get_msglevel,
 980         .set_msglevel           = ena_set_msglevel,
 981         .get_link               = ethtool_op_get_link,
 982         .get_coalesce           = ena_get_coalesce,
 983         .set_coalesce           = ena_set_coalesce,
 984         .get_ringparam          = ena_get_ringparam,
 985         .set_ringparam          = ena_set_ringparam,
 986         .get_sset_count         = ena_get_sset_count,
 987         .get_strings            = ena_get_ethtool_strings,
 988         .get_ethtool_stats      = ena_get_ethtool_stats,
 989         .get_rxnfc              = ena_get_rxnfc,
 990         .set_rxnfc              = ena_set_rxnfc,
 991         .get_rxfh_indir_size    = ena_get_rxfh_indir_size,
 992         .get_rxfh_key_size      = ena_get_rxfh_key_size,
 993         .get_rxfh               = ena_get_rxfh,
 994         .set_rxfh               = ena_set_rxfh,
 995         .get_channels           = ena_get_channels,
 996         .set_channels           = ena_set_channels,
 997         .get_tunable            = ena_get_tunable,
 998         .set_tunable            = ena_set_tunable,
 999         .get_ts_info            = ethtool_op_get_ts_info,
1000 };
1001
1002 void ena_set_ethtool_ops(struct net_device *netdev)
1003 {
1004         netdev->ethtool_ops = &ena_ethtool_ops;
1005 }
1006
1007 static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf)
1008 {
1009         struct net_device *netdev = adapter->netdev;
1010         u8 *strings_buf;
1011         u64 *data_buf;
1012         int strings_num;
1013         int i, rc;
1014
1015         strings_num = ena_get_sw_stats_count(adapter);
1016         if (strings_num <= 0) {
1017                 netif_err(adapter, drv, netdev, "Can't get stats num\n");
1018                 return;
1019         }
1020
1021         strings_buf = devm_kcalloc(&adapter->pdev->dev,
1022                                    ETH_GSTRING_LEN, strings_num,
1023                                    GFP_ATOMIC);
1024         if (!strings_buf) {
1025                 netif_err(adapter, drv, netdev,
1026                           "Failed to allocate strings_buf\n");
1027                 return;
1028         }
1029
1030         data_buf = devm_kcalloc(&adapter->pdev->dev,
1031                                 strings_num, sizeof(u64),
1032                                 GFP_ATOMIC);
1033         if (!data_buf) {
1034                 netif_err(adapter, drv, netdev,
1035                           "Failed to allocate data buf\n");
1036                 devm_kfree(&adapter->pdev->dev, strings_buf);
1037                 return;
1038         }
1039
1040         ena_get_strings(adapter, strings_buf, false);
1041         ena_get_stats(adapter, data_buf, false);
1042
1043         /* If there is a buffer, dump stats, otherwise print them to dmesg */
1044         if (buf)
1045                 for (i = 0; i < strings_num; i++) {
1046                         rc = snprintf(buf, ETH_GSTRING_LEN + sizeof(u64),
1047                                       "%s %llu\n",
1048                                       strings_buf + i * ETH_GSTRING_LEN,
1049                                       data_buf[i]);
1050                         buf += rc;
1051                 }
1052         else
1053                 for (i = 0; i < strings_num; i++)
1054                         netif_err(adapter, drv, netdev, "%s: %llu\n",
1055                                   strings_buf + i * ETH_GSTRING_LEN,
1056                                   data_buf[i]);
1057
1058         devm_kfree(&adapter->pdev->dev, strings_buf);
1059         devm_kfree(&adapter->pdev->dev, data_buf);
1060 }
1061
1062 void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf)
1063 {
1064         if (!buf)
1065                 return;
1066
1067         ena_dump_stats_ex(adapter, buf);
1068 }
1069
1070 void ena_dump_stats_to_dmesg(struct ena_adapter *adapter)
1071 {
1072         ena_dump_stats_ex(adapter, NULL);
1073 }