drivers/net/ethernet/amazon/ena/ena_ethtool.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
   4  */
   5
   6 #include <linux/ethtool.h>
   7 #include <linux/pci.h>
   8
   9 #include "ena_netdev.h"
  10 #include "ena_xdp.h"
  11
  12 struct ena_stats {
  13         char name[ETH_GSTRING_LEN];
  14         int stat_offset;
  15 };
  16
  17 #define ENA_STAT_ENA_COM_ENTRY(stat) { \
  18         .name = #stat, \
  19         .stat_offset = offsetof(struct ena_com_stats_admin, stat) / sizeof(u64) \
  20 }
  21
  22 #define ENA_STAT_ENTRY(stat, stat_type) { \
  23         .name = #stat, \
  24         .stat_offset = offsetof(struct ena_stats_##stat_type, stat) / sizeof(u64) \
  25 }
  26
  27 #define ENA_STAT_HW_ENTRY(stat, stat_type) { \
  28         .name = #stat, \
  29         .stat_offset = offsetof(struct ena_admin_##stat_type, stat) / sizeof(u64) \
  30 }
  31
  32 #define ENA_STAT_RX_ENTRY(stat) \
  33         ENA_STAT_ENTRY(stat, rx)
  34
  35 #define ENA_STAT_TX_ENTRY(stat) \
  36         ENA_STAT_ENTRY(stat, tx)
  37
  38 #define ENA_STAT_GLOBAL_ENTRY(stat) \
  39         ENA_STAT_ENTRY(stat, dev)
  40
  41 #define ENA_STAT_ENI_ENTRY(stat) \
  42         ENA_STAT_HW_ENTRY(stat, eni_stats)
  43
  44 static const struct ena_stats ena_stats_global_strings[] = {
  45         ENA_STAT_GLOBAL_ENTRY(tx_timeout),
  46         ENA_STAT_GLOBAL_ENTRY(suspend),
  47         ENA_STAT_GLOBAL_ENTRY(resume),
  48         ENA_STAT_GLOBAL_ENTRY(wd_expired),
  49         ENA_STAT_GLOBAL_ENTRY(interface_up),
  50         ENA_STAT_GLOBAL_ENTRY(interface_down),
  51         ENA_STAT_GLOBAL_ENTRY(admin_q_pause),
  52 };
  53
  54 static const struct ena_stats ena_stats_eni_strings[] = {
  55         ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded),
  56         ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded),
  57         ENA_STAT_ENI_ENTRY(pps_allowance_exceeded),
  58         ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded),
  59         ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded),
  60 };
  61
  62 static const struct ena_stats ena_stats_tx_strings[] = {
  63         ENA_STAT_TX_ENTRY(cnt),
  64         ENA_STAT_TX_ENTRY(bytes),
  65         ENA_STAT_TX_ENTRY(queue_stop),
  66         ENA_STAT_TX_ENTRY(queue_wakeup),
  67         ENA_STAT_TX_ENTRY(dma_mapping_err),
  68         ENA_STAT_TX_ENTRY(linearize),
  69         ENA_STAT_TX_ENTRY(linearize_failed),
  70         ENA_STAT_TX_ENTRY(napi_comp),
  71         ENA_STAT_TX_ENTRY(tx_poll),
  72         ENA_STAT_TX_ENTRY(doorbells),
  73         ENA_STAT_TX_ENTRY(prepare_ctx_err),
  74         ENA_STAT_TX_ENTRY(bad_req_id),
  75         ENA_STAT_TX_ENTRY(llq_buffer_copy),
  76         ENA_STAT_TX_ENTRY(missed_tx),
  77         ENA_STAT_TX_ENTRY(unmask_interrupt),
  78 };
  79
  80 static const struct ena_stats ena_stats_rx_strings[] = {
  81         ENA_STAT_RX_ENTRY(cnt),
  82         ENA_STAT_RX_ENTRY(bytes),
  83         ENA_STAT_RX_ENTRY(rx_copybreak_pkt),
  84         ENA_STAT_RX_ENTRY(csum_good),
  85         ENA_STAT_RX_ENTRY(refil_partial),
  86         ENA_STAT_RX_ENTRY(csum_bad),
  87         ENA_STAT_RX_ENTRY(page_alloc_fail),
  88         ENA_STAT_RX_ENTRY(skb_alloc_fail),
  89         ENA_STAT_RX_ENTRY(dma_mapping_err),
  90         ENA_STAT_RX_ENTRY(bad_desc_num),
  91         ENA_STAT_RX_ENTRY(bad_req_id),
  92         ENA_STAT_RX_ENTRY(empty_rx_ring),
  93         ENA_STAT_RX_ENTRY(csum_unchecked),
  94         ENA_STAT_RX_ENTRY(xdp_aborted),
  95         ENA_STAT_RX_ENTRY(xdp_drop),
  96         ENA_STAT_RX_ENTRY(xdp_pass),
  97         ENA_STAT_RX_ENTRY(xdp_tx),
  98         ENA_STAT_RX_ENTRY(xdp_invalid),
  99         ENA_STAT_RX_ENTRY(xdp_redirect),
 100 };
 101
 102 static const struct ena_stats ena_stats_ena_com_strings[] = {
 103         ENA_STAT_ENA_COM_ENTRY(aborted_cmd),
 104         ENA_STAT_ENA_COM_ENTRY(submitted_cmd),
 105         ENA_STAT_ENA_COM_ENTRY(completed_cmd),
 106         ENA_STAT_ENA_COM_ENTRY(out_of_space),
 107         ENA_STAT_ENA_COM_ENTRY(no_completion),
 108 };
 109
 110 #define ENA_STATS_ARRAY_GLOBAL          ARRAY_SIZE(ena_stats_global_strings)
 111 #define ENA_STATS_ARRAY_TX              ARRAY_SIZE(ena_stats_tx_strings)
 112 #define ENA_STATS_ARRAY_RX              ARRAY_SIZE(ena_stats_rx_strings)
 113 #define ENA_STATS_ARRAY_ENA_COM         ARRAY_SIZE(ena_stats_ena_com_strings)
 114 #define ENA_STATS_ARRAY_ENI(adapter)    ARRAY_SIZE(ena_stats_eni_strings)
 115
 116 static void ena_safe_update_stat(u64 *src, u64 *dst,
 117                                  struct u64_stats_sync *syncp)
 118 {
 119         unsigned int start;
 120
 121         do {
 122                 start = u64_stats_fetch_begin(syncp);
 123                 *(dst) = *src;
 124         } while (u64_stats_fetch_retry(syncp, start));
 125 }
 126
 127 static void ena_queue_stats(struct ena_adapter *adapter, u64 **data)
 128 {
 129         const struct ena_stats *ena_stats;
 130         struct ena_ring *ring;
 131
 132         u64 *ptr;
 133         int i, j;
 134
 135         for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
 136                 /* Tx stats */
 137                 ring = &adapter->tx_ring[i];
 138
 139                 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
 140                         ena_stats = &ena_stats_tx_strings[j];
 141
 142                         ptr = (u64 *)&ring->tx_stats + ena_stats->stat_offset;
 143
 144                         ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
 145                 }
 146                 /* XDP TX queues don't have a RX queue counterpart */
 147                 if (!ENA_IS_XDP_INDEX(adapter, i)) {
 148                         /* Rx stats */
 149                         ring = &adapter->rx_ring[i];
 150
 151                         for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
 152                                 ena_stats = &ena_stats_rx_strings[j];
 153
 154                                 ptr = (u64 *)&ring->rx_stats +
 155                                         ena_stats->stat_offset;
 156
 157                                 ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
 158                         }
 159                 }
 160         }
 161 }
 162
 163 static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data)
 164 {
 165         const struct ena_stats *ena_stats;
 166         u64 *ptr;
 167         int i;
 168
 169         for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
 170                 ena_stats = &ena_stats_ena_com_strings[i];
 171
 172                 ptr = (u64 *)&adapter->ena_dev->admin_queue.stats +
 173                         ena_stats->stat_offset;
 174
 175                 *(*data)++ = *ptr;
 176         }
 177 }
 178
 179 static void ena_get_stats(struct ena_adapter *adapter,
 180                           u64 *data,
 181                           bool eni_stats_needed)
 182 {
 183         const struct ena_stats *ena_stats;
 184         u64 *ptr;
 185         int i;
 186
 187         for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
 188                 ena_stats = &ena_stats_global_strings[i];
 189
 190                 ptr = (u64 *)&adapter->dev_stats + ena_stats->stat_offset;
 191
 192                 ena_safe_update_stat(ptr, data++, &adapter->syncp);
 193         }
 194
 195         if (eni_stats_needed) {
 196                 ena_update_hw_stats(adapter);
 197                 for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) {
 198                         ena_stats = &ena_stats_eni_strings[i];
 199
 200                         ptr = (u64 *)&adapter->eni_stats +
 201                                 ena_stats->stat_offset;
 202
 203                         ena_safe_update_stat(ptr, data++, &adapter->syncp);
 204                 }
 205         }
 206
 207         ena_queue_stats(adapter, &data);
 208         ena_dev_admin_queue_stats(adapter, &data);
 209 }
 210
 211 static void ena_get_ethtool_stats(struct net_device *netdev,
 212                                   struct ethtool_stats *stats,
 213                                   u64 *data)
 214 {
 215         struct ena_adapter *adapter = netdev_priv(netdev);
 216         struct ena_com_dev *dev = adapter->ena_dev;
 217
 218         ena_get_stats(adapter, data, ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS));
 219 }
 220
 221 static int ena_get_sw_stats_count(struct ena_adapter *adapter)
 222 {
 223         return adapter->num_io_queues * (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX)
 224                 + adapter->xdp_num_queues * ENA_STATS_ARRAY_TX
 225                 + ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM;
 226 }
 227
 228 static int ena_get_hw_stats_count(struct ena_adapter *adapter)
 229 {
 230         bool supported = ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENI_STATS);
 231
 232         return ENA_STATS_ARRAY_ENI(adapter) * supported;
 233 }
 234
 235 int ena_get_sset_count(struct net_device *netdev, int sset)
 236 {
 237         struct ena_adapter *adapter = netdev_priv(netdev);
 238
 239         switch (sset) {
 240         case ETH_SS_STATS:
 241                 return ena_get_sw_stats_count(adapter) +
 242                        ena_get_hw_stats_count(adapter);
 243         }
 244
 245         return -EOPNOTSUPP;
 246 }
 247
 248 static void ena_queue_strings(struct ena_adapter *adapter, u8 **data)
 249 {
 250         const struct ena_stats *ena_stats;
 251         bool is_xdp;
 252         int i, j;
 253
 254         for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
 255                 is_xdp = ENA_IS_XDP_INDEX(adapter, i);
 256                 /* Tx stats */
 257                 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
 258                         ena_stats = &ena_stats_tx_strings[j];
 259
 260                         ethtool_sprintf(data,
 261                                         "queue_%u_%s_%s", i,
 262                                         is_xdp ? "xdp_tx" : "tx",
 263                                         ena_stats->name);
 264                 }
 265
 266                 /* In XDP there isn't an RX queue counterpart */
 267                 if (is_xdp)
 268                         continue;
 269
 270                 for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
 271                         ena_stats = &ena_stats_rx_strings[j];
 272
 273                         ethtool_sprintf(data, "queue_%u_rx_%s", i, ena_stats->name);
 274                 }
 275         }
 276 }
 277
 278 static void ena_com_dev_strings(u8 **data)
 279 {
 280         const struct ena_stats *ena_stats;
 281         int i;
 282
 283         for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
 284                 ena_stats = &ena_stats_ena_com_strings[i];
 285
 286                 ethtool_sprintf(data,
 287                                 "ena_admin_q_%s", ena_stats->name);
 288         }
 289 }
 290
 291 static void ena_get_strings(struct ena_adapter *adapter,
 292                             u8 *data,
 293                             bool eni_stats_needed)
 294 {
 295         const struct ena_stats *ena_stats;
 296         int i;
 297
 298         for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
 299                 ena_stats = &ena_stats_global_strings[i];
 300                 ethtool_puts(&data, ena_stats->name);
 301         }
 302
 303         if (eni_stats_needed) {
 304                 for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) {
 305                         ena_stats = &ena_stats_eni_strings[i];
 306                         ethtool_puts(&data, ena_stats->name);
 307                 }
 308         }
 309
 310         ena_queue_strings(adapter, &data);
 311         ena_com_dev_strings(&data);
 312 }
 313
 314 static void ena_get_ethtool_strings(struct net_device *netdev,
 315                                     u32 sset,
 316                                     u8 *data)
 317 {
 318         struct ena_adapter *adapter = netdev_priv(netdev);
 319         struct ena_com_dev *dev = adapter->ena_dev;
 320
 321         switch (sset) {
 322         case ETH_SS_STATS:
 323                 ena_get_strings(adapter, data, ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS));
 324                 break;
 325         }
 326 }
 327
 328 static int ena_get_link_ksettings(struct net_device *netdev,
 329                                   struct ethtool_link_ksettings *link_ksettings)
 330 {
 331         struct ena_adapter *adapter = netdev_priv(netdev);
 332         struct ena_com_dev *ena_dev = adapter->ena_dev;
 333         struct ena_admin_get_feature_link_desc *link;
 334         struct ena_admin_get_feat_resp feat_resp;
 335         int rc;
 336
 337         rc = ena_com_get_link_params(ena_dev, &feat_resp);
 338         if (rc)
 339                 return rc;
 340
 341         link = &feat_resp.u.link;
 342         link_ksettings->base.speed = link->speed;
 343
 344         if (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) {
 345                 ethtool_link_ksettings_add_link_mode(link_ksettings,
 346                                                      supported, Autoneg);
 347                 ethtool_link_ksettings_add_link_mode(link_ksettings,
 348                                                      supported, Autoneg);
 349         }
 350
 351         link_ksettings->base.autoneg =
 352                 (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) ?
 353                 AUTONEG_ENABLE : AUTONEG_DISABLE;
 354
 355         link_ksettings->base.duplex = DUPLEX_FULL;
 356
 357         return 0;
 358 }
 359
 360 static int ena_get_coalesce(struct net_device *net_dev,
 361                             struct ethtool_coalesce *coalesce,
 362                             struct kernel_ethtool_coalesce *kernel_coal,
 363                             struct netlink_ext_ack *extack)
 364 {
 365         struct ena_adapter *adapter = netdev_priv(net_dev);
 366         struct ena_com_dev *ena_dev = adapter->ena_dev;
 367
 368         if (!ena_com_interrupt_moderation_supported(ena_dev))
 369                 return -EOPNOTSUPP;
 370
 371         coalesce->tx_coalesce_usecs =
 372                 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) *
 373                         ena_dev->intr_delay_resolution;
 374
 375         coalesce->rx_coalesce_usecs =
 376                 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev)
 377                 * ena_dev->intr_delay_resolution;
 378
 379         coalesce->use_adaptive_rx_coalesce =
 380                 ena_com_get_adaptive_moderation_enabled(ena_dev);
 381
 382         return 0;
 383 }
 384
 385 static void ena_update_tx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter)
 386 {
 387         unsigned int val;
 388         int i;
 389
 390         val = ena_com_get_nonadaptive_moderation_interval_tx(adapter->ena_dev);
 391
 392         for (i = 0; i < adapter->num_io_queues; i++)
 393                 adapter->tx_ring[i].smoothed_interval = val;
 394 }
 395
 396 static void ena_update_rx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter)
 397 {
 398         unsigned int val;
 399         int i;
 400
 401         val = ena_com_get_nonadaptive_moderation_interval_rx(adapter->ena_dev);
 402
 403         for (i = 0; i < adapter->num_io_queues; i++)
 404                 adapter->rx_ring[i].smoothed_interval = val;
 405 }
 406
 407 static int ena_set_coalesce(struct net_device *net_dev,
 408                             struct ethtool_coalesce *coalesce,
 409                             struct kernel_ethtool_coalesce *kernel_coal,
 410                             struct netlink_ext_ack *extack)
 411 {
 412         struct ena_adapter *adapter = netdev_priv(net_dev);
 413         struct ena_com_dev *ena_dev = adapter->ena_dev;
 414         int rc;
 415
 416         if (!ena_com_interrupt_moderation_supported(ena_dev))
 417                 return -EOPNOTSUPP;
 418
 419         rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev,
 420                                                                coalesce->tx_coalesce_usecs);
 421         if (rc)
 422                 return rc;
 423
 424         ena_update_tx_rings_nonadaptive_intr_moderation(adapter);
 425
 426         rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
 427                                                                coalesce->rx_coalesce_usecs);
 428         if (rc)
 429                 return rc;
 430
 431         ena_update_rx_rings_nonadaptive_intr_moderation(adapter);
 432
 433         if (coalesce->use_adaptive_rx_coalesce &&
 434             !ena_com_get_adaptive_moderation_enabled(ena_dev))
 435                 ena_com_enable_adaptive_moderation(ena_dev);
 436
 437         if (!coalesce->use_adaptive_rx_coalesce &&
 438             ena_com_get_adaptive_moderation_enabled(ena_dev))
 439                 ena_com_disable_adaptive_moderation(ena_dev);
 440
 441         return 0;
 442 }
 443
 444 static u32 ena_get_msglevel(struct net_device *netdev)
 445 {
 446         struct ena_adapter *adapter = netdev_priv(netdev);
 447
 448         return adapter->msg_enable;
 449 }
 450
 451 static void ena_set_msglevel(struct net_device *netdev, u32 value)
 452 {
 453         struct ena_adapter *adapter = netdev_priv(netdev);
 454
 455         adapter->msg_enable = value;
 456 }
 457
 458 static void ena_get_drvinfo(struct net_device *dev,
 459                             struct ethtool_drvinfo *info)
 460 {
 461         struct ena_adapter *adapter = netdev_priv(dev);
 462
 463         strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 464         strscpy(info->bus_info, pci_name(adapter->pdev),
 465                 sizeof(info->bus_info));
 466 }
 467
 468 static void ena_get_ringparam(struct net_device *netdev,
 469                               struct ethtool_ringparam *ring,
 470                               struct kernel_ethtool_ringparam *kernel_ring,
 471                               struct netlink_ext_ack *extack)
 472 {
 473         struct ena_adapter *adapter = netdev_priv(netdev);
 474
 475         ring->tx_max_pending = adapter->max_tx_ring_size;
 476         ring->rx_max_pending = adapter->max_rx_ring_size;
 477         if (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
 478                 bool large_llq_supported = adapter->large_llq_header_supported;
 479
 480                 kernel_ring->tx_push = true;
 481                 kernel_ring->tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
 482                 if (large_llq_supported)
 483                         kernel_ring->tx_push_buf_max_len = ENA_LLQ_LARGE_HEADER;
 484                 else
 485                         kernel_ring->tx_push_buf_max_len = ENA_LLQ_HEADER;
 486         } else {
 487                 kernel_ring->tx_push = false;
 488                 kernel_ring->tx_push_buf_max_len = 0;
 489                 kernel_ring->tx_push_buf_len = 0;
 490         }
 491
 492         ring->tx_pending = adapter->tx_ring[0].ring_size;
 493         ring->rx_pending = adapter->rx_ring[0].ring_size;
 494 }
 495
 496 static int ena_set_ringparam(struct net_device *netdev,
 497                              struct ethtool_ringparam *ring,
 498                              struct kernel_ethtool_ringparam *kernel_ring,
 499                              struct netlink_ext_ack *extack)
 500 {
 501         struct ena_adapter *adapter = netdev_priv(netdev);
 502         u32 new_tx_size, new_rx_size, new_tx_push_buf_len;
 503         bool changed = false;
 504
 505         new_tx_size = ring->tx_pending < ENA_MIN_RING_SIZE ?
 506                         ENA_MIN_RING_SIZE : ring->tx_pending;
 507         new_tx_size = rounddown_pow_of_two(new_tx_size);
 508
 509         new_rx_size = ring->rx_pending < ENA_MIN_RING_SIZE ?
 510                         ENA_MIN_RING_SIZE : ring->rx_pending;
 511         new_rx_size = rounddown_pow_of_two(new_rx_size);
 512
 513         changed |= new_tx_size != adapter->requested_tx_ring_size ||
 514                    new_rx_size != adapter->requested_rx_ring_size;
 515
 516         /* This value is ignored if LLQ is not supported */
 517         new_tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
 518
 519         if ((adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) !=
 520             kernel_ring->tx_push) {
 521                 NL_SET_ERR_MSG_MOD(extack, "Push mode state cannot be modified");
 522                 return -EINVAL;
 523         }
 524
 525         /* Validate that the push buffer is supported on the underlying device */
 526         if (kernel_ring->tx_push_buf_len) {
 527                 enum ena_admin_placement_policy_type placement;
 528
 529                 new_tx_push_buf_len = kernel_ring->tx_push_buf_len;
 530
 531                 placement = adapter->ena_dev->tx_mem_queue_type;
 532                 if (placement == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 533                         return -EOPNOTSUPP;
 534
 535                 if (new_tx_push_buf_len != ENA_LLQ_HEADER &&
 536                     new_tx_push_buf_len != ENA_LLQ_LARGE_HEADER) {
 537                         bool large_llq_sup = adapter->large_llq_header_supported;
 538                         char large_llq_size_str[40];
 539
 540                         snprintf(large_llq_size_str, 40, ", %lu", ENA_LLQ_LARGE_HEADER);
 541
 542                         NL_SET_ERR_MSG_FMT_MOD(extack,
 543                                                "Supported tx push buff values: [%lu%s]",
 544                                                ENA_LLQ_HEADER,
 545                                                large_llq_sup ? large_llq_size_str : "");
 546
 547                         return -EINVAL;
 548                 }
 549
 550                 changed |= new_tx_push_buf_len != adapter->ena_dev->tx_max_header_size;
 551         }
 552
 553         if (!changed)
 554                 return 0;
 555
 556         return ena_update_queue_params(adapter, new_tx_size, new_rx_size,
 557                                        new_tx_push_buf_len);
 558 }
 559
 560 static u32 ena_flow_hash_to_flow_type(u16 hash_fields)
 561 {
 562         u32 data = 0;
 563
 564         if (hash_fields & ENA_ADMIN_RSS_L2_DA)
 565                 data |= RXH_L2DA;
 566
 567         if (hash_fields & ENA_ADMIN_RSS_L3_DA)
 568                 data |= RXH_IP_DST;
 569
 570         if (hash_fields & ENA_ADMIN_RSS_L3_SA)
 571                 data |= RXH_IP_SRC;
 572
 573         if (hash_fields & ENA_ADMIN_RSS_L4_DP)
 574                 data |= RXH_L4_B_2_3;
 575
 576         if (hash_fields & ENA_ADMIN_RSS_L4_SP)
 577                 data |= RXH_L4_B_0_1;
 578
 579         return data;
 580 }
 581
 582 static u16 ena_flow_data_to_flow_hash(u32 hash_fields)
 583 {
 584         u16 data = 0;
 585
 586         if (hash_fields & RXH_L2DA)
 587                 data |= ENA_ADMIN_RSS_L2_DA;
 588
 589         if (hash_fields & RXH_IP_DST)
 590                 data |= ENA_ADMIN_RSS_L3_DA;
 591
 592         if (hash_fields & RXH_IP_SRC)
 593                 data |= ENA_ADMIN_RSS_L3_SA;
 594
 595         if (hash_fields & RXH_L4_B_2_3)
 596                 data |= ENA_ADMIN_RSS_L4_DP;
 597
 598         if (hash_fields & RXH_L4_B_0_1)
 599                 data |= ENA_ADMIN_RSS_L4_SP;
 600
 601         return data;
 602 }
 603
 604 static int ena_get_rss_hash(struct ena_com_dev *ena_dev,
 605                             struct ethtool_rxnfc *cmd)
 606 {
 607         enum ena_admin_flow_hash_proto proto;
 608         u16 hash_fields;
 609         int rc;
 610
 611         cmd->data = 0;
 612
 613         switch (cmd->flow_type) {
 614         case TCP_V4_FLOW:
 615                 proto = ENA_ADMIN_RSS_TCP4;
 616                 break;
 617         case UDP_V4_FLOW:
 618                 proto = ENA_ADMIN_RSS_UDP4;
 619                 break;
 620         case TCP_V6_FLOW:
 621                 proto = ENA_ADMIN_RSS_TCP6;
 622                 break;
 623         case UDP_V6_FLOW:
 624                 proto = ENA_ADMIN_RSS_UDP6;
 625                 break;
 626         case IPV4_FLOW:
 627                 proto = ENA_ADMIN_RSS_IP4;
 628                 break;
 629         case IPV6_FLOW:
 630                 proto = ENA_ADMIN_RSS_IP6;
 631                 break;
 632         case ETHER_FLOW:
 633                 proto = ENA_ADMIN_RSS_NOT_IP;
 634                 break;
 635         case AH_V4_FLOW:
 636         case ESP_V4_FLOW:
 637         case AH_V6_FLOW:
 638         case ESP_V6_FLOW:
 639         case SCTP_V4_FLOW:
 640         case AH_ESP_V4_FLOW:
 641                 return -EOPNOTSUPP;
 642         default:
 643                 return -EINVAL;
 644         }
 645
 646         rc = ena_com_get_hash_ctrl(ena_dev, proto, &hash_fields);
 647         if (rc)
 648                 return rc;
 649
 650         cmd->data = ena_flow_hash_to_flow_type(hash_fields);
 651
 652         return 0;
 653 }
 654
 655 static int ena_set_rss_hash(struct ena_com_dev *ena_dev,
 656                             struct ethtool_rxnfc *cmd)
 657 {
 658         enum ena_admin_flow_hash_proto proto;
 659         u16 hash_fields;
 660
 661         switch (cmd->flow_type) {
 662         case TCP_V4_FLOW:
 663                 proto = ENA_ADMIN_RSS_TCP4;
 664                 break;
 665         case UDP_V4_FLOW:
 666                 proto = ENA_ADMIN_RSS_UDP4;
 667                 break;
 668         case TCP_V6_FLOW:
 669                 proto = ENA_ADMIN_RSS_TCP6;
 670                 break;
 671         case UDP_V6_FLOW:
 672                 proto = ENA_ADMIN_RSS_UDP6;
 673                 break;
 674         case IPV4_FLOW:
 675                 proto = ENA_ADMIN_RSS_IP4;
 676                 break;
 677         case IPV6_FLOW:
 678                 proto = ENA_ADMIN_RSS_IP6;
 679                 break;
 680         case ETHER_FLOW:
 681                 proto = ENA_ADMIN_RSS_NOT_IP;
 682                 break;
 683         case AH_V4_FLOW:
 684         case ESP_V4_FLOW:
 685         case AH_V6_FLOW:
 686         case ESP_V6_FLOW:
 687         case SCTP_V4_FLOW:
 688         case AH_ESP_V4_FLOW:
 689                 return -EOPNOTSUPP;
 690         default:
 691                 return -EINVAL;
 692         }
 693
 694         hash_fields = ena_flow_data_to_flow_hash(cmd->data);
 695
 696         return ena_com_fill_hash_ctrl(ena_dev, proto, hash_fields);
 697 }
 698
 699 static int ena_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info)
 700 {
 701         struct ena_adapter *adapter = netdev_priv(netdev);
 702         int rc = 0;
 703
 704         switch (info->cmd) {
 705         case ETHTOOL_SRXFH:
 706                 rc = ena_set_rss_hash(adapter->ena_dev, info);
 707                 break;
 708         case ETHTOOL_SRXCLSRLDEL:
 709         case ETHTOOL_SRXCLSRLINS:
 710         default:
 711                 netif_err(adapter, drv, netdev,
 712                           "Command parameter %d is not supported\n", info->cmd);
 713                 rc = -EOPNOTSUPP;
 714         }
 715
 716         return rc;
 717 }
 718
 719 static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
 720                          u32 *rules)
 721 {
 722         struct ena_adapter *adapter = netdev_priv(netdev);
 723         int rc = 0;
 724
 725         switch (info->cmd) {
 726         case ETHTOOL_GRXRINGS:
 727                 info->data = adapter->num_io_queues;
 728                 rc = 0;
 729                 break;
 730         case ETHTOOL_GRXFH:
 731                 rc = ena_get_rss_hash(adapter->ena_dev, info);
 732                 break;
 733         case ETHTOOL_GRXCLSRLCNT:
 734         case ETHTOOL_GRXCLSRULE:
 735         case ETHTOOL_GRXCLSRLALL:
 736         default:
 737                 netif_err(adapter, drv, netdev,
 738                           "Command parameter %d is not supported\n", info->cmd);
 739                 rc = -EOPNOTSUPP;
 740         }
 741
 742         return rc;
 743 }
 744
 745 static u32 ena_get_rxfh_indir_size(struct net_device *netdev)
 746 {
 747         return ENA_RX_RSS_TABLE_SIZE;
 748 }
 749
 750 static u32 ena_get_rxfh_key_size(struct net_device *netdev)
 751 {
 752         return ENA_HASH_KEY_SIZE;
 753 }
 754
 755 static int ena_indirection_table_set(struct ena_adapter *adapter,
 756                                      const u32 *indir)
 757 {
 758         struct ena_com_dev *ena_dev = adapter->ena_dev;
 759         int i, rc;
 760
 761         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
 762                 rc = ena_com_indirect_table_fill_entry(ena_dev,
 763                                                        i,
 764                                                        ENA_IO_RXQ_IDX(indir[i]));
 765                 if (unlikely(rc)) {
 766                         netif_err(adapter, drv, adapter->netdev,
 767                                   "Cannot fill indirect table (index is too large)\n");
 768                         return rc;
 769                 }
 770         }
 771
 772         rc = ena_com_indirect_table_set(ena_dev);
 773         if (rc) {
 774                 netif_err(adapter, drv, adapter->netdev,
 775                           "Cannot set indirect table\n");
 776                 return rc == -EPERM ? -EOPNOTSUPP : rc;
 777         }
 778         return rc;
 779 }
 780
 781 static int ena_indirection_table_get(struct ena_adapter *adapter, u32 *indir)
 782 {
 783         struct ena_com_dev *ena_dev = adapter->ena_dev;
 784         int i, rc;
 785
 786         if (!indir)
 787                 return 0;
 788
 789         rc = ena_com_indirect_table_get(ena_dev, indir);
 790         if (rc)
 791                 return rc;
 792
 793         /* Our internal representation of the indices is: even indices
 794          * for Tx and uneven indices for Rx. We need to convert the Rx
 795          * indices to be consecutive
 796          */
 797         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++)
 798                 indir[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(indir[i]);
 799
 800         return rc;
 801 }
 802
 803 static int ena_get_rxfh(struct net_device *netdev,
 804                         struct ethtool_rxfh_param *rxfh)
 805 {
 806         struct ena_adapter *adapter = netdev_priv(netdev);
 807         enum ena_admin_hash_functions ena_func;
 808         u8 func;
 809         int rc;
 810
 811         rc = ena_indirection_table_get(adapter, rxfh->indir);
 812         if (rc)
 813                 return rc;
 814
 815         /* We call this function in order to check if the device
 816          * supports getting/setting the hash function.
 817          */
 818         rc = ena_com_get_hash_function(adapter->ena_dev, &ena_func);
 819         if (rc) {
 820                 if (rc == -EOPNOTSUPP)
 821                         rc = 0;
 822
 823                 return rc;
 824         }
 825
 826         rc = ena_com_get_hash_key(adapter->ena_dev, rxfh->key);
 827         if (rc)
 828                 return rc;
 829
 830         switch (ena_func) {
 831         case ENA_ADMIN_TOEPLITZ:
 832                 func = ETH_RSS_HASH_TOP;
 833                 break;
 834         case ENA_ADMIN_CRC32:
 835                 func = ETH_RSS_HASH_CRC32;
 836                 break;
 837         default:
 838                 netif_err(adapter, drv, netdev,
 839                           "Command parameter is not supported\n");
 840                 return -EOPNOTSUPP;
 841         }
 842
 843         rxfh->hfunc = func;
 844
 845         return 0;
 846 }
 847
 848 static int ena_set_rxfh(struct net_device *netdev,
 849                         struct ethtool_rxfh_param *rxfh,
 850                         struct netlink_ext_ack *extack)
 851 {
 852         struct ena_adapter *adapter = netdev_priv(netdev);
 853         struct ena_com_dev *ena_dev = adapter->ena_dev;
 854         enum ena_admin_hash_functions func = 0;
 855         int rc;
 856
 857         if (rxfh->indir) {
 858                 rc = ena_indirection_table_set(adapter, rxfh->indir);
 859                 if (rc)
 860                         return rc;
 861         }
 862
 863         switch (rxfh->hfunc) {
 864         case ETH_RSS_HASH_NO_CHANGE:
 865                 func = ena_com_get_current_hash_function(ena_dev);
 866                 break;
 867         case ETH_RSS_HASH_TOP:
 868                 func = ENA_ADMIN_TOEPLITZ;
 869                 break;
 870         case ETH_RSS_HASH_CRC32:
 871                 func = ENA_ADMIN_CRC32;
 872                 break;
 873         default:
 874                 netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n",
 875                           rxfh->hfunc);
 876                 return -EOPNOTSUPP;
 877         }
 878
 879         if (rxfh->key || func) {
 880                 rc = ena_com_fill_hash_function(ena_dev, func, rxfh->key,
 881                                                 ENA_HASH_KEY_SIZE,
 882                                                 0xFFFFFFFF);
 883                 if (unlikely(rc)) {
 884                         netif_err(adapter, drv, netdev, "Cannot fill key\n");
 885                         return rc == -EPERM ? -EOPNOTSUPP : rc;
 886                 }
 887         }
 888
 889         return 0;
 890 }
 891
 892 static void ena_get_channels(struct net_device *netdev,
 893                              struct ethtool_channels *channels)
 894 {
 895         struct ena_adapter *adapter = netdev_priv(netdev);
 896
 897         channels->max_combined = adapter->max_num_io_queues;
 898         channels->combined_count = adapter->num_io_queues;
 899 }
 900
 901 static int ena_set_channels(struct net_device *netdev,
 902                             struct ethtool_channels *channels)
 903 {
 904         struct ena_adapter *adapter = netdev_priv(netdev);
 905         u32 count = channels->combined_count;
 906         /* The check for max value is already done in ethtool */
 907         if (count < ENA_MIN_NUM_IO_QUEUES)
 908                 return -EINVAL;
 909
 910         if (!ena_xdp_legal_queue_count(adapter, count)) {
 911                 if (ena_xdp_present(adapter))
 912                         return -EINVAL;
 913
 914                 xdp_clear_features_flag(netdev);
 915         } else {
 916                 xdp_set_features_flag(netdev,
 917                                       NETDEV_XDP_ACT_BASIC |
 918                                       NETDEV_XDP_ACT_REDIRECT);
 919         }
 920
 921         return ena_update_queue_count(adapter, count);
 922 }
 923
 924 static int ena_get_tunable(struct net_device *netdev,
 925                            const struct ethtool_tunable *tuna, void *data)
 926 {
 927         struct ena_adapter *adapter = netdev_priv(netdev);
 928         int ret = 0;
 929
 930         switch (tuna->id) {
 931         case ETHTOOL_RX_COPYBREAK:
 932                 *(u32 *)data = adapter->rx_copybreak;
 933                 break;
 934         default:
 935                 ret = -EINVAL;
 936                 break;
 937         }
 938
 939         return ret;
 940 }
 941
 942 static int ena_set_tunable(struct net_device *netdev,
 943                            const struct ethtool_tunable *tuna,
 944                            const void *data)
 945 {
 946         struct ena_adapter *adapter = netdev_priv(netdev);
 947         int ret = 0;
 948         u32 len;
 949
 950         switch (tuna->id) {
 951         case ETHTOOL_RX_COPYBREAK:
 952                 len = *(u32 *)data;
 953                 ret = ena_set_rx_copybreak(adapter, len);
 954                 break;
 955         default:
 956                 ret = -EINVAL;
 957                 break;
 958         }
 959
 960         return ret;
 961 }
 962
 963 static const struct ethtool_ops ena_ethtool_ops = {
 964         .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
 965                                      ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
 966         .supported_ring_params  = ETHTOOL_RING_USE_TX_PUSH_BUF_LEN |
 967                                   ETHTOOL_RING_USE_TX_PUSH,
 968         .get_link_ksettings     = ena_get_link_ksettings,
 969         .get_drvinfo            = ena_get_drvinfo,
 970         .get_msglevel           = ena_get_msglevel,
 971         .set_msglevel           = ena_set_msglevel,
 972         .get_link               = ethtool_op_get_link,
 973         .get_coalesce           = ena_get_coalesce,
 974         .set_coalesce           = ena_set_coalesce,
 975         .get_ringparam          = ena_get_ringparam,
 976         .set_ringparam          = ena_set_ringparam,
 977         .get_sset_count         = ena_get_sset_count,
 978         .get_strings            = ena_get_ethtool_strings,
 979         .get_ethtool_stats      = ena_get_ethtool_stats,
 980         .get_rxnfc              = ena_get_rxnfc,
 981         .set_rxnfc              = ena_set_rxnfc,
 982         .get_rxfh_indir_size    = ena_get_rxfh_indir_size,
 983         .get_rxfh_key_size      = ena_get_rxfh_key_size,
 984         .get_rxfh               = ena_get_rxfh,
 985         .set_rxfh               = ena_set_rxfh,
 986         .get_channels           = ena_get_channels,
 987         .set_channels           = ena_set_channels,
 988         .get_tunable            = ena_get_tunable,
 989         .set_tunable            = ena_set_tunable,
 990         .get_ts_info            = ethtool_op_get_ts_info,
 991 };
 992
 993 void ena_set_ethtool_ops(struct net_device *netdev)
 994 {
 995         netdev->ethtool_ops = &ena_ethtool_ops;
 996 }
 997
 998 static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf)
 999 {
1000         struct net_device *netdev = adapter->netdev;
1001         u8 *strings_buf;
1002         u64 *data_buf;
1003         int strings_num;
1004         int i, rc;
1005
1006         strings_num = ena_get_sw_stats_count(adapter);
1007         if (strings_num <= 0) {
1008                 netif_err(adapter, drv, netdev, "Can't get stats num\n");
1009                 return;
1010         }
1011
1012         strings_buf = devm_kcalloc(&adapter->pdev->dev,
1013                                    ETH_GSTRING_LEN, strings_num,
1014                                    GFP_ATOMIC);
1015         if (!strings_buf) {
1016                 netif_err(adapter, drv, netdev,
1017                           "Failed to allocate strings_buf\n");
1018                 return;
1019         }
1020
1021         data_buf = devm_kcalloc(&adapter->pdev->dev,
1022                                 strings_num, sizeof(u64),
1023                                 GFP_ATOMIC);
1024         if (!data_buf) {
1025                 netif_err(adapter, drv, netdev,
1026                           "Failed to allocate data buf\n");
1027                 devm_kfree(&adapter->pdev->dev, strings_buf);
1028                 return;
1029         }
1030
1031         ena_get_strings(adapter, strings_buf, false);
1032         ena_get_stats(adapter, data_buf, false);
1033
1034         /* If there is a buffer, dump stats, otherwise print them to dmesg */
1035         if (buf)
1036                 for (i = 0; i < strings_num; i++) {
1037                         rc = snprintf(buf, ETH_GSTRING_LEN + sizeof(u64),
1038                                       "%s %llu\n",
1039                                       strings_buf + i * ETH_GSTRING_LEN,
1040                                       data_buf[i]);
1041                         buf += rc;
1042                 }
1043         else
1044                 for (i = 0; i < strings_num; i++)
1045                         netif_err(adapter, drv, netdev, "%s: %llu\n",
1046                                   strings_buf + i * ETH_GSTRING_LEN,
1047                                   data_buf[i]);
1048
1049         devm_kfree(&adapter->pdev->dev, strings_buf);
1050         devm_kfree(&adapter->pdev->dev, data_buf);
1051 }
1052
1053 void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf)
1054 {
1055         if (!buf)
1056                 return;
1057
1058         ena_dump_stats_ex(adapter, buf);
1059 }
1060
1061 void ena_dump_stats_to_dmesg(struct ena_adapter *adapter)
1062 {
1063         ena_dump_stats_ex(adapter, NULL);
1064 }