drivers/net/ethernet/pensando/ionic/ionic_lif.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ethtool.h>
   5 #include <linux/printk.h>
   6 #include <linux/dynamic_debug.h>
   7 #include <linux/netdevice.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/if_vlan.h>
  10 #include <linux/rtnetlink.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/pci.h>
  13 #include <linux/cpumask.h>
  14 #include <linux/crash_dump.h>
  15 #include <linux/vmalloc.h>
  16 #include <net/page_pool/helpers.h>
  17
  18 #include "ionic.h"
  19 #include "ionic_bus.h"
  20 #include "ionic_dev.h"
  21 #include "ionic_lif.h"
  22 #include "ionic_txrx.h"
  23 #include "ionic_ethtool.h"
  24 #include "ionic_debugfs.h"
  25
  26 /* queuetype support level */
  27 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
  28         [IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
  29         [IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
  30         [IONIC_QTYPE_RXQ]     = 2,   /* 0 = Base version with CQ+SG support
  31                                       * 2 =       ... with CMB rings
  32                                       */
  33         [IONIC_QTYPE_TXQ]     = 3,   /* 0 = Base version with CQ+SG support
  34                                       * 1 =       ... with Tx SG version 1
  35                                       * 3 =       ... with CMB rings
  36                                       */
  37 };
  38
  39 static void ionic_link_status_check(struct ionic_lif *lif);
  40 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
  41 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
  42 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
  43
  44 static void ionic_txrx_deinit(struct ionic_lif *lif);
  45 static int ionic_txrx_init(struct ionic_lif *lif);
  46 static int ionic_start_queues(struct ionic_lif *lif);
  47 static void ionic_stop_queues(struct ionic_lif *lif);
  48 static void ionic_lif_queue_identify(struct ionic_lif *lif);
  49
  50 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif);
  51 static void ionic_unregister_rxq_info(struct ionic_queue *q);
  52 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id);
  53
  54 static void ionic_dim_work(struct work_struct *work)
  55 {
  56         struct dim *dim = container_of(work, struct dim, work);
  57         struct dim_cq_moder cur_moder;
  58         struct ionic_intr_info *intr;
  59         struct ionic_qcq *qcq;
  60         struct ionic_lif *lif;
  61         struct ionic_queue *q;
  62         u32 new_coal;
  63
  64         qcq = container_of(dim, struct ionic_qcq, dim);
  65         q = &qcq->q;
  66         if (q->type == IONIC_QTYPE_RXQ)
  67                 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
  68         else
  69                 cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
  70         lif = q->lif;
  71         new_coal = ionic_coal_usec_to_hw(lif->ionic, cur_moder.usec);
  72         new_coal = new_coal ? new_coal : 1;
  73
  74         intr = &qcq->intr;
  75         if (intr->dim_coal_hw != new_coal) {
  76                 intr->dim_coal_hw = new_coal;
  77
  78                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
  79                                      intr->index, intr->dim_coal_hw);
  80         }
  81
  82         dim->state = DIM_START_MEASURE;
  83 }
  84
  85 static void ionic_lif_deferred_work(struct work_struct *work)
  86 {
  87         struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
  88         struct ionic_deferred *def = &lif->deferred;
  89         struct ionic_deferred_work *w = NULL;
  90
  91         do {
  92                 spin_lock_bh(&def->lock);
  93                 if (!list_empty(&def->list)) {
  94                         w = list_first_entry(&def->list,
  95                                              struct ionic_deferred_work, list);
  96                         list_del(&w->list);
  97                 }
  98                 spin_unlock_bh(&def->lock);
  99
 100                 if (!w)
 101                         break;
 102
 103                 switch (w->type) {
 104                 case IONIC_DW_TYPE_RX_MODE:
 105                         ionic_lif_rx_mode(lif);
 106                         break;
 107                 case IONIC_DW_TYPE_LINK_STATUS:
 108                         ionic_link_status_check(lif);
 109                         break;
 110                 case IONIC_DW_TYPE_LIF_RESET:
 111                         if (w->fw_status) {
 112                                 ionic_lif_handle_fw_up(lif);
 113                         } else {
 114                                 ionic_lif_handle_fw_down(lif);
 115
 116                                 /* Fire off another watchdog to see
 117                                  * if the FW is already back rather than
 118                                  * waiting another whole cycle
 119                                  */
 120                                 mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
 121                         }
 122                         break;
 123                 default:
 124                         break;
 125                 }
 126                 kfree(w);
 127                 w = NULL;
 128         } while (true);
 129 }
 130
 131 void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
 132                                 struct ionic_deferred_work *work)
 133 {
 134         spin_lock_bh(&lif->deferred.lock);
 135         list_add_tail(&work->list, &lif->deferred.list);
 136         spin_unlock_bh(&lif->deferred.lock);
 137         queue_work(lif->ionic->wq, &lif->deferred.work);
 138 }
 139
 140 static void ionic_link_status_check(struct ionic_lif *lif)
 141 {
 142         struct net_device *netdev = lif->netdev;
 143         u16 link_status;
 144         bool link_up;
 145
 146         if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 147                 return;
 148
 149         /* Don't put carrier back up if we're in a broken state */
 150         if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
 151                 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 152                 return;
 153         }
 154
 155         link_status = le16_to_cpu(lif->info->status.link_status);
 156         link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 157
 158         if (link_up) {
 159                 int err = 0;
 160
 161                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 162                         mutex_lock(&lif->queue_lock);
 163                         err = ionic_start_queues(lif);
 164                         if (err && err != -EBUSY) {
 165                                 netdev_err(netdev,
 166                                            "Failed to start queues: %d\n", err);
 167                                 set_bit(IONIC_LIF_F_BROKEN, lif->state);
 168                                 netif_carrier_off(lif->netdev);
 169                         }
 170                         mutex_unlock(&lif->queue_lock);
 171                 }
 172
 173                 if (!err && !netif_carrier_ok(netdev)) {
 174                         ionic_port_identify(lif->ionic);
 175                         netdev_info(netdev, "Link up - %d Gbps\n",
 176                                     le32_to_cpu(lif->info->status.link_speed) / 1000);
 177                         netif_carrier_on(netdev);
 178                 }
 179         } else {
 180                 if (netif_carrier_ok(netdev)) {
 181                         lif->link_down_count++;
 182                         netdev_info(netdev, "Link down\n");
 183                         netif_carrier_off(netdev);
 184                 }
 185
 186                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 187                         mutex_lock(&lif->queue_lock);
 188                         ionic_stop_queues(lif);
 189                         mutex_unlock(&lif->queue_lock);
 190                 }
 191         }
 192
 193         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 194 }
 195
 196 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
 197 {
 198         struct ionic_deferred_work *work;
 199
 200         /* we only need one request outstanding at a time */
 201         if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 202                 return;
 203
 204         if (!can_sleep) {
 205                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
 206                 if (!work) {
 207                         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 208                         return;
 209                 }
 210
 211                 work->type = IONIC_DW_TYPE_LINK_STATUS;
 212                 ionic_lif_deferred_enqueue(lif, work);
 213         } else {
 214                 ionic_link_status_check(lif);
 215         }
 216 }
 217
 218 static irqreturn_t ionic_isr(int irq, void *data)
 219 {
 220         struct napi_struct *napi = data;
 221
 222         napi_schedule_irqoff(napi);
 223
 224         return IRQ_HANDLED;
 225 }
 226
 227 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
 228 {
 229         struct ionic_intr_info *intr = &qcq->intr;
 230         struct device *dev = lif->ionic->dev;
 231         struct ionic_queue *q = &qcq->q;
 232         const char *name;
 233
 234         if (lif->registered)
 235                 name = netdev_name(lif->netdev);
 236         else
 237                 name = dev_name(dev);
 238
 239         snprintf(intr->name, sizeof(intr->name),
 240                  "%.5s-%.16s-%.8s", IONIC_DRV_NAME, name, q->name);
 241
 242         return devm_request_irq(dev, intr->vector, ionic_isr,
 243                                 0, intr->name, &qcq->napi);
 244 }
 245
 246 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 247 {
 248         struct ionic *ionic = lif->ionic;
 249         int index;
 250
 251         index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
 252         if (index == ionic->nintrs) {
 253                 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
 254                             __func__, index, ionic->nintrs);
 255                 return -ENOSPC;
 256         }
 257
 258         set_bit(index, ionic->intrs);
 259         ionic_intr_init(&ionic->idev, intr, index);
 260
 261         return 0;
 262 }
 263
 264 static void ionic_intr_free(struct ionic *ionic, int index)
 265 {
 266         if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
 267                 clear_bit(index, ionic->intrs);
 268 }
 269
 270 static void ionic_irq_aff_notify(struct irq_affinity_notify *notify,
 271                                  const cpumask_t *mask)
 272 {
 273         struct ionic_intr_info *intr = container_of(notify, struct ionic_intr_info, aff_notify);
 274
 275         cpumask_copy(*intr->affinity_mask, mask);
 276 }
 277
 278 static void ionic_irq_aff_release(struct kref __always_unused *ref)
 279 {
 280 }
 281
 282 static int ionic_qcq_enable(struct ionic_qcq *qcq)
 283 {
 284         struct ionic_queue *q = &qcq->q;
 285         struct ionic_lif *lif = q->lif;
 286         struct ionic_dev *idev;
 287         struct device *dev;
 288
 289         struct ionic_admin_ctx ctx = {
 290                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 291                 .cmd.q_control = {
 292                         .opcode = IONIC_CMD_Q_CONTROL,
 293                         .lif_index = cpu_to_le16(lif->index),
 294                         .type = q->type,
 295                         .index = cpu_to_le32(q->index),
 296                         .oper = IONIC_Q_ENABLE,
 297                 },
 298         };
 299         int ret;
 300
 301         idev = &lif->ionic->idev;
 302         dev = lif->ionic->dev;
 303
 304         dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
 305                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 306
 307         if (qcq->flags & IONIC_QCQ_F_INTR)
 308                 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
 309
 310         ret = ionic_adminq_post_wait(lif, &ctx);
 311         if (ret)
 312                 return ret;
 313
 314         if (qcq->flags & IONIC_QCQ_F_INTR) {
 315                 napi_enable(&qcq->napi);
 316                 irq_set_affinity_notifier(qcq->intr.vector,
 317                                           &qcq->intr.aff_notify);
 318                 irq_set_affinity_hint(qcq->intr.vector,
 319                                       *qcq->intr.affinity_mask);
 320                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 321                                 IONIC_INTR_MASK_CLEAR);
 322         }
 323
 324         return 0;
 325 }
 326
 327 static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
 328 {
 329         struct ionic_queue *q;
 330
 331         struct ionic_admin_ctx ctx = {
 332                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 333                 .cmd.q_control = {
 334                         .opcode = IONIC_CMD_Q_CONTROL,
 335                         .oper = IONIC_Q_DISABLE,
 336                 },
 337         };
 338
 339         if (!qcq) {
 340                 netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
 341                 return -ENXIO;
 342         }
 343
 344         q = &qcq->q;
 345
 346         if (qcq->flags & IONIC_QCQ_F_INTR) {
 347                 struct ionic_dev *idev = &lif->ionic->idev;
 348
 349                 if (lif->doorbell_wa)
 350                         cancel_work_sync(&qcq->doorbell_napi_work);
 351                 cancel_work_sync(&qcq->dim.work);
 352                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 353                                 IONIC_INTR_MASK_SET);
 354                 synchronize_irq(qcq->intr.vector);
 355                 irq_set_affinity_notifier(qcq->intr.vector, NULL);
 356                 irq_set_affinity_hint(qcq->intr.vector, NULL);
 357                 napi_disable(&qcq->napi);
 358         }
 359
 360         /* If there was a previous fw communcation error, don't bother with
 361          * sending the adminq command and just return the same error value.
 362          */
 363         if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
 364                 return fw_err;
 365
 366         ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
 367         ctx.cmd.q_control.type = q->type;
 368         ctx.cmd.q_control.index = cpu_to_le32(q->index);
 369         dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
 370                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 371
 372         return ionic_adminq_post_wait(lif, &ctx);
 373 }
 374
 375 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 376 {
 377         struct ionic_dev *idev = &lif->ionic->idev;
 378
 379         if (!qcq)
 380                 return;
 381
 382         if (!(qcq->flags & IONIC_QCQ_F_INITED))
 383                 return;
 384
 385         ionic_unregister_rxq_info(&qcq->q);
 386         if (qcq->flags & IONIC_QCQ_F_INTR) {
 387                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 388                                 IONIC_INTR_MASK_SET);
 389                 netif_napi_del(&qcq->napi);
 390         }
 391
 392         qcq->flags &= ~IONIC_QCQ_F_INITED;
 393 }
 394
 395 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 396 {
 397         if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
 398                 return;
 399
 400         irq_set_affinity_hint(qcq->intr.vector, NULL);
 401         devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
 402         qcq->intr.vector = 0;
 403         ionic_intr_free(lif->ionic, qcq->intr.index);
 404         qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 405 }
 406
 407 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 408 {
 409         struct device *dev = lif->ionic->dev;
 410
 411         if (!qcq)
 412                 return;
 413
 414         ionic_debugfs_del_qcq(qcq);
 415
 416         if (qcq->q_base) {
 417                 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
 418                 qcq->q_base = NULL;
 419                 qcq->q_base_pa = 0;
 420         }
 421
 422         if (qcq->cmb_q_base) {
 423                 iounmap(qcq->cmb_q_base);
 424                 ionic_put_cmb(lif, qcq->cmb_pgid, qcq->cmb_order);
 425                 qcq->cmb_pgid = 0;
 426                 qcq->cmb_order = 0;
 427                 qcq->cmb_q_base = NULL;
 428                 qcq->cmb_q_base_pa = 0;
 429         }
 430
 431         if (qcq->cq_base) {
 432                 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
 433                 qcq->cq_base = NULL;
 434                 qcq->cq_base_pa = 0;
 435         }
 436
 437         if (qcq->sg_base) {
 438                 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
 439                 qcq->sg_base = NULL;
 440                 qcq->sg_base_pa = 0;
 441         }
 442
 443         page_pool_destroy(qcq->q.page_pool);
 444         qcq->q.page_pool = NULL;
 445
 446         ionic_qcq_intr_free(lif, qcq);
 447         vfree(qcq->q.info);
 448         qcq->q.info = NULL;
 449 }
 450
 451 void ionic_qcqs_free(struct ionic_lif *lif)
 452 {
 453         struct device *dev = lif->ionic->dev;
 454         struct ionic_qcq *adminqcq;
 455         unsigned long irqflags;
 456
 457         if (lif->notifyqcq) {
 458                 ionic_qcq_free(lif, lif->notifyqcq);
 459                 devm_kfree(dev, lif->notifyqcq);
 460                 lif->notifyqcq = NULL;
 461         }
 462
 463         if (lif->adminqcq) {
 464                 spin_lock_irqsave(&lif->adminq_lock, irqflags);
 465                 adminqcq = READ_ONCE(lif->adminqcq);
 466                 lif->adminqcq = NULL;
 467                 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
 468                 if (adminqcq) {
 469                         ionic_qcq_free(lif, adminqcq);
 470                         devm_kfree(dev, adminqcq);
 471                 }
 472         }
 473
 474         if (lif->rxqcqs) {
 475                 devm_kfree(dev, lif->rxqstats);
 476                 lif->rxqstats = NULL;
 477                 devm_kfree(dev, lif->rxqcqs);
 478                 lif->rxqcqs = NULL;
 479         }
 480
 481         if (lif->txqcqs) {
 482                 devm_kfree(dev, lif->txqstats);
 483                 lif->txqstats = NULL;
 484                 devm_kfree(dev, lif->txqcqs);
 485                 lif->txqcqs = NULL;
 486         }
 487 }
 488
 489 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
 490                                       struct ionic_qcq *n_qcq)
 491 {
 492         n_qcq->intr.vector = src_qcq->intr.vector;
 493         n_qcq->intr.index = src_qcq->intr.index;
 494 }
 495
 496 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
 497 {
 498         cpumask_var_t *affinity_mask;
 499         int err;
 500
 501         if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
 502                 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 503                 return 0;
 504         }
 505
 506         err = ionic_intr_alloc(lif, &qcq->intr);
 507         if (err) {
 508                 netdev_warn(lif->netdev, "no intr for %s: %d\n",
 509                             qcq->q.name, err);
 510                 goto err_out;
 511         }
 512
 513         err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
 514         if (err < 0) {
 515                 netdev_warn(lif->netdev, "no vector for %s: %d\n",
 516                             qcq->q.name, err);
 517                 goto err_out_free_intr;
 518         }
 519         qcq->intr.vector = err;
 520         ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
 521                                IONIC_INTR_MASK_SET);
 522
 523         err = ionic_request_irq(lif, qcq);
 524         if (err) {
 525                 netdev_warn(lif->netdev, "irq request failed %d\n", err);
 526                 goto err_out_free_intr;
 527         }
 528
 529         /* try to get the irq on the local numa node first */
 530         affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];
 531         if (cpumask_empty(*affinity_mask)) {
 532                 unsigned int cpu;
 533
 534                 cpu = cpumask_local_spread(qcq->intr.index,
 535                                            dev_to_node(lif->ionic->dev));
 536                 if (cpu != -1)
 537                         cpumask_set_cpu(cpu, *affinity_mask);
 538         }
 539
 540         qcq->intr.affinity_mask = affinity_mask;
 541         qcq->intr.aff_notify.notify = ionic_irq_aff_notify;
 542         qcq->intr.aff_notify.release = ionic_irq_aff_release;
 543
 544         netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
 545         return 0;
 546
 547 err_out_free_intr:
 548         ionic_intr_free(lif->ionic, qcq->intr.index);
 549 err_out:
 550         return err;
 551 }
 552
 553 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
 554                            unsigned int index,
 555                            const char *name, unsigned int flags,
 556                            unsigned int num_descs, unsigned int desc_size,
 557                            unsigned int cq_desc_size,
 558                            unsigned int sg_desc_size,
 559                            unsigned int desc_info_size,
 560                            unsigned int pid, struct bpf_prog *xdp_prog,
 561                            struct ionic_qcq **qcq)
 562 {
 563         struct ionic_dev *idev = &lif->ionic->idev;
 564         struct device *dev = lif->ionic->dev;
 565         struct ionic_qcq *new;
 566         int err;
 567
 568         *qcq = NULL;
 569
 570         new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
 571         if (!new) {
 572                 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
 573                 err = -ENOMEM;
 574                 goto err_out;
 575         }
 576
 577         new->q.dev = dev;
 578         new->flags = flags;
 579
 580         new->q.info = vcalloc(num_descs, desc_info_size);
 581         if (!new->q.info) {
 582                 netdev_err(lif->netdev, "Cannot allocate queue info\n");
 583                 err = -ENOMEM;
 584                 goto err_out_free_qcq;
 585         }
 586
 587         if (type == IONIC_QTYPE_RXQ) {
 588                 struct page_pool_params pp_params = {
 589                         .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
 590                         .order = 0,
 591                         .pool_size = num_descs,
 592                         .nid = NUMA_NO_NODE,
 593                         .dev = lif->ionic->dev,
 594                         .napi = &new->napi,
 595                         .dma_dir = DMA_FROM_DEVICE,
 596                         .max_len = PAGE_SIZE,
 597                         .netdev = lif->netdev,
 598                 };
 599
 600                 if (xdp_prog)
 601                         pp_params.dma_dir = DMA_BIDIRECTIONAL;
 602
 603                 new->q.page_pool = page_pool_create(&pp_params);
 604                 if (IS_ERR(new->q.page_pool)) {
 605                         netdev_err(lif->netdev, "Cannot create page_pool\n");
 606                         err = PTR_ERR(new->q.page_pool);
 607                         new->q.page_pool = NULL;
 608                         goto err_out_free_q_info;
 609                 }
 610         }
 611
 612         new->q.type = type;
 613         new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
 614
 615         err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
 616                            desc_size, sg_desc_size, pid);
 617         if (err) {
 618                 netdev_err(lif->netdev, "Cannot initialize queue\n");
 619                 goto err_out_free_page_pool;
 620         }
 621
 622         err = ionic_alloc_qcq_interrupt(lif, new);
 623         if (err)
 624                 goto err_out_free_page_pool;
 625
 626         err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
 627         if (err) {
 628                 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
 629                 goto err_out_free_irq;
 630         }
 631
 632         if (flags & IONIC_QCQ_F_NOTIFYQ) {
 633                 int q_size;
 634
 635                 /* q & cq need to be contiguous in NotifyQ, so alloc it all in q
 636                  * and don't alloc qc.  We leave new->qc_size and new->qc_base
 637                  * as 0 to be sure we don't try to free it later.
 638                  */
 639                 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
 640                 new->q_size = PAGE_SIZE + q_size +
 641                               ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
 642                 new->q_base = dma_alloc_coherent(dev, new->q_size,
 643                                                  &new->q_base_pa, GFP_KERNEL);
 644                 if (!new->q_base) {
 645                         netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
 646                         err = -ENOMEM;
 647                         goto err_out_free_irq;
 648                 }
 649                 new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 650                 new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 651
 652                 /* Base the NotifyQ cq.base off of the ALIGNed q.base */
 653                 new->cq.base = PTR_ALIGN(new->q.base + q_size, PAGE_SIZE);
 654                 new->cq.base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
 655                 new->cq.bound_q = &new->q;
 656         } else {
 657                 /* regular DMA q descriptors */
 658                 new->q_size = PAGE_SIZE + (num_descs * desc_size);
 659                 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
 660                                                  GFP_KERNEL);
 661                 if (!new->q_base) {
 662                         netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
 663                         err = -ENOMEM;
 664                         goto err_out_free_irq;
 665                 }
 666                 new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 667                 new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 668
 669                 if (flags & IONIC_QCQ_F_CMB_RINGS) {
 670                         /* on-chip CMB q descriptors */
 671                         new->cmb_q_size = num_descs * desc_size;
 672                         new->cmb_order = order_base_2(new->cmb_q_size / PAGE_SIZE);
 673
 674                         err = ionic_get_cmb(lif, &new->cmb_pgid, &new->cmb_q_base_pa,
 675                                             new->cmb_order);
 676                         if (err) {
 677                                 netdev_err(lif->netdev,
 678                                            "Cannot allocate queue order %d from cmb: err %d\n",
 679                                            new->cmb_order, err);
 680                                 goto err_out_free_q;
 681                         }
 682
 683                         new->cmb_q_base = ioremap_wc(new->cmb_q_base_pa, new->cmb_q_size);
 684                         if (!new->cmb_q_base) {
 685                                 netdev_err(lif->netdev, "Cannot map queue from cmb\n");
 686                                 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
 687                                 err = -ENOMEM;
 688                                 goto err_out_free_q;
 689                         }
 690
 691                         new->cmb_q_base_pa -= idev->phy_cmb_pages;
 692                         new->q.cmb_base = new->cmb_q_base;
 693                         new->q.cmb_base_pa = new->cmb_q_base_pa;
 694                 }
 695
 696                 /* cq DMA descriptors */
 697                 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
 698                 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
 699                                                   GFP_KERNEL);
 700                 if (!new->cq_base) {
 701                         netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
 702                         err = -ENOMEM;
 703                         goto err_out_free_q;
 704                 }
 705                 new->cq.base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
 706                 new->cq.base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
 707                 new->cq.bound_q = &new->q;
 708         }
 709
 710         if (flags & IONIC_QCQ_F_SG) {
 711                 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
 712                 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
 713                                                   GFP_KERNEL);
 714                 if (!new->sg_base) {
 715                         netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
 716                         err = -ENOMEM;
 717                         goto err_out_free_cq;
 718                 }
 719                 new->q.sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
 720                 new->q.sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
 721         }
 722
 723         INIT_WORK(&new->dim.work, ionic_dim_work);
 724         new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
 725         if (lif->doorbell_wa)
 726                 INIT_WORK(&new->doorbell_napi_work, ionic_doorbell_napi_work);
 727
 728         *qcq = new;
 729
 730         return 0;
 731
 732 err_out_free_cq:
 733         dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
 734 err_out_free_q:
 735         if (new->cmb_q_base) {
 736                 iounmap(new->cmb_q_base);
 737                 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
 738         }
 739         dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
 740 err_out_free_irq:
 741         if (flags & IONIC_QCQ_F_INTR) {
 742                 devm_free_irq(dev, new->intr.vector, &new->napi);
 743                 ionic_intr_free(lif->ionic, new->intr.index);
 744         }
 745 err_out_free_page_pool:
 746         page_pool_destroy(new->q.page_pool);
 747 err_out_free_q_info:
 748         vfree(new->q.info);
 749 err_out_free_qcq:
 750         devm_kfree(dev, new);
 751 err_out:
 752         dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
 753         return err;
 754 }
 755
 756 static int ionic_qcqs_alloc(struct ionic_lif *lif)
 757 {
 758         struct device *dev = lif->ionic->dev;
 759         unsigned int flags;
 760         int err;
 761
 762         flags = IONIC_QCQ_F_INTR;
 763         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 764                               IONIC_ADMINQ_LENGTH,
 765                               sizeof(struct ionic_admin_cmd),
 766                               sizeof(struct ionic_admin_comp),
 767                               0,
 768                               sizeof(struct ionic_admin_desc_info),
 769                               lif->kern_pid, NULL, &lif->adminqcq);
 770         if (err)
 771                 return err;
 772         ionic_debugfs_add_qcq(lif, lif->adminqcq);
 773
 774         if (lif->ionic->nnqs_per_lif) {
 775                 flags = IONIC_QCQ_F_NOTIFYQ;
 776                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
 777                                       flags, IONIC_NOTIFYQ_LENGTH,
 778                                       sizeof(struct ionic_notifyq_cmd),
 779                                       sizeof(union ionic_notifyq_comp),
 780                                       0,
 781                                       sizeof(struct ionic_admin_desc_info),
 782                                       lif->kern_pid, NULL, &lif->notifyqcq);
 783                 if (err)
 784                         goto err_out;
 785                 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
 786
 787                 /* Let the notifyq ride on the adminq interrupt */
 788                 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
 789         }
 790
 791         err = -ENOMEM;
 792         lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
 793                                    sizeof(*lif->txqcqs), GFP_KERNEL);
 794         if (!lif->txqcqs)
 795                 goto err_out;
 796         lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
 797                                    sizeof(*lif->rxqcqs), GFP_KERNEL);
 798         if (!lif->rxqcqs)
 799                 goto err_out;
 800
 801         lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
 802                                      sizeof(*lif->txqstats), GFP_KERNEL);
 803         if (!lif->txqstats)
 804                 goto err_out;
 805         lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
 806                                      sizeof(*lif->rxqstats), GFP_KERNEL);
 807         if (!lif->rxqstats)
 808                 goto err_out;
 809
 810         return 0;
 811
 812 err_out:
 813         ionic_qcqs_free(lif);
 814         return err;
 815 }
 816
 817 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
 818 {
 819         qcq->q.tail_idx = 0;
 820         qcq->q.head_idx = 0;
 821         qcq->cq.tail_idx = 0;
 822         qcq->cq.done_color = 1;
 823         memset(qcq->q_base, 0, qcq->q_size);
 824         if (qcq->cmb_q_base)
 825                 memset_io(qcq->cmb_q_base, 0, qcq->cmb_q_size);
 826         memset(qcq->cq_base, 0, qcq->cq_size);
 827         memset(qcq->sg_base, 0, qcq->sg_size);
 828 }
 829
 830 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 831 {
 832         struct device *dev = lif->ionic->dev;
 833         struct ionic_queue *q = &qcq->q;
 834         struct ionic_cq *cq = &qcq->cq;
 835         struct ionic_admin_ctx ctx = {
 836                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 837                 .cmd.q_init = {
 838                         .opcode = IONIC_CMD_Q_INIT,
 839                         .lif_index = cpu_to_le16(lif->index),
 840                         .type = q->type,
 841                         .ver = lif->qtype_info[q->type].version,
 842                         .index = cpu_to_le32(q->index),
 843                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 844                                              IONIC_QINIT_F_SG),
 845                         .intr_index = cpu_to_le16(qcq->intr.index),
 846                         .pid = cpu_to_le16(q->pid),
 847                         .ring_size = ilog2(q->num_descs),
 848                         .ring_base = cpu_to_le64(q->base_pa),
 849                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 850                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 851                         .features = cpu_to_le64(q->features),
 852                 },
 853         };
 854         int err;
 855
 856         if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
 857                 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
 858                 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
 859         }
 860
 861         dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
 862         dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
 863         dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 864         dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 865         dev_dbg(dev, "txq_init.cq_ring_base 0x%llx\n", ctx.cmd.q_init.cq_ring_base);
 866         dev_dbg(dev, "txq_init.sg_ring_base 0x%llx\n", ctx.cmd.q_init.sg_ring_base);
 867         dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 868         dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
 869         dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 870
 871         ionic_qcq_sanitize(qcq);
 872
 873         err = ionic_adminq_post_wait(lif, &ctx);
 874         if (err)
 875                 return err;
 876
 877         q->hw_type = ctx.comp.q_init.hw_type;
 878         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 879         q->dbval = IONIC_DBELL_QID(q->hw_index);
 880
 881         dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
 882         dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 883
 884         q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
 885         q->dbell_jiffies = jiffies;
 886
 887         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 888                 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
 889
 890         qcq->flags |= IONIC_QCQ_F_INITED;
 891
 892         return 0;
 893 }
 894
 895 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 896 {
 897         struct device *dev = lif->ionic->dev;
 898         struct ionic_queue *q = &qcq->q;
 899         struct ionic_cq *cq = &qcq->cq;
 900         struct ionic_admin_ctx ctx = {
 901                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 902                 .cmd.q_init = {
 903                         .opcode = IONIC_CMD_Q_INIT,
 904                         .lif_index = cpu_to_le16(lif->index),
 905                         .type = q->type,
 906                         .ver = lif->qtype_info[q->type].version,
 907                         .index = cpu_to_le32(q->index),
 908                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ),
 909                         .intr_index = cpu_to_le16(cq->bound_intr->index),
 910                         .pid = cpu_to_le16(q->pid),
 911                         .ring_size = ilog2(q->num_descs),
 912                         .ring_base = cpu_to_le64(q->base_pa),
 913                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 914                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 915                         .features = cpu_to_le64(q->features),
 916                 },
 917         };
 918         int err;
 919
 920         q->partner = &lif->txqcqs[q->index]->q;
 921         q->partner->partner = q;
 922
 923         if (!lif->xdp_prog ||
 924             (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags))
 925                 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
 926
 927         if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
 928                 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
 929                 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
 930         }
 931
 932         dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
 933         dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
 934         dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 935         dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 936         dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 937         dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
 938         dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 939
 940         ionic_qcq_sanitize(qcq);
 941
 942         err = ionic_adminq_post_wait(lif, &ctx);
 943         if (err)
 944                 return err;
 945
 946         q->hw_type = ctx.comp.q_init.hw_type;
 947         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 948         q->dbval = IONIC_DBELL_QID(q->hw_index);
 949
 950         dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
 951         dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 952
 953         q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
 954         q->dbell_jiffies = jiffies;
 955
 956         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 957                 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
 958         else
 959                 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
 960         err = ionic_register_rxq_info(q, qcq->napi.napi_id);
 961         if (err) {
 962                 netif_napi_del(&qcq->napi);
 963                 return err;
 964         }
 965
 966         qcq->flags |= IONIC_QCQ_F_INITED;
 967
 968         return 0;
 969 }
 970
 971 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
 972 {
 973         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 974         unsigned int txq_i, flags;
 975         struct ionic_qcq *txq;
 976         u64 features;
 977         int err;
 978
 979         if (lif->hwstamp_txq)
 980                 return 0;
 981
 982         features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
 983
 984         num_desc = IONIC_MIN_TXRX_DESC;
 985         desc_sz = sizeof(struct ionic_txq_desc);
 986         comp_sz = 2 * sizeof(struct ionic_txq_comp);
 987
 988         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
 989             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
 990                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
 991         else
 992                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
 993
 994         txq_i = lif->ionic->ntxqs_per_lif;
 995         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
 996
 997         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
 998                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 999                               sizeof(struct ionic_tx_desc_info),
1000                               lif->kern_pid, NULL, &txq);
1001         if (err)
1002                 goto err_qcq_alloc;
1003
1004         txq->q.features = features;
1005
1006         ionic_link_qcq_interrupts(lif->adminqcq, txq);
1007         ionic_debugfs_add_qcq(lif, txq);
1008
1009         lif->hwstamp_txq = txq;
1010
1011         if (netif_running(lif->netdev)) {
1012                 err = ionic_lif_txq_init(lif, txq);
1013                 if (err)
1014                         goto err_qcq_init;
1015
1016                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1017                         err = ionic_qcq_enable(txq);
1018                         if (err)
1019                                 goto err_qcq_enable;
1020                 }
1021         }
1022
1023         return 0;
1024
1025 err_qcq_enable:
1026         ionic_lif_qcq_deinit(lif, txq);
1027 err_qcq_init:
1028         lif->hwstamp_txq = NULL;
1029         ionic_debugfs_del_qcq(txq);
1030         ionic_qcq_free(lif, txq);
1031         devm_kfree(lif->ionic->dev, txq);
1032 err_qcq_alloc:
1033         return err;
1034 }
1035
1036 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
1037 {
1038         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
1039         unsigned int rxq_i, flags;
1040         struct ionic_qcq *rxq;
1041         u64 features;
1042         int err;
1043
1044         if (lif->hwstamp_rxq)
1045                 return 0;
1046
1047         features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1048
1049         num_desc = IONIC_MIN_TXRX_DESC;
1050         desc_sz = sizeof(struct ionic_rxq_desc);
1051         comp_sz = 2 * sizeof(struct ionic_rxq_comp);
1052         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
1053
1054         rxq_i = lif->ionic->nrxqs_per_lif;
1055         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
1056
1057         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
1058                               num_desc, desc_sz, comp_sz, sg_desc_sz,
1059                               sizeof(struct ionic_rx_desc_info),
1060                               lif->kern_pid, NULL, &rxq);
1061         if (err)
1062                 goto err_qcq_alloc;
1063
1064         rxq->q.features = features;
1065
1066         ionic_link_qcq_interrupts(lif->adminqcq, rxq);
1067         ionic_debugfs_add_qcq(lif, rxq);
1068
1069         lif->hwstamp_rxq = rxq;
1070
1071         if (netif_running(lif->netdev)) {
1072                 err = ionic_lif_rxq_init(lif, rxq);
1073                 if (err)
1074                         goto err_qcq_init;
1075
1076                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1077                         ionic_rx_fill(&rxq->q, NULL);
1078                         err = ionic_qcq_enable(rxq);
1079                         if (err)
1080                                 goto err_qcq_enable;
1081                 }
1082         }
1083
1084         return 0;
1085
1086 err_qcq_enable:
1087         ionic_lif_qcq_deinit(lif, rxq);
1088 err_qcq_init:
1089         lif->hwstamp_rxq = NULL;
1090         ionic_debugfs_del_qcq(rxq);
1091         ionic_qcq_free(lif, rxq);
1092         devm_kfree(lif->ionic->dev, rxq);
1093 err_qcq_alloc:
1094         return err;
1095 }
1096
1097 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
1098 {
1099         struct ionic_queue_params qparam;
1100
1101         ionic_init_queue_params(lif, &qparam);
1102
1103         if (rx_all)
1104                 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1105         else
1106                 qparam.rxq_features = 0;
1107
1108         /* if we're not running, just set the values and return */
1109         if (!netif_running(lif->netdev)) {
1110                 lif->rxq_features = qparam.rxq_features;
1111                 return 0;
1112         }
1113
1114         return ionic_reconfigure_queues(lif, &qparam);
1115 }
1116
1117 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1118 {
1119         struct ionic_admin_ctx ctx = {
1120                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1121                 .cmd.lif_setattr = {
1122                         .opcode = IONIC_CMD_LIF_SETATTR,
1123                         .index = cpu_to_le16(lif->index),
1124                         .attr = IONIC_LIF_ATTR_TXSTAMP,
1125                         .txstamp_mode = cpu_to_le16(txstamp_mode),
1126                 },
1127         };
1128
1129         return ionic_adminq_post_wait(lif, &ctx);
1130 }
1131
1132 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1133 {
1134         struct ionic_admin_ctx ctx = {
1135                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1136                 .cmd.rx_filter_del = {
1137                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1138                         .lif_index = cpu_to_le16(lif->index),
1139                 },
1140         };
1141         struct ionic_rx_filter *f;
1142         u32 filter_id;
1143         int err;
1144
1145         spin_lock_bh(&lif->rx_filters.lock);
1146
1147         f = ionic_rx_filter_rxsteer(lif);
1148         if (!f) {
1149                 spin_unlock_bh(&lif->rx_filters.lock);
1150                 return;
1151         }
1152
1153         filter_id = f->filter_id;
1154         ionic_rx_filter_free(lif, f);
1155
1156         spin_unlock_bh(&lif->rx_filters.lock);
1157
1158         netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1159
1160         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1161
1162         err = ionic_adminq_post_wait(lif, &ctx);
1163         if (err && err != -EEXIST)
1164                 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1165 }
1166
1167 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1168 {
1169         struct ionic_admin_ctx ctx = {
1170                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1171                 .cmd.rx_filter_add = {
1172                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1173                         .lif_index = cpu_to_le16(lif->index),
1174                         .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1175                         .pkt_class = cpu_to_le64(pkt_class),
1176                 },
1177         };
1178         u8 qtype;
1179         u32 qid;
1180         int err;
1181
1182         if (!lif->hwstamp_rxq)
1183                 return -EINVAL;
1184
1185         qtype = lif->hwstamp_rxq->q.type;
1186         ctx.cmd.rx_filter_add.qtype = qtype;
1187
1188         qid = lif->hwstamp_rxq->q.index;
1189         ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1190
1191         netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1192         err = ionic_adminq_post_wait(lif, &ctx);
1193         if (err && err != -EEXIST)
1194                 return err;
1195
1196         spin_lock_bh(&lif->rx_filters.lock);
1197         err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
1198         spin_unlock_bh(&lif->rx_filters.lock);
1199
1200         return err;
1201 }
1202
1203 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1204 {
1205         ionic_lif_del_hwstamp_rxfilt(lif);
1206
1207         if (!pkt_class)
1208                 return 0;
1209
1210         return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1211 }
1212
1213 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1214 {
1215         struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1216         struct ionic_lif *lif = napi_to_cq(napi)->lif;
1217         struct ionic_dev *idev = &lif->ionic->idev;
1218         unsigned long irqflags;
1219         unsigned int flags = 0;
1220         int rx_work = 0;
1221         int tx_work = 0;
1222         int n_work = 0;
1223         int a_work = 0;
1224         int work_done;
1225         int credits;
1226
1227         if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1228                 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1229                                           ionic_notifyq_service, NULL, NULL);
1230
1231         spin_lock_irqsave(&lif->adminq_lock, irqflags);
1232         if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1233                 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1234                                           ionic_adminq_service, NULL, NULL);
1235
1236         spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1237
1238         if (lif->hwstamp_rxq)
1239                 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1240                                            ionic_rx_service, NULL, NULL);
1241
1242         if (lif->hwstamp_txq)
1243                 tx_work = ionic_tx_cq_service(&lif->hwstamp_txq->cq, budget, !!budget);
1244
1245         work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1246         if (work_done < budget && napi_complete_done(napi, work_done)) {
1247                 flags |= IONIC_INTR_CRED_UNMASK;
1248                 intr->rearm_count++;
1249         }
1250
1251         if (work_done || flags) {
1252                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1253                 credits = n_work + a_work + rx_work + tx_work;
1254                 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1255         }
1256
1257         if (lif->doorbell_wa) {
1258                 if (!a_work)
1259                         ionic_adminq_poke_doorbell(&lif->adminqcq->q);
1260                 if (lif->hwstamp_rxq && !rx_work)
1261                         ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q);
1262                 if (lif->hwstamp_txq && !tx_work)
1263                         ionic_txq_poke_doorbell(&lif->hwstamp_txq->q);
1264         }
1265
1266         return work_done;
1267 }
1268
1269 void ionic_get_stats64(struct net_device *netdev,
1270                        struct rtnl_link_stats64 *ns)
1271 {
1272         struct ionic_lif *lif = netdev_priv(netdev);
1273         struct ionic_lif_stats *ls;
1274
1275         memset(ns, 0, sizeof(*ns));
1276         ls = &lif->info->stats;
1277
1278         ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1279                          le64_to_cpu(ls->rx_mcast_packets) +
1280                          le64_to_cpu(ls->rx_bcast_packets);
1281
1282         ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1283                          le64_to_cpu(ls->tx_mcast_packets) +
1284                          le64_to_cpu(ls->tx_bcast_packets);
1285
1286         ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1287                        le64_to_cpu(ls->rx_mcast_bytes) +
1288                        le64_to_cpu(ls->rx_bcast_bytes);
1289
1290         ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1291                        le64_to_cpu(ls->tx_mcast_bytes) +
1292                        le64_to_cpu(ls->tx_bcast_bytes);
1293
1294         ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1295                          le64_to_cpu(ls->rx_mcast_drop_packets) +
1296                          le64_to_cpu(ls->rx_bcast_drop_packets);
1297
1298         ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1299                          le64_to_cpu(ls->tx_mcast_drop_packets) +
1300                          le64_to_cpu(ls->tx_bcast_drop_packets);
1301
1302         ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1303
1304         ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1305
1306         ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1307                                le64_to_cpu(ls->rx_queue_disabled) +
1308                                le64_to_cpu(ls->rx_desc_fetch_error) +
1309                                le64_to_cpu(ls->rx_desc_data_error);
1310
1311         ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1312                                 le64_to_cpu(ls->tx_queue_disabled) +
1313                                 le64_to_cpu(ls->tx_desc_fetch_error) +
1314                                 le64_to_cpu(ls->tx_desc_data_error);
1315
1316         ns->rx_errors = ns->rx_over_errors +
1317                         ns->rx_missed_errors;
1318
1319         ns->tx_errors = ns->tx_aborted_errors;
1320 }
1321
1322 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1323 {
1324         return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
1325 }
1326
1327 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1328 {
1329         /* Don't delete our own address from the uc list */
1330         if (ether_addr_equal(addr, netdev->dev_addr))
1331                 return 0;
1332
1333         return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
1334 }
1335
1336 void ionic_lif_rx_mode(struct ionic_lif *lif)
1337 {
1338         struct net_device *netdev = lif->netdev;
1339         unsigned int nfilters;
1340         unsigned int nd_flags;
1341         char buf[128];
1342         u16 rx_mode;
1343         int i;
1344 #define REMAIN(__x) (sizeof(buf) - (__x))
1345
1346         mutex_lock(&lif->config_lock);
1347
1348         /* grab the flags once for local use */
1349         nd_flags = netdev->flags;
1350
1351         rx_mode = IONIC_RX_MODE_F_UNICAST;
1352         rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1353         rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1354         rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1355         rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1356
1357         /* sync the filters */
1358         ionic_rx_filter_sync(lif);
1359
1360         /* check for overflow state
1361          *    if so, we track that we overflowed and enable NIC PROMISC
1362          *    else if the overflow is set and not needed
1363          *       we remove our overflow flag and check the netdev flags
1364          *       to see if we can disable NIC PROMISC
1365          */
1366         nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1367
1368         if (((lif->nucast + lif->nmcast) >= nfilters) ||
1369             (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
1370                 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1371                 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1372         } else {
1373                 if (!(nd_flags & IFF_PROMISC))
1374                         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1375                 if (!(nd_flags & IFF_ALLMULTI))
1376                         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1377         }
1378
1379         i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1380                       lif->rx_mode, rx_mode);
1381         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1382                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1383         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1384                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1385         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1386                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1387         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1388                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1389         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1390                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1391         if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1392                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1393         netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1394
1395         if (lif->rx_mode != rx_mode) {
1396                 struct ionic_admin_ctx ctx = {
1397                         .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1398                         .cmd.rx_mode_set = {
1399                                 .opcode = IONIC_CMD_RX_MODE_SET,
1400                                 .lif_index = cpu_to_le16(lif->index),
1401                         },
1402                 };
1403                 int err;
1404
1405                 ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1406                 err = ionic_adminq_post_wait(lif, &ctx);
1407                 if (err)
1408                         netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1409                                     rx_mode, err);
1410                 else
1411                         lif->rx_mode = rx_mode;
1412         }
1413
1414         mutex_unlock(&lif->config_lock);
1415 }
1416
1417 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1418 {
1419         struct ionic_lif *lif = netdev_priv(netdev);
1420         struct ionic_deferred_work *work;
1421
1422         /* Sync the kernel filter list with the driver filter list */
1423         __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1424         __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1425
1426         /* Shove off the rest of the rxmode work to the work task
1427          * which will include syncing the filters to the firmware.
1428          */
1429         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1430         if (!work) {
1431                 netdev_err(lif->netdev, "rxmode change dropped\n");
1432                 return;
1433         }
1434         work->type = IONIC_DW_TYPE_RX_MODE;
1435         netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1436         ionic_lif_deferred_enqueue(lif, work);
1437 }
1438
1439 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1440 {
1441         u64 wanted = 0;
1442
1443         if (features & NETIF_F_HW_VLAN_CTAG_TX)
1444                 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1445         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1446                 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1447         if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1448                 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1449         if (features & NETIF_F_RXHASH)
1450                 wanted |= IONIC_ETH_HW_RX_HASH;
1451         if (features & NETIF_F_RXCSUM)
1452                 wanted |= IONIC_ETH_HW_RX_CSUM;
1453         if (features & NETIF_F_SG)
1454                 wanted |= IONIC_ETH_HW_TX_SG;
1455         if (features & NETIF_F_HW_CSUM)
1456                 wanted |= IONIC_ETH_HW_TX_CSUM;
1457         if (features & NETIF_F_TSO)
1458                 wanted |= IONIC_ETH_HW_TSO;
1459         if (features & NETIF_F_TSO6)
1460                 wanted |= IONIC_ETH_HW_TSO_IPV6;
1461         if (features & NETIF_F_TSO_ECN)
1462                 wanted |= IONIC_ETH_HW_TSO_ECN;
1463         if (features & NETIF_F_GSO_GRE)
1464                 wanted |= IONIC_ETH_HW_TSO_GRE;
1465         if (features & NETIF_F_GSO_GRE_CSUM)
1466                 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1467         if (features & NETIF_F_GSO_IPXIP4)
1468                 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1469         if (features & NETIF_F_GSO_IPXIP6)
1470                 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1471         if (features & NETIF_F_GSO_UDP_TUNNEL)
1472                 wanted |= IONIC_ETH_HW_TSO_UDP;
1473         if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1474                 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1475
1476         return cpu_to_le64(wanted);
1477 }
1478
1479 static int ionic_set_nic_features(struct ionic_lif *lif,
1480                                   netdev_features_t features)
1481 {
1482         struct device *dev = lif->ionic->dev;
1483         struct ionic_admin_ctx ctx = {
1484                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1485                 .cmd.lif_setattr = {
1486                         .opcode = IONIC_CMD_LIF_SETATTR,
1487                         .index = cpu_to_le16(lif->index),
1488                         .attr = IONIC_LIF_ATTR_FEATURES,
1489                 },
1490         };
1491         u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1492                          IONIC_ETH_HW_VLAN_RX_STRIP |
1493                          IONIC_ETH_HW_VLAN_RX_FILTER;
1494         u64 old_hw_features;
1495         int err;
1496
1497         ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1498
1499         if (lif->phc)
1500                 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1501
1502         err = ionic_adminq_post_wait(lif, &ctx);
1503         if (err)
1504                 return err;
1505
1506         old_hw_features = lif->hw_features;
1507         lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1508                                        ctx.comp.lif_setattr.features);
1509
1510         if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1511                 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1512
1513         if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&
1514             !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1515                 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1516
1517         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1518                 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1519         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1520                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1521         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1522                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1523         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1524                 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1525         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1526                 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1527         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1528                 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1529         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1530                 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1531         if (lif->hw_features & IONIC_ETH_HW_TSO)
1532                 dev_dbg(dev, "feature ETH_HW_TSO\n");
1533         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1534                 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1535         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1536                 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1537         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1538                 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1539         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1540                 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1541         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1542                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1543         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1544                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1545         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1546                 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1547         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1548                 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1549         if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1550                 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1551
1552         return 0;
1553 }
1554
1555 static int ionic_init_nic_features(struct ionic_lif *lif)
1556 {
1557         struct net_device *netdev = lif->netdev;
1558         netdev_features_t features;
1559         int err;
1560
1561         /* set up what we expect to support by default */
1562         features = NETIF_F_HW_VLAN_CTAG_TX |
1563                    NETIF_F_HW_VLAN_CTAG_RX |
1564                    NETIF_F_HW_VLAN_CTAG_FILTER |
1565                    NETIF_F_SG |
1566                    NETIF_F_HW_CSUM |
1567                    NETIF_F_RXCSUM |
1568                    NETIF_F_TSO |
1569                    NETIF_F_TSO6 |
1570                    NETIF_F_TSO_ECN |
1571                    NETIF_F_GSO_GRE |
1572                    NETIF_F_GSO_GRE_CSUM |
1573                    NETIF_F_GSO_IPXIP4 |
1574                    NETIF_F_GSO_IPXIP6 |
1575                    NETIF_F_GSO_UDP_TUNNEL |
1576                    NETIF_F_GSO_UDP_TUNNEL_CSUM;
1577
1578         if (lif->nxqs > 1)
1579                 features |= NETIF_F_RXHASH;
1580
1581         err = ionic_set_nic_features(lif, features);
1582         if (err)
1583                 return err;
1584
1585         /* tell the netdev what we actually can support */
1586         netdev->features |= NETIF_F_HIGHDMA;
1587
1588         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1589                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1590         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1591                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1592         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1593                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1594         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1595                 netdev->hw_features |= NETIF_F_RXHASH;
1596         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1597                 netdev->hw_features |= NETIF_F_SG;
1598
1599         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1600                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1601         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1602                 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1603         if (lif->hw_features & IONIC_ETH_HW_TSO)
1604                 netdev->hw_enc_features |= NETIF_F_TSO;
1605         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1606                 netdev->hw_enc_features |= NETIF_F_TSO6;
1607         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1608                 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1609         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1610                 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1611         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1612                 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1613         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1614                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1615         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1616                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1617         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1618                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1619         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1620                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1621
1622         netdev->hw_features |= netdev->hw_enc_features;
1623         netdev->features |= netdev->hw_features;
1624         netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1625
1626         netdev->priv_flags |= IFF_UNICAST_FLT |
1627                               IFF_LIVE_ADDR_CHANGE;
1628
1629         netdev->xdp_features = NETDEV_XDP_ACT_BASIC    |
1630                                NETDEV_XDP_ACT_REDIRECT |
1631                                NETDEV_XDP_ACT_RX_SG    |
1632                                NETDEV_XDP_ACT_NDO_XMIT |
1633                                NETDEV_XDP_ACT_NDO_XMIT_SG;
1634
1635         return 0;
1636 }
1637
1638 static int ionic_set_features(struct net_device *netdev,
1639                               netdev_features_t features)
1640 {
1641         struct ionic_lif *lif = netdev_priv(netdev);
1642         int err;
1643
1644         netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1645                    __func__, (u64)lif->netdev->features, (u64)features);
1646
1647         err = ionic_set_nic_features(lif, features);
1648
1649         return err;
1650 }
1651
1652 static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)
1653 {
1654         struct ionic_admin_ctx ctx = {
1655                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1656                 .cmd.lif_setattr = {
1657                         .opcode = IONIC_CMD_LIF_SETATTR,
1658                         .index = cpu_to_le16(lif->index),
1659                         .attr = IONIC_LIF_ATTR_MAC,
1660                 },
1661         };
1662
1663         ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);
1664         return ionic_adminq_post_wait(lif, &ctx);
1665 }
1666
1667 static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)
1668 {
1669         struct ionic_admin_ctx ctx = {
1670                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1671                 .cmd.lif_getattr = {
1672                         .opcode = IONIC_CMD_LIF_GETATTR,
1673                         .index = cpu_to_le16(lif->index),
1674                         .attr = IONIC_LIF_ATTR_MAC,
1675                 },
1676         };
1677         int err;
1678
1679         err = ionic_adminq_post_wait(lif, &ctx);
1680         if (err)
1681                 return err;
1682
1683         ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);
1684         return 0;
1685 }
1686
1687 static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)
1688 {
1689         u8  get_mac[ETH_ALEN];
1690         int err;
1691
1692         err = ionic_set_attr_mac(lif, mac);
1693         if (err)
1694                 return err;
1695
1696         err = ionic_get_attr_mac(lif, get_mac);
1697         if (err)
1698                 return err;
1699
1700         /* To deal with older firmware that silently ignores the set attr mac:
1701          * doesn't actually change the mac and doesn't return an error, so we
1702          * do the get attr to verify whether or not the set actually happened
1703          */
1704         if (!ether_addr_equal(get_mac, mac))
1705                 return 1;
1706
1707         return 0;
1708 }
1709
1710 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1711 {
1712         struct ionic_lif *lif = netdev_priv(netdev);
1713         struct sockaddr *addr = sa;
1714         u8 *mac;
1715         int err;
1716
1717         mac = (u8 *)addr->sa_data;
1718         if (ether_addr_equal(netdev->dev_addr, mac))
1719                 return 0;
1720
1721         err = ionic_program_mac(lif, mac);
1722         if (err < 0)
1723                 return err;
1724
1725         if (err > 0)
1726                 netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
1727                            __func__);
1728
1729         err = eth_prepare_mac_addr_change(netdev, addr);
1730         if (err)
1731                 return err;
1732
1733         if (!is_zero_ether_addr(netdev->dev_addr)) {
1734                 netdev_info(netdev, "deleting mac addr %pM\n",
1735                             netdev->dev_addr);
1736                 ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
1737         }
1738
1739         eth_commit_mac_addr_change(netdev, addr);
1740         netdev_info(netdev, "updating mac addr %pM\n", mac);
1741
1742         return ionic_lif_addr_add(netdev_priv(netdev), mac);
1743 }
1744
1745 void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1746 {
1747         /* Stop and clean the queues before reconfiguration */
1748         netif_device_detach(lif->netdev);
1749         ionic_stop_queues(lif);
1750         ionic_txrx_deinit(lif);
1751 }
1752
1753 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1754 {
1755         int err;
1756
1757         /* Re-init the queues after reconfiguration */
1758
1759         /* The only way txrx_init can fail here is if communication
1760          * with FW is suddenly broken.  There's not much we can do
1761          * at this point - error messages have already been printed,
1762          * so we can continue on and the user can eventually do a
1763          * DOWN and UP to try to reset and clear the issue.
1764          */
1765         err = ionic_txrx_init(lif);
1766         ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1767         netif_device_attach(lif->netdev);
1768
1769         return err;
1770 }
1771
1772 static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,
1773                                    struct bpf_prog *xdp_prog)
1774 {
1775         if (!xdp_prog)
1776                 return true;
1777
1778         if (mtu <= IONIC_XDP_MAX_LINEAR_MTU)
1779                 return true;
1780
1781         if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags)
1782                 return true;
1783
1784         return false;
1785 }
1786
1787 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1788 {
1789         struct ionic_lif *lif = netdev_priv(netdev);
1790         struct ionic_admin_ctx ctx = {
1791                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1792                 .cmd.lif_setattr = {
1793                         .opcode = IONIC_CMD_LIF_SETATTR,
1794                         .index = cpu_to_le16(lif->index),
1795                         .attr = IONIC_LIF_ATTR_MTU,
1796                         .mtu = cpu_to_le32(new_mtu),
1797                 },
1798         };
1799         struct bpf_prog *xdp_prog;
1800         int err;
1801
1802         xdp_prog = READ_ONCE(lif->xdp_prog);
1803         if (!ionic_xdp_is_valid_mtu(lif, new_mtu, xdp_prog))
1804                 return -EINVAL;
1805
1806         err = ionic_adminq_post_wait(lif, &ctx);
1807         if (err)
1808                 return err;
1809
1810         /* if we're not running, nothing more to do */
1811         if (!netif_running(netdev)) {
1812                 WRITE_ONCE(netdev->mtu, new_mtu);
1813                 return 0;
1814         }
1815
1816         mutex_lock(&lif->queue_lock);
1817         ionic_stop_queues_reconfig(lif);
1818         WRITE_ONCE(netdev->mtu, new_mtu);
1819         err = ionic_start_queues_reconfig(lif);
1820         mutex_unlock(&lif->queue_lock);
1821
1822         return err;
1823 }
1824
1825 static void ionic_tx_timeout_work(struct work_struct *ws)
1826 {
1827         struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1828         int err;
1829
1830         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1831                 return;
1832
1833         /* if we were stopped before this scheduled job was launched,
1834          * don't bother the queues as they are already stopped.
1835          */
1836         if (!netif_running(lif->netdev))
1837                 return;
1838
1839         mutex_lock(&lif->queue_lock);
1840         ionic_stop_queues_reconfig(lif);
1841         err = ionic_start_queues_reconfig(lif);
1842         mutex_unlock(&lif->queue_lock);
1843
1844         if (err)
1845                 dev_err(lif->ionic->dev, "%s: Restarting queues failed\n", __func__);
1846 }
1847
1848 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1849 {
1850         struct ionic_lif *lif = netdev_priv(netdev);
1851
1852         netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1853         schedule_work(&lif->tx_timeout_work);
1854 }
1855
1856 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1857                                  u16 vid)
1858 {
1859         struct ionic_lif *lif = netdev_priv(netdev);
1860         int err;
1861
1862         err = ionic_lif_vlan_add(lif, vid);
1863         if (err)
1864                 return err;
1865
1866         ionic_lif_rx_mode(lif);
1867
1868         return 0;
1869 }
1870
1871 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1872                                   u16 vid)
1873 {
1874         struct ionic_lif *lif = netdev_priv(netdev);
1875         int err;
1876
1877         err = ionic_lif_vlan_del(lif, vid);
1878         if (err)
1879                 return err;
1880
1881         ionic_lif_rx_mode(lif);
1882
1883         return 0;
1884 }
1885
1886 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1887                          const u8 *key, const u32 *indir)
1888 {
1889         struct ionic_admin_ctx ctx = {
1890                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1891                 .cmd.lif_setattr = {
1892                         .opcode = IONIC_CMD_LIF_SETATTR,
1893                         .attr = IONIC_LIF_ATTR_RSS,
1894                         .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1895                 },
1896         };
1897         unsigned int i, tbl_sz;
1898
1899         if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1900                 lif->rss_types = types;
1901                 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1902         }
1903
1904         if (key)
1905                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1906
1907         if (indir) {
1908                 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1909                 for (i = 0; i < tbl_sz; i++)
1910                         lif->rss_ind_tbl[i] = indir[i];
1911         }
1912
1913         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1914                IONIC_RSS_HASH_KEY_SIZE);
1915
1916         return ionic_adminq_post_wait(lif, &ctx);
1917 }
1918
1919 static int ionic_lif_rss_init(struct ionic_lif *lif)
1920 {
1921         unsigned int tbl_sz;
1922         unsigned int i;
1923
1924         lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1925                          IONIC_RSS_TYPE_IPV4_TCP |
1926                          IONIC_RSS_TYPE_IPV4_UDP |
1927                          IONIC_RSS_TYPE_IPV6     |
1928                          IONIC_RSS_TYPE_IPV6_TCP |
1929                          IONIC_RSS_TYPE_IPV6_UDP;
1930
1931         /* Fill indirection table with 'default' values */
1932         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1933         for (i = 0; i < tbl_sz; i++)
1934                 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1935
1936         return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1937 }
1938
1939 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1940 {
1941         int tbl_sz;
1942
1943         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1944         memset(lif->rss_ind_tbl, 0, tbl_sz);
1945         memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1946
1947         ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1948 }
1949
1950 static void ionic_lif_quiesce(struct ionic_lif *lif)
1951 {
1952         struct ionic_admin_ctx ctx = {
1953                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1954                 .cmd.lif_setattr = {
1955                         .opcode = IONIC_CMD_LIF_SETATTR,
1956                         .index = cpu_to_le16(lif->index),
1957                         .attr = IONIC_LIF_ATTR_STATE,
1958                         .state = IONIC_LIF_QUIESCE,
1959                 },
1960         };
1961         int err;
1962
1963         err = ionic_adminq_post_wait(lif, &ctx);
1964         if (err)
1965                 netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
1966 }
1967
1968 static void ionic_txrx_disable(struct ionic_lif *lif)
1969 {
1970         unsigned int i;
1971         int err = 0;
1972
1973         if (lif->txqcqs) {
1974                 for (i = 0; i < lif->nxqs; i++)
1975                         err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
1976         }
1977
1978         if (lif->hwstamp_txq)
1979                 err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
1980
1981         if (lif->rxqcqs) {
1982                 for (i = 0; i < lif->nxqs; i++)
1983                         err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
1984         }
1985
1986         if (lif->hwstamp_rxq)
1987                 err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
1988
1989         ionic_lif_quiesce(lif);
1990 }
1991
1992 static void ionic_txrx_deinit(struct ionic_lif *lif)
1993 {
1994         unsigned int i;
1995
1996         if (lif->txqcqs) {
1997                 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1998                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1999                         ionic_tx_flush(&lif->txqcqs[i]->cq);
2000                         ionic_tx_empty(&lif->txqcqs[i]->q);
2001                 }
2002         }
2003
2004         if (lif->rxqcqs) {
2005                 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
2006                         ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2007                         ionic_rx_empty(&lif->rxqcqs[i]->q);
2008                 }
2009         }
2010         lif->rx_mode = 0;
2011
2012         if (lif->hwstamp_txq) {
2013                 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
2014                 ionic_tx_flush(&lif->hwstamp_txq->cq);
2015                 ionic_tx_empty(&lif->hwstamp_txq->q);
2016         }
2017
2018         if (lif->hwstamp_rxq) {
2019                 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
2020                 ionic_rx_empty(&lif->hwstamp_rxq->q);
2021         }
2022 }
2023
2024 void ionic_txrx_free(struct ionic_lif *lif)
2025 {
2026         unsigned int i;
2027
2028         if (lif->txqcqs) {
2029                 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
2030                         ionic_qcq_free(lif, lif->txqcqs[i]);
2031                         devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
2032                         lif->txqcqs[i] = NULL;
2033                 }
2034         }
2035
2036         if (lif->rxqcqs) {
2037                 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2038                         ionic_qcq_free(lif, lif->rxqcqs[i]);
2039                         devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
2040                         lif->rxqcqs[i] = NULL;
2041                 }
2042         }
2043
2044         if (lif->hwstamp_txq) {
2045                 ionic_qcq_free(lif, lif->hwstamp_txq);
2046                 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2047                 lif->hwstamp_txq = NULL;
2048         }
2049
2050         if (lif->hwstamp_rxq) {
2051                 ionic_qcq_free(lif, lif->hwstamp_rxq);
2052                 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2053                 lif->hwstamp_rxq = NULL;
2054         }
2055 }
2056
2057 static int ionic_txrx_alloc(struct ionic_lif *lif)
2058 {
2059         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2060         unsigned int flags, i;
2061         int err = 0;
2062
2063         num_desc = lif->ntxq_descs;
2064         desc_sz = sizeof(struct ionic_txq_desc);
2065         comp_sz = sizeof(struct ionic_txq_comp);
2066
2067         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2068             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2069                                           sizeof(struct ionic_txq_sg_desc_v1))
2070                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2071         else
2072                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2073
2074         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2075
2076         if (test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state))
2077                 flags |= IONIC_QCQ_F_CMB_RINGS;
2078
2079         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2080                 flags |= IONIC_QCQ_F_INTR;
2081
2082         for (i = 0; i < lif->nxqs; i++) {
2083                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2084                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2085                                       sizeof(struct ionic_tx_desc_info),
2086                                       lif->kern_pid, NULL, &lif->txqcqs[i]);
2087                 if (err)
2088                         goto err_out;
2089
2090                 if (flags & IONIC_QCQ_F_INTR) {
2091                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2092                                              lif->txqcqs[i]->intr.index,
2093                                              lif->tx_coalesce_hw);
2094                         if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2095                                 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2096                 }
2097
2098                 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2099         }
2100
2101         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2102
2103         if (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state))
2104                 flags |= IONIC_QCQ_F_CMB_RINGS;
2105
2106         num_desc = lif->nrxq_descs;
2107         desc_sz = sizeof(struct ionic_rxq_desc);
2108         comp_sz = sizeof(struct ionic_rxq_comp);
2109         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2110
2111         if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2112                 comp_sz *= 2;
2113
2114         for (i = 0; i < lif->nxqs; i++) {
2115                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2116                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2117                                       sizeof(struct ionic_rx_desc_info),
2118                                       lif->kern_pid, lif->xdp_prog,
2119                                       &lif->rxqcqs[i]);
2120                 if (err)
2121                         goto err_out;
2122
2123                 lif->rxqcqs[i]->q.features = lif->rxq_features;
2124
2125                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2126                                      lif->rxqcqs[i]->intr.index,
2127                                      lif->rx_coalesce_hw);
2128                 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2129                         lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2130
2131                 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2132                         ionic_link_qcq_interrupts(lif->rxqcqs[i],
2133                                                   lif->txqcqs[i]);
2134
2135                 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2136         }
2137
2138         return 0;
2139
2140 err_out:
2141         ionic_txrx_free(lif);
2142
2143         return err;
2144 }
2145
2146 static int ionic_txrx_init(struct ionic_lif *lif)
2147 {
2148         unsigned int i;
2149         int err;
2150
2151         for (i = 0; i < lif->nxqs; i++) {
2152                 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2153                 if (err)
2154                         goto err_out;
2155
2156                 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2157                 if (err) {
2158                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2159                         goto err_out;
2160                 }
2161         }
2162
2163         if (lif->netdev->features & NETIF_F_RXHASH)
2164                 ionic_lif_rss_init(lif);
2165
2166         ionic_lif_rx_mode(lif);
2167
2168         return 0;
2169
2170 err_out:
2171         while (i--) {
2172                 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2173                 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2174         }
2175
2176         return err;
2177 }
2178
2179 static int ionic_txrx_enable(struct ionic_lif *lif)
2180 {
2181         int derr = 0;
2182         int i, err;
2183
2184         ionic_xdp_rxqs_prog_update(lif);
2185
2186         for (i = 0; i < lif->nxqs; i++) {
2187                 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2188                         dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2189                         err = -ENXIO;
2190                         goto err_out;
2191                 }
2192
2193                 ionic_rx_fill(&lif->rxqcqs[i]->q,
2194                               READ_ONCE(lif->rxqcqs[i]->q.xdp_prog));
2195                 err = ionic_qcq_enable(lif->rxqcqs[i]);
2196                 if (err)
2197                         goto err_out;
2198
2199                 err = ionic_qcq_enable(lif->txqcqs[i]);
2200                 if (err) {
2201                         derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
2202                         goto err_out;
2203                 }
2204         }
2205
2206         if (lif->hwstamp_rxq) {
2207                 ionic_rx_fill(&lif->hwstamp_rxq->q, NULL);
2208                 err = ionic_qcq_enable(lif->hwstamp_rxq);
2209                 if (err)
2210                         goto err_out_hwstamp_rx;
2211         }
2212
2213         if (lif->hwstamp_txq) {
2214                 err = ionic_qcq_enable(lif->hwstamp_txq);
2215                 if (err)
2216                         goto err_out_hwstamp_tx;
2217         }
2218
2219         return 0;
2220
2221 err_out_hwstamp_tx:
2222         if (lif->hwstamp_rxq)
2223                 derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
2224 err_out_hwstamp_rx:
2225         i = lif->nxqs;
2226 err_out:
2227         while (i--) {
2228                 derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
2229                 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
2230         }
2231
2232         ionic_xdp_rxqs_prog_update(lif);
2233
2234         return err;
2235 }
2236
2237 static int ionic_start_queues(struct ionic_lif *lif)
2238 {
2239         int err;
2240
2241         if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2242                 return -EIO;
2243
2244         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2245                 return -EBUSY;
2246
2247         if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2248                 return 0;
2249
2250         err = ionic_txrx_enable(lif);
2251         if (err) {
2252                 clear_bit(IONIC_LIF_F_UP, lif->state);
2253                 return err;
2254         }
2255         netif_tx_wake_all_queues(lif->netdev);
2256
2257         return 0;
2258 }
2259
2260 static int ionic_open(struct net_device *netdev)
2261 {
2262         struct ionic_lif *lif = netdev_priv(netdev);
2263         int err;
2264
2265         /* If recovering from a broken state, clear the bit and we'll try again */
2266         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2267                 netdev_info(netdev, "clearing broken state\n");
2268
2269         mutex_lock(&lif->queue_lock);
2270
2271         err = ionic_txrx_alloc(lif);
2272         if (err)
2273                 goto err_unlock;
2274
2275         err = ionic_txrx_init(lif);
2276         if (err)
2277                 goto err_txrx_free;
2278
2279         err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2280         if (err)
2281                 goto err_txrx_deinit;
2282
2283         err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2284         if (err)
2285                 goto err_txrx_deinit;
2286
2287         /* don't start the queues until we have link */
2288         if (netif_carrier_ok(netdev)) {
2289                 err = ionic_start_queues(lif);
2290                 if (err)
2291                         goto err_txrx_deinit;
2292         }
2293
2294         /* If hardware timestamping is enabled, but the queues were freed by
2295          * ionic_stop, those need to be reallocated and initialized, too.
2296          */
2297         ionic_lif_hwstamp_recreate_queues(lif);
2298
2299         mutex_unlock(&lif->queue_lock);
2300
2301         return 0;
2302
2303 err_txrx_deinit:
2304         ionic_txrx_deinit(lif);
2305 err_txrx_free:
2306         ionic_txrx_free(lif);
2307 err_unlock:
2308         mutex_unlock(&lif->queue_lock);
2309         return err;
2310 }
2311
2312 static void ionic_stop_queues(struct ionic_lif *lif)
2313 {
2314         if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2315                 return;
2316
2317         netif_tx_disable(lif->netdev);
2318         ionic_txrx_disable(lif);
2319 }
2320
2321 static int ionic_stop(struct net_device *netdev)
2322 {
2323         struct ionic_lif *lif = netdev_priv(netdev);
2324
2325         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2326                 return 0;
2327
2328         mutex_lock(&lif->queue_lock);
2329         ionic_stop_queues(lif);
2330         ionic_txrx_deinit(lif);
2331         ionic_txrx_free(lif);
2332         mutex_unlock(&lif->queue_lock);
2333
2334         return 0;
2335 }
2336
2337 static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2338 {
2339         struct ionic_lif *lif = netdev_priv(netdev);
2340
2341         switch (cmd) {
2342         case SIOCSHWTSTAMP:
2343                 return ionic_lif_hwstamp_set(lif, ifr);
2344         case SIOCGHWTSTAMP:
2345                 return ionic_lif_hwstamp_get(lif, ifr);
2346         default:
2347                 return -EOPNOTSUPP;
2348         }
2349 }
2350
2351 static int ionic_get_vf_config(struct net_device *netdev,
2352                                int vf, struct ifla_vf_info *ivf)
2353 {
2354         struct ionic_lif *lif = netdev_priv(netdev);
2355         struct ionic *ionic = lif->ionic;
2356         int ret = 0;
2357
2358         if (!netif_device_present(netdev))
2359                 return -EBUSY;
2360
2361         down_read(&ionic->vf_op_lock);
2362
2363         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2364                 ret = -EINVAL;
2365         } else {
2366                 struct ionic_vf *vfdata = &ionic->vfs[vf];
2367
2368                 ivf->vf           = vf;
2369                 ivf->qos          = 0;
2370                 ivf->vlan         = le16_to_cpu(vfdata->vlanid);
2371                 ivf->spoofchk     = vfdata->spoofchk;
2372                 ivf->linkstate    = vfdata->linkstate;
2373                 ivf->max_tx_rate  = le32_to_cpu(vfdata->maxrate);
2374                 ivf->trusted      = vfdata->trusted;
2375                 ether_addr_copy(ivf->mac, vfdata->macaddr);
2376         }
2377
2378         up_read(&ionic->vf_op_lock);
2379         return ret;
2380 }
2381
2382 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2383                               struct ifla_vf_stats *vf_stats)
2384 {
2385         struct ionic_lif *lif = netdev_priv(netdev);
2386         struct ionic *ionic = lif->ionic;
2387         struct ionic_lif_stats *vs;
2388         int ret = 0;
2389
2390         if (!netif_device_present(netdev))
2391                 return -EBUSY;
2392
2393         down_read(&ionic->vf_op_lock);
2394
2395         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2396                 ret = -EINVAL;
2397         } else {
2398                 memset(vf_stats, 0, sizeof(*vf_stats));
2399                 vs = &ionic->vfs[vf].stats;
2400
2401                 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2402                 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2403                 vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2404                 vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2405                 vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2406                 vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2407                 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2408                                        le64_to_cpu(vs->rx_mcast_drop_packets) +
2409                                        le64_to_cpu(vs->rx_bcast_drop_packets);
2410                 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2411                                        le64_to_cpu(vs->tx_mcast_drop_packets) +
2412                                        le64_to_cpu(vs->tx_bcast_drop_packets);
2413         }
2414
2415         up_read(&ionic->vf_op_lock);
2416         return ret;
2417 }
2418
2419 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2420 {
2421         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
2422         struct ionic_lif *lif = netdev_priv(netdev);
2423         struct ionic *ionic = lif->ionic;
2424         int ret;
2425
2426         if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2427                 return -EINVAL;
2428
2429         if (!netif_device_present(netdev))
2430                 return -EBUSY;
2431
2432         down_write(&ionic->vf_op_lock);
2433
2434         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2435                 ret = -EINVAL;
2436         } else {
2437                 ether_addr_copy(vfc.macaddr, mac);
2438                 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
2439                         __func__, vf, vfc.macaddr);
2440
2441                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2442                 if (!ret)
2443                         ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2444         }
2445
2446         up_write(&ionic->vf_op_lock);
2447         return ret;
2448 }
2449
2450 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2451                              u8 qos, __be16 proto)
2452 {
2453         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
2454         struct ionic_lif *lif = netdev_priv(netdev);
2455         struct ionic *ionic = lif->ionic;
2456         int ret;
2457
2458         /* until someday when we support qos */
2459         if (qos)
2460                 return -EINVAL;
2461
2462         if (vlan > 4095)
2463                 return -EINVAL;
2464
2465         if (proto != htons(ETH_P_8021Q))
2466                 return -EPROTONOSUPPORT;
2467
2468         if (!netif_device_present(netdev))
2469                 return -EBUSY;
2470
2471         down_write(&ionic->vf_op_lock);
2472
2473         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2474                 ret = -EINVAL;
2475         } else {
2476                 vfc.vlanid = cpu_to_le16(vlan);
2477                 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
2478                         __func__, vf, le16_to_cpu(vfc.vlanid));
2479
2480                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2481                 if (!ret)
2482                         ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2483         }
2484
2485         up_write(&ionic->vf_op_lock);
2486         return ret;
2487 }
2488
2489 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2490                              int tx_min, int tx_max)
2491 {
2492         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
2493         struct ionic_lif *lif = netdev_priv(netdev);
2494         struct ionic *ionic = lif->ionic;
2495         int ret;
2496
2497         /* setting the min just seems silly */
2498         if (tx_min)
2499                 return -EINVAL;
2500
2501         if (!netif_device_present(netdev))
2502                 return -EBUSY;
2503
2504         down_write(&ionic->vf_op_lock);
2505
2506         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2507                 ret = -EINVAL;
2508         } else {
2509                 vfc.maxrate = cpu_to_le32(tx_max);
2510                 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
2511                         __func__, vf, le32_to_cpu(vfc.maxrate));
2512
2513                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2514                 if (!ret)
2515                         ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2516         }
2517
2518         up_write(&ionic->vf_op_lock);
2519         return ret;
2520 }
2521
2522 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2523 {
2524         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
2525         struct ionic_lif *lif = netdev_priv(netdev);
2526         struct ionic *ionic = lif->ionic;
2527         int ret;
2528
2529         if (!netif_device_present(netdev))
2530                 return -EBUSY;
2531
2532         down_write(&ionic->vf_op_lock);
2533
2534         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2535                 ret = -EINVAL;
2536         } else {
2537                 vfc.spoofchk = set;
2538                 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
2539                         __func__, vf, vfc.spoofchk);
2540
2541                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2542                 if (!ret)
2543                         ionic->vfs[vf].spoofchk = set;
2544         }
2545
2546         up_write(&ionic->vf_op_lock);
2547         return ret;
2548 }
2549
2550 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2551 {
2552         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
2553         struct ionic_lif *lif = netdev_priv(netdev);
2554         struct ionic *ionic = lif->ionic;
2555         int ret;
2556
2557         if (!netif_device_present(netdev))
2558                 return -EBUSY;
2559
2560         down_write(&ionic->vf_op_lock);
2561
2562         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2563                 ret = -EINVAL;
2564         } else {
2565                 vfc.trust = set;
2566                 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
2567                         __func__, vf, vfc.trust);
2568
2569                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2570                 if (!ret)
2571                         ionic->vfs[vf].trusted = set;
2572         }
2573
2574         up_write(&ionic->vf_op_lock);
2575         return ret;
2576 }
2577
2578 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2579 {
2580         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
2581         struct ionic_lif *lif = netdev_priv(netdev);
2582         struct ionic *ionic = lif->ionic;
2583         u8 vfls;
2584         int ret;
2585
2586         switch (set) {
2587         case IFLA_VF_LINK_STATE_ENABLE:
2588                 vfls = IONIC_VF_LINK_STATUS_UP;
2589                 break;
2590         case IFLA_VF_LINK_STATE_DISABLE:
2591                 vfls = IONIC_VF_LINK_STATUS_DOWN;
2592                 break;
2593         case IFLA_VF_LINK_STATE_AUTO:
2594                 vfls = IONIC_VF_LINK_STATUS_AUTO;
2595                 break;
2596         default:
2597                 return -EINVAL;
2598         }
2599
2600         if (!netif_device_present(netdev))
2601                 return -EBUSY;
2602
2603         down_write(&ionic->vf_op_lock);
2604
2605         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2606                 ret = -EINVAL;
2607         } else {
2608                 vfc.linkstate = vfls;
2609                 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
2610                         __func__, vf, vfc.linkstate);
2611
2612                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2613                 if (!ret)
2614                         ionic->vfs[vf].linkstate = set;
2615         }
2616
2617         up_write(&ionic->vf_op_lock);
2618         return ret;
2619 }
2620
2621 static void ionic_vf_attr_replay(struct ionic_lif *lif)
2622 {
2623         struct ionic_vf_setattr_cmd vfc = { };
2624         struct ionic *ionic = lif->ionic;
2625         struct ionic_vf *v;
2626         int i;
2627
2628         if (!ionic->vfs)
2629                 return;
2630
2631         down_read(&ionic->vf_op_lock);
2632
2633         for (i = 0; i < ionic->num_vfs; i++) {
2634                 v = &ionic->vfs[i];
2635
2636                 if (v->stats_pa) {
2637                         vfc.attr = IONIC_VF_ATTR_STATSADDR;
2638                         vfc.stats_pa = cpu_to_le64(v->stats_pa);
2639                         ionic_set_vf_config(ionic, i, &vfc);
2640                         vfc.stats_pa = 0;
2641                 }
2642
2643                 if (!is_zero_ether_addr(v->macaddr)) {
2644                         vfc.attr = IONIC_VF_ATTR_MAC;
2645                         ether_addr_copy(vfc.macaddr, v->macaddr);
2646                         ionic_set_vf_config(ionic, i, &vfc);
2647                         eth_zero_addr(vfc.macaddr);
2648                 }
2649
2650                 if (v->vlanid) {
2651                         vfc.attr = IONIC_VF_ATTR_VLAN;
2652                         vfc.vlanid = v->vlanid;
2653                         ionic_set_vf_config(ionic, i, &vfc);
2654                         vfc.vlanid = 0;
2655                 }
2656
2657                 if (v->maxrate) {
2658                         vfc.attr = IONIC_VF_ATTR_RATE;
2659                         vfc.maxrate = v->maxrate;
2660                         ionic_set_vf_config(ionic, i, &vfc);
2661                         vfc.maxrate = 0;
2662                 }
2663
2664                 if (v->spoofchk) {
2665                         vfc.attr = IONIC_VF_ATTR_SPOOFCHK;
2666                         vfc.spoofchk = v->spoofchk;
2667                         ionic_set_vf_config(ionic, i, &vfc);
2668                         vfc.spoofchk = 0;
2669                 }
2670
2671                 if (v->trusted) {
2672                         vfc.attr = IONIC_VF_ATTR_TRUST;
2673                         vfc.trust = v->trusted;
2674                         ionic_set_vf_config(ionic, i, &vfc);
2675                         vfc.trust = 0;
2676                 }
2677
2678                 if (v->linkstate) {
2679                         vfc.attr = IONIC_VF_ATTR_LINKSTATE;
2680                         vfc.linkstate = v->linkstate;
2681                         ionic_set_vf_config(ionic, i, &vfc);
2682                         vfc.linkstate = 0;
2683                 }
2684         }
2685
2686         up_read(&ionic->vf_op_lock);
2687
2688         ionic_vf_start(ionic);
2689 }
2690
2691 static void ionic_unregister_rxq_info(struct ionic_queue *q)
2692 {
2693         struct xdp_rxq_info *xi;
2694
2695         if (!q->xdp_rxq_info)
2696                 return;
2697
2698         xi = q->xdp_rxq_info;
2699         q->xdp_rxq_info = NULL;
2700
2701         xdp_rxq_info_unreg(xi);
2702         kfree(xi);
2703 }
2704
2705 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)
2706 {
2707         struct xdp_rxq_info *rxq_info;
2708         int err;
2709
2710         rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
2711         if (!rxq_info)
2712                 return -ENOMEM;
2713
2714         err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id);
2715         if (err) {
2716                 netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n",
2717                            q->index, err);
2718                 goto err_out;
2719         }
2720
2721         err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool);
2722         if (err) {
2723                 netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n",
2724                            q->index, err);
2725                 xdp_rxq_info_unreg(rxq_info);
2726                 goto err_out;
2727         }
2728
2729         q->xdp_rxq_info = rxq_info;
2730
2731         return 0;
2732
2733 err_out:
2734         kfree(rxq_info);
2735         return err;
2736 }
2737
2738 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif)
2739 {
2740         struct bpf_prog *xdp_prog;
2741         unsigned int i;
2742
2743         if (!lif->rxqcqs)
2744                 return;
2745
2746         xdp_prog = READ_ONCE(lif->xdp_prog);
2747         for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2748                 struct ionic_queue *q = &lif->rxqcqs[i]->q;
2749
2750                 WRITE_ONCE(q->xdp_prog, xdp_prog);
2751         }
2752 }
2753
2754 static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
2755 {
2756         struct ionic_lif *lif = netdev_priv(netdev);
2757         struct bpf_prog *old_prog;
2758         u32 maxfs;
2759
2760         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
2761 #define XDP_ERR_SPLIT "XDP not available with split Tx/Rx interrupts"
2762                 NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_SPLIT);
2763                 netdev_info(lif->netdev, XDP_ERR_SPLIT);
2764                 return -EOPNOTSUPP;
2765         }
2766
2767         if (!ionic_xdp_is_valid_mtu(lif, netdev->mtu, bpf->prog)) {
2768 #define XDP_ERR_MTU "MTU is too large for XDP without frags support"
2769                 NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_MTU);
2770                 netdev_info(lif->netdev, XDP_ERR_MTU);
2771                 return -EINVAL;
2772         }
2773
2774         maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
2775         if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags))
2776                 maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU);
2777         netdev->max_mtu = maxfs;
2778
2779         if (!netif_running(netdev)) {
2780                 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2781         } else if (lif->xdp_prog && bpf->prog) {
2782                 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2783                 ionic_xdp_rxqs_prog_update(lif);
2784         } else {
2785                 struct ionic_queue_params qparams;
2786
2787                 ionic_init_queue_params(lif, &qparams);
2788                 qparams.xdp_prog = bpf->prog;
2789                 mutex_lock(&lif->queue_lock);
2790                 ionic_reconfigure_queues(lif, &qparams);
2791                 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2792                 mutex_unlock(&lif->queue_lock);
2793         }
2794
2795         if (old_prog)
2796                 bpf_prog_put(old_prog);
2797
2798         return 0;
2799 }
2800
2801 static int ionic_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
2802 {
2803         switch (bpf->command) {
2804         case XDP_SETUP_PROG:
2805                 return ionic_xdp_config(netdev, bpf);
2806         default:
2807                 return -EINVAL;
2808         }
2809 }
2810
2811 static const struct net_device_ops ionic_netdev_ops = {
2812         .ndo_open               = ionic_open,
2813         .ndo_stop               = ionic_stop,
2814         .ndo_eth_ioctl          = ionic_eth_ioctl,
2815         .ndo_start_xmit         = ionic_start_xmit,
2816         .ndo_bpf                = ionic_xdp,
2817         .ndo_xdp_xmit           = ionic_xdp_xmit,
2818         .ndo_get_stats64        = ionic_get_stats64,
2819         .ndo_set_rx_mode        = ionic_ndo_set_rx_mode,
2820         .ndo_set_features       = ionic_set_features,
2821         .ndo_set_mac_address    = ionic_set_mac_address,
2822         .ndo_validate_addr      = eth_validate_addr,
2823         .ndo_tx_timeout         = ionic_tx_timeout,
2824         .ndo_change_mtu         = ionic_change_mtu,
2825         .ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2826         .ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2827         .ndo_set_vf_vlan        = ionic_set_vf_vlan,
2828         .ndo_set_vf_trust       = ionic_set_vf_trust,
2829         .ndo_set_vf_mac         = ionic_set_vf_mac,
2830         .ndo_set_vf_rate        = ionic_set_vf_rate,
2831         .ndo_set_vf_spoofchk    = ionic_set_vf_spoofchk,
2832         .ndo_get_vf_config      = ionic_get_vf_config,
2833         .ndo_set_vf_link_state  = ionic_set_vf_link_state,
2834         .ndo_get_vf_stats       = ionic_get_vf_stats,
2835 };
2836
2837 static int ionic_cmb_reconfig(struct ionic_lif *lif,
2838                               struct ionic_queue_params *qparam)
2839 {
2840         struct ionic_queue_params start_qparams;
2841         int err = 0;
2842
2843         /* When changing CMB queue parameters, we're using limited
2844          * on-device memory and don't have extra memory to use for
2845          * duplicate allocations, so we free it all first then
2846          * re-allocate with the new parameters.
2847          */
2848
2849         /* Checkpoint for possible unwind */
2850         ionic_init_queue_params(lif, &start_qparams);
2851
2852         /* Stop and free the queues */
2853         ionic_stop_queues_reconfig(lif);
2854         ionic_txrx_free(lif);
2855
2856         /* Set up new qparams */
2857         ionic_set_queue_params(lif, qparam);
2858
2859         if (netif_running(lif->netdev)) {
2860                 /* Alloc and start the new configuration */
2861                 err = ionic_txrx_alloc(lif);
2862                 if (err) {
2863                         dev_warn(lif->ionic->dev,
2864                                  "CMB reconfig failed, restoring values: %d\n", err);
2865
2866                         /* Back out the changes */
2867                         ionic_set_queue_params(lif, &start_qparams);
2868                         err = ionic_txrx_alloc(lif);
2869                         if (err) {
2870                                 dev_err(lif->ionic->dev,
2871                                         "CMB restore failed: %d\n", err);
2872                                 goto err_out;
2873                         }
2874                 }
2875
2876                 err = ionic_start_queues_reconfig(lif);
2877                 if (err) {
2878                         dev_err(lif->ionic->dev,
2879                                 "CMB reconfig failed: %d\n", err);
2880                         goto err_out;
2881                 }
2882         }
2883
2884 err_out:
2885         /* This was detached in ionic_stop_queues_reconfig() */
2886         netif_device_attach(lif->netdev);
2887
2888         return err;
2889 }
2890
2891 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2892 {
2893         /* only swapping the queues and napi, not flags or other stuff */
2894         swap(a->napi,         b->napi);
2895
2896         if (a->q.type == IONIC_QTYPE_RXQ) {
2897                 swap(a->q.page_pool, b->q.page_pool);
2898                 a->q.page_pool->p.napi = &a->napi;
2899                 if (b->q.page_pool)  /* is NULL when increasing queue count */
2900                         b->q.page_pool->p.napi = &b->napi;
2901         }
2902
2903         swap(a->q.features,   b->q.features);
2904         swap(a->q.num_descs,  b->q.num_descs);
2905         swap(a->q.desc_size,  b->q.desc_size);
2906         swap(a->q.base,       b->q.base);
2907         swap(a->q.base_pa,    b->q.base_pa);
2908         swap(a->q.info,       b->q.info);
2909         swap(a->q.xdp_prog,   b->q.xdp_prog);
2910         swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
2911         swap(a->q.partner,    b->q.partner);
2912         swap(a->q_base,       b->q_base);
2913         swap(a->q_base_pa,    b->q_base_pa);
2914         swap(a->q_size,       b->q_size);
2915
2916         swap(a->q.sg_desc_size, b->q.sg_desc_size);
2917         swap(a->q.sg_base,    b->q.sg_base);
2918         swap(a->q.sg_base_pa, b->q.sg_base_pa);
2919         swap(a->sg_base,      b->sg_base);
2920         swap(a->sg_base_pa,   b->sg_base_pa);
2921         swap(a->sg_size,      b->sg_size);
2922
2923         swap(a->cq.num_descs, b->cq.num_descs);
2924         swap(a->cq.desc_size, b->cq.desc_size);
2925         swap(a->cq.base,      b->cq.base);
2926         swap(a->cq.base_pa,   b->cq.base_pa);
2927         swap(a->cq_base,      b->cq_base);
2928         swap(a->cq_base_pa,   b->cq_base_pa);
2929         swap(a->cq_size,      b->cq_size);
2930
2931         ionic_debugfs_del_qcq(a);
2932         ionic_debugfs_add_qcq(a->q.lif, a);
2933 }
2934
2935 int ionic_reconfigure_queues(struct ionic_lif *lif,
2936                              struct ionic_queue_params *qparam)
2937 {
2938         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2939         struct ionic_qcq **tx_qcqs = NULL;
2940         struct ionic_qcq **rx_qcqs = NULL;
2941         unsigned int flags, i;
2942         int err = 0;
2943
2944         /* Are we changing q params while CMB is on */
2945         if ((test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state) && qparam->cmb_tx) ||
2946             (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state) && qparam->cmb_rx))
2947                 return ionic_cmb_reconfig(lif, qparam);
2948
2949         /* allocate temporary qcq arrays to hold new queue structs */
2950         if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2951                 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2952                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2953                 if (!tx_qcqs) {
2954                         err = -ENOMEM;
2955                         goto err_out;
2956                 }
2957         }
2958         if (qparam->nxqs != lif->nxqs ||
2959             qparam->nrxq_descs != lif->nrxq_descs ||
2960             qparam->rxq_features != lif->rxq_features ||
2961             qparam->xdp_prog != lif->xdp_prog) {
2962                 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2963                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2964                 if (!rx_qcqs) {
2965                         err = -ENOMEM;
2966                         goto err_out;
2967                 }
2968         }
2969
2970         /* allocate new desc_info and rings, but leave the interrupt setup
2971          * until later so as to not mess with the still-running queues
2972          */
2973         if (tx_qcqs) {
2974                 num_desc = qparam->ntxq_descs;
2975                 desc_sz = sizeof(struct ionic_txq_desc);
2976                 comp_sz = sizeof(struct ionic_txq_comp);
2977
2978                 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2979                     lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2980                     sizeof(struct ionic_txq_sg_desc_v1))
2981                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2982                 else
2983                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2984
2985                 for (i = 0; i < qparam->nxqs; i++) {
2986                         /* If missing, short placeholder qcq needed for swap */
2987                         if (!lif->txqcqs[i]) {
2988                                 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2989                                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2990                                                       4, desc_sz, comp_sz, sg_desc_sz,
2991                                                       sizeof(struct ionic_tx_desc_info),
2992                                                       lif->kern_pid, NULL, &lif->txqcqs[i]);
2993                                 if (err)
2994                                         goto err_out;
2995                         }
2996
2997                         flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2998                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2999                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
3000                                               sizeof(struct ionic_tx_desc_info),
3001                                               lif->kern_pid, NULL, &tx_qcqs[i]);
3002                         if (err)
3003                                 goto err_out;
3004                 }
3005         }
3006
3007         if (rx_qcqs) {
3008                 num_desc = qparam->nrxq_descs;
3009                 desc_sz = sizeof(struct ionic_rxq_desc);
3010                 comp_sz = sizeof(struct ionic_rxq_comp);
3011                 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
3012
3013                 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
3014                         comp_sz *= 2;
3015
3016                 for (i = 0; i < qparam->nxqs; i++) {
3017                         /* If missing, short placeholder qcq needed for swap */
3018                         if (!lif->rxqcqs[i]) {
3019                                 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
3020                                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3021                                                       4, desc_sz, comp_sz, sg_desc_sz,
3022                                                       sizeof(struct ionic_rx_desc_info),
3023                                                       lif->kern_pid, NULL, &lif->rxqcqs[i]);
3024                                 if (err)
3025                                         goto err_out;
3026                         }
3027
3028                         flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
3029                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3030                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
3031                                               sizeof(struct ionic_rx_desc_info),
3032                                               lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]);
3033                         if (err)
3034                                 goto err_out;
3035
3036                         rx_qcqs[i]->q.features = qparam->rxq_features;
3037                         rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog;
3038                 }
3039         }
3040
3041         /* stop and clean the queues */
3042         ionic_stop_queues_reconfig(lif);
3043
3044         if (qparam->nxqs != lif->nxqs) {
3045                 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
3046                 if (err)
3047                         goto err_out_reinit_unlock;
3048                 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
3049                 if (err) {
3050                         netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
3051                         goto err_out_reinit_unlock;
3052                 }
3053         }
3054
3055         /* swap new desc_info and rings, keeping existing interrupt config */
3056         if (tx_qcqs) {
3057                 lif->ntxq_descs = qparam->ntxq_descs;
3058                 for (i = 0; i < qparam->nxqs; i++)
3059                         ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
3060         }
3061
3062         if (rx_qcqs) {
3063                 lif->nrxq_descs = qparam->nrxq_descs;
3064                 for (i = 0; i < qparam->nxqs; i++)
3065                         ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
3066         }
3067
3068         /* if we need to change the interrupt layout, this is the time */
3069         if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
3070             qparam->nxqs != lif->nxqs) {
3071                 if (qparam->intr_split) {
3072                         set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3073                 } else {
3074                         clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3075                         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3076                         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3077                 }
3078
3079                 /* Clear existing interrupt assignments.  We check for NULL here
3080                  * because we're checking the whole array for potential qcqs, not
3081                  * just those qcqs that have just been set up.
3082                  */
3083                 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
3084                         if (lif->txqcqs[i])
3085                                 ionic_qcq_intr_free(lif, lif->txqcqs[i]);
3086                         if (lif->rxqcqs[i])
3087                                 ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
3088                 }
3089
3090                 /* re-assign the interrupts */
3091                 for (i = 0; i < qparam->nxqs; i++) {
3092                         lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3093                         err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
3094                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3095                                              lif->rxqcqs[i]->intr.index,
3096                                              lif->rx_coalesce_hw);
3097
3098                         if (qparam->intr_split) {
3099                                 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3100                                 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
3101                                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3102                                                      lif->txqcqs[i]->intr.index,
3103                                                      lif->tx_coalesce_hw);
3104                                 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
3105                                         lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
3106                         } else {
3107                                 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3108                                 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
3109                         }
3110                 }
3111         }
3112
3113         /* now we can rework the debugfs mappings */
3114         if (tx_qcqs) {
3115                 for (i = 0; i < qparam->nxqs; i++) {
3116                         ionic_debugfs_del_qcq(lif->txqcqs[i]);
3117                         ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
3118                 }
3119         }
3120
3121         if (rx_qcqs) {
3122                 for (i = 0; i < qparam->nxqs; i++) {
3123                         ionic_debugfs_del_qcq(lif->rxqcqs[i]);
3124                         ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
3125                 }
3126         }
3127
3128         swap(lif->nxqs, qparam->nxqs);
3129         swap(lif->rxq_features, qparam->rxq_features);
3130
3131 err_out_reinit_unlock:
3132         /* re-init the queues, but don't lose an error code */
3133         if (err)
3134                 ionic_start_queues_reconfig(lif);
3135         else
3136                 err = ionic_start_queues_reconfig(lif);
3137
3138 err_out:
3139         /* free old allocs without cleaning intr */
3140         for (i = 0; i < qparam->nxqs; i++) {
3141                 if (tx_qcqs && tx_qcqs[i]) {
3142                         tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3143                         ionic_qcq_free(lif, tx_qcqs[i]);
3144                         devm_kfree(lif->ionic->dev, tx_qcqs[i]);
3145                         tx_qcqs[i] = NULL;
3146                 }
3147                 if (rx_qcqs && rx_qcqs[i]) {
3148                         rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3149                         ionic_qcq_free(lif, rx_qcqs[i]);
3150                         devm_kfree(lif->ionic->dev, rx_qcqs[i]);
3151                         rx_qcqs[i] = NULL;
3152                 }
3153         }
3154
3155         /* free q array */
3156         if (rx_qcqs) {
3157                 devm_kfree(lif->ionic->dev, rx_qcqs);
3158                 rx_qcqs = NULL;
3159         }
3160         if (tx_qcqs) {
3161                 devm_kfree(lif->ionic->dev, tx_qcqs);
3162                 tx_qcqs = NULL;
3163         }
3164
3165         /* clean the unused dma and info allocations when new set is smaller
3166          * than the full array, but leave the qcq shells in place
3167          */
3168         for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
3169                 if (lif->txqcqs && lif->txqcqs[i]) {
3170                         lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3171                         ionic_qcq_free(lif, lif->txqcqs[i]);
3172                 }
3173
3174                 if (lif->rxqcqs && lif->rxqcqs[i]) {
3175                         lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3176                         ionic_qcq_free(lif, lif->rxqcqs[i]);
3177                 }
3178         }
3179
3180         if (err)
3181                 netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
3182
3183         return err;
3184 }
3185
3186 static int ionic_affinity_masks_alloc(struct ionic *ionic)
3187 {
3188         cpumask_var_t *affinity_masks;
3189         int nintrs = ionic->nintrs;
3190         int i;
3191
3192         affinity_masks = kcalloc(nintrs, sizeof(cpumask_var_t), GFP_KERNEL);
3193         if (!affinity_masks)
3194                 return -ENOMEM;
3195
3196         for (i = 0; i < nintrs; i++) {
3197                 if (!zalloc_cpumask_var_node(&affinity_masks[i], GFP_KERNEL,
3198                                              dev_to_node(ionic->dev)))
3199                         goto err_out;
3200         }
3201
3202         ionic->affinity_masks = affinity_masks;
3203
3204         return 0;
3205
3206 err_out:
3207         for (--i; i >= 0; i--)
3208                 free_cpumask_var(affinity_masks[i]);
3209         kfree(affinity_masks);
3210
3211         return -ENOMEM;
3212 }
3213
3214 static void ionic_affinity_masks_free(struct ionic *ionic)
3215 {
3216         int i;
3217
3218         for (i = 0; i < ionic->nintrs; i++)
3219                 free_cpumask_var(ionic->affinity_masks[i]);
3220         kfree(ionic->affinity_masks);
3221         ionic->affinity_masks = NULL;
3222 }
3223
3224 int ionic_lif_alloc(struct ionic *ionic)
3225 {
3226         struct device *dev = ionic->dev;
3227         union ionic_lif_identity *lid;
3228         struct net_device *netdev;
3229         struct ionic_lif *lif;
3230         int tbl_sz;
3231         int err;
3232
3233         lid = kzalloc(sizeof(*lid), GFP_KERNEL);
3234         if (!lid)
3235                 return -ENOMEM;
3236
3237         netdev = alloc_etherdev_mqs(sizeof(*lif),
3238                                     ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
3239         if (!netdev) {
3240                 dev_err(dev, "Cannot allocate netdev, aborting\n");
3241                 err = -ENOMEM;
3242                 goto err_out_free_lid;
3243         }
3244
3245         SET_NETDEV_DEV(netdev, dev);
3246
3247         lif = netdev_priv(netdev);
3248         lif->netdev = netdev;
3249         ionic->lif = lif;
3250         lif->ionic = ionic;
3251         netdev->netdev_ops = &ionic_netdev_ops;
3252         ionic_ethtool_set_ops(netdev);
3253
3254         netdev->watchdog_timeo = 5 * HZ;
3255         netif_carrier_off(netdev);
3256
3257         lif->identity = lid;
3258         lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
3259         err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
3260         if (err) {
3261                 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
3262                         lif->lif_type, err);
3263                 goto err_out_free_netdev;
3264         }
3265         lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
3266                                      le32_to_cpu(lif->identity->eth.min_frame_size));
3267         lif->netdev->max_mtu =
3268                 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
3269
3270         lif->neqs = ionic->neqs_per_lif;
3271         lif->nxqs = ionic->ntxqs_per_lif;
3272
3273         lif->index = 0;
3274
3275         if (is_kdump_kernel()) {
3276                 lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
3277                 lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
3278         } else {
3279                 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
3280                 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
3281         }
3282
3283         /* Convert the default coalesce value to actual hw resolution */
3284         lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
3285         lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
3286                                                     lif->rx_coalesce_usecs);
3287         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3288         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3289         set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
3290         set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
3291
3292         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
3293
3294         mutex_init(&lif->queue_lock);
3295         mutex_init(&lif->config_lock);
3296
3297         spin_lock_init(&lif->adminq_lock);
3298
3299         spin_lock_init(&lif->deferred.lock);
3300         INIT_LIST_HEAD(&lif->deferred.list);
3301         INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
3302
3303         /* allocate lif info */
3304         lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
3305         lif->info = dma_alloc_coherent(dev, lif->info_sz,
3306                                        &lif->info_pa, GFP_KERNEL);
3307         if (!lif->info) {
3308                 dev_err(dev, "Failed to allocate lif info, aborting\n");
3309                 err = -ENOMEM;
3310                 goto err_out_free_mutex;
3311         }
3312
3313         ionic_debugfs_add_lif(lif);
3314
3315         err = ionic_affinity_masks_alloc(ionic);
3316         if (err)
3317                 goto err_out_free_lif_info;
3318
3319         /* allocate control queues and txrx queue arrays */
3320         ionic_lif_queue_identify(lif);
3321         err = ionic_qcqs_alloc(lif);
3322         if (err)
3323                 goto err_out_free_affinity_masks;
3324
3325         /* allocate rss indirection table */
3326         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
3327         lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
3328         lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
3329                                               &lif->rss_ind_tbl_pa,
3330                                               GFP_KERNEL);
3331
3332         if (!lif->rss_ind_tbl) {
3333                 err = -ENOMEM;
3334                 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
3335                 goto err_out_free_qcqs;
3336         }
3337         netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
3338
3339         ionic_lif_alloc_phc(lif);
3340
3341         return 0;
3342
3343 err_out_free_qcqs:
3344         ionic_qcqs_free(lif);
3345 err_out_free_affinity_masks:
3346         ionic_affinity_masks_free(lif->ionic);
3347 err_out_free_lif_info:
3348         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3349         lif->info = NULL;
3350         lif->info_pa = 0;
3351 err_out_free_mutex:
3352         mutex_destroy(&lif->config_lock);
3353         mutex_destroy(&lif->queue_lock);
3354 err_out_free_netdev:
3355         free_netdev(lif->netdev);
3356         lif = NULL;
3357 err_out_free_lid:
3358         kfree(lid);
3359
3360         return err;
3361 }
3362
3363 static void ionic_lif_reset(struct ionic_lif *lif)
3364 {
3365         struct ionic_dev *idev = &lif->ionic->idev;
3366
3367         if (!ionic_is_fw_running(idev))
3368                 return;
3369
3370         mutex_lock(&lif->ionic->dev_cmd_lock);
3371         ionic_dev_cmd_lif_reset(idev, lif->index);
3372         ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3373         mutex_unlock(&lif->ionic->dev_cmd_lock);
3374 }
3375
3376 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
3377 {
3378         struct ionic *ionic = lif->ionic;
3379
3380         if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
3381                 return;
3382
3383         dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
3384
3385         netif_device_detach(lif->netdev);
3386
3387         mutex_lock(&lif->queue_lock);
3388         if (test_bit(IONIC_LIF_F_UP, lif->state)) {
3389                 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
3390                 ionic_stop_queues(lif);
3391         }
3392
3393         if (netif_running(lif->netdev)) {
3394                 ionic_txrx_deinit(lif);
3395                 ionic_txrx_free(lif);
3396         }
3397         ionic_lif_deinit(lif);
3398         ionic_reset(ionic);
3399         ionic_qcqs_free(lif);
3400
3401         mutex_unlock(&lif->queue_lock);
3402
3403         clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
3404         dev_info(ionic->dev, "FW Down: LIFs stopped\n");
3405 }
3406
3407 int ionic_restart_lif(struct ionic_lif *lif)
3408 {
3409         struct ionic *ionic = lif->ionic;
3410         int err;
3411
3412         mutex_lock(&lif->queue_lock);
3413
3414         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
3415                 dev_info(ionic->dev, "FW Up: clearing broken state\n");
3416
3417         err = ionic_qcqs_alloc(lif);
3418         if (err)
3419                 goto err_unlock;
3420
3421         err = ionic_lif_init(lif);
3422         if (err)
3423                 goto err_qcqs_free;
3424
3425         ionic_vf_attr_replay(lif);
3426
3427         if (lif->registered)
3428                 ionic_lif_set_netdev_info(lif);
3429
3430         ionic_rx_filter_replay(lif);
3431
3432         if (netif_running(lif->netdev)) {
3433                 err = ionic_txrx_alloc(lif);
3434                 if (err)
3435                         goto err_lifs_deinit;
3436
3437                 err = ionic_txrx_init(lif);
3438                 if (err)
3439                         goto err_txrx_free;
3440         }
3441
3442         mutex_unlock(&lif->queue_lock);
3443
3444         clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3445         ionic_link_status_check_request(lif, CAN_SLEEP);
3446         netif_device_attach(lif->netdev);
3447         ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
3448
3449         return 0;
3450
3451 err_txrx_free:
3452         ionic_txrx_free(lif);
3453 err_lifs_deinit:
3454         ionic_lif_deinit(lif);
3455 err_qcqs_free:
3456         ionic_qcqs_free(lif);
3457 err_unlock:
3458         mutex_unlock(&lif->queue_lock);
3459
3460         return err;
3461 }
3462
3463 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
3464 {
3465         struct ionic *ionic = lif->ionic;
3466         int err;
3467
3468         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3469                 return;
3470
3471         dev_info(ionic->dev, "FW Up: restarting LIFs\n");
3472
3473         /* This is a little different from what happens at
3474          * probe time because the LIF already exists so we
3475          * just need to reanimate it.
3476          */
3477         ionic_init_devinfo(ionic);
3478         ionic_reset(ionic);
3479         err = ionic_identify(ionic);
3480         if (err)
3481                 goto err_out;
3482         err = ionic_port_identify(ionic);
3483         if (err)
3484                 goto err_out;
3485         err = ionic_port_init(ionic);
3486         if (err)
3487                 goto err_out;
3488
3489         err = ionic_restart_lif(lif);
3490         if (err)
3491                 goto err_out;
3492
3493         dev_info(ionic->dev, "FW Up: LIFs restarted\n");
3494
3495         /* restore the hardware timestamping queues */
3496         ionic_lif_hwstamp_replay(lif);
3497
3498         return;
3499
3500 err_out:
3501         dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3502 }
3503
3504 void ionic_lif_free(struct ionic_lif *lif)
3505 {
3506         struct device *dev = lif->ionic->dev;
3507
3508         ionic_lif_free_phc(lif);
3509
3510         /* free rss indirection table */
3511         dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3512                           lif->rss_ind_tbl_pa);
3513         lif->rss_ind_tbl = NULL;
3514         lif->rss_ind_tbl_pa = 0;
3515
3516         /* free queues */
3517         ionic_qcqs_free(lif);
3518         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3519                 ionic_lif_reset(lif);
3520
3521         ionic_affinity_masks_free(lif->ionic);
3522
3523         /* free lif info */
3524         kfree(lif->identity);
3525         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3526         lif->info = NULL;
3527         lif->info_pa = 0;
3528
3529         /* unmap doorbell page */
3530         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3531         lif->kern_dbpage = NULL;
3532
3533         mutex_destroy(&lif->config_lock);
3534         mutex_destroy(&lif->queue_lock);
3535
3536         /* free netdev & lif */
3537         ionic_debugfs_del_lif(lif);
3538         free_netdev(lif->netdev);
3539 }
3540
3541 void ionic_lif_deinit(struct ionic_lif *lif)
3542 {
3543         if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3544                 return;
3545
3546         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3547                 cancel_work_sync(&lif->deferred.work);
3548                 cancel_work_sync(&lif->tx_timeout_work);
3549                 ionic_rx_filters_deinit(lif);
3550                 if (lif->netdev->features & NETIF_F_RXHASH)
3551                         ionic_lif_rss_deinit(lif);
3552         }
3553
3554         napi_disable(&lif->adminqcq->napi);
3555         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3556         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3557
3558         ionic_lif_reset(lif);
3559 }
3560
3561 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3562 {
3563         struct device *dev = lif->ionic->dev;
3564         struct ionic_q_init_comp comp;
3565         struct ionic_dev *idev;
3566         struct ionic_qcq *qcq;
3567         struct ionic_queue *q;
3568         int err;
3569
3570         idev = &lif->ionic->idev;
3571         qcq = lif->adminqcq;
3572         q = &qcq->q;
3573
3574         mutex_lock(&lif->ionic->dev_cmd_lock);
3575         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3576         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3577         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3578         mutex_unlock(&lif->ionic->dev_cmd_lock);
3579         if (err) {
3580                 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3581                 return err;
3582         }
3583
3584         q->hw_type = comp.hw_type;
3585         q->hw_index = le32_to_cpu(comp.hw_index);
3586         q->dbval = IONIC_DBELL_QID(q->hw_index);
3587
3588         dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3589         dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3590
3591         q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;
3592         q->dbell_jiffies = jiffies;
3593
3594         netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
3595
3596         napi_enable(&qcq->napi);
3597
3598         if (qcq->flags & IONIC_QCQ_F_INTR) {
3599                 irq_set_affinity_hint(qcq->intr.vector,
3600                                       *qcq->intr.affinity_mask);
3601                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3602                                 IONIC_INTR_MASK_CLEAR);
3603         }
3604
3605         qcq->flags |= IONIC_QCQ_F_INITED;
3606
3607         return 0;
3608 }
3609
3610 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3611 {
3612         struct ionic_qcq *qcq = lif->notifyqcq;
3613         struct device *dev = lif->ionic->dev;
3614         struct ionic_queue *q = &qcq->q;
3615         int err;
3616
3617         struct ionic_admin_ctx ctx = {
3618                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3619                 .cmd.q_init = {
3620                         .opcode = IONIC_CMD_Q_INIT,
3621                         .lif_index = cpu_to_le16(lif->index),
3622                         .type = q->type,
3623                         .ver = lif->qtype_info[q->type].version,
3624                         .index = cpu_to_le32(q->index),
3625                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3626                                              IONIC_QINIT_F_ENA),
3627                         .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3628                         .pid = cpu_to_le16(q->pid),
3629                         .ring_size = ilog2(q->num_descs),
3630                         .ring_base = cpu_to_le64(q->base_pa),
3631                 }
3632         };
3633
3634         dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3635         dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3636         dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3637         dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3638
3639         err = ionic_adminq_post_wait(lif, &ctx);
3640         if (err)
3641                 return err;
3642
3643         lif->last_eid = 0;
3644         q->hw_type = ctx.comp.q_init.hw_type;
3645         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3646         q->dbval = IONIC_DBELL_QID(q->hw_index);
3647
3648         dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3649         dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3650
3651         /* preset the callback info */
3652         q->admin_info[0].ctx = lif;
3653
3654         qcq->flags |= IONIC_QCQ_F_INITED;
3655
3656         return 0;
3657 }
3658
3659 static int ionic_station_set(struct ionic_lif *lif)
3660 {
3661         struct net_device *netdev = lif->netdev;
3662         struct ionic_admin_ctx ctx = {
3663                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3664                 .cmd.lif_getattr = {
3665                         .opcode = IONIC_CMD_LIF_GETATTR,
3666                         .index = cpu_to_le16(lif->index),
3667                         .attr = IONIC_LIF_ATTR_MAC,
3668                 },
3669         };
3670         u8 mac_address[ETH_ALEN];
3671         struct sockaddr addr;
3672         int err;
3673
3674         err = ionic_adminq_post_wait(lif, &ctx);
3675         if (err)
3676                 return err;
3677         netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3678                    ctx.comp.lif_getattr.mac);
3679         ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);
3680
3681         if (is_zero_ether_addr(mac_address)) {
3682                 eth_hw_addr_random(netdev);
3683                 netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);
3684                 ether_addr_copy(mac_address, netdev->dev_addr);
3685
3686                 err = ionic_program_mac(lif, mac_address);
3687                 if (err < 0)
3688                         return err;
3689
3690                 if (err > 0) {
3691                         netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",
3692                                    __func__);
3693                         return 0;
3694                 }
3695         }
3696
3697         if (!is_zero_ether_addr(netdev->dev_addr)) {
3698                 /* If the netdev mac is non-zero and doesn't match the default
3699                  * device address, it was set by something earlier and we're
3700                  * likely here again after a fw-upgrade reset.  We need to be
3701                  * sure the netdev mac is in our filter list.
3702                  */
3703                 if (!ether_addr_equal(mac_address, netdev->dev_addr))
3704                         ionic_lif_addr_add(lif, netdev->dev_addr);
3705         } else {
3706                 /* Update the netdev mac with the device's mac */
3707                 ether_addr_copy(addr.sa_data, mac_address);
3708                 addr.sa_family = AF_INET;
3709                 err = eth_prepare_mac_addr_change(netdev, &addr);
3710                 if (err) {
3711                         netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3712                                     addr.sa_data, err);
3713                         return 0;
3714                 }
3715
3716                 eth_commit_mac_addr_change(netdev, &addr);
3717         }
3718
3719         netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3720                    netdev->dev_addr);
3721         ionic_lif_addr_add(lif, netdev->dev_addr);
3722
3723         return 0;
3724 }
3725
3726 int ionic_lif_init(struct ionic_lif *lif)
3727 {
3728         struct ionic_dev *idev = &lif->ionic->idev;
3729         struct device *dev = lif->ionic->dev;
3730         struct ionic_lif_init_comp comp;
3731         int dbpage_num;
3732         int err;
3733
3734         mutex_lock(&lif->ionic->dev_cmd_lock);
3735         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3736         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3737         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3738         mutex_unlock(&lif->ionic->dev_cmd_lock);
3739         if (err)
3740                 return err;
3741
3742         lif->hw_index = le16_to_cpu(comp.hw_index);
3743
3744         /* now that we have the hw_index we can figure out our doorbell page */
3745         lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3746         if (!lif->dbid_count) {
3747                 dev_err(dev, "No doorbell pages, aborting\n");
3748                 return -EINVAL;
3749         }
3750
3751         lif->kern_pid = 0;
3752         dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3753         lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3754         if (!lif->kern_dbpage) {
3755                 dev_err(dev, "Cannot map dbpage, aborting\n");
3756                 return -ENOMEM;
3757         }
3758
3759         err = ionic_lif_adminq_init(lif);
3760         if (err)
3761                 goto err_out_adminq_deinit;
3762
3763         if (lif->ionic->nnqs_per_lif) {
3764                 err = ionic_lif_notifyq_init(lif);
3765                 if (err)
3766                         goto err_out_notifyq_deinit;
3767         }
3768
3769         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3770                 err = ionic_set_nic_features(lif, lif->netdev->features);
3771         else
3772                 err = ionic_init_nic_features(lif);
3773         if (err)
3774                 goto err_out_notifyq_deinit;
3775
3776         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3777                 err = ionic_rx_filters_init(lif);
3778                 if (err)
3779                         goto err_out_notifyq_deinit;
3780         }
3781
3782         err = ionic_station_set(lif);
3783         if (err)
3784                 goto err_out_notifyq_deinit;
3785
3786         lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3787         lif->doorbell_wa = ionic_doorbell_wa(lif->ionic);
3788
3789         set_bit(IONIC_LIF_F_INITED, lif->state);
3790
3791         INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3792
3793         return 0;
3794
3795 err_out_notifyq_deinit:
3796         napi_disable(&lif->adminqcq->napi);
3797         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3798 err_out_adminq_deinit:
3799         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3800         ionic_lif_reset(lif);
3801         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3802         lif->kern_dbpage = NULL;
3803
3804         return err;
3805 }
3806
3807 static void ionic_lif_notify_work(struct work_struct *ws)
3808 {
3809 }
3810
3811 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3812 {
3813         struct ionic_admin_ctx ctx = {
3814                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3815                 .cmd.lif_setattr = {
3816                         .opcode = IONIC_CMD_LIF_SETATTR,
3817                         .index = cpu_to_le16(lif->index),
3818                         .attr = IONIC_LIF_ATTR_NAME,
3819                 },
3820         };
3821
3822         strscpy(ctx.cmd.lif_setattr.name, netdev_name(lif->netdev),
3823                 sizeof(ctx.cmd.lif_setattr.name));
3824
3825         ionic_adminq_post_wait(lif, &ctx);
3826 }
3827
3828 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3829 {
3830         if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3831                 return NULL;
3832
3833         return netdev_priv(netdev);
3834 }
3835
3836 static int ionic_lif_notify(struct notifier_block *nb,
3837                             unsigned long event, void *info)
3838 {
3839         struct net_device *ndev = netdev_notifier_info_to_dev(info);
3840         struct ionic *ionic = container_of(nb, struct ionic, nb);
3841         struct ionic_lif *lif = ionic_netdev_lif(ndev);
3842
3843         if (!lif || lif->ionic != ionic)
3844                 return NOTIFY_DONE;
3845
3846         switch (event) {
3847         case NETDEV_CHANGENAME:
3848                 ionic_lif_set_netdev_info(lif);
3849                 break;
3850         }
3851
3852         return NOTIFY_DONE;
3853 }
3854
3855 int ionic_lif_register(struct ionic_lif *lif)
3856 {
3857         int err;
3858
3859         ionic_lif_register_phc(lif);
3860
3861         INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3862
3863         lif->ionic->nb.notifier_call = ionic_lif_notify;
3864
3865         err = register_netdevice_notifier(&lif->ionic->nb);
3866         if (err)
3867                 lif->ionic->nb.notifier_call = NULL;
3868
3869         /* only register LIF0 for now */
3870         err = register_netdev(lif->netdev);
3871         if (err) {
3872                 dev_err(lif->ionic->dev, "Cannot register net device: %d, aborting\n", err);
3873                 ionic_lif_unregister(lif);
3874                 return err;
3875         }
3876
3877         ionic_link_status_check_request(lif, CAN_SLEEP);
3878         lif->registered = true;
3879         ionic_lif_set_netdev_info(lif);
3880
3881         return 0;
3882 }
3883
3884 void ionic_lif_unregister(struct ionic_lif *lif)
3885 {
3886         if (lif->ionic->nb.notifier_call) {
3887                 unregister_netdevice_notifier(&lif->ionic->nb);
3888                 cancel_work_sync(&lif->ionic->nb_work);
3889                 lif->ionic->nb.notifier_call = NULL;
3890         }
3891
3892         if (lif->netdev->reg_state == NETREG_REGISTERED)
3893                 unregister_netdev(lif->netdev);
3894
3895         ionic_lif_unregister_phc(lif);
3896
3897         lif->registered = false;
3898 }
3899
3900 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3901 {
3902         union ionic_q_identity __iomem *q_ident;
3903         struct ionic *ionic = lif->ionic;
3904         struct ionic_dev *idev;
3905         u16 max_frags;
3906         int qtype;
3907         int err;
3908
3909         idev = &lif->ionic->idev;
3910         q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3911
3912         for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3913                 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3914
3915                 /* filter out the ones we know about */
3916                 switch (qtype) {
3917                 case IONIC_QTYPE_ADMINQ:
3918                 case IONIC_QTYPE_NOTIFYQ:
3919                 case IONIC_QTYPE_RXQ:
3920                 case IONIC_QTYPE_TXQ:
3921                         break;
3922                 default:
3923                         continue;
3924                 }
3925
3926                 memset(qti, 0, sizeof(*qti));
3927
3928                 mutex_lock(&ionic->dev_cmd_lock);
3929                 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3930                                              ionic_qtype_versions[qtype]);
3931                 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3932                 if (!err) {
3933                         qti->version   = readb(&q_ident->version);
3934                         qti->supported = readb(&q_ident->supported);
3935                         qti->features  = readq(&q_ident->features);
3936                         qti->desc_sz   = readw(&q_ident->desc_sz);
3937                         qti->comp_sz   = readw(&q_ident->comp_sz);
3938                         qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3939                         qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3940                         qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3941                 }
3942                 mutex_unlock(&ionic->dev_cmd_lock);
3943
3944                 if (err == -EINVAL) {
3945                         dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3946                         continue;
3947                 } else if (err == -EIO) {
3948                         dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3949                         return;
3950                 } else if (err) {
3951                         dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3952                                 qtype, err);
3953                         return;
3954                 }
3955
3956                 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3957                         qtype, qti->version);
3958                 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3959                         qtype, qti->supported);
3960                 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3961                         qtype, qti->features);
3962                 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3963                         qtype, qti->desc_sz);
3964                 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3965                         qtype, qti->comp_sz);
3966                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3967                         qtype, qti->sg_desc_sz);
3968                 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3969                         qtype, qti->max_sg_elems);
3970                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3971                         qtype, qti->sg_desc_stride);
3972
3973                 if (qtype == IONIC_QTYPE_TXQ)
3974                         max_frags = IONIC_TX_MAX_FRAGS;
3975                 else if (qtype == IONIC_QTYPE_RXQ)
3976                         max_frags = IONIC_RX_MAX_FRAGS;
3977                 else
3978                         max_frags = 1;
3979
3980                 qti->max_sg_elems = min_t(u16, max_frags - 1, MAX_SKB_FRAGS);
3981                 dev_dbg(ionic->dev, "qtype %d max_sg_elems %d\n",
3982                         qtype, qti->max_sg_elems);
3983         }
3984 }
3985
3986 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3987                        union ionic_lif_identity *lid)
3988 {
3989         struct ionic_dev *idev = &ionic->idev;
3990         size_t sz;
3991         int err;
3992
3993         sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3994
3995         mutex_lock(&ionic->dev_cmd_lock);
3996         ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3997         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3998         memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3999         mutex_unlock(&ionic->dev_cmd_lock);
4000         if (err)
4001                 return (err);
4002
4003         dev_dbg(ionic->dev, "capabilities 0x%llx\n",
4004                 le64_to_cpu(lid->capabilities));
4005
4006         dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
4007                 le32_to_cpu(lid->eth.max_ucast_filters));
4008         dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
4009                 le32_to_cpu(lid->eth.max_mcast_filters));
4010         dev_dbg(ionic->dev, "eth.features 0x%llx\n",
4011                 le64_to_cpu(lid->eth.config.features));
4012         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
4013                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
4014         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
4015                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
4016         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
4017                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
4018         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
4019                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
4020         dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
4021         dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
4022         dev_dbg(ionic->dev, "eth.config.mtu %d\n",
4023                 le32_to_cpu(lid->eth.config.mtu));
4024
4025         return 0;
4026 }
4027
4028 int ionic_lif_size(struct ionic *ionic)
4029 {
4030         struct ionic_identity *ident = &ionic->ident;
4031         unsigned int nintrs, dev_nintrs;
4032         union ionic_lif_config *lc;
4033         unsigned int ntxqs_per_lif;
4034         unsigned int nrxqs_per_lif;
4035         unsigned int neqs_per_lif;
4036         unsigned int nnqs_per_lif;
4037         unsigned int nxqs, neqs;
4038         unsigned int min_intrs;
4039         int err;
4040
4041         /* retrieve basic values from FW */
4042         lc = &ident->lif.eth.config;
4043         dev_nintrs = le32_to_cpu(ident->dev.nintrs);
4044         neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
4045         nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
4046         ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
4047         nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
4048
4049         /* limit values to play nice with kdump */
4050         if (is_kdump_kernel()) {
4051                 dev_nintrs = 2;
4052                 neqs_per_lif = 0;
4053                 nnqs_per_lif = 0;
4054                 ntxqs_per_lif = 1;
4055                 nrxqs_per_lif = 1;
4056         }
4057
4058         /* reserve last queue id for hardware timestamping */
4059         if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
4060                 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
4061                         lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
4062                 } else {
4063                         ntxqs_per_lif -= 1;
4064                         nrxqs_per_lif -= 1;
4065                 }
4066         }
4067
4068         nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
4069         nxqs = min(nxqs, num_online_cpus());
4070         neqs = min(neqs_per_lif, num_online_cpus());
4071
4072 try_again:
4073         /* interrupt usage:
4074          *    1 for master lif adminq/notifyq
4075          *    1 for each CPU for master lif TxRx queue pairs
4076          *    whatever's left is for RDMA queues
4077          */
4078         nintrs = 1 + nxqs + neqs;
4079         min_intrs = 2;  /* adminq + 1 TxRx queue pair */
4080
4081         if (nintrs > dev_nintrs)
4082                 goto try_fewer;
4083
4084         err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
4085         if (err < 0 && err != -ENOSPC) {
4086                 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
4087                 return err;
4088         }
4089         if (err == -ENOSPC)
4090                 goto try_fewer;
4091
4092         if (err != nintrs) {
4093                 ionic_bus_free_irq_vectors(ionic);
4094                 goto try_fewer;
4095         }
4096
4097         ionic->nnqs_per_lif = nnqs_per_lif;
4098         ionic->neqs_per_lif = neqs;
4099         ionic->ntxqs_per_lif = nxqs;
4100         ionic->nrxqs_per_lif = nxqs;
4101         ionic->nintrs = nintrs;
4102
4103         ionic_debugfs_add_sizes(ionic);
4104
4105         return 0;
4106
4107 try_fewer:
4108         if (nnqs_per_lif > 1) {
4109                 nnqs_per_lif >>= 1;
4110                 goto try_again;
4111         }
4112         if (neqs > 1) {
4113                 neqs >>= 1;
4114                 goto try_again;
4115         }
4116         if (nxqs > 1) {
4117                 nxqs >>= 1;
4118                 goto try_again;
4119         }
4120         dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
4121         return -ENOSPC;
4122 }