drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: [email protected]
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/module.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/pci.h>
  21 #include <linux/firmware.h>
  22 #include <net/vxlan.h>
  23 #include <linux/kthread.h>
  24 #include "liquidio_common.h"
  25 #include "octeon_droq.h"
  26 #include "octeon_iq.h"
  27 #include "response_manager.h"
  28 #include "octeon_device.h"
  29 #include "octeon_nic.h"
  30 #include "octeon_main.h"
  31 #include "octeon_network.h"
  32 #include "cn66xx_regs.h"
  33 #include "cn66xx_device.h"
  34 #include "cn68xx_device.h"
  35 #include "cn23xx_pf_device.h"
  36 #include "liquidio_image.h"
  37
  38 MODULE_AUTHOR("Cavium Networks, <[email protected]>");
  39 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  40 MODULE_LICENSE("GPL");
  41 MODULE_VERSION(LIQUIDIO_VERSION);
  42 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX);
  43 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX);
  44 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX);
  45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME LIO_FW_NAME_SUFFIX);
  46
  47 static int ddr_timeout = 10000;
  48 module_param(ddr_timeout, int, 0644);
  49 MODULE_PARM_DESC(ddr_timeout,
  50                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  51
  52 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  53
  54 static int debug = -1;
  55 module_param(debug, int, 0644);
  56 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  57
  58 static char fw_type[LIO_MAX_FW_TYPE_LEN];
  59 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
  60 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
  61
  62 static int ptp_enable = 1;
  63
  64 /* Polling interval for determining when NIC application is alive */
  65 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  66
  67 /* runtime link query interval */
  68 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  69
  70 struct liquidio_if_cfg_context {
  71         int octeon_id;
  72
  73         wait_queue_head_t wc;
  74
  75         int cond;
  76 };
  77
  78 struct liquidio_if_cfg_resp {
  79         u64 rh;
  80         struct liquidio_if_cfg_info cfg_info;
  81         u64 status;
  82 };
  83
  84 struct liquidio_rx_ctl_context {
  85         int octeon_id;
  86
  87         wait_queue_head_t wc;
  88
  89         int cond;
  90 };
  91
  92 struct oct_link_status_resp {
  93         u64 rh;
  94         struct oct_link_info link_info;
  95         u64 status;
  96 };
  97
  98 struct oct_timestamp_resp {
  99         u64 rh;
 100         u64 timestamp;
 101         u64 status;
 102 };
 103
 104 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 105
 106 union tx_info {
 107         u64 u64;
 108         struct {
 109 #ifdef __BIG_ENDIAN_BITFIELD
 110                 u16 gso_size;
 111                 u16 gso_segs;
 112                 u32 reserved;
 113 #else
 114                 u32 reserved;
 115                 u16 gso_segs;
 116                 u16 gso_size;
 117 #endif
 118         } s;
 119 };
 120
 121 /** Octeon device properties to be used by the NIC module.
 122  * Each octeon device in the system will be represented
 123  * by this structure in the NIC module.
 124  */
 125
 126 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
 127
 128 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 129 #define OCTNIC_GSO_MAX_SIZE                                                    \
 130         (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 131
 132 /** Structure of a node in list of gather components maintained by
 133  * NIC driver for each network device.
 134  */
 135 struct octnic_gather {
 136         /** List manipulation. Next and prev pointers. */
 137         struct list_head list;
 138
 139         /** Size of the gather component at sg in bytes. */
 140         int sg_size;
 141
 142         /** Number of bytes that sg was adjusted to make it 8B-aligned. */
 143         int adjust;
 144
 145         /** Gather component that can accommodate max sized fragment list
 146          *  received from the IP layer.
 147          */
 148         struct octeon_sg_entry *sg;
 149
 150         dma_addr_t sg_dma_ptr;
 151 };
 152
 153 struct handshake {
 154         struct completion init;
 155         struct completion started;
 156         struct pci_dev *pci_dev;
 157         int init_ok;
 158         int started_ok;
 159 };
 160
 161 struct octeon_device_priv {
 162         /** Tasklet structures for this device. */
 163         struct tasklet_struct droq_tasklet;
 164         unsigned long napi_mask;
 165 };
 166
 167 #ifdef CONFIG_PCI_IOV
 168 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
 169 #endif
 170
 171 static int octeon_device_init(struct octeon_device *);
 172 static int liquidio_stop(struct net_device *netdev);
 173 static void liquidio_remove(struct pci_dev *pdev);
 174 static int liquidio_probe(struct pci_dev *pdev,
 175                           const struct pci_device_id *ent);
 176 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
 177                                       int linkstate);
 178
 179 static struct handshake handshake[MAX_OCTEON_DEVICES];
 180 static struct completion first_stage;
 181
 182 static void octeon_droq_bh(unsigned long pdev)
 183 {
 184         int q_no;
 185         int reschedule = 0;
 186         struct octeon_device *oct = (struct octeon_device *)pdev;
 187         struct octeon_device_priv *oct_priv =
 188                 (struct octeon_device_priv *)oct->priv;
 189
 190         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
 191                 if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
 192                         continue;
 193                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 194                                                           MAX_PACKET_BUDGET);
 195                 lio_enable_irq(oct->droq[q_no], NULL);
 196
 197                 if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
 198                         /* set time and cnt interrupt thresholds for this DROQ
 199                          * for NAPI
 200                          */
 201                         int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
 202
 203                         octeon_write_csr64(
 204                             oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
 205                             0x5700000040ULL);
 206                         octeon_write_csr64(
 207                             oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
 208                 }
 209         }
 210
 211         if (reschedule)
 212                 tasklet_schedule(&oct_priv->droq_tasklet);
 213 }
 214
 215 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 216 {
 217         struct octeon_device_priv *oct_priv =
 218                 (struct octeon_device_priv *)oct->priv;
 219         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 220         int i;
 221
 222         do {
 223                 pending_pkts = 0;
 224
 225                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
 226                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
 227                                 continue;
 228                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
 229                 }
 230                 if (pkt_cnt > 0) {
 231                         pending_pkts += pkt_cnt;
 232                         tasklet_schedule(&oct_priv->droq_tasklet);
 233                 }
 234                 pkt_cnt = 0;
 235                 schedule_timeout_uninterruptible(1);
 236
 237         } while (retry-- && pending_pkts);
 238
 239         return pkt_cnt;
 240 }
 241
 242 /**
 243  * \brief Forces all IO queues off on a given device
 244  * @param oct Pointer to Octeon device
 245  */
 246 static void force_io_queues_off(struct octeon_device *oct)
 247 {
 248         if ((oct->chip_id == OCTEON_CN66XX) ||
 249             (oct->chip_id == OCTEON_CN68XX)) {
 250                 /* Reset the Enable bits for Input Queues. */
 251                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 252
 253                 /* Reset the Enable bits for Output Queues. */
 254                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 255         }
 256 }
 257
 258 /**
 259  * \brief wait for all pending requests to complete
 260  * @param oct Pointer to Octeon device
 261  *
 262  * Called during shutdown sequence
 263  */
 264 static int wait_for_pending_requests(struct octeon_device *oct)
 265 {
 266         int i, pcount = 0;
 267
 268         for (i = 0; i < 100; i++) {
 269                 pcount =
 270                         atomic_read(&oct->response_list
 271                                 [OCTEON_ORDERED_SC_LIST].pending_req_count);
 272                 if (pcount)
 273                         schedule_timeout_uninterruptible(HZ / 10);
 274                 else
 275                         break;
 276         }
 277
 278         if (pcount)
 279                 return 1;
 280
 281         return 0;
 282 }
 283
 284 /**
 285  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 286  * @param oct Pointer to Octeon device
 287  */
 288 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 289 {
 290         int i;
 291
 292         /* Disable the input and output queues now. No more packets will
 293          * arrive from Octeon, but we should wait for all packet processing
 294          * to finish.
 295          */
 296         force_io_queues_off(oct);
 297
 298         /* To allow for in-flight requests */
 299         schedule_timeout_uninterruptible(100);
 300
 301         if (wait_for_pending_requests(oct))
 302                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 303
 304         /* Force all requests waiting to be fetched by OCTEON to complete. */
 305         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 306                 struct octeon_instr_queue *iq;
 307
 308                 if (!(oct->io_qmask.iq & BIT_ULL(i)))
 309                         continue;
 310                 iq = oct->instr_queue[i];
 311
 312                 if (atomic_read(&iq->instr_pending)) {
 313                         spin_lock_bh(&iq->lock);
 314                         iq->fill_cnt = 0;
 315                         iq->octeon_read_index = iq->host_write_index;
 316                         iq->stats.instr_processed +=
 317                                 atomic_read(&iq->instr_pending);
 318                         lio_process_iq_request_list(oct, iq, 0);
 319                         spin_unlock_bh(&iq->lock);
 320                 }
 321         }
 322
 323         /* Force all pending ordered list requests to time out. */
 324         lio_process_ordered_list(oct, 1);
 325
 326         /* We do not need to wait for output queue packets to be processed. */
 327 }
 328
 329 /**
 330  * \brief Cleanup PCI AER uncorrectable error status
 331  * @param dev Pointer to PCI device
 332  */
 333 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 334 {
 335         int pos = 0x100;
 336         u32 status, mask;
 337
 338         pr_info("%s :\n", __func__);
 339
 340         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 341         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 342         if (dev->error_state == pci_channel_io_normal)
 343                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 344         else
 345                 status &= mask;         /* Clear corresponding fatal bits */
 346         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 347 }
 348
 349 /**
 350  * \brief Stop all PCI IO to a given device
 351  * @param dev Pointer to Octeon device
 352  */
 353 static void stop_pci_io(struct octeon_device *oct)
 354 {
 355         /* No more instructions will be forwarded. */
 356         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 357
 358         pci_disable_device(oct->pci_dev);
 359
 360         /* Disable interrupts  */
 361         oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
 362
 363         pcierror_quiesce_device(oct);
 364
 365         /* Release the interrupt line */
 366         free_irq(oct->pci_dev->irq, oct);
 367
 368         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 369                 pci_disable_msi(oct->pci_dev);
 370
 371         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 372                 lio_get_state_string(&oct->status));
 373
 374         /* making it a common function for all OCTEON models */
 375         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 376 }
 377
 378 /**
 379  * \brief called when PCI error is detected
 380  * @param pdev Pointer to PCI device
 381  * @param state The current pci connection state
 382  *
 383  * This function is called after a PCI bus error affecting
 384  * this device has been detected.
 385  */
 386 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 387                                                      pci_channel_state_t state)
 388 {
 389         struct octeon_device *oct = pci_get_drvdata(pdev);
 390
 391         /* Non-correctable Non-fatal errors */
 392         if (state == pci_channel_io_normal) {
 393                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 394                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 395                 return PCI_ERS_RESULT_CAN_RECOVER;
 396         }
 397
 398         /* Non-correctable Fatal errors */
 399         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 400         stop_pci_io(oct);
 401
 402         /* Always return a DISCONNECT. There is no support for recovery but only
 403          * for a clean shutdown.
 404          */
 405         return PCI_ERS_RESULT_DISCONNECT;
 406 }
 407
 408 /**
 409  * \brief mmio handler
 410  * @param pdev Pointer to PCI device
 411  */
 412 static pci_ers_result_t liquidio_pcie_mmio_enabled(
 413                                 struct pci_dev *pdev __attribute__((unused)))
 414 {
 415         /* We should never hit this since we never ask for a reset for a Fatal
 416          * Error. We always return DISCONNECT in io_error above.
 417          * But play safe and return RECOVERED for now.
 418          */
 419         return PCI_ERS_RESULT_RECOVERED;
 420 }
 421
 422 /**
 423  * \brief called after the pci bus has been reset.
 424  * @param pdev Pointer to PCI device
 425  *
 426  * Restart the card from scratch, as if from a cold-boot. Implementation
 427  * resembles the first-half of the octeon_resume routine.
 428  */
 429 static pci_ers_result_t liquidio_pcie_slot_reset(
 430                                 struct pci_dev *pdev __attribute__((unused)))
 431 {
 432         /* We should never hit this since we never ask for a reset for a Fatal
 433          * Error. We always return DISCONNECT in io_error above.
 434          * But play safe and return RECOVERED for now.
 435          */
 436         return PCI_ERS_RESULT_RECOVERED;
 437 }
 438
 439 /**
 440  * \brief called when traffic can start flowing again.
 441  * @param pdev Pointer to PCI device
 442  *
 443  * This callback is called when the error recovery driver tells us that
 444  * its OK to resume normal operation. Implementation resembles the
 445  * second-half of the octeon_resume routine.
 446  */
 447 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
 448 {
 449         /* Nothing to be done here. */
 450 }
 451
 452 #ifdef CONFIG_PM
 453 /**
 454  * \brief called when suspending
 455  * @param pdev Pointer to PCI device
 456  * @param state state to suspend to
 457  */
 458 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
 459                             pm_message_t state __attribute__((unused)))
 460 {
 461         return 0;
 462 }
 463
 464 /**
 465  * \brief called when resuming
 466  * @param pdev Pointer to PCI device
 467  */
 468 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
 469 {
 470         return 0;
 471 }
 472 #endif
 473
 474 /* For PCI-E Advanced Error Recovery (AER) Interface */
 475 static const struct pci_error_handlers liquidio_err_handler = {
 476         .error_detected = liquidio_pcie_error_detected,
 477         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 478         .slot_reset     = liquidio_pcie_slot_reset,
 479         .resume         = liquidio_pcie_resume,
 480 };
 481
 482 static const struct pci_device_id liquidio_pci_tbl[] = {
 483         {       /* 68xx */
 484                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 485         },
 486         {       /* 66xx */
 487                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 488         },
 489         {       /* 23xx pf */
 490                 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 491         },
 492         {
 493                 0, 0, 0, 0, 0, 0, 0
 494         }
 495 };
 496 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 497
 498 static struct pci_driver liquidio_pci_driver = {
 499         .name           = "LiquidIO",
 500         .id_table       = liquidio_pci_tbl,
 501         .probe          = liquidio_probe,
 502         .remove         = liquidio_remove,
 503         .err_handler    = &liquidio_err_handler,    /* For AER */
 504
 505 #ifdef CONFIG_PM
 506         .suspend        = liquidio_suspend,
 507         .resume         = liquidio_resume,
 508 #endif
 509 #ifdef CONFIG_PCI_IOV
 510         .sriov_configure = liquidio_enable_sriov,
 511 #endif
 512 };
 513
 514 /**
 515  * \brief register PCI driver
 516  */
 517 static int liquidio_init_pci(void)
 518 {
 519         return pci_register_driver(&liquidio_pci_driver);
 520 }
 521
 522 /**
 523  * \brief unregister PCI driver
 524  */
 525 static void liquidio_deinit_pci(void)
 526 {
 527         pci_unregister_driver(&liquidio_pci_driver);
 528 }
 529
 530 /**
 531  * \brief Stop Tx queues
 532  * @param netdev network device
 533  */
 534 static inline void txqs_stop(struct net_device *netdev)
 535 {
 536         if (netif_is_multiqueue(netdev)) {
 537                 int i;
 538
 539                 for (i = 0; i < netdev->num_tx_queues; i++)
 540                         netif_stop_subqueue(netdev, i);
 541         } else {
 542                 netif_stop_queue(netdev);
 543         }
 544 }
 545
 546 /**
 547  * \brief Start Tx queues
 548  * @param netdev network device
 549  */
 550 static inline void txqs_start(struct net_device *netdev)
 551 {
 552         if (netif_is_multiqueue(netdev)) {
 553                 int i;
 554
 555                 for (i = 0; i < netdev->num_tx_queues; i++)
 556                         netif_start_subqueue(netdev, i);
 557         } else {
 558                 netif_start_queue(netdev);
 559         }
 560 }
 561
 562 /**
 563  * \brief Wake Tx queues
 564  * @param netdev network device
 565  */
 566 static inline void txqs_wake(struct net_device *netdev)
 567 {
 568         struct lio *lio = GET_LIO(netdev);
 569
 570         if (netif_is_multiqueue(netdev)) {
 571                 int i;
 572
 573                 for (i = 0; i < netdev->num_tx_queues; i++) {
 574                         int qno = lio->linfo.txpciq[i %
 575                                 (lio->linfo.num_txpciq)].s.q_no;
 576
 577                         if (__netif_subqueue_stopped(netdev, i)) {
 578                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
 579                                                           tx_restart, 1);
 580                                 netif_wake_subqueue(netdev, i);
 581                         }
 582                 }
 583         } else {
 584                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 585                                           tx_restart, 1);
 586                 netif_wake_queue(netdev);
 587         }
 588 }
 589
 590 /**
 591  * \brief Stop Tx queue
 592  * @param netdev network device
 593  */
 594 static void stop_txq(struct net_device *netdev)
 595 {
 596         txqs_stop(netdev);
 597 }
 598
 599 /**
 600  * \brief Start Tx queue
 601  * @param netdev network device
 602  */
 603 static void start_txq(struct net_device *netdev)
 604 {
 605         struct lio *lio = GET_LIO(netdev);
 606
 607         if (lio->linfo.link.s.link_up) {
 608                 txqs_start(netdev);
 609                 return;
 610         }
 611 }
 612
 613 /**
 614  * \brief Wake a queue
 615  * @param netdev network device
 616  * @param q which queue to wake
 617  */
 618 static inline void wake_q(struct net_device *netdev, int q)
 619 {
 620         if (netif_is_multiqueue(netdev))
 621                 netif_wake_subqueue(netdev, q);
 622         else
 623                 netif_wake_queue(netdev);
 624 }
 625
 626 /**
 627  * \brief Stop a queue
 628  * @param netdev network device
 629  * @param q which queue to stop
 630  */
 631 static inline void stop_q(struct net_device *netdev, int q)
 632 {
 633         if (netif_is_multiqueue(netdev))
 634                 netif_stop_subqueue(netdev, q);
 635         else
 636                 netif_stop_queue(netdev);
 637 }
 638
 639 /**
 640  * \brief Check Tx queue status, and take appropriate action
 641  * @param lio per-network private data
 642  * @returns 0 if full, number of queues woken up otherwise
 643  */
 644 static inline int check_txq_status(struct lio *lio)
 645 {
 646         int ret_val = 0;
 647
 648         if (netif_is_multiqueue(lio->netdev)) {
 649                 int numqs = lio->netdev->num_tx_queues;
 650                 int q, iq = 0;
 651
 652                 /* check each sub-queue state */
 653                 for (q = 0; q < numqs; q++) {
 654                         iq = lio->linfo.txpciq[q %
 655                                 (lio->linfo.num_txpciq)].s.q_no;
 656                         if (octnet_iq_is_full(lio->oct_dev, iq))
 657                                 continue;
 658                         if (__netif_subqueue_stopped(lio->netdev, q)) {
 659                                 wake_q(lio->netdev, q);
 660                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
 661                                                           tx_restart, 1);
 662                                 ret_val++;
 663                         }
 664                 }
 665         } else {
 666                 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
 667                         return 0;
 668                 wake_q(lio->netdev, lio->txq);
 669                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 670                                           tx_restart, 1);
 671                 ret_val = 1;
 672         }
 673         return ret_val;
 674 }
 675
 676 /**
 677  * Remove the node at the head of the list. The list would be empty at
 678  * the end of this call if there are no more nodes in the list.
 679  */
 680 static inline struct list_head *list_delete_head(struct list_head *root)
 681 {
 682         struct list_head *node;
 683
 684         if ((root->prev == root) && (root->next == root))
 685                 node = NULL;
 686         else
 687                 node = root->next;
 688
 689         if (node)
 690                 list_del(node);
 691
 692         return node;
 693 }
 694
 695 /**
 696  * \brief Delete gather lists
 697  * @param lio per-network private data
 698  */
 699 static void delete_glists(struct lio *lio)
 700 {
 701         struct octnic_gather *g;
 702         int i;
 703
 704         kfree(lio->glist_lock);
 705         lio->glist_lock = NULL;
 706
 707         if (!lio->glist)
 708                 return;
 709
 710         for (i = 0; i < lio->linfo.num_txpciq; i++) {
 711                 do {
 712                         g = (struct octnic_gather *)
 713                                 list_delete_head(&lio->glist[i]);
 714                         if (g)
 715                                 kfree(g);
 716                 } while (g);
 717
 718                 if (lio->glists_virt_base && lio->glists_virt_base[i] &&
 719                     lio->glists_dma_base && lio->glists_dma_base[i]) {
 720                         lio_dma_free(lio->oct_dev,
 721                                      lio->glist_entry_size * lio->tx_qsize,
 722                                      lio->glists_virt_base[i],
 723                                      lio->glists_dma_base[i]);
 724                 }
 725         }
 726
 727         kfree(lio->glists_virt_base);
 728         lio->glists_virt_base = NULL;
 729
 730         kfree(lio->glists_dma_base);
 731         lio->glists_dma_base = NULL;
 732
 733         kfree(lio->glist);
 734         lio->glist = NULL;
 735 }
 736
 737 /**
 738  * \brief Setup gather lists
 739  * @param lio per-network private data
 740  */
 741 static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
 742 {
 743         int i, j;
 744         struct octnic_gather *g;
 745
 746         lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
 747                                   GFP_KERNEL);
 748         if (!lio->glist_lock)
 749                 return -ENOMEM;
 750
 751         lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
 752                              GFP_KERNEL);
 753         if (!lio->glist) {
 754                 kfree(lio->glist_lock);
 755                 lio->glist_lock = NULL;
 756                 return -ENOMEM;
 757         }
 758
 759         lio->glist_entry_size =
 760                 ROUNDUP8((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
 761
 762         /* allocate memory to store virtual and dma base address of
 763          * per glist consistent memory
 764          */
 765         lio->glists_virt_base = kcalloc(num_iqs, sizeof(*lio->glists_virt_base),
 766                                         GFP_KERNEL);
 767         lio->glists_dma_base = kcalloc(num_iqs, sizeof(*lio->glists_dma_base),
 768                                        GFP_KERNEL);
 769
 770         if (!lio->glists_virt_base || !lio->glists_dma_base) {
 771                 delete_glists(lio);
 772                 return -ENOMEM;
 773         }
 774
 775         for (i = 0; i < num_iqs; i++) {
 776                 int numa_node = dev_to_node(&oct->pci_dev->dev);
 777
 778                 spin_lock_init(&lio->glist_lock[i]);
 779
 780                 INIT_LIST_HEAD(&lio->glist[i]);
 781
 782                 lio->glists_virt_base[i] =
 783                         lio_dma_alloc(oct,
 784                                       lio->glist_entry_size * lio->tx_qsize,
 785                                       &lio->glists_dma_base[i]);
 786
 787                 if (!lio->glists_virt_base[i]) {
 788                         delete_glists(lio);
 789                         return -ENOMEM;
 790                 }
 791
 792                 for (j = 0; j < lio->tx_qsize; j++) {
 793                         g = kzalloc_node(sizeof(*g), GFP_KERNEL,
 794                                          numa_node);
 795                         if (!g)
 796                                 g = kzalloc(sizeof(*g), GFP_KERNEL);
 797                         if (!g)
 798                                 break;
 799
 800                         g->sg = lio->glists_virt_base[i] +
 801                                 (j * lio->glist_entry_size);
 802
 803                         g->sg_dma_ptr = lio->glists_dma_base[i] +
 804                                         (j * lio->glist_entry_size);
 805
 806                         list_add_tail(&g->list, &lio->glist[i]);
 807                 }
 808
 809                 if (j != lio->tx_qsize) {
 810                         delete_glists(lio);
 811                         return -ENOMEM;
 812                 }
 813         }
 814
 815         return 0;
 816 }
 817
 818 /**
 819  * \brief Print link information
 820  * @param netdev network device
 821  */
 822 static void print_link_info(struct net_device *netdev)
 823 {
 824         struct lio *lio = GET_LIO(netdev);
 825
 826         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
 827                 struct oct_link_info *linfo = &lio->linfo;
 828
 829                 if (linfo->link.s.link_up) {
 830                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 831                                    linfo->link.s.speed,
 832                                    (linfo->link.s.duplex) ? "Full" : "Half");
 833                 } else {
 834                         netif_info(lio, link, lio->netdev, "Link Down\n");
 835                 }
 836         }
 837 }
 838
 839 /**
 840  * \brief Routine to notify MTU change
 841  * @param work work_struct data structure
 842  */
 843 static void octnet_link_status_change(struct work_struct *work)
 844 {
 845         struct cavium_wk *wk = (struct cavium_wk *)work;
 846         struct lio *lio = (struct lio *)wk->ctxptr;
 847
 848         rtnl_lock();
 849         call_netdevice_notifiers(NETDEV_CHANGEMTU, lio->netdev);
 850         rtnl_unlock();
 851 }
 852
 853 /**
 854  * \brief Sets up the mtu status change work
 855  * @param netdev network device
 856  */
 857 static inline int setup_link_status_change_wq(struct net_device *netdev)
 858 {
 859         struct lio *lio = GET_LIO(netdev);
 860         struct octeon_device *oct = lio->oct_dev;
 861
 862         lio->link_status_wq.wq = alloc_workqueue("link-status",
 863                                                  WQ_MEM_RECLAIM, 0);
 864         if (!lio->link_status_wq.wq) {
 865                 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
 866                 return -1;
 867         }
 868         INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
 869                           octnet_link_status_change);
 870         lio->link_status_wq.wk.ctxptr = lio;
 871
 872         return 0;
 873 }
 874
 875 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
 876 {
 877         struct lio *lio = GET_LIO(netdev);
 878
 879         if (lio->link_status_wq.wq) {
 880                 cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
 881                 destroy_workqueue(lio->link_status_wq.wq);
 882         }
 883 }
 884
 885 /**
 886  * \brief Update link status
 887  * @param netdev network device
 888  * @param ls link status structure
 889  *
 890  * Called on receipt of a link status response from the core application to
 891  * update each interface's link status.
 892  */
 893 static inline void update_link_status(struct net_device *netdev,
 894                                       union oct_link_status *ls)
 895 {
 896         struct lio *lio = GET_LIO(netdev);
 897         int changed = (lio->linfo.link.u64 != ls->u64);
 898
 899         lio->linfo.link.u64 = ls->u64;
 900
 901         if ((lio->intf_open) && (changed)) {
 902                 print_link_info(netdev);
 903                 lio->link_changes++;
 904
 905                 if (lio->linfo.link.s.link_up) {
 906                         netif_carrier_on(netdev);
 907                         txqs_wake(netdev);
 908                 } else {
 909                         netif_carrier_off(netdev);
 910                         stop_txq(netdev);
 911                 }
 912         }
 913 }
 914
 915 /* Runs in interrupt context. */
 916 static void update_txq_status(struct octeon_device *oct, int iq_num)
 917 {
 918         struct net_device *netdev;
 919         struct lio *lio;
 920         struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
 921
 922         netdev = oct->props[iq->ifidx].netdev;
 923
 924         /* This is needed because the first IQ does not have
 925          * a netdev associated with it.
 926          */
 927         if (!netdev)
 928                 return;
 929
 930         lio = GET_LIO(netdev);
 931         if (netif_is_multiqueue(netdev)) {
 932                 if (__netif_subqueue_stopped(netdev, iq->q_index) &&
 933                     lio->linfo.link.s.link_up &&
 934                     (!octnet_iq_is_full(oct, iq_num))) {
 935                         INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
 936                                                   tx_restart, 1);
 937                         netif_wake_subqueue(netdev, iq->q_index);
 938                 }
 939         } else if (netif_queue_stopped(netdev) &&
 940                    lio->linfo.link.s.link_up &&
 941                    (!octnet_iq_is_full(oct, lio->txq))) {
 942                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
 943                                           lio->txq, tx_restart, 1);
 944                 netif_wake_queue(netdev);
 945         }
 946 }
 947
 948 static
 949 int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret)
 950 {
 951         struct octeon_device *oct = droq->oct_dev;
 952         struct octeon_device_priv *oct_priv =
 953             (struct octeon_device_priv *)oct->priv;
 954
 955         if (droq->ops.poll_mode) {
 956                 droq->ops.napi_fn(droq);
 957         } else {
 958                 if (ret & MSIX_PO_INT) {
 959                         tasklet_schedule(&oct_priv->droq_tasklet);
 960                         return 1;
 961                 }
 962                 /* this will be flushed periodically by check iq db */
 963                 if (ret & MSIX_PI_INT)
 964                         return 0;
 965         }
 966         return 0;
 967 }
 968
 969 /**
 970  * \brief Droq packet processor sceduler
 971  * @param oct octeon device
 972  */
 973 static void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 974 {
 975         struct octeon_device_priv *oct_priv =
 976                 (struct octeon_device_priv *)oct->priv;
 977         u64 oq_no;
 978         struct octeon_droq *droq;
 979
 980         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
 981                 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
 982                      oq_no++) {
 983                         if (!(oct->droq_intr & BIT_ULL(oq_no)))
 984                                 continue;
 985
 986                         droq = oct->droq[oq_no];
 987
 988                         if (droq->ops.poll_mode) {
 989                                 droq->ops.napi_fn(droq);
 990                                 oct_priv->napi_mask |= (1 << oq_no);
 991                         } else {
 992                                 tasklet_schedule(&oct_priv->droq_tasklet);
 993                         }
 994                 }
 995         }
 996 }
 997
 998 static irqreturn_t
 999 liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev)
1000 {
1001         u64 ret;
1002         struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
1003         struct octeon_device *oct = ioq_vector->oct_dev;
1004         struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
1005
1006         ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
1007
1008         if ((ret & MSIX_PO_INT) || (ret & MSIX_PI_INT))
1009                 liquidio_schedule_msix_droq_pkt_handler(droq, ret);
1010
1011         return IRQ_HANDLED;
1012 }
1013
1014 /**
1015  * \brief Interrupt handler for octeon
1016  * @param irq unused
1017  * @param dev octeon device
1018  */
1019 static
1020 irqreturn_t liquidio_legacy_intr_handler(int irq __attribute__((unused)),
1021                                          void *dev)
1022 {
1023         struct octeon_device *oct = (struct octeon_device *)dev;
1024         irqreturn_t ret;
1025
1026         /* Disable our interrupts for the duration of ISR */
1027         oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1028
1029         ret = oct->fn_list.process_interrupt_regs(oct);
1030
1031         if (ret == IRQ_HANDLED)
1032                 liquidio_schedule_droq_pkt_handlers(oct);
1033
1034         /* Re-enable our interrupts  */
1035         if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
1036                 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
1037
1038         return ret;
1039 }
1040
1041 /**
1042  * \brief Setup interrupt for octeon device
1043  * @param oct octeon device
1044  *
1045  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
1046  */
1047 static int octeon_setup_interrupt(struct octeon_device *oct)
1048 {
1049         int irqret, err;
1050         struct msix_entry *msix_entries;
1051         int i;
1052         int num_ioq_vectors;
1053         int num_alloc_ioq_vectors;
1054         char *queue_irq_names = NULL;
1055         char *aux_irq_name = NULL;
1056
1057         if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
1058                 oct->num_msix_irqs = oct->sriov_info.num_pf_rings;
1059                 /* one non ioq interrupt for handling sli_mac_pf_int_sum */
1060                 oct->num_msix_irqs += 1;
1061
1062                 /* allocate storage for the names assigned to each irq */
1063                 oct->irq_name_storage =
1064                         kcalloc((MAX_IOQ_INTERRUPTS_PER_PF + 1), INTRNAMSIZ,
1065                                 GFP_KERNEL);
1066                 if (!oct->irq_name_storage) {
1067                         dev_err(&oct->pci_dev->dev, "Irq name storage alloc failed...\n");
1068                         return -ENOMEM;
1069                 }
1070
1071                 queue_irq_names = oct->irq_name_storage;
1072                 aux_irq_name = &queue_irq_names
1073                                 [IRQ_NAME_OFF(MAX_IOQ_INTERRUPTS_PER_PF)];
1074
1075                 oct->msix_entries = kcalloc(
1076                     oct->num_msix_irqs, sizeof(struct msix_entry), GFP_KERNEL);
1077                 if (!oct->msix_entries) {
1078                         dev_err(&oct->pci_dev->dev, "Memory Alloc failed...\n");
1079                         kfree(oct->irq_name_storage);
1080                         oct->irq_name_storage = NULL;
1081                         return -ENOMEM;
1082                 }
1083
1084                 msix_entries = (struct msix_entry *)oct->msix_entries;
1085                 /*Assumption is that pf msix vectors start from pf srn to pf to
1086                  * trs and not from 0. if not change this code
1087                  */
1088                 for (i = 0; i < oct->num_msix_irqs - 1; i++)
1089                         msix_entries[i].entry = oct->sriov_info.pf_srn + i;
1090                 msix_entries[oct->num_msix_irqs - 1].entry =
1091                     oct->sriov_info.trs;
1092                 num_alloc_ioq_vectors = pci_enable_msix_range(
1093                                                 oct->pci_dev, msix_entries,
1094                                                 oct->num_msix_irqs,
1095                                                 oct->num_msix_irqs);
1096                 if (num_alloc_ioq_vectors < 0) {
1097                         dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n");
1098                         kfree(oct->msix_entries);
1099                         oct->msix_entries = NULL;
1100                         kfree(oct->irq_name_storage);
1101                         oct->irq_name_storage = NULL;
1102                         return num_alloc_ioq_vectors;
1103                 }
1104                 dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n");
1105
1106                 num_ioq_vectors = oct->num_msix_irqs;
1107
1108                 /** For PF, there is one non-ioq interrupt handler */
1109                 num_ioq_vectors -= 1;
1110
1111                 snprintf(aux_irq_name, INTRNAMSIZ,
1112                          "LiquidIO%u-pf%u-aux", oct->octeon_id, oct->pf_num);
1113                 irqret = request_irq(msix_entries[num_ioq_vectors].vector,
1114                                      liquidio_legacy_intr_handler, 0,
1115                                      aux_irq_name, oct);
1116                 if (irqret) {
1117                         dev_err(&oct->pci_dev->dev,
1118                                 "OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
1119                                 irqret);
1120                         pci_disable_msix(oct->pci_dev);
1121                         kfree(oct->msix_entries);
1122                         oct->msix_entries = NULL;
1123                         kfree(oct->irq_name_storage);
1124                         oct->irq_name_storage = NULL;
1125                         return irqret;
1126                 }
1127
1128                 for (i = 0; i < num_ioq_vectors; i++) {
1129                         snprintf(&queue_irq_names[IRQ_NAME_OFF(i)], INTRNAMSIZ,
1130                                  "LiquidIO%u-pf%u-rxtx-%u",
1131                                  oct->octeon_id, oct->pf_num, i);
1132
1133                         irqret = request_irq(msix_entries[i].vector,
1134                                              liquidio_msix_intr_handler, 0,
1135                                              &queue_irq_names[IRQ_NAME_OFF(i)],
1136                                              &oct->ioq_vector[i]);
1137                         if (irqret) {
1138                                 dev_err(&oct->pci_dev->dev,
1139                                         "OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
1140                                         irqret);
1141                                 /** Freeing the non-ioq irq vector here . */
1142                                 free_irq(msix_entries[num_ioq_vectors].vector,
1143                                          oct);
1144
1145                                 while (i) {
1146                                         i--;
1147                                         /** clearing affinity mask. */
1148                                         irq_set_affinity_hint(
1149                                                 msix_entries[i].vector, NULL);
1150                                         free_irq(msix_entries[i].vector,
1151                                                  &oct->ioq_vector[i]);
1152                                 }
1153                                 pci_disable_msix(oct->pci_dev);
1154                                 kfree(oct->msix_entries);
1155                                 oct->msix_entries = NULL;
1156                                 kfree(oct->irq_name_storage);
1157                                 oct->irq_name_storage = NULL;
1158                                 return irqret;
1159                         }
1160                         oct->ioq_vector[i].vector = msix_entries[i].vector;
1161                         /* assign the cpu mask for this msix interrupt vector */
1162                         irq_set_affinity_hint(
1163                                         msix_entries[i].vector,
1164                                         (&oct->ioq_vector[i].affinity_mask));
1165                 }
1166                 dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: MSI-X enabled\n",
1167                         oct->octeon_id);
1168         } else {
1169                 err = pci_enable_msi(oct->pci_dev);
1170                 if (err)
1171                         dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
1172                                  err);
1173                 else
1174                         oct->flags |= LIO_FLAG_MSI_ENABLED;
1175
1176                 /* allocate storage for the names assigned to the irq */
1177                 oct->irq_name_storage = kcalloc(1, INTRNAMSIZ, GFP_KERNEL);
1178                 if (!oct->irq_name_storage)
1179                         return -ENOMEM;
1180
1181                 queue_irq_names = oct->irq_name_storage;
1182
1183                 snprintf(&queue_irq_names[IRQ_NAME_OFF(0)], INTRNAMSIZ,
1184                          "LiquidIO%u-pf%u-rxtx-%u",
1185                          oct->octeon_id, oct->pf_num, 0);
1186
1187                 irqret = request_irq(oct->pci_dev->irq,
1188                                      liquidio_legacy_intr_handler,
1189                                      IRQF_SHARED,
1190                                      &queue_irq_names[IRQ_NAME_OFF(0)], oct);
1191                 if (irqret) {
1192                         if (oct->flags & LIO_FLAG_MSI_ENABLED)
1193                                 pci_disable_msi(oct->pci_dev);
1194                         dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
1195                                 irqret);
1196                         kfree(oct->irq_name_storage);
1197                         oct->irq_name_storage = NULL;
1198                         return irqret;
1199                 }
1200         }
1201         return 0;
1202 }
1203
1204 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
1205 {
1206         struct octeon_device *other_oct;
1207
1208         other_oct = lio_get_device(oct->octeon_id + 1);
1209
1210         if (other_oct && other_oct->pci_dev) {
1211                 int oct_busnum, other_oct_busnum;
1212
1213                 oct_busnum = oct->pci_dev->bus->number;
1214                 other_oct_busnum = other_oct->pci_dev->bus->number;
1215
1216                 if (oct_busnum == other_oct_busnum) {
1217                         int oct_slot, other_oct_slot;
1218
1219                         oct_slot = PCI_SLOT(oct->pci_dev->devfn);
1220                         other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
1221
1222                         if (oct_slot == other_oct_slot)
1223                                 return other_oct;
1224                 }
1225         }
1226
1227         return NULL;
1228 }
1229
1230 static void disable_all_vf_links(struct octeon_device *oct)
1231 {
1232         struct net_device *netdev;
1233         int max_vfs, vf, i;
1234
1235         if (!oct)
1236                 return;
1237
1238         max_vfs = oct->sriov_info.max_vfs;
1239
1240         for (i = 0; i < oct->ifcount; i++) {
1241                 netdev = oct->props[i].netdev;
1242                 if (!netdev)
1243                         continue;
1244
1245                 for (vf = 0; vf < max_vfs; vf++)
1246                         liquidio_set_vf_link_state(netdev, vf,
1247                                                    IFLA_VF_LINK_STATE_DISABLE);
1248         }
1249 }
1250
1251 static int liquidio_watchdog(void *param)
1252 {
1253         bool err_msg_was_printed[LIO_MAX_CORES];
1254         u16 mask_of_crashed_or_stuck_cores = 0;
1255         bool all_vf_links_are_disabled = false;
1256         struct octeon_device *oct = param;
1257         struct octeon_device *other_oct;
1258 #ifdef CONFIG_MODULE_UNLOAD
1259         long refcount, vfs_referencing_pf;
1260         u64 vfs_mask1, vfs_mask2;
1261 #endif
1262         int core;
1263
1264         memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
1265
1266         while (!kthread_should_stop()) {
1267                 /* sleep for a couple of seconds so that we don't hog the CPU */
1268                 set_current_state(TASK_INTERRUPTIBLE);
1269                 schedule_timeout(msecs_to_jiffies(2000));
1270
1271                 mask_of_crashed_or_stuck_cores =
1272                     (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
1273
1274                 if (!mask_of_crashed_or_stuck_cores)
1275                         continue;
1276
1277                 WRITE_ONCE(oct->cores_crashed, true);
1278                 other_oct = get_other_octeon_device(oct);
1279                 if (other_oct)
1280                         WRITE_ONCE(other_oct->cores_crashed, true);
1281
1282                 for (core = 0; core < LIO_MAX_CORES; core++) {
1283                         bool core_crashed_or_got_stuck;
1284
1285                         core_crashed_or_got_stuck =
1286                                                 (mask_of_crashed_or_stuck_cores
1287                                                  >> core) & 1;
1288
1289                         if (core_crashed_or_got_stuck &&
1290                             !err_msg_was_printed[core]) {
1291                                 dev_err(&oct->pci_dev->dev,
1292                                         "ERROR: Octeon core %d crashed or got stuck!  See oct-fwdump for details.\n",
1293                                         core);
1294                                         err_msg_was_printed[core] = true;
1295                         }
1296                 }
1297
1298                 if (all_vf_links_are_disabled)
1299                         continue;
1300
1301                 disable_all_vf_links(oct);
1302                 disable_all_vf_links(other_oct);
1303                 all_vf_links_are_disabled = true;
1304
1305 #ifdef CONFIG_MODULE_UNLOAD
1306                 vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
1307                 vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
1308
1309                 vfs_referencing_pf  = hweight64(vfs_mask1);
1310                 vfs_referencing_pf += hweight64(vfs_mask2);
1311
1312                 refcount = module_refcount(THIS_MODULE);
1313                 if (refcount >= vfs_referencing_pf) {
1314                         while (vfs_referencing_pf) {
1315                                 module_put(THIS_MODULE);
1316                                 vfs_referencing_pf--;
1317                         }
1318                 }
1319 #endif
1320         }
1321
1322         return 0;
1323 }
1324
1325 /**
1326  * \brief PCI probe handler
1327  * @param pdev PCI device structure
1328  * @param ent unused
1329  */
1330 static int
1331 liquidio_probe(struct pci_dev *pdev,
1332                const struct pci_device_id *ent __attribute__((unused)))
1333 {
1334         struct octeon_device *oct_dev = NULL;
1335         struct handshake *hs;
1336
1337         oct_dev = octeon_allocate_device(pdev->device,
1338                                          sizeof(struct octeon_device_priv));
1339         if (!oct_dev) {
1340                 dev_err(&pdev->dev, "Unable to allocate device\n");
1341                 return -ENOMEM;
1342         }
1343
1344         if (pdev->device == OCTEON_CN23XX_PF_VID)
1345                 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
1346
1347         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1348                  (u32)pdev->vendor, (u32)pdev->device);
1349
1350         /* Assign octeon_device for this device to the private data area. */
1351         pci_set_drvdata(pdev, oct_dev);
1352
1353         /* set linux specific device pointer */
1354         oct_dev->pci_dev = (void *)pdev;
1355
1356         hs = &handshake[oct_dev->octeon_id];
1357         init_completion(&hs->init);
1358         init_completion(&hs->started);
1359         hs->pci_dev = pdev;
1360
1361         if (oct_dev->octeon_id == 0)
1362                 /* first LiquidIO NIC is detected */
1363                 complete(&first_stage);
1364
1365         if (octeon_device_init(oct_dev)) {
1366                 complete(&hs->init);
1367                 liquidio_remove(pdev);
1368                 return -ENOMEM;
1369         }
1370
1371         if (OCTEON_CN23XX_PF(oct_dev)) {
1372                 u64 scratch1;
1373                 u8 bus, device, function;
1374
1375                 scratch1 = octeon_read_csr64(oct_dev, CN23XX_SLI_SCRATCH1);
1376                 if (!(scratch1 & 4ULL)) {
1377                         /* Bit 2 of SLI_SCRATCH_1 is a flag that indicates that
1378                          * the lio watchdog kernel thread is running for this
1379                          * NIC.  Each NIC gets one watchdog kernel thread.
1380                          */
1381                         scratch1 |= 4ULL;
1382                         octeon_write_csr64(oct_dev, CN23XX_SLI_SCRATCH1,
1383                                            scratch1);
1384
1385                         bus = pdev->bus->number;
1386                         device = PCI_SLOT(pdev->devfn);
1387                         function = PCI_FUNC(pdev->devfn);
1388                         oct_dev->watchdog_task = kthread_create(
1389                             liquidio_watchdog, oct_dev,
1390                             "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
1391                         if (!IS_ERR(oct_dev->watchdog_task)) {
1392                                 wake_up_process(oct_dev->watchdog_task);
1393                         } else {
1394                                 oct_dev->watchdog_task = NULL;
1395                                 dev_err(&oct_dev->pci_dev->dev,
1396                                         "failed to create kernel_thread\n");
1397                                 liquidio_remove(pdev);
1398                                 return -1;
1399                         }
1400                 }
1401         }
1402
1403         oct_dev->rx_pause = 1;
1404         oct_dev->tx_pause = 1;
1405
1406         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1407
1408         return 0;
1409 }
1410
1411 static bool fw_type_is_none(void)
1412 {
1413         return strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
1414                        sizeof(LIO_FW_NAME_TYPE_NONE)) == 0;
1415 }
1416
1417 /**
1418  *\brief Destroy resources associated with octeon device
1419  * @param pdev PCI device structure
1420  * @param ent unused
1421  */
1422 static void octeon_destroy_resources(struct octeon_device *oct)
1423 {
1424         int i, refcount;
1425         struct msix_entry *msix_entries;
1426         struct octeon_device_priv *oct_priv =
1427                 (struct octeon_device_priv *)oct->priv;
1428
1429         struct handshake *hs;
1430
1431         switch (atomic_read(&oct->status)) {
1432         case OCT_DEV_RUNNING:
1433         case OCT_DEV_CORE_OK:
1434
1435                 /* No more instructions will be forwarded. */
1436                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
1437
1438                 oct->app_mode = CVM_DRV_INVALID_APP;
1439                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1440                         lio_get_state_string(&oct->status));
1441
1442                 schedule_timeout_uninterruptible(HZ / 10);
1443
1444                 /* fallthrough */
1445         case OCT_DEV_HOST_OK:
1446
1447                 /* fallthrough */
1448         case OCT_DEV_CONSOLE_INIT_DONE:
1449                 /* Remove any consoles */
1450                 octeon_remove_consoles(oct);
1451
1452                 /* fallthrough */
1453         case OCT_DEV_IO_QUEUES_DONE:
1454                 if (wait_for_pending_requests(oct))
1455                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1456
1457                 if (lio_wait_for_instr_fetch(oct))
1458                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1459
1460                 /* Disable the input and output queues now. No more packets will
1461                  * arrive from Octeon, but we should wait for all packet
1462                  * processing to finish.
1463                  */
1464                 oct->fn_list.disable_io_queues(oct);
1465
1466                 if (lio_wait_for_oq_pkts(oct))
1467                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1468
1469         /* fallthrough */
1470         case OCT_DEV_INTR_SET_DONE:
1471                 /* Disable interrupts  */
1472                 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1473
1474                 if (oct->msix_on) {
1475                         msix_entries = (struct msix_entry *)oct->msix_entries;
1476                         for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1477                                 /* clear the affinity_cpumask */
1478                                 irq_set_affinity_hint(msix_entries[i].vector,
1479                                                       NULL);
1480                                 free_irq(msix_entries[i].vector,
1481                                          &oct->ioq_vector[i]);
1482                         }
1483                         /* non-iov vector's argument is oct struct */
1484                         free_irq(msix_entries[i].vector, oct);
1485
1486                         pci_disable_msix(oct->pci_dev);
1487                         kfree(oct->msix_entries);
1488                         oct->msix_entries = NULL;
1489                 } else {
1490                         /* Release the interrupt line */
1491                         free_irq(oct->pci_dev->irq, oct);
1492
1493                         if (oct->flags & LIO_FLAG_MSI_ENABLED)
1494                                 pci_disable_msi(oct->pci_dev);
1495                 }
1496
1497                 kfree(oct->irq_name_storage);
1498                 oct->irq_name_storage = NULL;
1499
1500         /* fallthrough */
1501         case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
1502                 if (OCTEON_CN23XX_PF(oct))
1503                         octeon_free_ioq_vector(oct);
1504
1505         /* fallthrough */
1506         case OCT_DEV_MBOX_SETUP_DONE:
1507                 if (OCTEON_CN23XX_PF(oct))
1508                         oct->fn_list.free_mbox(oct);
1509
1510         /* fallthrough */
1511         case OCT_DEV_IN_RESET:
1512         case OCT_DEV_DROQ_INIT_DONE:
1513                 /* Wait for any pending operations */
1514                 mdelay(100);
1515                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1516                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
1517                                 continue;
1518                         octeon_delete_droq(oct, i);
1519                 }
1520
1521                 /* Force any pending handshakes to complete */
1522                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1523                         hs = &handshake[i];
1524
1525                         if (hs->pci_dev) {
1526                                 handshake[oct->octeon_id].init_ok = 0;
1527                                 complete(&handshake[oct->octeon_id].init);
1528                                 handshake[oct->octeon_id].started_ok = 0;
1529                                 complete(&handshake[oct->octeon_id].started);
1530                         }
1531                 }
1532
1533                 /* fallthrough */
1534         case OCT_DEV_RESP_LIST_INIT_DONE:
1535                 octeon_delete_response_list(oct);
1536
1537                 /* fallthrough */
1538         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1539                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1540                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
1541                                 continue;
1542                         octeon_delete_instr_queue(oct, i);
1543                 }
1544 #ifdef CONFIG_PCI_IOV
1545                 if (oct->sriov_info.sriov_enabled)
1546                         pci_disable_sriov(oct->pci_dev);
1547 #endif
1548                 /* fallthrough */
1549         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1550                 octeon_free_sc_buffer_pool(oct);
1551
1552                 /* fallthrough */
1553         case OCT_DEV_DISPATCH_INIT_DONE:
1554                 octeon_delete_dispatch_list(oct);
1555                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1556
1557                 /* fallthrough */
1558         case OCT_DEV_PCI_MAP_DONE:
1559                 refcount = octeon_deregister_device(oct);
1560
1561                 if (!fw_type_is_none()) {
1562                         /* Soft reset the octeon device before exiting.
1563                          * Implementation note: here, we reset the device
1564                          * if it is a CN6XXX OR the last CN23XX device.
1565                          */
1566                         if (OCTEON_CN6XXX(oct) || !refcount)
1567                                 oct->fn_list.soft_reset(oct);
1568                 }
1569
1570                 octeon_unmap_pci_barx(oct, 0);
1571                 octeon_unmap_pci_barx(oct, 1);
1572
1573                 /* fallthrough */
1574         case OCT_DEV_PCI_ENABLE_DONE:
1575                 pci_clear_master(oct->pci_dev);
1576                 /* Disable the device, releasing the PCI INT */
1577                 pci_disable_device(oct->pci_dev);
1578
1579                 /* fallthrough */
1580         case OCT_DEV_BEGIN_STATE:
1581                 /* Nothing to be done here either */
1582                 break;
1583         }                       /* end switch (oct->status) */
1584
1585         tasklet_kill(&oct_priv->droq_tasklet);
1586 }
1587
1588 /**
1589  * \brief Callback for rx ctrl
1590  * @param status status of request
1591  * @param buf pointer to resp structure
1592  */
1593 static void rx_ctl_callback(struct octeon_device *oct,
1594                             u32 status,
1595                             void *buf)
1596 {
1597         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1598         struct liquidio_rx_ctl_context *ctx;
1599
1600         ctx  = (struct liquidio_rx_ctl_context *)sc->ctxptr;
1601
1602         oct = lio_get_device(ctx->octeon_id);
1603         if (status)
1604                 dev_err(&oct->pci_dev->dev, "rx ctl instruction failed. Status: %llx\n",
1605                         CVM_CAST64(status));
1606         WRITE_ONCE(ctx->cond, 1);
1607
1608         /* This barrier is required to be sure that the response has been
1609          * written fully before waking up the handler
1610          */
1611         wmb();
1612
1613         wake_up_interruptible(&ctx->wc);
1614 }
1615
1616 /**
1617  * \brief Send Rx control command
1618  * @param lio per-network private data
1619  * @param start_stop whether to start or stop
1620  */
1621 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1622 {
1623         struct octeon_soft_command *sc;
1624         struct liquidio_rx_ctl_context *ctx;
1625         union octnet_cmd *ncmd;
1626         int ctx_size = sizeof(struct liquidio_rx_ctl_context);
1627         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1628         int retval;
1629
1630         if (oct->props[lio->ifidx].rx_on == start_stop)
1631                 return;
1632
1633         sc = (struct octeon_soft_command *)
1634                 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1635                                           16, ctx_size);
1636
1637         ncmd = (union octnet_cmd *)sc->virtdptr;
1638         ctx  = (struct liquidio_rx_ctl_context *)sc->ctxptr;
1639
1640         WRITE_ONCE(ctx->cond, 0);
1641         ctx->octeon_id = lio_get_device_id(oct);
1642         init_waitqueue_head(&ctx->wc);
1643
1644         ncmd->u64 = 0;
1645         ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1646         ncmd->s.param1 = start_stop;
1647
1648         octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1649
1650         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1651
1652         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1653                                     OPCODE_NIC_CMD, 0, 0, 0);
1654
1655         sc->callback = rx_ctl_callback;
1656         sc->callback_arg = sc;
1657         sc->wait_time = 5000;
1658
1659         retval = octeon_send_soft_command(oct, sc);
1660         if (retval == IQ_SEND_FAILED) {
1661                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1662         } else {
1663                 /* Sleep on a wait queue till the cond flag indicates that the
1664                  * response arrived or timed-out.
1665                  */
1666                 if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR)
1667                         return;
1668                 oct->props[lio->ifidx].rx_on = start_stop;
1669         }
1670
1671         octeon_free_soft_command(oct, sc);
1672 }
1673
1674 /**
1675  * \brief Destroy NIC device interface
1676  * @param oct octeon device
1677  * @param ifidx which interface to destroy
1678  *
1679  * Cleanup associated with each interface for an Octeon device  when NIC
1680  * module is being unloaded or if initialization fails during load.
1681  */
1682 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1683 {
1684         struct net_device *netdev = oct->props[ifidx].netdev;
1685         struct lio *lio;
1686         struct napi_struct *napi, *n;
1687
1688         if (!netdev) {
1689                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1690                         __func__, ifidx);
1691                 return;
1692         }
1693
1694         lio = GET_LIO(netdev);
1695
1696         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1697
1698         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1699                 liquidio_stop(netdev);
1700
1701         if (fw_type_is_none()) {
1702                 struct octnic_ctrl_pkt nctrl;
1703
1704                 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1705                 nctrl.ncmd.s.cmd = OCTNET_CMD_RESET_PF;
1706                 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1707                 octnet_send_nic_ctrl_pkt(oct, &nctrl);
1708         }
1709
1710         if (oct->props[lio->ifidx].napi_enabled == 1) {
1711                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1712                         napi_disable(napi);
1713
1714                 oct->props[lio->ifidx].napi_enabled = 0;
1715
1716                 if (OCTEON_CN23XX_PF(oct))
1717                         oct->droq[0]->ops.poll_mode = 0;
1718         }
1719
1720         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1721                 unregister_netdev(netdev);
1722
1723         cleanup_link_status_change_wq(netdev);
1724
1725         cleanup_rx_oom_poll_fn(netdev);
1726
1727         delete_glists(lio);
1728
1729         free_netdev(netdev);
1730
1731         oct->props[ifidx].gmxport = -1;
1732
1733         oct->props[ifidx].netdev = NULL;
1734 }
1735
1736 /**
1737  * \brief Stop complete NIC functionality
1738  * @param oct octeon device
1739  */
1740 static int liquidio_stop_nic_module(struct octeon_device *oct)
1741 {
1742         int i, j;
1743         struct lio *lio;
1744
1745         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1746         if (!oct->ifcount) {
1747                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1748                 return 1;
1749         }
1750
1751         spin_lock_bh(&oct->cmd_resp_wqlock);
1752         oct->cmd_resp_state = OCT_DRV_OFFLINE;
1753         spin_unlock_bh(&oct->cmd_resp_wqlock);
1754
1755         for (i = 0; i < oct->ifcount; i++) {
1756                 lio = GET_LIO(oct->props[i].netdev);
1757                 for (j = 0; j < lio->linfo.num_rxpciq; j++)
1758                         octeon_unregister_droq_ops(oct,
1759                                                    lio->linfo.rxpciq[j].s.q_no);
1760         }
1761
1762         for (i = 0; i < oct->ifcount; i++)
1763                 liquidio_destroy_nic_device(oct, i);
1764
1765         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1766         return 0;
1767 }
1768
1769 /**
1770  * \brief Cleans up resources at unload time
1771  * @param pdev PCI device structure
1772  */
1773 static void liquidio_remove(struct pci_dev *pdev)
1774 {
1775         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1776
1777         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1778
1779         if (oct_dev->watchdog_task)
1780                 kthread_stop(oct_dev->watchdog_task);
1781
1782         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1783                 liquidio_stop_nic_module(oct_dev);
1784
1785         /* Reset the octeon device and cleanup all memory allocated for
1786          * the octeon device by driver.
1787          */
1788         octeon_destroy_resources(oct_dev);
1789
1790         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1791
1792         /* This octeon device has been removed. Update the global
1793          * data structure to reflect this. Free the device structure.
1794          */
1795         octeon_free_device_mem(oct_dev);
1796 }
1797
1798 /**
1799  * \brief Identify the Octeon device and to map the BAR address space
1800  * @param oct octeon device
1801  */
1802 static int octeon_chip_specific_setup(struct octeon_device *oct)
1803 {
1804         u32 dev_id, rev_id;
1805         int ret = 1;
1806         char *s;
1807
1808         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1809         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1810         oct->rev_id = rev_id & 0xff;
1811
1812         switch (dev_id) {
1813         case OCTEON_CN68XX_PCIID:
1814                 oct->chip_id = OCTEON_CN68XX;
1815                 ret = lio_setup_cn68xx_octeon_device(oct);
1816                 s = "CN68XX";
1817                 break;
1818
1819         case OCTEON_CN66XX_PCIID:
1820                 oct->chip_id = OCTEON_CN66XX;
1821                 ret = lio_setup_cn66xx_octeon_device(oct);
1822                 s = "CN66XX";
1823                 break;
1824
1825         case OCTEON_CN23XX_PCIID_PF:
1826                 oct->chip_id = OCTEON_CN23XX_PF_VID;
1827                 ret = setup_cn23xx_octeon_pf_device(oct);
1828                 s = "CN23XX";
1829                 break;
1830
1831         default:
1832                 s = "?";
1833                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1834                         dev_id);
1835         }
1836
1837         if (!ret)
1838                 dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1839                          OCTEON_MAJOR_REV(oct),
1840                          OCTEON_MINOR_REV(oct),
1841                          octeon_get_conf(oct)->card_name,
1842                          LIQUIDIO_VERSION);
1843
1844         return ret;
1845 }
1846
1847 /**
1848  * \brief PCI initialization for each Octeon device.
1849  * @param oct octeon device
1850  */
1851 static int octeon_pci_os_setup(struct octeon_device *oct)
1852 {
1853         /* setup PCI stuff first */
1854         if (pci_enable_device(oct->pci_dev)) {
1855                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1856                 return 1;
1857         }
1858
1859         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1860                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1861                 pci_disable_device(oct->pci_dev);
1862                 return 1;
1863         }
1864
1865         /* Enable PCI DMA Master. */
1866         pci_set_master(oct->pci_dev);
1867
1868         return 0;
1869 }
1870
1871 static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
1872 {
1873         int q = 0;
1874
1875         if (netif_is_multiqueue(lio->netdev))
1876                 q = skb->queue_mapping % lio->linfo.num_txpciq;
1877
1878         return q;
1879 }
1880
1881 /**
1882  * \brief Check Tx queue state for a given network buffer
1883  * @param lio per-network private data
1884  * @param skb network buffer
1885  */
1886 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1887 {
1888         int q = 0, iq = 0;
1889
1890         if (netif_is_multiqueue(lio->netdev)) {
1891                 q = skb->queue_mapping;
1892                 iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
1893         } else {
1894                 iq = lio->txq;
1895                 q = iq;
1896         }
1897
1898         if (octnet_iq_is_full(lio->oct_dev, iq))
1899                 return 0;
1900
1901         if (__netif_subqueue_stopped(lio->netdev, q)) {
1902                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1);
1903                 wake_q(lio->netdev, q);
1904         }
1905         return 1;
1906 }
1907
1908 /**
1909  * \brief Unmap and free network buffer
1910  * @param buf buffer
1911  */
1912 static void free_netbuf(void *buf)
1913 {
1914         struct sk_buff *skb;
1915         struct octnet_buf_free_info *finfo;
1916         struct lio *lio;
1917
1918         finfo = (struct octnet_buf_free_info *)buf;
1919         skb = finfo->skb;
1920         lio = finfo->lio;
1921
1922         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1923                          DMA_TO_DEVICE);
1924
1925         check_txq_state(lio, skb);
1926
1927         tx_buffer_free(skb);
1928 }
1929
1930 /**
1931  * \brief Unmap and free gather buffer
1932  * @param buf buffer
1933  */
1934 static void free_netsgbuf(void *buf)
1935 {
1936         struct octnet_buf_free_info *finfo;
1937         struct sk_buff *skb;
1938         struct lio *lio;
1939         struct octnic_gather *g;
1940         int i, frags, iq;
1941
1942         finfo = (struct octnet_buf_free_info *)buf;
1943         skb = finfo->skb;
1944         lio = finfo->lio;
1945         g = finfo->g;
1946         frags = skb_shinfo(skb)->nr_frags;
1947
1948         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1949                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1950                          DMA_TO_DEVICE);
1951
1952         i = 1;
1953         while (frags--) {
1954                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1955
1956                 pci_unmap_page((lio->oct_dev)->pci_dev,
1957                                g->sg[(i >> 2)].ptr[(i & 3)],
1958                                frag->size, DMA_TO_DEVICE);
1959                 i++;
1960         }
1961
1962         iq = skb_iq(lio, skb);
1963         spin_lock(&lio->glist_lock[iq]);
1964         list_add_tail(&g->list, &lio->glist[iq]);
1965         spin_unlock(&lio->glist_lock[iq]);
1966
1967         check_txq_state(lio, skb);     /* mq support: sub-queue state check */
1968
1969         tx_buffer_free(skb);
1970 }
1971
1972 /**
1973  * \brief Unmap and free gather buffer with response
1974  * @param buf buffer
1975  */
1976 static void free_netsgbuf_with_resp(void *buf)
1977 {
1978         struct octeon_soft_command *sc;
1979         struct octnet_buf_free_info *finfo;
1980         struct sk_buff *skb;
1981         struct lio *lio;
1982         struct octnic_gather *g;
1983         int i, frags, iq;
1984
1985         sc = (struct octeon_soft_command *)buf;
1986         skb = (struct sk_buff *)sc->callback_arg;
1987         finfo = (struct octnet_buf_free_info *)&skb->cb;
1988
1989         lio = finfo->lio;
1990         g = finfo->g;
1991         frags = skb_shinfo(skb)->nr_frags;
1992
1993         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1994                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1995                          DMA_TO_DEVICE);
1996
1997         i = 1;
1998         while (frags--) {
1999                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
2000
2001                 pci_unmap_page((lio->oct_dev)->pci_dev,
2002                                g->sg[(i >> 2)].ptr[(i & 3)],
2003                                frag->size, DMA_TO_DEVICE);
2004                 i++;
2005         }
2006
2007         iq = skb_iq(lio, skb);
2008
2009         spin_lock(&lio->glist_lock[iq]);
2010         list_add_tail(&g->list, &lio->glist[iq]);
2011         spin_unlock(&lio->glist_lock[iq]);
2012
2013         /* Don't free the skb yet */
2014
2015         check_txq_state(lio, skb);
2016 }
2017
2018 /**
2019  * \brief Adjust ptp frequency
2020  * @param ptp PTP clock info
2021  * @param ppb how much to adjust by, in parts-per-billion
2022  */
2023 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
2024 {
2025         struct lio *lio = container_of(ptp, struct lio, ptp_info);
2026         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2027         u64 comp, delta;
2028         unsigned long flags;
2029         bool neg_adj = false;
2030
2031         if (ppb < 0) {
2032                 neg_adj = true;
2033                 ppb = -ppb;
2034         }
2035
2036         /* The hardware adds the clock compensation value to the
2037          * PTP clock on every coprocessor clock cycle, so we
2038          * compute the delta in terms of coprocessor clocks.
2039          */
2040         delta = (u64)ppb << 32;
2041         do_div(delta, oct->coproc_clock_rate);
2042
2043         spin_lock_irqsave(&lio->ptp_lock, flags);
2044         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
2045         if (neg_adj)
2046                 comp -= delta;
2047         else
2048                 comp += delta;
2049         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
2050         spin_unlock_irqrestore(&lio->ptp_lock, flags);
2051
2052         return 0;
2053 }
2054
2055 /**
2056  * \brief Adjust ptp time
2057  * @param ptp PTP clock info
2058  * @param delta how much to adjust by, in nanosecs
2059  */
2060 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
2061 {
2062         unsigned long flags;
2063         struct lio *lio = container_of(ptp, struct lio, ptp_info);
2064
2065         spin_lock_irqsave(&lio->ptp_lock, flags);
2066         lio->ptp_adjust += delta;
2067         spin_unlock_irqrestore(&lio->ptp_lock, flags);
2068
2069         return 0;
2070 }
2071
2072 /**
2073  * \brief Get hardware clock time, including any adjustment
2074  * @param ptp PTP clock info
2075  * @param ts timespec
2076  */
2077 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
2078                                 struct timespec64 *ts)
2079 {
2080         u64 ns;
2081         unsigned long flags;
2082         struct lio *lio = container_of(ptp, struct lio, ptp_info);
2083         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2084
2085         spin_lock_irqsave(&lio->ptp_lock, flags);
2086         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
2087         ns += lio->ptp_adjust;
2088         spin_unlock_irqrestore(&lio->ptp_lock, flags);
2089
2090         *ts = ns_to_timespec64(ns);
2091
2092         return 0;
2093 }
2094
2095 /**
2096  * \brief Set hardware clock time. Reset adjustment
2097  * @param ptp PTP clock info
2098  * @param ts timespec
2099  */
2100 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
2101                                 const struct timespec64 *ts)
2102 {
2103         u64 ns;
2104         unsigned long flags;
2105         struct lio *lio = container_of(ptp, struct lio, ptp_info);
2106         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2107
2108         ns = timespec_to_ns(ts);
2109
2110         spin_lock_irqsave(&lio->ptp_lock, flags);
2111         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
2112         lio->ptp_adjust = 0;
2113         spin_unlock_irqrestore(&lio->ptp_lock, flags);
2114
2115         return 0;
2116 }
2117
2118 /**
2119  * \brief Check if PTP is enabled
2120  * @param ptp PTP clock info
2121  * @param rq request
2122  * @param on is it on
2123  */
2124 static int
2125 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
2126                     struct ptp_clock_request *rq __attribute__((unused)),
2127                     int on __attribute__((unused)))
2128 {
2129         return -EOPNOTSUPP;
2130 }
2131
2132 /**
2133  * \brief Open PTP clock source
2134  * @param netdev network device
2135  */
2136 static void oct_ptp_open(struct net_device *netdev)
2137 {
2138         struct lio *lio = GET_LIO(netdev);
2139         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2140
2141         spin_lock_init(&lio->ptp_lock);
2142
2143         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
2144         lio->ptp_info.owner = THIS_MODULE;
2145         lio->ptp_info.max_adj = 250000000;
2146         lio->ptp_info.n_alarm = 0;
2147         lio->ptp_info.n_ext_ts = 0;
2148         lio->ptp_info.n_per_out = 0;
2149         lio->ptp_info.pps = 0;
2150         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
2151         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
2152         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
2153         lio->ptp_info.settime64 = liquidio_ptp_settime;
2154         lio->ptp_info.enable = liquidio_ptp_enable;
2155
2156         lio->ptp_adjust = 0;
2157
2158         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
2159                                              &oct->pci_dev->dev);
2160
2161         if (IS_ERR(lio->ptp_clock))
2162                 lio->ptp_clock = NULL;
2163 }
2164
2165 /**
2166  * \brief Init PTP clock
2167  * @param oct octeon device
2168  */
2169 static void liquidio_ptp_init(struct octeon_device *oct)
2170 {
2171         u64 clock_comp, cfg;
2172
2173         clock_comp = (u64)NSEC_PER_SEC << 32;
2174         do_div(clock_comp, oct->coproc_clock_rate);
2175         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
2176
2177         /* Enable */
2178         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
2179         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
2180 }
2181
2182 /**
2183  * \brief Load firmware to device
2184  * @param oct octeon device
2185  *
2186  * Maps device to firmware filename, requests firmware, and downloads it
2187  */
2188 static int load_firmware(struct octeon_device *oct)
2189 {
2190         int ret = 0;
2191         const struct firmware *fw;
2192         char fw_name[LIO_MAX_FW_FILENAME_LEN];
2193         char *tmp_fw_type;
2194
2195         if (fw_type_is_none()) {
2196                 dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
2197                 return ret;
2198         }
2199
2200         if (fw_type[0] == '\0')
2201                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
2202         else
2203                 tmp_fw_type = fw_type;
2204
2205         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
2206                 octeon_get_conf(oct)->card_name, tmp_fw_type,
2207                 LIO_FW_NAME_SUFFIX);
2208
2209         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
2210         if (ret) {
2211                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
2212                         fw_name);
2213                 release_firmware(fw);
2214                 return ret;
2215         }
2216
2217         ret = octeon_download_firmware(oct, fw->data, fw->size);
2218
2219         release_firmware(fw);
2220
2221         return ret;
2222 }
2223
2224 /**
2225  * \brief Setup output queue
2226  * @param oct octeon device
2227  * @param q_no which queue
2228  * @param num_descs how many descriptors
2229  * @param desc_size size of each descriptor
2230  * @param app_ctx application context
2231  */
2232 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
2233                              int desc_size, void *app_ctx)
2234 {
2235         int ret_val = 0;
2236
2237         dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
2238         /* droq creation and local register settings. */
2239         ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
2240         if (ret_val < 0)
2241                 return ret_val;
2242
2243         if (ret_val == 1) {
2244                 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
2245                 return 0;
2246         }
2247         /* tasklet creation for the droq */
2248
2249         /* Enable the droq queues */
2250         octeon_set_droq_pkt_op(oct, q_no, 1);
2251
2252         /* Send Credit for Octeon Output queues. Credits are always
2253          * sent after the output queue is enabled.
2254          */
2255         writel(oct->droq[q_no]->max_count,
2256                oct->droq[q_no]->pkts_credit_reg);
2257
2258         return ret_val;
2259 }
2260
2261 /**
2262  * \brief Callback for getting interface configuration
2263  * @param status status of request
2264  * @param buf pointer to resp structure
2265  */
2266 static void if_cfg_callback(struct octeon_device *oct,
2267                             u32 status __attribute__((unused)),
2268                             void *buf)
2269 {
2270         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
2271         struct liquidio_if_cfg_resp *resp;
2272         struct liquidio_if_cfg_context *ctx;
2273
2274         resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
2275         ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
2276
2277         oct = lio_get_device(ctx->octeon_id);
2278         if (resp->status)
2279                 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: 0x%llx (0x%08x)\n",
2280                         CVM_CAST64(resp->status), status);
2281         WRITE_ONCE(ctx->cond, 1);
2282
2283         snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
2284                  resp->cfg_info.liquidio_firmware_version);
2285
2286         /* This barrier is required to be sure that the response has been
2287          * written fully before waking up the handler
2288          */
2289         wmb();
2290
2291         wake_up_interruptible(&ctx->wc);
2292 }
2293
2294 /** Routine to push packets arriving on Octeon interface upto network layer.
2295  * @param oct_id   - octeon device id.
2296  * @param skbuff   - skbuff struct to be passed to network layer.
2297  * @param len      - size of total data received.
2298  * @param rh       - Control header associated with the packet
2299  * @param param    - additional control data with the packet
2300  * @param arg      - farg registered in droq_ops
2301  */
2302 static void
2303 liquidio_push_packet(u32 octeon_id __attribute__((unused)),
2304                      void *skbuff,
2305                      u32 len,
2306                      union octeon_rh *rh,
2307                      void *param,
2308                      void *arg)
2309 {
2310         struct napi_struct *napi = param;
2311         struct sk_buff *skb = (struct sk_buff *)skbuff;
2312         struct skb_shared_hwtstamps *shhwtstamps;
2313         u64 ns;
2314         u16 vtag = 0;
2315         u32 r_dh_off;
2316         struct net_device *netdev = (struct net_device *)arg;
2317         struct octeon_droq *droq = container_of(param, struct octeon_droq,
2318                                                 napi);
2319         if (netdev) {
2320                 int packet_was_received;
2321                 struct lio *lio = GET_LIO(netdev);
2322                 struct octeon_device *oct = lio->oct_dev;
2323
2324                 /* Do not proceed if the interface is not in RUNNING state. */
2325                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
2326                         recv_buffer_free(skb);
2327                         droq->stats.rx_dropped++;
2328                         return;
2329                 }
2330
2331                 skb->dev = netdev;
2332
2333                 skb_record_rx_queue(skb, droq->q_no);
2334                 if (likely(len > MIN_SKB_SIZE)) {
2335                         struct octeon_skb_page_info *pg_info;
2336                         unsigned char *va;
2337
2338                         pg_info = ((struct octeon_skb_page_info *)(skb->cb));
2339                         if (pg_info->page) {
2340                                 /* For Paged allocation use the frags */
2341                                 va = page_address(pg_info->page) +
2342                                         pg_info->page_offset;
2343                                 memcpy(skb->data, va, MIN_SKB_SIZE);
2344                                 skb_put(skb, MIN_SKB_SIZE);
2345                                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
2346                                                 pg_info->page,
2347                                                 pg_info->page_offset +
2348                                                 MIN_SKB_SIZE,
2349                                                 len - MIN_SKB_SIZE,
2350                                                 LIO_RXBUFFER_SZ);
2351                         }
2352                 } else {
2353                         struct octeon_skb_page_info *pg_info =
2354                                 ((struct octeon_skb_page_info *)(skb->cb));
2355                         skb_copy_to_linear_data(skb, page_address(pg_info->page)
2356                                                 + pg_info->page_offset, len);
2357                         skb_put(skb, len);
2358                         put_page(pg_info->page);
2359                 }
2360
2361                 r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT;
2362
2363                 if (((oct->chip_id == OCTEON_CN66XX) ||
2364                      (oct->chip_id == OCTEON_CN68XX)) &&
2365                     ptp_enable) {
2366                         if (rh->r_dh.has_hwtstamp) {
2367                                 /* timestamp is included from the hardware at
2368                                  * the beginning of the packet.
2369                                  */
2370                                 if (ifstate_check
2371                                     (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
2372                                         /* Nanoseconds are in the first 64-bits
2373                                          * of the packet.
2374                                          */
2375                                         memcpy(&ns, (skb->data + r_dh_off),
2376                                                sizeof(ns));
2377                                         r_dh_off -= BYTES_PER_DHLEN_UNIT;
2378                                         shhwtstamps = skb_hwtstamps(skb);
2379                                         shhwtstamps->hwtstamp =
2380                                                 ns_to_ktime(ns +
2381                                                             lio->ptp_adjust);
2382                                 }
2383                         }
2384                 }
2385
2386                 if (rh->r_dh.has_hash) {
2387                         __be32 *hash_be = (__be32 *)(skb->data + r_dh_off);
2388                         u32 hash = be32_to_cpu(*hash_be);
2389
2390                         skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
2391                         r_dh_off -= BYTES_PER_DHLEN_UNIT;
2392                 }
2393
2394                 skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT);
2395
2396                 skb->protocol = eth_type_trans(skb, skb->dev);
2397                 if ((netdev->features & NETIF_F_RXCSUM) &&
2398                     (((rh->r_dh.encap_on) &&
2399                       (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
2400                      (!(rh->r_dh.encap_on) &&
2401                       (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
2402                         /* checksum has already been verified */
2403                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2404                 else
2405                         skb->ip_summed = CHECKSUM_NONE;
2406
2407                 /* Setting Encapsulation field on basis of status received
2408                  * from the firmware
2409                  */
2410                 if (rh->r_dh.encap_on) {
2411                         skb->encapsulation = 1;
2412                         skb->csum_level = 1;
2413                         droq->stats.rx_vxlan++;
2414                 }
2415
2416                 /* inbound VLAN tag */
2417                 if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
2418                     (rh->r_dh.vlan != 0)) {
2419                         u16 vid = rh->r_dh.vlan;
2420                         u16 priority = rh->r_dh.priority;
2421
2422                         vtag = priority << 13 | vid;
2423                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
2424                 }
2425
2426                 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
2427
2428                 if (packet_was_received) {
2429                         droq->stats.rx_bytes_received += len;
2430                         droq->stats.rx_pkts_received++;
2431                 } else {
2432                         droq->stats.rx_dropped++;
2433                         netif_info(lio, rx_err, lio->netdev,
2434                                    "droq:%d  error rx_dropped:%llu\n",
2435                                    droq->q_no, droq->stats.rx_dropped);
2436                 }
2437
2438         } else {
2439                 recv_buffer_free(skb);
2440         }
2441 }
2442
2443 /**
2444  * \brief wrapper for calling napi_schedule
2445  * @param param parameters to pass to napi_schedule
2446  *
2447  * Used when scheduling on different CPUs
2448  */
2449 static void napi_schedule_wrapper(void *param)
2450 {
2451         struct napi_struct *napi = param;
2452
2453         napi_schedule(napi);
2454 }
2455
2456 /**
2457  * \brief callback when receive interrupt occurs and we are in NAPI mode
2458  * @param arg pointer to octeon output queue
2459  */
2460 static void liquidio_napi_drv_callback(void *arg)
2461 {
2462         struct octeon_device *oct;
2463         struct octeon_droq *droq = arg;
2464         int this_cpu = smp_processor_id();
2465
2466         oct = droq->oct_dev;
2467
2468         if (OCTEON_CN23XX_PF(oct) || droq->cpu_id == this_cpu) {
2469                 napi_schedule_irqoff(&droq->napi);
2470         } else {
2471                 struct call_single_data *csd = &droq->csd;
2472
2473                 csd->func = napi_schedule_wrapper;
2474                 csd->info = &droq->napi;
2475                 csd->flags = 0;
2476
2477                 smp_call_function_single_async(droq->cpu_id, csd);
2478         }
2479 }
2480
2481 /**
2482  * \brief Entry point for NAPI polling
2483  * @param napi NAPI structure
2484  * @param budget maximum number of items to process
2485  */
2486 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
2487 {
2488         struct octeon_droq *droq;
2489         int work_done;
2490         int tx_done = 0, iq_no;
2491         struct octeon_instr_queue *iq;
2492         struct octeon_device *oct;
2493
2494         droq = container_of(napi, struct octeon_droq, napi);
2495         oct = droq->oct_dev;
2496         iq_no = droq->q_no;
2497         /* Handle Droq descriptors */
2498         work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
2499                                                  POLL_EVENT_PROCESS_PKTS,
2500                                                  budget);
2501
2502         /* Flush the instruction queue */
2503         iq = oct->instr_queue[iq_no];
2504         if (iq) {
2505                 if (atomic_read(&iq->instr_pending))
2506                         /* Process iq buffers with in the budget limits */
2507                         tx_done = octeon_flush_iq(oct, iq, budget);
2508                 else
2509                         tx_done = 1;
2510                 /* Update iq read-index rather than waiting for next interrupt.
2511                  * Return back if tx_done is false.
2512                  */
2513                 update_txq_status(oct, iq_no);
2514         } else {
2515                 dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
2516                         __func__, iq_no);
2517         }
2518
2519         /* force enable interrupt if reg cnts are high to avoid wraparound */
2520         if ((work_done < budget && tx_done) ||
2521             (iq && iq->pkt_in_done >= MAX_REG_CNT) ||
2522             (droq->pkt_count >= MAX_REG_CNT)) {
2523                 tx_done = 1;
2524                 napi_complete_done(napi, work_done);
2525                 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
2526                                              POLL_EVENT_ENABLE_INTR, 0);
2527                 return 0;
2528         }
2529
2530         return (!tx_done) ? (budget) : (work_done);
2531 }
2532
2533 /**
2534  * \brief Setup input and output queues
2535  * @param octeon_dev octeon device
2536  * @param ifidx  Interface Index
2537  *
2538  * Note: Queues are with respect to the octeon device. Thus
2539  * an input queue is for egress packets, and output queues
2540  * are for ingress packets.
2541  */
2542 static inline int setup_io_queues(struct octeon_device *octeon_dev,
2543                                   int ifidx)
2544 {
2545         struct octeon_droq_ops droq_ops;
2546         struct net_device *netdev;
2547         static int cpu_id;
2548         static int cpu_id_modulus;
2549         struct octeon_droq *droq;
2550         struct napi_struct *napi;
2551         int q, q_no, retval = 0;
2552         struct lio *lio;
2553         int num_tx_descs;
2554
2555         netdev = octeon_dev->props[ifidx].netdev;
2556
2557         lio = GET_LIO(netdev);
2558
2559         memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
2560
2561         droq_ops.fptr = liquidio_push_packet;
2562         droq_ops.farg = (void *)netdev;
2563
2564         droq_ops.poll_mode = 1;
2565         droq_ops.napi_fn = liquidio_napi_drv_callback;
2566         cpu_id = 0;
2567         cpu_id_modulus = num_present_cpus();
2568
2569         /* set up DROQs. */
2570         for (q = 0; q < lio->linfo.num_rxpciq; q++) {
2571                 q_no = lio->linfo.rxpciq[q].s.q_no;
2572                 dev_dbg(&octeon_dev->pci_dev->dev,
2573                         "setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
2574                         q, q_no);
2575                 retval = octeon_setup_droq(octeon_dev, q_no,
2576                                            CFG_GET_NUM_RX_DESCS_NIC_IF
2577                                                    (octeon_get_conf(octeon_dev),
2578                                                    lio->ifidx),
2579                                            CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
2580                                                    (octeon_get_conf(octeon_dev),
2581                                                    lio->ifidx), NULL);
2582                 if (retval) {
2583                         dev_err(&octeon_dev->pci_dev->dev,
2584                                 "%s : Runtime DROQ(RxQ) creation failed.\n",
2585                                 __func__);
2586                         return 1;
2587                 }
2588
2589                 droq = octeon_dev->droq[q_no];
2590                 napi = &droq->napi;
2591                 dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx pf_num:%d\n",
2592                         (u64)netdev, (u64)octeon_dev, octeon_dev->pf_num);
2593                 netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
2594
2595                 /* designate a CPU for this droq */
2596                 droq->cpu_id = cpu_id;
2597                 cpu_id++;
2598                 if (cpu_id >= cpu_id_modulus)
2599                         cpu_id = 0;
2600
2601                 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
2602         }
2603
2604         if (OCTEON_CN23XX_PF(octeon_dev)) {
2605                 /* 23XX PF can receive control messages (via the first PF-owned
2606                  * droq) from the firmware even if the ethX interface is down,
2607                  * so that's why poll_mode must be off for the first droq.
2608                  */
2609                 octeon_dev->droq[0]->ops.poll_mode = 0;
2610         }
2611
2612         /* set up IQs. */
2613         for (q = 0; q < lio->linfo.num_txpciq; q++) {
2614                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
2615                                                            (octeon_dev),
2616                                                            lio->ifidx);
2617                 retval = octeon_setup_iq(octeon_dev, ifidx, q,
2618                                          lio->linfo.txpciq[q], num_tx_descs,
2619                                          netdev_get_tx_queue(netdev, q));
2620                 if (retval) {
2621                         dev_err(&octeon_dev->pci_dev->dev,
2622                                 " %s : Runtime IQ(TxQ) creation failed.\n",
2623                                 __func__);
2624                         return 1;
2625                 }
2626
2627                 if (octeon_dev->ioq_vector) {
2628                         struct octeon_ioq_vector *ioq_vector;
2629
2630                         ioq_vector = &octeon_dev->ioq_vector[q];
2631                         netif_set_xps_queue(netdev,
2632                                             &ioq_vector->affinity_mask,
2633                                             ioq_vector->iq_index);
2634                 }
2635         }
2636
2637         return 0;
2638 }
2639
2640 /**
2641  * \brief Poll routine for checking transmit queue status
2642  * @param work work_struct data structure
2643  */
2644 static void octnet_poll_check_txq_status(struct work_struct *work)
2645 {
2646         struct cavium_wk *wk = (struct cavium_wk *)work;
2647         struct lio *lio = (struct lio *)wk->ctxptr;
2648
2649         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2650                 return;
2651
2652         check_txq_status(lio);
2653         queue_delayed_work(lio->txq_status_wq.wq,
2654                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2655 }
2656
2657 /**
2658  * \brief Sets up the txq poll check
2659  * @param netdev network device
2660  */
2661 static inline int setup_tx_poll_fn(struct net_device *netdev)
2662 {
2663         struct lio *lio = GET_LIO(netdev);
2664         struct octeon_device *oct = lio->oct_dev;
2665
2666         lio->txq_status_wq.wq = alloc_workqueue("txq-status",
2667                                                 WQ_MEM_RECLAIM, 0);
2668         if (!lio->txq_status_wq.wq) {
2669                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
2670                 return -1;
2671         }
2672         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2673                           octnet_poll_check_txq_status);
2674         lio->txq_status_wq.wk.ctxptr = lio;
2675         queue_delayed_work(lio->txq_status_wq.wq,
2676                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2677         return 0;
2678 }
2679
2680 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
2681 {
2682         struct lio *lio = GET_LIO(netdev);
2683
2684         if (lio->txq_status_wq.wq) {
2685                 cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2686                 destroy_workqueue(lio->txq_status_wq.wq);
2687         }
2688 }
2689
2690 /**
2691  * \brief Net device open for LiquidIO
2692  * @param netdev network device
2693  */
2694 static int liquidio_open(struct net_device *netdev)
2695 {
2696         struct lio *lio = GET_LIO(netdev);
2697         struct octeon_device *oct = lio->oct_dev;
2698         struct napi_struct *napi, *n;
2699
2700         if (oct->props[lio->ifidx].napi_enabled == 0) {
2701                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2702                         napi_enable(napi);
2703
2704                 oct->props[lio->ifidx].napi_enabled = 1;
2705
2706                 if (OCTEON_CN23XX_PF(oct))
2707                         oct->droq[0]->ops.poll_mode = 1;
2708         }
2709
2710         if ((oct->chip_id == OCTEON_CN66XX || oct->chip_id == OCTEON_CN68XX) &&
2711             ptp_enable)
2712                 oct_ptp_open(netdev);
2713
2714         ifstate_set(lio, LIO_IFSTATE_RUNNING);
2715
2716         /* Ready for link status updates */
2717         lio->intf_open = 1;
2718
2719         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2720
2721         if (OCTEON_CN23XX_PF(oct)) {
2722                 if (!oct->msix_on)
2723                         if (setup_tx_poll_fn(netdev))
2724                                 return -1;
2725         } else {
2726                 if (setup_tx_poll_fn(netdev))
2727                         return -1;
2728         }
2729
2730         start_txq(netdev);
2731
2732         /* tell Octeon to start forwarding packets to host */
2733         send_rx_ctrl_cmd(lio, 1);
2734
2735         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2736                  netdev->name);
2737
2738         return 0;
2739 }
2740
2741 /**
2742  * \brief Net device stop for LiquidIO
2743  * @param netdev network device
2744  */
2745 static int liquidio_stop(struct net_device *netdev)
2746 {
2747         struct lio *lio = GET_LIO(netdev);
2748         struct octeon_device *oct = lio->oct_dev;
2749
2750         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2751
2752         netif_tx_disable(netdev);
2753
2754         /* Inform that netif carrier is down */
2755         netif_carrier_off(netdev);
2756         lio->intf_open = 0;
2757         lio->linfo.link.s.link_up = 0;
2758         lio->link_changes++;
2759
2760         /* Tell Octeon that nic interface is down. */
2761         send_rx_ctrl_cmd(lio, 0);
2762
2763         if (OCTEON_CN23XX_PF(oct)) {
2764                 if (!oct->msix_on)
2765                         cleanup_tx_poll_fn(netdev);
2766         } else {
2767                 cleanup_tx_poll_fn(netdev);
2768         }
2769
2770         if (lio->ptp_clock) {
2771                 ptp_clock_unregister(lio->ptp_clock);
2772                 lio->ptp_clock = NULL;
2773         }
2774
2775         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
2776
2777         return 0;
2778 }
2779
2780 /**
2781  * \brief Converts a mask based on net device flags
2782  * @param netdev network device
2783  *
2784  * This routine generates a octnet_ifflags mask from the net device flags
2785  * received from the OS.
2786  */
2787 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2788 {
2789         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2790
2791         if (netdev->flags & IFF_PROMISC)
2792                 f |= OCTNET_IFFLAG_PROMISC;
2793
2794         if (netdev->flags & IFF_ALLMULTI)
2795                 f |= OCTNET_IFFLAG_ALLMULTI;
2796
2797         if (netdev->flags & IFF_MULTICAST) {
2798                 f |= OCTNET_IFFLAG_MULTICAST;
2799
2800                 /* Accept all multicast addresses if there are more than we
2801                  * can handle
2802                  */
2803                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2804                         f |= OCTNET_IFFLAG_ALLMULTI;
2805         }
2806
2807         if (netdev->flags & IFF_BROADCAST)
2808                 f |= OCTNET_IFFLAG_BROADCAST;
2809
2810         return f;
2811 }
2812
2813 /**
2814  * \brief Net device set_multicast_list
2815  * @param netdev network device
2816  */
2817 static void liquidio_set_mcast_list(struct net_device *netdev)
2818 {
2819         struct lio *lio = GET_LIO(netdev);
2820         struct octeon_device *oct = lio->oct_dev;
2821         struct octnic_ctrl_pkt nctrl;
2822         struct netdev_hw_addr *ha;
2823         u64 *mc;
2824         int ret;
2825         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2826
2827         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2828
2829         /* Create a ctrl pkt command to be sent to core app. */
2830         nctrl.ncmd.u64 = 0;
2831         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
2832         nctrl.ncmd.s.param1 = get_new_flags(netdev);
2833         nctrl.ncmd.s.param2 = mc_count;
2834         nctrl.ncmd.s.more = mc_count;
2835         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2836         nctrl.netpndev = (u64)netdev;
2837         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2838
2839         /* copy all the addresses into the udd */
2840         mc = &nctrl.udd[0];
2841         netdev_for_each_mc_addr(ha, netdev) {
2842                 *mc = 0;
2843                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2844                 /* no need to swap bytes */
2845
2846                 if (++mc > &nctrl.udd[mc_count])
2847                         break;
2848         }
2849
2850         /* Apparently, any activity in this call from the kernel has to
2851          * be atomic. So we won't wait for response.
2852          */
2853         nctrl.wait_time = 0;
2854
2855         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2856         if (ret < 0) {
2857                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2858                         ret);
2859         }
2860 }
2861
2862 /**
2863  * \brief Net device set_mac_address
2864  * @param netdev network device
2865  */
2866 static int liquidio_set_mac(struct net_device *netdev, void *p)
2867 {
2868         int ret = 0;
2869         struct lio *lio = GET_LIO(netdev);
2870         struct octeon_device *oct = lio->oct_dev;
2871         struct sockaddr *addr = (struct sockaddr *)p;
2872         struct octnic_ctrl_pkt nctrl;
2873
2874         if (!is_valid_ether_addr(addr->sa_data))
2875                 return -EADDRNOTAVAIL;
2876
2877         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2878
2879         nctrl.ncmd.u64 = 0;
2880         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2881         nctrl.ncmd.s.param1 = 0;
2882         nctrl.ncmd.s.more = 1;
2883         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2884         nctrl.netpndev = (u64)netdev;
2885         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2886         nctrl.wait_time = 100;
2887
2888         nctrl.udd[0] = 0;
2889         /* The MAC Address is presented in network byte order. */
2890         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2891
2892         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2893         if (ret < 0) {
2894                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2895                 return -ENOMEM;
2896         }
2897         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2898         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2899
2900         return 0;
2901 }
2902
2903 /**
2904  * \brief Net device get_stats
2905  * @param netdev network device
2906  */
2907 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2908 {
2909         struct lio *lio = GET_LIO(netdev);
2910         struct net_device_stats *stats = &netdev->stats;
2911         struct octeon_device *oct;
2912         u64 pkts = 0, drop = 0, bytes = 0;
2913         struct oct_droq_stats *oq_stats;
2914         struct oct_iq_stats *iq_stats;
2915         int i, iq_no, oq_no;
2916
2917         oct = lio->oct_dev;
2918
2919         for (i = 0; i < lio->linfo.num_txpciq; i++) {
2920                 iq_no = lio->linfo.txpciq[i].s.q_no;
2921                 iq_stats = &oct->instr_queue[iq_no]->stats;
2922                 pkts += iq_stats->tx_done;
2923                 drop += iq_stats->tx_dropped;
2924                 bytes += iq_stats->tx_tot_bytes;
2925         }
2926
2927         stats->tx_packets = pkts;
2928         stats->tx_bytes = bytes;
2929         stats->tx_dropped = drop;
2930
2931         pkts = 0;
2932         drop = 0;
2933         bytes = 0;
2934
2935         for (i = 0; i < lio->linfo.num_rxpciq; i++) {
2936                 oq_no = lio->linfo.rxpciq[i].s.q_no;
2937                 oq_stats = &oct->droq[oq_no]->stats;
2938                 pkts += oq_stats->rx_pkts_received;
2939                 drop += (oq_stats->rx_dropped +
2940                          oq_stats->dropped_nodispatch +
2941                          oq_stats->dropped_toomany +
2942                          oq_stats->dropped_nomem);
2943                 bytes += oq_stats->rx_bytes_received;
2944         }
2945
2946         stats->rx_bytes = bytes;
2947         stats->rx_packets = pkts;
2948         stats->rx_dropped = drop;
2949
2950         return stats;
2951 }
2952
2953 /**
2954  * \brief Net device change_mtu
2955  * @param netdev network device
2956  */
2957 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2958 {
2959         struct lio *lio = GET_LIO(netdev);
2960         struct octeon_device *oct = lio->oct_dev;
2961         struct octnic_ctrl_pkt nctrl;
2962         int ret = 0;
2963
2964         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2965
2966         nctrl.ncmd.u64 = 0;
2967         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
2968         nctrl.ncmd.s.param1 = new_mtu;
2969         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2970         nctrl.wait_time = 100;
2971         nctrl.netpndev = (u64)netdev;
2972         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2973
2974         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2975         if (ret < 0) {
2976                 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2977                 return -1;
2978         }
2979
2980         lio->mtu = new_mtu;
2981
2982         return 0;
2983 }
2984
2985 /**
2986  * \brief Handler for SIOCSHWTSTAMP ioctl
2987  * @param netdev network device
2988  * @param ifr interface request
2989  * @param cmd command
2990  */
2991 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2992 {
2993         struct hwtstamp_config conf;
2994         struct lio *lio = GET_LIO(netdev);
2995
2996         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2997                 return -EFAULT;
2998
2999         if (conf.flags)
3000                 return -EINVAL;
3001
3002         switch (conf.tx_type) {
3003         case HWTSTAMP_TX_ON:
3004         case HWTSTAMP_TX_OFF:
3005                 break;
3006         default:
3007                 return -ERANGE;
3008         }
3009
3010         switch (conf.rx_filter) {
3011         case HWTSTAMP_FILTER_NONE:
3012                 break;
3013         case HWTSTAMP_FILTER_ALL:
3014         case HWTSTAMP_FILTER_SOME:
3015         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
3016         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
3017         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
3018         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
3019         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
3020         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
3021         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
3022         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
3023         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
3024         case HWTSTAMP_FILTER_PTP_V2_EVENT:
3025         case HWTSTAMP_FILTER_PTP_V2_SYNC:
3026         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
3027         case HWTSTAMP_FILTER_NTP_ALL:
3028                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
3029                 break;
3030         default:
3031                 return -ERANGE;
3032         }
3033
3034         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
3035                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
3036
3037         else
3038                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
3039
3040         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
3041 }
3042
3043 /**
3044  * \brief ioctl handler
3045  * @param netdev network device
3046  * @param ifr interface request
3047  * @param cmd command
3048  */
3049 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
3050 {
3051         struct lio *lio = GET_LIO(netdev);
3052
3053         switch (cmd) {
3054         case SIOCSHWTSTAMP:
3055                 if ((lio->oct_dev->chip_id == OCTEON_CN66XX ||
3056                      lio->oct_dev->chip_id == OCTEON_CN68XX) && ptp_enable)
3057                         return hwtstamp_ioctl(netdev, ifr);
3058         default:
3059                 return -EOPNOTSUPP;
3060         }
3061 }
3062
3063 /**
3064  * \brief handle a Tx timestamp response
3065  * @param status response status
3066  * @param buf pointer to skb
3067  */
3068 static void handle_timestamp(struct octeon_device *oct,
3069                              u32 status,
3070                              void *buf)
3071 {
3072         struct octnet_buf_free_info *finfo;
3073         struct octeon_soft_command *sc;
3074         struct oct_timestamp_resp *resp;
3075         struct lio *lio;
3076         struct sk_buff *skb = (struct sk_buff *)buf;
3077
3078         finfo = (struct octnet_buf_free_info *)skb->cb;
3079         lio = finfo->lio;
3080         sc = finfo->sc;
3081         oct = lio->oct_dev;
3082         resp = (struct oct_timestamp_resp *)sc->virtrptr;
3083
3084         if (status != OCTEON_REQUEST_DONE) {
3085                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
3086                         CVM_CAST64(status));
3087                 resp->timestamp = 0;
3088         }
3089
3090         octeon_swap_8B_data(&resp->timestamp, 1);
3091
3092         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
3093                 struct skb_shared_hwtstamps ts;
3094                 u64 ns = resp->timestamp;
3095
3096                 netif_info(lio, tx_done, lio->netdev,
3097                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
3098                            skb, (unsigned long long)ns);
3099                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
3100                 skb_tstamp_tx(skb, &ts);
3101         }
3102
3103         octeon_free_soft_command(oct, sc);
3104         tx_buffer_free(skb);
3105 }
3106
3107 /* \brief Send a data packet that will be timestamped
3108  * @param oct octeon device
3109  * @param ndata pointer to network data
3110  * @param finfo pointer to private network data
3111  */
3112 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
3113                                          struct octnic_data_pkt *ndata,
3114                                          struct octnet_buf_free_info *finfo)
3115 {
3116         int retval;
3117         struct octeon_soft_command *sc;
3118         struct lio *lio;
3119         int ring_doorbell;
3120         u32 len;
3121
3122         lio = finfo->lio;
3123
3124         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
3125                                             sizeof(struct oct_timestamp_resp));
3126         finfo->sc = sc;
3127
3128         if (!sc) {
3129                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
3130                 return IQ_SEND_FAILED;
3131         }
3132
3133         if (ndata->reqtype == REQTYPE_NORESP_NET)
3134                 ndata->reqtype = REQTYPE_RESP_NET;
3135         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
3136                 ndata->reqtype = REQTYPE_RESP_NET_SG;
3137
3138         sc->callback = handle_timestamp;
3139         sc->callback_arg = finfo->skb;
3140         sc->iq_no = ndata->q_no;
3141
3142         if (OCTEON_CN23XX_PF(oct))
3143                 len = (u32)((struct octeon_instr_ih3 *)
3144                             (&sc->cmd.cmd3.ih3))->dlengsz;
3145         else
3146                 len = (u32)((struct octeon_instr_ih2 *)
3147                             (&sc->cmd.cmd2.ih2))->dlengsz;
3148
3149         ring_doorbell = 1;
3150
3151         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
3152                                      sc, len, ndata->reqtype);
3153
3154         if (retval == IQ_SEND_FAILED) {
3155                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
3156                         retval);
3157                 octeon_free_soft_command(oct, sc);
3158         } else {
3159                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
3160         }
3161
3162         return retval;
3163 }
3164
3165 /** \brief Transmit networks packets to the Octeon interface
3166  * @param skbuff   skbuff struct to be passed to network layer.
3167  * @param netdev    pointer to network device
3168  * @returns whether the packet was transmitted to the device okay or not
3169  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
3170  */
3171 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
3172 {
3173         struct lio *lio;
3174         struct octnet_buf_free_info *finfo;
3175         union octnic_cmd_setup cmdsetup;
3176         struct octnic_data_pkt ndata;
3177         struct octeon_device *oct;
3178         struct oct_iq_stats *stats;
3179         struct octeon_instr_irh *irh;
3180         union tx_info *tx_info;
3181         int status = 0;
3182         int q_idx = 0, iq_no = 0;
3183         int j;
3184         u64 dptr = 0;
3185         u32 tag = 0;
3186
3187         lio = GET_LIO(netdev);
3188         oct = lio->oct_dev;
3189
3190         if (netif_is_multiqueue(netdev)) {
3191                 q_idx = skb->queue_mapping;
3192                 q_idx = (q_idx % (lio->linfo.num_txpciq));
3193                 tag = q_idx;
3194                 iq_no = lio->linfo.txpciq[q_idx].s.q_no;
3195         } else {
3196                 iq_no = lio->txq;
3197         }
3198
3199         stats = &oct->instr_queue[iq_no]->stats;
3200
3201         /* Check for all conditions in which the current packet cannot be
3202          * transmitted.
3203          */
3204         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
3205             (!lio->linfo.link.s.link_up) ||
3206             (skb->len <= 0)) {
3207                 netif_info(lio, tx_err, lio->netdev,
3208                            "Transmit failed link_status : %d\n",
3209                            lio->linfo.link.s.link_up);
3210                 goto lio_xmit_failed;
3211         }
3212
3213         /* Use space in skb->cb to store info used to unmap and
3214          * free the buffers.
3215          */
3216         finfo = (struct octnet_buf_free_info *)skb->cb;
3217         finfo->lio = lio;
3218         finfo->skb = skb;
3219         finfo->sc = NULL;
3220
3221         /* Prepare the attributes for the data to be passed to OSI. */
3222         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
3223
3224         ndata.buf = (void *)finfo;
3225
3226         ndata.q_no = iq_no;
3227
3228         if (netif_is_multiqueue(netdev)) {
3229                 if (octnet_iq_is_full(oct, ndata.q_no)) {
3230                         /* defer sending if queue is full */
3231                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
3232                                    ndata.q_no);
3233                         stats->tx_iq_busy++;
3234                         return NETDEV_TX_BUSY;
3235                 }
3236         } else {
3237                 if (octnet_iq_is_full(oct, lio->txq)) {
3238                         /* defer sending if queue is full */
3239                         stats->tx_iq_busy++;
3240                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
3241                                    lio->txq);
3242                         return NETDEV_TX_BUSY;
3243                 }
3244         }
3245         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
3246          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
3247          */
3248
3249         ndata.datasize = skb->len;
3250
3251         cmdsetup.u64 = 0;
3252         cmdsetup.s.iq_no = iq_no;
3253
3254         if (skb->ip_summed == CHECKSUM_PARTIAL) {
3255                 if (skb->encapsulation) {
3256                         cmdsetup.s.tnl_csum = 1;
3257                         stats->tx_vxlan++;
3258                 } else {
3259                         cmdsetup.s.transport_csum = 1;
3260                 }
3261         }
3262         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3263                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3264                 cmdsetup.s.timestamp = 1;
3265         }
3266
3267         if (skb_shinfo(skb)->nr_frags == 0) {
3268                 cmdsetup.s.u.datasize = skb->len;
3269                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
3270
3271                 /* Offload checksum calculation for TCP/UDP packets */
3272                 dptr = dma_map_single(&oct->pci_dev->dev,
3273                                       skb->data,
3274                                       skb->len,
3275                                       DMA_TO_DEVICE);
3276                 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
3277                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
3278                                 __func__);
3279                         return NETDEV_TX_BUSY;
3280                 }
3281
3282                 if (OCTEON_CN23XX_PF(oct))
3283                         ndata.cmd.cmd3.dptr = dptr;
3284                 else
3285                         ndata.cmd.cmd2.dptr = dptr;
3286                 finfo->dptr = dptr;
3287                 ndata.reqtype = REQTYPE_NORESP_NET;
3288
3289         } else {
3290                 int i, frags;
3291                 struct skb_frag_struct *frag;
3292                 struct octnic_gather *g;
3293
3294                 spin_lock(&lio->glist_lock[q_idx]);
3295                 g = (struct octnic_gather *)
3296                         list_delete_head(&lio->glist[q_idx]);
3297                 spin_unlock(&lio->glist_lock[q_idx]);
3298
3299                 if (!g) {
3300                         netif_info(lio, tx_err, lio->netdev,
3301                                    "Transmit scatter gather: glist null!\n");
3302                         goto lio_xmit_failed;
3303                 }
3304
3305                 cmdsetup.s.gather = 1;
3306                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
3307                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
3308
3309                 memset(g->sg, 0, g->sg_size);
3310
3311                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
3312                                                  skb->data,
3313                                                  (skb->len - skb->data_len),
3314                                                  DMA_TO_DEVICE);
3315                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
3316                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
3317                                 __func__);
3318                         return NETDEV_TX_BUSY;
3319                 }
3320                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
3321
3322                 frags = skb_shinfo(skb)->nr_frags;
3323                 i = 1;
3324                 while (frags--) {
3325                         frag = &skb_shinfo(skb)->frags[i - 1];
3326
3327                         g->sg[(i >> 2)].ptr[(i & 3)] =
3328                                 dma_map_page(&oct->pci_dev->dev,
3329                                              frag->page.p,
3330                                              frag->page_offset,
3331                                              frag->size,
3332                                              DMA_TO_DEVICE);
3333
3334                         if (dma_mapping_error(&oct->pci_dev->dev,
3335                                               g->sg[i >> 2].ptr[i & 3])) {
3336                                 dma_unmap_single(&oct->pci_dev->dev,
3337                                                  g->sg[0].ptr[0],
3338                                                  skb->len - skb->data_len,
3339                                                  DMA_TO_DEVICE);
3340                                 for (j = 1; j < i; j++) {
3341                                         frag = &skb_shinfo(skb)->frags[j - 1];
3342                                         dma_unmap_page(&oct->pci_dev->dev,
3343                                                        g->sg[j >> 2].ptr[j & 3],
3344                                                        frag->size,
3345                                                        DMA_TO_DEVICE);
3346                                 }
3347                                 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
3348                                         __func__);
3349                                 return NETDEV_TX_BUSY;
3350                         }
3351
3352                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
3353                         i++;
3354                 }
3355
3356                 dptr = g->sg_dma_ptr;
3357
3358                 if (OCTEON_CN23XX_PF(oct))
3359                         ndata.cmd.cmd3.dptr = dptr;
3360                 else
3361                         ndata.cmd.cmd2.dptr = dptr;
3362                 finfo->dptr = dptr;
3363                 finfo->g = g;
3364
3365                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
3366         }
3367
3368         if (OCTEON_CN23XX_PF(oct)) {
3369                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
3370                 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
3371         } else {
3372                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
3373                 tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
3374         }
3375
3376         if (skb_shinfo(skb)->gso_size) {
3377                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
3378                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
3379                 stats->tx_gso++;
3380         }
3381
3382         /* HW insert VLAN tag */
3383         if (skb_vlan_tag_present(skb)) {
3384                 irh->priority = skb_vlan_tag_get(skb) >> 13;
3385                 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
3386         }
3387
3388         if (unlikely(cmdsetup.s.timestamp))
3389                 status = send_nic_timestamp_pkt(oct, &ndata, finfo);
3390         else
3391                 status = octnet_send_nic_data_pkt(oct, &ndata);
3392         if (status == IQ_SEND_FAILED)
3393                 goto lio_xmit_failed;
3394
3395         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
3396
3397         if (status == IQ_SEND_STOP)
3398                 stop_q(lio->netdev, q_idx);
3399
3400         netif_trans_update(netdev);
3401
3402         if (tx_info->s.gso_segs)
3403                 stats->tx_done += tx_info->s.gso_segs;
3404         else
3405                 stats->tx_done++;
3406         stats->tx_tot_bytes += ndata.datasize;
3407
3408         return NETDEV_TX_OK;
3409
3410 lio_xmit_failed:
3411         stats->tx_dropped++;
3412         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
3413                    iq_no, stats->tx_dropped);
3414         if (dptr)
3415                 dma_unmap_single(&oct->pci_dev->dev, dptr,
3416                                  ndata.datasize, DMA_TO_DEVICE);
3417         tx_buffer_free(skb);
3418         return NETDEV_TX_OK;
3419 }
3420
3421 /** \brief Network device Tx timeout
3422  * @param netdev    pointer to network device
3423  */
3424 static void liquidio_tx_timeout(struct net_device *netdev)
3425 {
3426         struct lio *lio;
3427
3428         lio = GET_LIO(netdev);
3429
3430         netif_info(lio, tx_err, lio->netdev,
3431                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
3432                    netdev->stats.tx_dropped);
3433         netif_trans_update(netdev);
3434         txqs_wake(netdev);
3435 }
3436
3437 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
3438                                     __be16 proto __attribute__((unused)),
3439                                     u16 vid)
3440 {
3441         struct lio *lio = GET_LIO(netdev);
3442         struct octeon_device *oct = lio->oct_dev;
3443         struct octnic_ctrl_pkt nctrl;
3444         int ret = 0;
3445
3446         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3447
3448         nctrl.ncmd.u64 = 0;
3449         nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
3450         nctrl.ncmd.s.param1 = vid;
3451         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3452         nctrl.wait_time = 100;
3453         nctrl.netpndev = (u64)netdev;
3454         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3455
3456         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3457         if (ret < 0) {
3458                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
3459                         ret);
3460         }
3461
3462         return ret;
3463 }
3464
3465 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
3466                                      __be16 proto __attribute__((unused)),
3467                                      u16 vid)
3468 {
3469         struct lio *lio = GET_LIO(netdev);
3470         struct octeon_device *oct = lio->oct_dev;
3471         struct octnic_ctrl_pkt nctrl;
3472         int ret = 0;
3473
3474         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3475
3476         nctrl.ncmd.u64 = 0;
3477         nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
3478         nctrl.ncmd.s.param1 = vid;
3479         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3480         nctrl.wait_time = 100;
3481         nctrl.netpndev = (u64)netdev;
3482         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3483
3484         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3485         if (ret < 0) {
3486                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
3487                         ret);
3488         }
3489         return ret;
3490 }
3491
3492 /** Sending command to enable/disable RX checksum offload
3493  * @param netdev                pointer to network device
3494  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
3495  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
3496  *                              OCTNET_CMD_RXCSUM_DISABLE
3497  * @returns                     SUCCESS or FAILURE
3498  */
3499 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
3500                                        u8 rx_cmd)
3501 {
3502         struct lio *lio = GET_LIO(netdev);
3503         struct octeon_device *oct = lio->oct_dev;
3504         struct octnic_ctrl_pkt nctrl;
3505         int ret = 0;
3506
3507         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3508
3509         nctrl.ncmd.u64 = 0;
3510         nctrl.ncmd.s.cmd = command;
3511         nctrl.ncmd.s.param1 = rx_cmd;
3512         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3513         nctrl.wait_time = 100;
3514         nctrl.netpndev = (u64)netdev;
3515         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3516
3517         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3518         if (ret < 0) {
3519                 dev_err(&oct->pci_dev->dev,
3520                         "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
3521                         ret);
3522         }
3523         return ret;
3524 }
3525
3526 /** Sending command to add/delete VxLAN UDP port to firmware
3527  * @param netdev                pointer to network device
3528  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
3529  * @param vxlan_port            VxLAN port to be added or deleted
3530  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
3531  *                              OCTNET_CMD_VXLAN_PORT_DEL
3532  * @returns                     SUCCESS or FAILURE
3533  */
3534 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
3535                                        u16 vxlan_port, u8 vxlan_cmd_bit)
3536 {
3537         struct lio *lio = GET_LIO(netdev);
3538         struct octeon_device *oct = lio->oct_dev;
3539         struct octnic_ctrl_pkt nctrl;
3540         int ret = 0;
3541
3542         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3543
3544         nctrl.ncmd.u64 = 0;
3545         nctrl.ncmd.s.cmd = command;
3546         nctrl.ncmd.s.more = vxlan_cmd_bit;
3547         nctrl.ncmd.s.param1 = vxlan_port;
3548         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3549         nctrl.wait_time = 100;
3550         nctrl.netpndev = (u64)netdev;
3551         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3552
3553         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3554         if (ret < 0) {
3555                 dev_err(&oct->pci_dev->dev,
3556                         "VxLAN port add/delete failed in core (ret:0x%x)\n",
3557                         ret);
3558         }
3559         return ret;
3560 }
3561
3562 /** \brief Net device fix features
3563  * @param netdev  pointer to network device
3564  * @param request features requested
3565  * @returns updated features list
3566  */
3567 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
3568                                                netdev_features_t request)
3569 {
3570         struct lio *lio = netdev_priv(netdev);
3571
3572         if ((request & NETIF_F_RXCSUM) &&
3573             !(lio->dev_capability & NETIF_F_RXCSUM))
3574                 request &= ~NETIF_F_RXCSUM;
3575
3576         if ((request & NETIF_F_HW_CSUM) &&
3577             !(lio->dev_capability & NETIF_F_HW_CSUM))
3578                 request &= ~NETIF_F_HW_CSUM;
3579
3580         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
3581                 request &= ~NETIF_F_TSO;
3582
3583         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
3584                 request &= ~NETIF_F_TSO6;
3585
3586         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
3587                 request &= ~NETIF_F_LRO;
3588
3589         /*Disable LRO if RXCSUM is off */
3590         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
3591             (lio->dev_capability & NETIF_F_LRO))
3592                 request &= ~NETIF_F_LRO;
3593
3594         if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3595             !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
3596                 request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
3597
3598         return request;
3599 }
3600
3601 /** \brief Net device set features
3602  * @param netdev  pointer to network device
3603  * @param features features to enable/disable
3604  */
3605 static int liquidio_set_features(struct net_device *netdev,
3606                                  netdev_features_t features)
3607 {
3608         struct lio *lio = netdev_priv(netdev);
3609
3610         if ((features & NETIF_F_LRO) &&
3611             (lio->dev_capability & NETIF_F_LRO) &&
3612             !(netdev->features & NETIF_F_LRO))
3613                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3614                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3615         else if (!(features & NETIF_F_LRO) &&
3616                  (lio->dev_capability & NETIF_F_LRO) &&
3617                  (netdev->features & NETIF_F_LRO))
3618                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
3619                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3620
3621         /* Sending command to firmware to enable/disable RX checksum
3622          * offload settings using ethtool
3623          */
3624         if (!(netdev->features & NETIF_F_RXCSUM) &&
3625             (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3626             (features & NETIF_F_RXCSUM))
3627                 liquidio_set_rxcsum_command(netdev,
3628                                             OCTNET_CMD_TNL_RX_CSUM_CTL,
3629                                             OCTNET_CMD_RXCSUM_ENABLE);
3630         else if ((netdev->features & NETIF_F_RXCSUM) &&
3631                  (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3632                  !(features & NETIF_F_RXCSUM))
3633                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3634                                             OCTNET_CMD_RXCSUM_DISABLE);
3635
3636         if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3637             (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3638             !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
3639                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3640                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
3641         else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3642                  (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3643                  (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
3644                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3645                                      OCTNET_CMD_VLAN_FILTER_DISABLE);
3646
3647         return 0;
3648 }
3649
3650 static void liquidio_add_vxlan_port(struct net_device *netdev,
3651                                     struct udp_tunnel_info *ti)
3652 {
3653         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3654                 return;
3655
3656         liquidio_vxlan_port_command(netdev,
3657                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3658                                     htons(ti->port),
3659                                     OCTNET_CMD_VXLAN_PORT_ADD);
3660 }
3661
3662 static void liquidio_del_vxlan_port(struct net_device *netdev,
3663                                     struct udp_tunnel_info *ti)
3664 {
3665         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3666                 return;
3667
3668         liquidio_vxlan_port_command(netdev,
3669                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3670                                     htons(ti->port),
3671                                     OCTNET_CMD_VXLAN_PORT_DEL);
3672 }
3673
3674 static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
3675                                  u8 *mac, bool is_admin_assigned)
3676 {
3677         struct lio *lio = GET_LIO(netdev);
3678         struct octeon_device *oct = lio->oct_dev;
3679         struct octnic_ctrl_pkt nctrl;
3680
3681         if (!is_valid_ether_addr(mac))
3682                 return -EINVAL;
3683
3684         if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
3685                 return -EINVAL;
3686
3687         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3688
3689         nctrl.ncmd.u64 = 0;
3690         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
3691         /* vfidx is 0 based, but vf_num (param1) is 1 based */
3692         nctrl.ncmd.s.param1 = vfidx + 1;
3693         nctrl.ncmd.s.param2 = (is_admin_assigned ? 1 : 0);
3694         nctrl.ncmd.s.more = 1;
3695         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3696         nctrl.netpndev = (u64)netdev;
3697         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3698         nctrl.wait_time = LIO_CMD_WAIT_TM;
3699
3700         nctrl.udd[0] = 0;
3701         /* The MAC Address is presented in network byte order. */
3702         ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
3703
3704         oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
3705
3706         octnet_send_nic_ctrl_pkt(oct, &nctrl);
3707
3708         return 0;
3709 }
3710
3711 static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
3712 {
3713         struct lio *lio = GET_LIO(netdev);
3714         struct octeon_device *oct = lio->oct_dev;
3715         int retval;
3716
3717         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3718                 return -EINVAL;
3719
3720         retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
3721         if (!retval)
3722                 cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
3723
3724         return retval;
3725 }
3726
3727 static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
3728                                 u16 vlan, u8 qos, __be16 vlan_proto)
3729 {
3730         struct lio *lio = GET_LIO(netdev);
3731         struct octeon_device *oct = lio->oct_dev;
3732         struct octnic_ctrl_pkt nctrl;
3733         u16 vlantci;
3734
3735         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3736                 return -EINVAL;
3737
3738         if (vlan_proto != htons(ETH_P_8021Q))
3739                 return -EPROTONOSUPPORT;
3740
3741         if (vlan >= VLAN_N_VID || qos > 7)
3742                 return -EINVAL;
3743
3744         if (vlan)
3745                 vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
3746         else
3747                 vlantci = 0;
3748
3749         if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
3750                 return 0;
3751
3752         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3753
3754         if (vlan)
3755                 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
3756         else
3757                 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
3758
3759         nctrl.ncmd.s.param1 = vlantci;
3760         nctrl.ncmd.s.param2 =
3761             vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
3762         nctrl.ncmd.s.more = 0;
3763         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3764         nctrl.cb_fn = 0;
3765         nctrl.wait_time = LIO_CMD_WAIT_TM;
3766
3767         octnet_send_nic_ctrl_pkt(oct, &nctrl);
3768
3769         oct->sriov_info.vf_vlantci[vfidx] = vlantci;
3770
3771         return 0;
3772 }
3773
3774 static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
3775                                   struct ifla_vf_info *ivi)
3776 {
3777         struct lio *lio = GET_LIO(netdev);
3778         struct octeon_device *oct = lio->oct_dev;
3779         u8 *macaddr;
3780
3781         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3782                 return -EINVAL;
3783
3784         ivi->vf = vfidx;
3785         macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
3786         ether_addr_copy(&ivi->mac[0], macaddr);
3787         ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
3788         ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
3789         ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
3790         return 0;
3791 }
3792
3793 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3794                                       int linkstate)
3795 {
3796         struct lio *lio = GET_LIO(netdev);
3797         struct octeon_device *oct = lio->oct_dev;
3798         struct octnic_ctrl_pkt nctrl;
3799
3800         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3801                 return -EINVAL;
3802
3803         if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3804                 return 0;
3805
3806         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3807         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3808         nctrl.ncmd.s.param1 =
3809             vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3810         nctrl.ncmd.s.param2 = linkstate;
3811         nctrl.ncmd.s.more = 0;
3812         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3813         nctrl.cb_fn = 0;
3814         nctrl.wait_time = LIO_CMD_WAIT_TM;
3815
3816         octnet_send_nic_ctrl_pkt(oct, &nctrl);
3817
3818         oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3819
3820         return 0;
3821 }
3822
3823 static const struct net_device_ops lionetdevops = {
3824         .ndo_open               = liquidio_open,
3825         .ndo_stop               = liquidio_stop,
3826         .ndo_start_xmit         = liquidio_xmit,
3827         .ndo_get_stats          = liquidio_get_stats,
3828         .ndo_set_mac_address    = liquidio_set_mac,
3829         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3830         .ndo_tx_timeout         = liquidio_tx_timeout,
3831
3832         .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3833         .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3834         .ndo_change_mtu         = liquidio_change_mtu,
3835         .ndo_do_ioctl           = liquidio_ioctl,
3836         .ndo_fix_features       = liquidio_fix_features,
3837         .ndo_set_features       = liquidio_set_features,
3838         .ndo_udp_tunnel_add     = liquidio_add_vxlan_port,
3839         .ndo_udp_tunnel_del     = liquidio_del_vxlan_port,
3840         .ndo_set_vf_mac         = liquidio_set_vf_mac,
3841         .ndo_set_vf_vlan        = liquidio_set_vf_vlan,
3842         .ndo_get_vf_config      = liquidio_get_vf_config,
3843         .ndo_set_vf_link_state  = liquidio_set_vf_link_state,
3844 };
3845
3846 /** \brief Entry point for the liquidio module
3847  */
3848 static int __init liquidio_init(void)
3849 {
3850         int i;
3851         struct handshake *hs;
3852
3853         init_completion(&first_stage);
3854
3855         octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
3856
3857         if (liquidio_init_pci())
3858                 return -EINVAL;
3859
3860         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3861
3862         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3863                 hs = &handshake[i];
3864                 if (hs->pci_dev) {
3865                         wait_for_completion(&hs->init);
3866                         if (!hs->init_ok) {
3867                                 /* init handshake failed */
3868                                 dev_err(&hs->pci_dev->dev,
3869                                         "Failed to init device\n");
3870                                 liquidio_deinit_pci();
3871                                 return -EIO;
3872                         }
3873                 }
3874         }
3875
3876         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3877                 hs = &handshake[i];
3878                 if (hs->pci_dev) {
3879                         wait_for_completion_timeout(&hs->started,
3880                                                     msecs_to_jiffies(30000));
3881                         if (!hs->started_ok) {
3882                                 /* starter handshake failed */
3883                                 dev_err(&hs->pci_dev->dev,
3884                                         "Firmware failed to start\n");
3885                                 liquidio_deinit_pci();
3886                                 return -EIO;
3887                         }
3888                 }
3889         }
3890
3891         return 0;
3892 }
3893
3894 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3895 {
3896         struct octeon_device *oct = (struct octeon_device *)buf;
3897         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3898         int gmxport = 0;
3899         union oct_link_status *ls;
3900         int i;
3901
3902         if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
3903                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3904                         recv_pkt->buffer_size[0],
3905                         recv_pkt->rh.r_nic_info.gmxport);
3906                 goto nic_info_err;
3907         }
3908
3909         gmxport = recv_pkt->rh.r_nic_info.gmxport;
3910         ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3911                 OCT_DROQ_INFO_SIZE);
3912
3913         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3914         for (i = 0; i < oct->ifcount; i++) {
3915                 if (oct->props[i].gmxport == gmxport) {
3916                         update_link_status(oct->props[i].netdev, ls);
3917                         break;
3918                 }
3919         }
3920
3921 nic_info_err:
3922         for (i = 0; i < recv_pkt->buffer_count; i++)
3923                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3924         octeon_free_recv_info(recv_info);
3925         return 0;
3926 }
3927
3928 /**
3929  * \brief Setup network interfaces
3930  * @param octeon_dev  octeon device
3931  *
3932  * Called during init time for each device. It assumes the NIC
3933  * is already up and running.  The link information for each
3934  * interface is passed in link_info.
3935  */
3936 static int setup_nic_devices(struct octeon_device *octeon_dev)
3937 {
3938         struct lio *lio = NULL;
3939         struct net_device *netdev;
3940         u8 mac[6], i, j;
3941         struct octeon_soft_command *sc;
3942         struct liquidio_if_cfg_context *ctx;
3943         struct liquidio_if_cfg_resp *resp;
3944         struct octdev_props *props;
3945         int retval, num_iqueues, num_oqueues;
3946         union oct_nic_if_cfg if_cfg;
3947         unsigned int base_queue;
3948         unsigned int gmx_port_id;
3949         u32 resp_size, ctx_size, data_size;
3950         u32 ifidx_or_pfnum;
3951         struct lio_version *vdata;
3952
3953         /* This is to handle link status changes */
3954         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3955                                     OPCODE_NIC_INFO,
3956                                     lio_nic_info, octeon_dev);
3957
3958         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3959          * They are handled directly.
3960          */
3961         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3962                                         free_netbuf);
3963
3964         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3965                                         free_netsgbuf);
3966
3967         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3968                                         free_netsgbuf_with_resp);
3969
3970         for (i = 0; i < octeon_dev->ifcount; i++) {
3971                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3972                 ctx_size = sizeof(struct liquidio_if_cfg_context);
3973                 data_size = sizeof(struct lio_version);
3974                 sc = (struct octeon_soft_command *)
3975                         octeon_alloc_soft_command(octeon_dev, data_size,
3976                                                   resp_size, ctx_size);
3977                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3978                 ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
3979                 vdata = (struct lio_version *)sc->virtdptr;
3980
3981                 *((u64 *)vdata) = 0;
3982                 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3983                 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3984                 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3985
3986                 if (OCTEON_CN23XX_PF(octeon_dev)) {
3987                         num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3988                         num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3989                         base_queue = octeon_dev->sriov_info.pf_srn;
3990
3991                         gmx_port_id = octeon_dev->pf_num;
3992                         ifidx_or_pfnum = octeon_dev->pf_num;
3993                 } else {
3994                         num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3995                                                 octeon_get_conf(octeon_dev), i);
3996                         num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3997                                                 octeon_get_conf(octeon_dev), i);
3998                         base_queue = CFG_GET_BASE_QUE_NIC_IF(
3999                                                 octeon_get_conf(octeon_dev), i);
4000                         gmx_port_id = CFG_GET_GMXID_NIC_IF(
4001                                                 octeon_get_conf(octeon_dev), i);
4002                         ifidx_or_pfnum = i;
4003                 }
4004
4005                 dev_dbg(&octeon_dev->pci_dev->dev,
4006                         "requesting config for interface %d, iqs %d, oqs %d\n",
4007                         ifidx_or_pfnum, num_iqueues, num_oqueues);
4008                 WRITE_ONCE(ctx->cond, 0);
4009                 ctx->octeon_id = lio_get_device_id(octeon_dev);
4010                 init_waitqueue_head(&ctx->wc);
4011
4012                 if_cfg.u64 = 0;
4013                 if_cfg.s.num_iqueues = num_iqueues;
4014                 if_cfg.s.num_oqueues = num_oqueues;
4015                 if_cfg.s.base_queue = base_queue;
4016                 if_cfg.s.gmx_port_id = gmx_port_id;
4017
4018                 sc->iq_no = 0;
4019
4020                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
4021                                             OPCODE_NIC_IF_CFG, 0,
4022                                             if_cfg.u64, 0);
4023
4024                 sc->callback = if_cfg_callback;
4025                 sc->callback_arg = sc;
4026                 sc->wait_time = 3000;
4027
4028                 retval = octeon_send_soft_command(octeon_dev, sc);
4029                 if (retval == IQ_SEND_FAILED) {
4030                         dev_err(&octeon_dev->pci_dev->dev,
4031                                 "iq/oq config failed status: %x\n",
4032                                 retval);
4033                         /* Soft instr is freed by driver in case of failure. */
4034                         goto setup_nic_dev_fail;
4035                 }
4036
4037                 /* Sleep on a wait queue till the cond flag indicates that the
4038                  * response arrived or timed-out.
4039                  */
4040                 if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR) {
4041                         dev_err(&octeon_dev->pci_dev->dev, "Wait interrupted\n");
4042                         goto setup_nic_wait_intr;
4043                 }
4044
4045                 retval = resp->status;
4046                 if (retval) {
4047                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
4048                         goto setup_nic_dev_fail;
4049                 }
4050
4051                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
4052                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
4053
4054                 num_iqueues = hweight64(resp->cfg_info.iqmask);
4055                 num_oqueues = hweight64(resp->cfg_info.oqmask);
4056
4057                 if (!(num_iqueues) || !(num_oqueues)) {
4058                         dev_err(&octeon_dev->pci_dev->dev,
4059                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
4060                                 resp->cfg_info.iqmask,
4061                                 resp->cfg_info.oqmask);
4062                         goto setup_nic_dev_fail;
4063                 }
4064                 dev_dbg(&octeon_dev->pci_dev->dev,
4065                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
4066                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
4067                         num_iqueues, num_oqueues);
4068                 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
4069
4070                 if (!netdev) {
4071                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
4072                         goto setup_nic_dev_fail;
4073                 }
4074
4075                 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
4076
4077                 /* Associate the routines that will handle different
4078                  * netdev tasks.
4079                  */
4080                 netdev->netdev_ops = &lionetdevops;
4081
4082                 lio = GET_LIO(netdev);
4083
4084                 memset(lio, 0, sizeof(struct lio));
4085
4086                 lio->ifidx = ifidx_or_pfnum;
4087
4088                 props = &octeon_dev->props[i];
4089                 props->gmxport = resp->cfg_info.linfo.gmxport;
4090                 props->netdev = netdev;
4091
4092                 lio->linfo.num_rxpciq = num_oqueues;
4093                 lio->linfo.num_txpciq = num_iqueues;
4094                 for (j = 0; j < num_oqueues; j++) {
4095                         lio->linfo.rxpciq[j].u64 =
4096                                 resp->cfg_info.linfo.rxpciq[j].u64;
4097                 }
4098                 for (j = 0; j < num_iqueues; j++) {
4099                         lio->linfo.txpciq[j].u64 =
4100                                 resp->cfg_info.linfo.txpciq[j].u64;
4101                 }
4102                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
4103                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
4104                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
4105
4106                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
4107
4108                 if (OCTEON_CN23XX_PF(octeon_dev) ||
4109                     OCTEON_CN6XXX(octeon_dev)) {
4110                         lio->dev_capability = NETIF_F_HIGHDMA
4111                                               | NETIF_F_IP_CSUM
4112                                               | NETIF_F_IPV6_CSUM
4113                                               | NETIF_F_SG | NETIF_F_RXCSUM
4114                                               | NETIF_F_GRO
4115                                               | NETIF_F_TSO | NETIF_F_TSO6
4116                                               | NETIF_F_LRO;
4117                 }
4118                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
4119
4120                 /*  Copy of transmit encapsulation capabilities:
4121                  *  TSO, TSO6, Checksums for this device
4122                  */
4123                 lio->enc_dev_capability = NETIF_F_IP_CSUM
4124                                           | NETIF_F_IPV6_CSUM
4125                                           | NETIF_F_GSO_UDP_TUNNEL
4126                                           | NETIF_F_HW_CSUM | NETIF_F_SG
4127                                           | NETIF_F_RXCSUM
4128                                           | NETIF_F_TSO | NETIF_F_TSO6
4129                                           | NETIF_F_LRO;
4130
4131                 netdev->hw_enc_features = (lio->enc_dev_capability &
4132                                            ~NETIF_F_LRO);
4133
4134                 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
4135
4136                 netdev->vlan_features = lio->dev_capability;
4137                 /* Add any unchangeable hw features */
4138                 lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
4139                                         NETIF_F_HW_VLAN_CTAG_RX |
4140                                         NETIF_F_HW_VLAN_CTAG_TX;
4141
4142                 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
4143
4144                 netdev->hw_features = lio->dev_capability;
4145                 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
4146                 netdev->hw_features = netdev->hw_features &
4147                         ~NETIF_F_HW_VLAN_CTAG_RX;
4148
4149                 /* MTU range: 68 - 16000 */
4150                 netdev->min_mtu = LIO_MIN_MTU_SIZE;
4151                 netdev->max_mtu = LIO_MAX_MTU_SIZE;
4152
4153                 /* Point to the  properties for octeon device to which this
4154                  * interface belongs.
4155                  */
4156                 lio->oct_dev = octeon_dev;
4157                 lio->octprops = props;
4158                 lio->netdev = netdev;
4159
4160                 dev_dbg(&octeon_dev->pci_dev->dev,
4161                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
4162                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
4163
4164                 for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
4165                         u8 vfmac[ETH_ALEN];
4166
4167                         random_ether_addr(&vfmac[0]);
4168                         if (__liquidio_set_vf_mac(netdev, j,
4169                                                   &vfmac[0], false)) {
4170                                 dev_err(&octeon_dev->pci_dev->dev,
4171                                         "Error setting VF%d MAC address\n",
4172                                         j);
4173                                 goto setup_nic_dev_fail;
4174                         }
4175                 }
4176
4177                 /* 64-bit swap required on LE machines */
4178                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
4179                 for (j = 0; j < 6; j++)
4180                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
4181
4182                 /* Copy MAC Address to OS network device structure */
4183
4184                 ether_addr_copy(netdev->dev_addr, mac);
4185
4186                 /* By default all interfaces on a single Octeon uses the same
4187                  * tx and rx queues
4188                  */
4189                 lio->txq = lio->linfo.txpciq[0].s.q_no;
4190                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
4191                 if (setup_io_queues(octeon_dev, i)) {
4192                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
4193                         goto setup_nic_dev_fail;
4194                 }
4195
4196                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
4197
4198                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
4199                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
4200
4201                 if (setup_glists(octeon_dev, lio, num_iqueues)) {
4202                         dev_err(&octeon_dev->pci_dev->dev,
4203                                 "Gather list allocation failed\n");
4204                         goto setup_nic_dev_fail;
4205                 }
4206
4207                 /* Register ethtool support */
4208                 liquidio_set_ethtool_ops(netdev);
4209                 if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
4210                         octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
4211                 else
4212                         octeon_dev->priv_flags = 0x0;
4213
4214                 if (netdev->features & NETIF_F_LRO)
4215                         liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
4216                                              OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
4217
4218                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
4219                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
4220
4221                 if ((debug != -1) && (debug & NETIF_MSG_HW))
4222                         liquidio_set_feature(netdev,
4223                                              OCTNET_CMD_VERBOSE_ENABLE, 0);
4224
4225                 if (setup_link_status_change_wq(netdev))
4226                         goto setup_nic_dev_fail;
4227
4228                 if (setup_rx_oom_poll_fn(netdev))
4229                         goto setup_nic_dev_fail;
4230
4231                 /* Register the network device with the OS */
4232                 if (register_netdev(netdev)) {
4233                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
4234                         goto setup_nic_dev_fail;
4235                 }
4236
4237                 dev_dbg(&octeon_dev->pci_dev->dev,
4238                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
4239                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
4240                 netif_carrier_off(netdev);
4241                 lio->link_changes++;
4242
4243                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
4244
4245                 /* Sending command to firmware to enable Rx checksum offload
4246                  * by default at the time of setup of Liquidio driver for
4247                  * this device
4248                  */
4249                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
4250                                             OCTNET_CMD_RXCSUM_ENABLE);
4251                 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
4252                                      OCTNET_CMD_TXCSUM_ENABLE);
4253
4254                 dev_dbg(&octeon_dev->pci_dev->dev,
4255                         "NIC ifidx:%d Setup successful\n", i);
4256
4257                 octeon_free_soft_command(octeon_dev, sc);
4258         }
4259
4260         return 0;
4261
4262 setup_nic_dev_fail:
4263
4264         octeon_free_soft_command(octeon_dev, sc);
4265
4266 setup_nic_wait_intr:
4267
4268         while (i--) {
4269                 dev_err(&octeon_dev->pci_dev->dev,
4270                         "NIC ifidx:%d Setup failed\n", i);
4271                 liquidio_destroy_nic_device(octeon_dev, i);
4272         }
4273         return -ENODEV;
4274 }
4275
4276 #ifdef CONFIG_PCI_IOV
4277 static int octeon_enable_sriov(struct octeon_device *oct)
4278 {
4279         unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
4280         struct pci_dev *vfdev;
4281         int err;
4282         u32 u;
4283
4284         if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
4285                 err = pci_enable_sriov(oct->pci_dev,
4286                                        oct->sriov_info.num_vfs_alloced);
4287                 if (err) {
4288                         dev_err(&oct->pci_dev->dev,
4289                                 "OCTEON: Failed to enable PCI sriov: %d\n",
4290                                 err);
4291                         oct->sriov_info.num_vfs_alloced = 0;
4292                         return err;
4293                 }
4294                 oct->sriov_info.sriov_enabled = 1;
4295
4296                 /* init lookup table that maps DPI ring number to VF pci_dev
4297                  * struct pointer
4298                  */
4299                 u = 0;
4300                 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
4301                                        OCTEON_CN23XX_VF_VID, NULL);
4302                 while (vfdev) {
4303                         if (vfdev->is_virtfn &&
4304                             (vfdev->physfn == oct->pci_dev)) {
4305                                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
4306                                         vfdev;
4307                                 u += oct->sriov_info.rings_per_vf;
4308                         }
4309                         vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
4310                                                OCTEON_CN23XX_VF_VID, vfdev);
4311                 }
4312         }
4313
4314         return num_vfs_alloced;
4315 }
4316
4317 static int lio_pci_sriov_disable(struct octeon_device *oct)
4318 {
4319         int u;
4320
4321         if (pci_vfs_assigned(oct->pci_dev)) {
4322                 dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
4323                 return -EPERM;
4324         }
4325
4326         pci_disable_sriov(oct->pci_dev);
4327
4328         u = 0;
4329         while (u < MAX_POSSIBLE_VFS) {
4330                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
4331                 u += oct->sriov_info.rings_per_vf;
4332         }
4333
4334         oct->sriov_info.num_vfs_alloced = 0;
4335         dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
4336                  oct->pf_num);
4337
4338         return 0;
4339 }
4340
4341 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
4342 {
4343         struct octeon_device *oct = pci_get_drvdata(dev);
4344         int ret = 0;
4345
4346         if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
4347             (oct->sriov_info.sriov_enabled)) {
4348                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
4349                          oct->pf_num, num_vfs);
4350                 return 0;
4351         }
4352
4353         if (!num_vfs) {
4354                 ret = lio_pci_sriov_disable(oct);
4355         } else if (num_vfs > oct->sriov_info.max_vfs) {
4356                 dev_err(&oct->pci_dev->dev,
4357                         "OCTEON: Max allowed VFs:%d user requested:%d",
4358                         oct->sriov_info.max_vfs, num_vfs);
4359                 ret = -EPERM;
4360         } else {
4361                 oct->sriov_info.num_vfs_alloced = num_vfs;
4362                 ret = octeon_enable_sriov(oct);
4363                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
4364                          oct->pf_num, num_vfs);
4365         }
4366
4367         return ret;
4368 }
4369 #endif
4370
4371 /**
4372  * \brief initialize the NIC
4373  * @param oct octeon device
4374  *
4375  * This initialization routine is called once the Octeon device application is
4376  * up and running
4377  */
4378 static int liquidio_init_nic_module(struct octeon_device *oct)
4379 {
4380         int i, retval = 0;
4381         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
4382
4383         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
4384
4385         /* only default iq and oq were initialized
4386          * initialize the rest as well
4387          */
4388         /* run port_config command for each port */
4389         oct->ifcount = num_nic_ports;
4390
4391         memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
4392
4393         for (i = 0; i < MAX_OCTEON_LINKS; i++)
4394                 oct->props[i].gmxport = -1;
4395
4396         retval = setup_nic_devices(oct);
4397         if (retval) {
4398                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
4399                 goto octnet_init_failure;
4400         }
4401
4402         liquidio_ptp_init(oct);
4403
4404         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
4405
4406         return retval;
4407
4408 octnet_init_failure:
4409
4410         oct->ifcount = 0;
4411
4412         return retval;
4413 }
4414
4415 /**
4416  * \brief starter callback that invokes the remaining initialization work after
4417  * the NIC is up and running.
4418  * @param octptr  work struct work_struct
4419  */
4420 static void nic_starter(struct work_struct *work)
4421 {
4422         struct octeon_device *oct;
4423         struct cavium_wk *wk = (struct cavium_wk *)work;
4424
4425         oct = (struct octeon_device *)wk->ctxptr;
4426
4427         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
4428                 return;
4429
4430         /* If the status of the device is CORE_OK, the core
4431          * application has reported its application type. Call
4432          * any registered handlers now and move to the RUNNING
4433          * state.
4434          */
4435         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
4436                 schedule_delayed_work(&oct->nic_poll_work.work,
4437                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4438                 return;
4439         }
4440
4441         atomic_set(&oct->status, OCT_DEV_RUNNING);
4442
4443         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
4444                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
4445
4446                 if (liquidio_init_nic_module(oct))
4447                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
4448                 else
4449                         handshake[oct->octeon_id].started_ok = 1;
4450         } else {
4451                 dev_err(&oct->pci_dev->dev,
4452                         "Unexpected application running on NIC (%d). Check firmware.\n",
4453                         oct->app_mode);
4454         }
4455
4456         complete(&handshake[oct->octeon_id].started);
4457 }
4458
4459 static int
4460 octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
4461 {
4462         struct octeon_device *oct = (struct octeon_device *)buf;
4463         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
4464         int i, notice, vf_idx;
4465         bool cores_crashed;
4466         u64 *data, vf_num;
4467
4468         notice = recv_pkt->rh.r.ossp;
4469         data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
4470
4471         /* the first 64-bit word of data is the vf_num */
4472         vf_num = data[0];
4473         octeon_swap_8B_data(&vf_num, 1);
4474         vf_idx = (int)vf_num - 1;
4475
4476         cores_crashed = READ_ONCE(oct->cores_crashed);
4477
4478         if (notice == VF_DRV_LOADED) {
4479                 if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
4480                         oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
4481                         dev_info(&oct->pci_dev->dev,
4482                                  "driver for VF%d was loaded\n", vf_idx);
4483                         if (!cores_crashed)
4484                                 try_module_get(THIS_MODULE);
4485                 }
4486         } else if (notice == VF_DRV_REMOVED) {
4487                 if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4488                         oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4489                         dev_info(&oct->pci_dev->dev,
4490                                  "driver for VF%d was removed\n", vf_idx);
4491                         if (!cores_crashed)
4492                                 module_put(THIS_MODULE);
4493                 }
4494         } else if (notice == VF_DRV_MACADDR_CHANGED) {
4495                 u8 *b = (u8 *)&data[1];
4496
4497                 oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4498                 dev_info(&oct->pci_dev->dev,
4499                          "VF driver changed VF%d's MAC address to %pM\n",
4500                          vf_idx, b + 2);
4501         }
4502
4503         for (i = 0; i < recv_pkt->buffer_count; i++)
4504                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
4505         octeon_free_recv_info(recv_info);
4506
4507         return 0;
4508 }
4509
4510 /**
4511  * \brief Device initialization for each Octeon device that is probed
4512  * @param octeon_dev  octeon device
4513  */
4514 static int octeon_device_init(struct octeon_device *octeon_dev)
4515 {
4516         int j, ret;
4517         int fw_loaded = 0;
4518         char bootcmd[] = "\n";
4519         struct octeon_device_priv *oct_priv =
4520                 (struct octeon_device_priv *)octeon_dev->priv;
4521         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4522
4523         /* Enable access to the octeon device and make its DMA capability
4524          * known to the OS.
4525          */
4526         if (octeon_pci_os_setup(octeon_dev))
4527                 return 1;
4528
4529         atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4530
4531         /* Identify the Octeon type and map the BAR address space. */
4532         if (octeon_chip_specific_setup(octeon_dev)) {
4533                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4534                 return 1;
4535         }
4536
4537         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4538
4539         /* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4540          * since that is what is required for the reference to be removed
4541          * during de-initialization (see 'octeon_destroy_resources').
4542          */
4543         octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4544                                PCI_SLOT(octeon_dev->pci_dev->devfn),
4545                                PCI_FUNC(octeon_dev->pci_dev->devfn),
4546                                true);
4547
4548         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4549
4550         if (OCTEON_CN23XX_PF(octeon_dev)) {
4551                 if (!cn23xx_fw_loaded(octeon_dev)) {
4552                         fw_loaded = 0;
4553                         if (!fw_type_is_none()) {
4554                                 /* Do a soft reset of the Octeon device. */
4555                                 if (octeon_dev->fn_list.soft_reset(octeon_dev))
4556                                         return 1;
4557                                 /* things might have changed */
4558                                 if (!cn23xx_fw_loaded(octeon_dev))
4559                                         fw_loaded = 0;
4560                                 else
4561                                         fw_loaded = 1;
4562                         }
4563                 } else {
4564                         fw_loaded = 1;
4565                 }
4566         } else if (octeon_dev->fn_list.soft_reset(octeon_dev)) {
4567                 return 1;
4568         }
4569
4570         /* Initialize the dispatch mechanism used to push packets arriving on
4571          * Octeon Output queues.
4572          */
4573         if (octeon_init_dispatch_list(octeon_dev))
4574                 return 1;
4575
4576         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4577                                     OPCODE_NIC_CORE_DRV_ACTIVE,
4578                                     octeon_core_drv_init,
4579                                     octeon_dev);
4580
4581         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4582                                     OPCODE_NIC_VF_DRV_NOTICE,
4583                                     octeon_recv_vf_drv_notice, octeon_dev);
4584         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4585         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4586         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4587                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4588
4589         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4590
4591         if (octeon_set_io_queues_off(octeon_dev)) {
4592                 dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4593                 return 1;
4594         }
4595
4596         if (OCTEON_CN23XX_PF(octeon_dev)) {
4597                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4598                 if (ret) {
4599                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4600                         return ret;
4601                 }
4602         }
4603
4604         /* Initialize soft command buffer pool
4605          */
4606         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4607                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4608                 return 1;
4609         }
4610         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4611
4612         /*  Setup the data structures that manage this Octeon's Input queues. */
4613         if (octeon_setup_instr_queues(octeon_dev)) {
4614                 dev_err(&octeon_dev->pci_dev->dev,
4615                         "instruction queue initialization failed\n");
4616                 return 1;
4617         }
4618         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4619
4620         /* Initialize lists to manage the requests of different types that
4621          * arrive from user & kernel applications for this octeon device.
4622          */
4623         if (octeon_setup_response_list(octeon_dev)) {
4624                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4625                 return 1;
4626         }
4627         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4628
4629         if (octeon_setup_output_queues(octeon_dev)) {
4630                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4631                 return 1;
4632         }
4633
4634         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4635
4636         if (OCTEON_CN23XX_PF(octeon_dev)) {
4637                 if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4638                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4639                         return 1;
4640                 }
4641                 atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4642
4643                 if (octeon_allocate_ioq_vector(octeon_dev)) {
4644                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4645                         return 1;
4646                 }
4647                 atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
4648
4649         } else {
4650                 /* The input and output queue registers were setup earlier (the
4651                  * queues were not enabled). Any additional registers
4652                  * that need to be programmed should be done now.
4653                  */
4654                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4655                 if (ret) {
4656                         dev_err(&octeon_dev->pci_dev->dev,
4657                                 "Failed to configure device registers\n");
4658                         return ret;
4659                 }
4660         }
4661
4662         /* Initialize the tasklet that handles output queue packet processing.*/
4663         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4664         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
4665                      (unsigned long)octeon_dev);
4666
4667         /* Setup the interrupt handler and record the INT SUM register address
4668          */
4669         if (octeon_setup_interrupt(octeon_dev))
4670                 return 1;
4671
4672         /* Enable Octeon device interrupts */
4673         octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4674
4675         atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4676
4677         /* Enable the input and output queues for this Octeon device */
4678         ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4679         if (ret) {
4680                 dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4681                 return ret;
4682         }
4683
4684         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4685
4686         if ((!OCTEON_CN23XX_PF(octeon_dev)) || !fw_loaded) {
4687                 dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4688                 if (!ddr_timeout) {
4689                         dev_info(&octeon_dev->pci_dev->dev,
4690                                  "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4691                 }
4692
4693                 schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4694
4695                 /* Wait for the octeon to initialize DDR after the soft-reset.*/
4696                 while (!ddr_timeout) {
4697                         set_current_state(TASK_INTERRUPTIBLE);
4698                         if (schedule_timeout(HZ / 10)) {
4699                                 /* user probably pressed Control-C */
4700                                 return 1;
4701                         }
4702                 }
4703                 ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4704                 if (ret) {
4705                         dev_err(&octeon_dev->pci_dev->dev,
4706                                 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4707                                 ret);
4708                         return 1;
4709                 }
4710
4711                 if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4712                         dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4713                         return 1;
4714                 }
4715
4716                 /* Divert uboot to take commands from host instead. */
4717                 ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4718
4719                 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4720                 ret = octeon_init_consoles(octeon_dev);
4721                 if (ret) {
4722                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4723                         return 1;
4724                 }
4725                 ret = octeon_add_console(octeon_dev, 0);
4726                 if (ret) {
4727                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4728                         return 1;
4729                 }
4730
4731                 atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4732
4733                 dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4734                 ret = load_firmware(octeon_dev);
4735                 if (ret) {
4736                         dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4737                         return 1;
4738                 }
4739                 /* set bit 1 of SLI_SCRATCH_1 to indicate that firmware is
4740                  * loaded
4741                  */
4742                 if (OCTEON_CN23XX_PF(octeon_dev))
4743                         octeon_write_csr64(octeon_dev, CN23XX_SLI_SCRATCH1,
4744                                            2ULL);
4745         }
4746
4747         handshake[octeon_dev->octeon_id].init_ok = 1;
4748         complete(&handshake[octeon_dev->octeon_id].init);
4749
4750         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4751
4752         /* Send Credit for Octeon Output queues. Credits are always sent after
4753          * the output queue is enabled.
4754          */
4755         for (j = 0; j < octeon_dev->num_oqs; j++)
4756                 writel(octeon_dev->droq[j]->max_count,
4757                        octeon_dev->droq[j]->pkts_credit_reg);
4758
4759         /* Packets can start arriving on the output queues from this point. */
4760         return 0;
4761 }
4762
4763 /**
4764  * \brief Exits the module
4765  */
4766 static void __exit liquidio_exit(void)
4767 {
4768         liquidio_deinit_pci();
4769
4770         pr_info("LiquidIO network module is now unloaded\n");
4771 }
4772
4773 module_init(liquidio_init);
4774 module_exit(liquidio_exit);