drivers/net/ethernet/meta/fbnic/fbnic_pci.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */
   3
   4 #include <linux/init.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/rtnetlink.h>
   8 #include <linux/types.h>
   9
  10 #include "fbnic.h"
  11 #include "fbnic_drvinfo.h"
  12 #include "fbnic_netdev.h"
  13
  14 char fbnic_driver_name[] = DRV_NAME;
  15
  16 MODULE_DESCRIPTION(DRV_SUMMARY);
  17 MODULE_LICENSE("GPL");
  18
  19 static const struct fbnic_info fbnic_asic_info = {
  20         .max_num_queues = FBNIC_MAX_QUEUES,
  21         .bar_mask = BIT(0) | BIT(4)
  22 };
  23
  24 static const struct fbnic_info *fbnic_info_tbl[] = {
  25         [fbnic_board_asic] = &fbnic_asic_info,
  26 };
  27
  28 static const struct pci_device_id fbnic_pci_tbl[] = {
  29         { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) },
  30         /* Required last entry */
  31         {0, }
  32 };
  33 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);
  34
  35 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)
  36 {
  37         u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
  38         u32 value;
  39
  40         if (!csr)
  41                 return ~0U;
  42
  43         value = readl(csr + reg);
  44
  45         /* If any bits are 0 value should be valid */
  46         if (~value)
  47                 return value;
  48
  49         /* All 1's may be valid if ZEROs register still works */
  50         if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))
  51                 return value;
  52
  53         /* Hardware is giving us all 1's reads, assume it is gone */
  54         WRITE_ONCE(fbd->uc_addr0, NULL);
  55         WRITE_ONCE(fbd->uc_addr4, NULL);
  56
  57         dev_err(fbd->dev,
  58                 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
  59                 reg, reg << 2);
  60
  61         /* Notify stack that device has lost (PCIe) link */
  62         if (!fbnic_init_failure(fbd))
  63                 netif_device_detach(fbd->netdev);
  64
  65         return ~0U;
  66 }
  67
  68 bool fbnic_fw_present(struct fbnic_dev *fbd)
  69 {
  70         return !!READ_ONCE(fbd->uc_addr4);
  71 }
  72
  73 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)
  74 {
  75         u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
  76
  77         if (csr)
  78                 writel(val, csr + reg);
  79 }
  80
  81 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg)
  82 {
  83         u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
  84         u32 value;
  85
  86         if (!csr)
  87                 return ~0U;
  88
  89         value = readl(csr + reg);
  90
  91         /* If any bits are 0 value should be valid */
  92         if (~value)
  93                 return value;
  94
  95         /* All 1's may be valid if ZEROs register still works */
  96         if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG))
  97                 return value;
  98
  99         /* Hardware is giving us all 1's reads, assume it is gone */
 100         WRITE_ONCE(fbd->uc_addr0, NULL);
 101         WRITE_ONCE(fbd->uc_addr4, NULL);
 102
 103         dev_err(fbd->dev,
 104                 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
 105                 reg, reg << 2);
 106
 107         /* Notify stack that device has lost (PCIe) link */
 108         if (!fbnic_init_failure(fbd))
 109                 netif_device_detach(fbd->netdev);
 110
 111         return ~0U;
 112 }
 113
 114 static void fbnic_service_task_start(struct fbnic_net *fbn)
 115 {
 116         struct fbnic_dev *fbd = fbn->fbd;
 117
 118         schedule_delayed_work(&fbd->service_task, HZ);
 119         phylink_resume(fbn->phylink);
 120 }
 121
 122 static void fbnic_service_task_stop(struct fbnic_net *fbn)
 123 {
 124         struct fbnic_dev *fbd = fbn->fbd;
 125
 126         phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd));
 127         cancel_delayed_work(&fbd->service_task);
 128 }
 129
 130 void fbnic_up(struct fbnic_net *fbn)
 131 {
 132         fbnic_enable(fbn);
 133
 134         fbnic_fill(fbn);
 135
 136         fbnic_rss_reinit_hw(fbn->fbd, fbn);
 137
 138         __fbnic_set_rx_mode(fbn->netdev);
 139
 140         /* Enable Tx/Rx processing */
 141         fbnic_napi_enable(fbn);
 142         netif_tx_start_all_queues(fbn->netdev);
 143
 144         fbnic_service_task_start(fbn);
 145 }
 146
 147 static void fbnic_down_noidle(struct fbnic_net *fbn)
 148 {
 149         fbnic_service_task_stop(fbn);
 150
 151         /* Disable Tx/Rx Processing */
 152         fbnic_napi_disable(fbn);
 153         netif_tx_disable(fbn->netdev);
 154
 155         fbnic_clear_rx_mode(fbn->netdev);
 156         fbnic_clear_rules(fbn->fbd);
 157         fbnic_rss_disable_hw(fbn->fbd);
 158         fbnic_disable(fbn);
 159 }
 160
 161 void fbnic_down(struct fbnic_net *fbn)
 162 {
 163         fbnic_down_noidle(fbn);
 164
 165         fbnic_wait_all_queues_idle(fbn->fbd, false);
 166
 167         fbnic_flush(fbn);
 168 }
 169
 170 static void fbnic_health_check(struct fbnic_dev *fbd)
 171 {
 172         struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];
 173
 174         /* As long as the heart is beating the FW is healty */
 175         if (fbd->fw_heartbeat_enabled)
 176                 return;
 177
 178         /* If the Tx mailbox still has messages sitting in it then there likely
 179          * isn't anything we can do. We will wait until the mailbox is empty to
 180          * report the fault so we can collect the crashlog.
 181          */
 182         if (tx_mbx->head != tx_mbx->tail)
 183                 return;
 184
 185         /* TBD: Need to add a more thorough recovery here.
 186          *      Specifically I need to verify what all the firmware will have
 187          *      changed since we had setup and it rebooted. May just need to
 188          *      perform a down/up. For now we will just reclaim ownership so
 189          *      the heartbeat can catch the next fault.
 190          */
 191         fbnic_fw_xmit_ownership_msg(fbd, true);
 192 }
 193
 194 static void fbnic_service_task(struct work_struct *work)
 195 {
 196         struct fbnic_dev *fbd = container_of(to_delayed_work(work),
 197                                              struct fbnic_dev, service_task);
 198
 199         rtnl_lock();
 200
 201         fbnic_fw_check_heartbeat(fbd);
 202
 203         fbnic_health_check(fbd);
 204
 205         if (netif_carrier_ok(fbd->netdev))
 206                 fbnic_napi_depletion_check(fbd->netdev);
 207
 208         if (netif_running(fbd->netdev))
 209                 schedule_delayed_work(&fbd->service_task, HZ);
 210
 211         rtnl_unlock();
 212 }
 213
 214 /**
 215  * fbnic_probe - Device Initialization Routine
 216  * @pdev: PCI device information struct
 217  * @ent: entry in fbnic_pci_tbl
 218  *
 219  * Initializes a PCI device identified by a pci_dev structure.
 220  * The OS initialization, configuring of the adapter private structure,
 221  * and a hardware reset occur.
 222  *
 223  * Return: 0 on success, negative on failure
 224  **/
 225 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 226 {
 227         const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data];
 228         struct net_device *netdev;
 229         struct fbnic_dev *fbd;
 230         int err;
 231
 232         if (pdev->error_state != pci_channel_io_normal) {
 233                 dev_err(&pdev->dev,
 234                         "PCI device still in an error state. Unable to load...\n");
 235                 return -EIO;
 236         }
 237
 238         err = pcim_enable_device(pdev);
 239         if (err) {
 240                 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
 241                 return err;
 242         }
 243
 244         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
 245         if (err)
 246                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 247         if (err) {
 248                 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
 249                 return err;
 250         }
 251
 252         err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name);
 253         if (err) {
 254                 dev_err(&pdev->dev,
 255                         "pci_request_selected_regions failed: %d\n", err);
 256                 return err;
 257         }
 258
 259         fbd = fbnic_devlink_alloc(pdev);
 260         if (!fbd) {
 261                 dev_err(&pdev->dev, "Devlink allocation failed\n");
 262                 return -ENOMEM;
 263         }
 264
 265         /* Populate driver with hardware-specific info and handlers */
 266         fbd->max_num_queues = info->max_num_queues;
 267
 268         pci_set_master(pdev);
 269         pci_save_state(pdev);
 270
 271         INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task);
 272
 273         err = fbnic_alloc_irqs(fbd);
 274         if (err)
 275                 goto free_fbd;
 276
 277         err = fbnic_mac_init(fbd);
 278         if (err) {
 279                 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);
 280                 goto free_irqs;
 281         }
 282
 283         err = fbnic_fw_enable_mbx(fbd);
 284         if (err) {
 285                 dev_err(&pdev->dev,
 286                         "Firmware mailbox initialization failure\n");
 287                 goto free_irqs;
 288         }
 289
 290         fbnic_devlink_register(fbd);
 291
 292         if (!fbd->dsn) {
 293                 dev_warn(&pdev->dev, "Reading serial number failed\n");
 294                 goto init_failure_mode;
 295         }
 296
 297         netdev = fbnic_netdev_alloc(fbd);
 298         if (!netdev) {
 299                 dev_err(&pdev->dev, "Netdev allocation failed\n");
 300                 goto init_failure_mode;
 301         }
 302
 303         err = fbnic_netdev_register(netdev);
 304         if (err) {
 305                 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
 306                 goto ifm_free_netdev;
 307         }
 308
 309         return 0;
 310
 311 ifm_free_netdev:
 312         fbnic_netdev_free(fbd);
 313 init_failure_mode:
 314         dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n");
 315          /* Always return 0 even on error so devlink is registered to allow
 316           * firmware updates for fixes.
 317           */
 318         return 0;
 319 free_irqs:
 320         fbnic_free_irqs(fbd);
 321 free_fbd:
 322         pci_disable_device(pdev);
 323         fbnic_devlink_free(fbd);
 324
 325         return err;
 326 }
 327
 328 /**
 329  * fbnic_remove - Device Removal Routine
 330  * @pdev: PCI device information struct
 331  *
 332  * Called by the PCI subsystem to alert the driver that it should release
 333  * a PCI device.  The could be caused by a Hot-Plug event, or because the
 334  * driver is going to be removed from memory.
 335  **/
 336 static void fbnic_remove(struct pci_dev *pdev)
 337 {
 338         struct fbnic_dev *fbd = pci_get_drvdata(pdev);
 339
 340         if (!fbnic_init_failure(fbd)) {
 341                 struct net_device *netdev = fbd->netdev;
 342
 343                 fbnic_netdev_unregister(netdev);
 344                 cancel_delayed_work_sync(&fbd->service_task);
 345                 fbnic_netdev_free(fbd);
 346         }
 347
 348         fbnic_devlink_unregister(fbd);
 349         fbnic_fw_disable_mbx(fbd);
 350         fbnic_free_irqs(fbd);
 351
 352         pci_disable_device(pdev);
 353         fbnic_devlink_free(fbd);
 354 }
 355
 356 static int fbnic_pm_suspend(struct device *dev)
 357 {
 358         struct fbnic_dev *fbd = dev_get_drvdata(dev);
 359         struct net_device *netdev = fbd->netdev;
 360
 361         if (fbnic_init_failure(fbd))
 362                 goto null_uc_addr;
 363
 364         rtnl_lock();
 365
 366         netif_device_detach(netdev);
 367
 368         if (netif_running(netdev))
 369                 netdev->netdev_ops->ndo_stop(netdev);
 370
 371         rtnl_unlock();
 372
 373 null_uc_addr:
 374         fbnic_fw_disable_mbx(fbd);
 375
 376         /* Free the IRQs so they aren't trying to occupy sleeping CPUs */
 377         fbnic_free_irqs(fbd);
 378
 379         /* Hardware is about to go away, so switch off MMIO access internally */
 380         WRITE_ONCE(fbd->uc_addr0, NULL);
 381         WRITE_ONCE(fbd->uc_addr4, NULL);
 382
 383         return 0;
 384 }
 385
 386 static int __fbnic_pm_resume(struct device *dev)
 387 {
 388         struct fbnic_dev *fbd = dev_get_drvdata(dev);
 389         struct net_device *netdev = fbd->netdev;
 390         void __iomem * const *iomap_table;
 391         struct fbnic_net *fbn;
 392         int err;
 393
 394         /* Restore MMIO access */
 395         iomap_table = pcim_iomap_table(to_pci_dev(dev));
 396         fbd->uc_addr0 = iomap_table[0];
 397         fbd->uc_addr4 = iomap_table[4];
 398
 399         /* Rerequest the IRQs */
 400         err = fbnic_alloc_irqs(fbd);
 401         if (err)
 402                 goto err_invalidate_uc_addr;
 403
 404         fbd->mac->init_regs(fbd);
 405
 406         /* Re-enable mailbox */
 407         err = fbnic_fw_enable_mbx(fbd);
 408         if (err)
 409                 goto err_free_irqs;
 410
 411         /* No netdev means there isn't a network interface to bring up */
 412         if (fbnic_init_failure(fbd))
 413                 return 0;
 414
 415         fbn = netdev_priv(netdev);
 416
 417         /* Reset the queues if needed */
 418         fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
 419
 420         rtnl_lock();
 421
 422         if (netif_running(netdev)) {
 423                 err = __fbnic_open(fbn);
 424                 if (err)
 425                         goto err_disable_mbx;
 426         }
 427
 428         rtnl_unlock();
 429
 430         return 0;
 431 err_disable_mbx:
 432         rtnl_unlock();
 433         fbnic_fw_disable_mbx(fbd);
 434 err_free_irqs:
 435         fbnic_free_irqs(fbd);
 436 err_invalidate_uc_addr:
 437         WRITE_ONCE(fbd->uc_addr0, NULL);
 438         WRITE_ONCE(fbd->uc_addr4, NULL);
 439         return err;
 440 }
 441
 442 static void __fbnic_pm_attach(struct device *dev)
 443 {
 444         struct fbnic_dev *fbd = dev_get_drvdata(dev);
 445         struct net_device *netdev = fbd->netdev;
 446         struct fbnic_net *fbn;
 447
 448         if (fbnic_init_failure(fbd))
 449                 return;
 450
 451         fbn = netdev_priv(netdev);
 452
 453         if (netif_running(netdev))
 454                 fbnic_up(fbn);
 455
 456         netif_device_attach(netdev);
 457 }
 458
 459 static int __maybe_unused fbnic_pm_resume(struct device *dev)
 460 {
 461         int err;
 462
 463         err = __fbnic_pm_resume(dev);
 464         if (!err)
 465                 __fbnic_pm_attach(dev);
 466
 467         return err;
 468 }
 469
 470 static const struct dev_pm_ops fbnic_pm_ops = {
 471         SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume)
 472 };
 473
 474 static void fbnic_shutdown(struct pci_dev *pdev)
 475 {
 476         fbnic_pm_suspend(&pdev->dev);
 477 }
 478
 479 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev,
 480                                                  pci_channel_state_t state)
 481 {
 482         /* Disconnect device if failure is not recoverable via reset */
 483         if (state == pci_channel_io_perm_failure)
 484                 return PCI_ERS_RESULT_DISCONNECT;
 485
 486         fbnic_pm_suspend(&pdev->dev);
 487
 488         /* Request a slot reset */
 489         return PCI_ERS_RESULT_NEED_RESET;
 490 }
 491
 492 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev)
 493 {
 494         int err;
 495
 496         pci_set_power_state(pdev, PCI_D0);
 497         pci_restore_state(pdev);
 498         pci_save_state(pdev);
 499
 500         if (pci_enable_device_mem(pdev)) {
 501                 dev_err(&pdev->dev,
 502                         "Cannot re-enable PCI device after reset.\n");
 503                 return PCI_ERS_RESULT_DISCONNECT;
 504         }
 505
 506         /* Restore device to previous state */
 507         err = __fbnic_pm_resume(&pdev->dev);
 508
 509         return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
 510 }
 511
 512 static void fbnic_err_resume(struct pci_dev *pdev)
 513 {
 514         __fbnic_pm_attach(&pdev->dev);
 515 }
 516
 517 static const struct pci_error_handlers fbnic_err_handler = {
 518         .error_detected = fbnic_err_error_detected,
 519         .slot_reset     = fbnic_err_slot_reset,
 520         .resume         = fbnic_err_resume,
 521 };
 522
 523 static struct pci_driver fbnic_driver = {
 524         .name           = fbnic_driver_name,
 525         .id_table       = fbnic_pci_tbl,
 526         .probe          = fbnic_probe,
 527         .remove         = fbnic_remove,
 528         .driver.pm      = &fbnic_pm_ops,
 529         .shutdown       = fbnic_shutdown,
 530         .err_handler    = &fbnic_err_handler,
 531 };
 532
 533 /**
 534  * fbnic_init_module - Driver Registration Routine
 535  *
 536  * The first routine called when the driver is loaded.  All it does is
 537  * register with the PCI subsystem.
 538  *
 539  * Return: 0 on success, negative on failure
 540  **/
 541 static int __init fbnic_init_module(void)
 542 {
 543         int err;
 544
 545         err = pci_register_driver(&fbnic_driver);
 546         if (err)
 547                 goto out;
 548
 549         pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name);
 550 out:
 551         return err;
 552 }
 553 module_init(fbnic_init_module);
 554
 555 /**
 556  * fbnic_exit_module - Driver Exit Cleanup Routine
 557  *
 558  * Called just before the driver is removed from memory.
 559  **/
 560 static void __exit fbnic_exit_module(void)
 561 {
 562         pci_unregister_driver(&fbnic_driver);
 563 }
 564 module_exit(fbnic_exit_module);