drivers/net/ethernet/meta/fbnic/fbnic_pci.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */
   3
   4 #include <linux/init.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/rtnetlink.h>
   8 #include <linux/types.h>
   9
  10 #include "fbnic.h"
  11 #include "fbnic_drvinfo.h"
  12 #include "fbnic_hw_stats.h"
  13 #include "fbnic_netdev.h"
  14
  15 char fbnic_driver_name[] = DRV_NAME;
  16
  17 MODULE_DESCRIPTION(DRV_SUMMARY);
  18 MODULE_LICENSE("GPL");
  19
  20 static const struct fbnic_info fbnic_asic_info = {
  21         .max_num_queues = FBNIC_MAX_QUEUES,
  22         .bar_mask = BIT(0) | BIT(4)
  23 };
  24
  25 static const struct fbnic_info *fbnic_info_tbl[] = {
  26         [fbnic_board_asic] = &fbnic_asic_info,
  27 };
  28
  29 static const struct pci_device_id fbnic_pci_tbl[] = {
  30         { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) },
  31         /* Required last entry */
  32         {0, }
  33 };
  34 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);
  35
  36 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)
  37 {
  38         u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
  39         u32 value;
  40
  41         if (!csr)
  42                 return ~0U;
  43
  44         value = readl(csr + reg);
  45
  46         /* If any bits are 0 value should be valid */
  47         if (~value)
  48                 return value;
  49
  50         /* All 1's may be valid if ZEROs register still works */
  51         if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))
  52                 return value;
  53
  54         /* Hardware is giving us all 1's reads, assume it is gone */
  55         WRITE_ONCE(fbd->uc_addr0, NULL);
  56         WRITE_ONCE(fbd->uc_addr4, NULL);
  57
  58         dev_err(fbd->dev,
  59                 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
  60                 reg, reg << 2);
  61
  62         /* Notify stack that device has lost (PCIe) link */
  63         if (!fbnic_init_failure(fbd))
  64                 netif_device_detach(fbd->netdev);
  65
  66         return ~0U;
  67 }
  68
  69 bool fbnic_fw_present(struct fbnic_dev *fbd)
  70 {
  71         return !!READ_ONCE(fbd->uc_addr4);
  72 }
  73
  74 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)
  75 {
  76         u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
  77
  78         if (csr)
  79                 writel(val, csr + reg);
  80 }
  81
  82 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg)
  83 {
  84         u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
  85         u32 value;
  86
  87         if (!csr)
  88                 return ~0U;
  89
  90         value = readl(csr + reg);
  91
  92         /* If any bits are 0 value should be valid */
  93         if (~value)
  94                 return value;
  95
  96         /* All 1's may be valid if ZEROs register still works */
  97         if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG))
  98                 return value;
  99
 100         /* Hardware is giving us all 1's reads, assume it is gone */
 101         WRITE_ONCE(fbd->uc_addr0, NULL);
 102         WRITE_ONCE(fbd->uc_addr4, NULL);
 103
 104         dev_err(fbd->dev,
 105                 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
 106                 reg, reg << 2);
 107
 108         /* Notify stack that device has lost (PCIe) link */
 109         if (!fbnic_init_failure(fbd))
 110                 netif_device_detach(fbd->netdev);
 111
 112         return ~0U;
 113 }
 114
 115 static void fbnic_service_task_start(struct fbnic_net *fbn)
 116 {
 117         struct fbnic_dev *fbd = fbn->fbd;
 118
 119         schedule_delayed_work(&fbd->service_task, HZ);
 120         phylink_resume(fbn->phylink);
 121 }
 122
 123 static void fbnic_service_task_stop(struct fbnic_net *fbn)
 124 {
 125         struct fbnic_dev *fbd = fbn->fbd;
 126
 127         phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd));
 128         cancel_delayed_work(&fbd->service_task);
 129 }
 130
 131 void fbnic_up(struct fbnic_net *fbn)
 132 {
 133         fbnic_enable(fbn);
 134
 135         fbnic_fill(fbn);
 136
 137         fbnic_rss_reinit_hw(fbn->fbd, fbn);
 138
 139         __fbnic_set_rx_mode(fbn->netdev);
 140
 141         /* Enable Tx/Rx processing */
 142         fbnic_napi_enable(fbn);
 143         netif_tx_start_all_queues(fbn->netdev);
 144
 145         fbnic_service_task_start(fbn);
 146 }
 147
 148 void fbnic_down_noidle(struct fbnic_net *fbn)
 149 {
 150         fbnic_service_task_stop(fbn);
 151
 152         /* Disable Tx/Rx Processing */
 153         fbnic_napi_disable(fbn);
 154         netif_tx_disable(fbn->netdev);
 155
 156         fbnic_clear_rx_mode(fbn->netdev);
 157         fbnic_clear_rules(fbn->fbd);
 158         fbnic_rss_disable_hw(fbn->fbd);
 159         fbnic_disable(fbn);
 160 }
 161
 162 void fbnic_down(struct fbnic_net *fbn)
 163 {
 164         fbnic_down_noidle(fbn);
 165
 166         fbnic_wait_all_queues_idle(fbn->fbd, false);
 167
 168         fbnic_flush(fbn);
 169 }
 170
 171 static void fbnic_health_check(struct fbnic_dev *fbd)
 172 {
 173         struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];
 174
 175         /* As long as the heart is beating the FW is healty */
 176         if (fbd->fw_heartbeat_enabled)
 177                 return;
 178
 179         /* If the Tx mailbox still has messages sitting in it then there likely
 180          * isn't anything we can do. We will wait until the mailbox is empty to
 181          * report the fault so we can collect the crashlog.
 182          */
 183         if (tx_mbx->head != tx_mbx->tail)
 184                 return;
 185
 186         /* TBD: Need to add a more thorough recovery here.
 187          *      Specifically I need to verify what all the firmware will have
 188          *      changed since we had setup and it rebooted. May just need to
 189          *      perform a down/up. For now we will just reclaim ownership so
 190          *      the heartbeat can catch the next fault.
 191          */
 192         fbnic_fw_xmit_ownership_msg(fbd, true);
 193 }
 194
 195 static void fbnic_service_task(struct work_struct *work)
 196 {
 197         struct fbnic_dev *fbd = container_of(to_delayed_work(work),
 198                                              struct fbnic_dev, service_task);
 199
 200         rtnl_lock();
 201
 202         fbnic_get_hw_stats32(fbd);
 203
 204         fbnic_fw_check_heartbeat(fbd);
 205
 206         fbnic_health_check(fbd);
 207
 208         if (netif_carrier_ok(fbd->netdev))
 209                 fbnic_napi_depletion_check(fbd->netdev);
 210
 211         if (netif_running(fbd->netdev))
 212                 schedule_delayed_work(&fbd->service_task, HZ);
 213
 214         rtnl_unlock();
 215 }
 216
 217 /**
 218  * fbnic_probe - Device Initialization Routine
 219  * @pdev: PCI device information struct
 220  * @ent: entry in fbnic_pci_tbl
 221  *
 222  * Initializes a PCI device identified by a pci_dev structure.
 223  * The OS initialization, configuring of the adapter private structure,
 224  * and a hardware reset occur.
 225  *
 226  * Return: 0 on success, negative on failure
 227  **/
 228 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 229 {
 230         const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data];
 231         struct net_device *netdev;
 232         struct fbnic_dev *fbd;
 233         int err;
 234
 235         if (pdev->error_state != pci_channel_io_normal) {
 236                 dev_err(&pdev->dev,
 237                         "PCI device still in an error state. Unable to load...\n");
 238                 return -EIO;
 239         }
 240
 241         err = pcim_enable_device(pdev);
 242         if (err) {
 243                 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
 244                 return err;
 245         }
 246
 247         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
 248         if (err)
 249                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 250         if (err) {
 251                 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
 252                 return err;
 253         }
 254
 255         err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name);
 256         if (err) {
 257                 dev_err(&pdev->dev,
 258                         "pci_request_selected_regions failed: %d\n", err);
 259                 return err;
 260         }
 261
 262         fbd = fbnic_devlink_alloc(pdev);
 263         if (!fbd) {
 264                 dev_err(&pdev->dev, "Devlink allocation failed\n");
 265                 return -ENOMEM;
 266         }
 267
 268         /* Populate driver with hardware-specific info and handlers */
 269         fbd->max_num_queues = info->max_num_queues;
 270
 271         pci_set_master(pdev);
 272         pci_save_state(pdev);
 273
 274         INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task);
 275
 276         err = fbnic_alloc_irqs(fbd);
 277         if (err)
 278                 goto free_fbd;
 279
 280         err = fbnic_mac_init(fbd);
 281         if (err) {
 282                 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);
 283                 goto free_irqs;
 284         }
 285
 286         err = fbnic_fw_enable_mbx(fbd);
 287         if (err) {
 288                 dev_err(&pdev->dev,
 289                         "Firmware mailbox initialization failure\n");
 290                 goto free_irqs;
 291         }
 292
 293         fbnic_devlink_register(fbd);
 294         fbnic_dbg_fbd_init(fbd);
 295
 296         /* Capture snapshot of hardware stats so netdev can calculate delta */
 297         fbnic_reset_hw_stats(fbd);
 298
 299         fbnic_hwmon_register(fbd);
 300
 301         if (!fbd->dsn) {
 302                 dev_warn(&pdev->dev, "Reading serial number failed\n");
 303                 goto init_failure_mode;
 304         }
 305
 306         netdev = fbnic_netdev_alloc(fbd);
 307         if (!netdev) {
 308                 dev_err(&pdev->dev, "Netdev allocation failed\n");
 309                 goto init_failure_mode;
 310         }
 311
 312         err = fbnic_ptp_setup(fbd);
 313         if (err)
 314                 goto ifm_free_netdev;
 315
 316         err = fbnic_netdev_register(netdev);
 317         if (err) {
 318                 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
 319                 goto ifm_destroy_ptp;
 320         }
 321
 322         return 0;
 323
 324 ifm_destroy_ptp:
 325         fbnic_ptp_destroy(fbd);
 326 ifm_free_netdev:
 327         fbnic_netdev_free(fbd);
 328 init_failure_mode:
 329         dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n");
 330          /* Always return 0 even on error so devlink is registered to allow
 331           * firmware updates for fixes.
 332           */
 333         return 0;
 334 free_irqs:
 335         fbnic_free_irqs(fbd);
 336 free_fbd:
 337         fbnic_devlink_free(fbd);
 338
 339         return err;
 340 }
 341
 342 /**
 343  * fbnic_remove - Device Removal Routine
 344  * @pdev: PCI device information struct
 345  *
 346  * Called by the PCI subsystem to alert the driver that it should release
 347  * a PCI device.  The could be caused by a Hot-Plug event, or because the
 348  * driver is going to be removed from memory.
 349  **/
 350 static void fbnic_remove(struct pci_dev *pdev)
 351 {
 352         struct fbnic_dev *fbd = pci_get_drvdata(pdev);
 353
 354         if (!fbnic_init_failure(fbd)) {
 355                 struct net_device *netdev = fbd->netdev;
 356
 357                 fbnic_netdev_unregister(netdev);
 358                 cancel_delayed_work_sync(&fbd->service_task);
 359                 fbnic_ptp_destroy(fbd);
 360                 fbnic_netdev_free(fbd);
 361         }
 362
 363         fbnic_hwmon_unregister(fbd);
 364         fbnic_dbg_fbd_exit(fbd);
 365         fbnic_devlink_unregister(fbd);
 366         fbnic_fw_disable_mbx(fbd);
 367         fbnic_free_irqs(fbd);
 368
 369         fbnic_devlink_free(fbd);
 370 }
 371
 372 static int fbnic_pm_suspend(struct device *dev)
 373 {
 374         struct fbnic_dev *fbd = dev_get_drvdata(dev);
 375         struct net_device *netdev = fbd->netdev;
 376
 377         if (fbnic_init_failure(fbd))
 378                 goto null_uc_addr;
 379
 380         rtnl_lock();
 381
 382         netif_device_detach(netdev);
 383
 384         if (netif_running(netdev))
 385                 netdev->netdev_ops->ndo_stop(netdev);
 386
 387         rtnl_unlock();
 388
 389 null_uc_addr:
 390         fbnic_fw_disable_mbx(fbd);
 391
 392         /* Free the IRQs so they aren't trying to occupy sleeping CPUs */
 393         fbnic_free_irqs(fbd);
 394
 395         /* Hardware is about to go away, so switch off MMIO access internally */
 396         WRITE_ONCE(fbd->uc_addr0, NULL);
 397         WRITE_ONCE(fbd->uc_addr4, NULL);
 398
 399         return 0;
 400 }
 401
 402 static int __fbnic_pm_resume(struct device *dev)
 403 {
 404         struct fbnic_dev *fbd = dev_get_drvdata(dev);
 405         struct net_device *netdev = fbd->netdev;
 406         void __iomem * const *iomap_table;
 407         struct fbnic_net *fbn;
 408         int err;
 409
 410         /* Restore MMIO access */
 411         iomap_table = pcim_iomap_table(to_pci_dev(dev));
 412         fbd->uc_addr0 = iomap_table[0];
 413         fbd->uc_addr4 = iomap_table[4];
 414
 415         /* Rerequest the IRQs */
 416         err = fbnic_alloc_irqs(fbd);
 417         if (err)
 418                 goto err_invalidate_uc_addr;
 419
 420         fbd->mac->init_regs(fbd);
 421
 422         /* Re-enable mailbox */
 423         err = fbnic_fw_enable_mbx(fbd);
 424         if (err)
 425                 goto err_free_irqs;
 426
 427         /* No netdev means there isn't a network interface to bring up */
 428         if (fbnic_init_failure(fbd))
 429                 return 0;
 430
 431         fbn = netdev_priv(netdev);
 432
 433         /* Reset the queues if needed */
 434         fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
 435
 436         rtnl_lock();
 437
 438         if (netif_running(netdev)) {
 439                 err = __fbnic_open(fbn);
 440                 if (err)
 441                         goto err_disable_mbx;
 442         }
 443
 444         rtnl_unlock();
 445
 446         return 0;
 447 err_disable_mbx:
 448         rtnl_unlock();
 449         fbnic_fw_disable_mbx(fbd);
 450 err_free_irqs:
 451         fbnic_free_irqs(fbd);
 452 err_invalidate_uc_addr:
 453         WRITE_ONCE(fbd->uc_addr0, NULL);
 454         WRITE_ONCE(fbd->uc_addr4, NULL);
 455         return err;
 456 }
 457
 458 static void __fbnic_pm_attach(struct device *dev)
 459 {
 460         struct fbnic_dev *fbd = dev_get_drvdata(dev);
 461         struct net_device *netdev = fbd->netdev;
 462         struct fbnic_net *fbn;
 463
 464         if (fbnic_init_failure(fbd))
 465                 return;
 466
 467         fbn = netdev_priv(netdev);
 468
 469         if (netif_running(netdev))
 470                 fbnic_up(fbn);
 471
 472         netif_device_attach(netdev);
 473 }
 474
 475 static int __maybe_unused fbnic_pm_resume(struct device *dev)
 476 {
 477         int err;
 478
 479         err = __fbnic_pm_resume(dev);
 480         if (!err)
 481                 __fbnic_pm_attach(dev);
 482
 483         return err;
 484 }
 485
 486 static const struct dev_pm_ops fbnic_pm_ops = {
 487         SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume)
 488 };
 489
 490 static void fbnic_shutdown(struct pci_dev *pdev)
 491 {
 492         fbnic_pm_suspend(&pdev->dev);
 493 }
 494
 495 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev,
 496                                                  pci_channel_state_t state)
 497 {
 498         /* Disconnect device if failure is not recoverable via reset */
 499         if (state == pci_channel_io_perm_failure)
 500                 return PCI_ERS_RESULT_DISCONNECT;
 501
 502         fbnic_pm_suspend(&pdev->dev);
 503
 504         /* Request a slot reset */
 505         return PCI_ERS_RESULT_NEED_RESET;
 506 }
 507
 508 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev)
 509 {
 510         int err;
 511
 512         pci_set_power_state(pdev, PCI_D0);
 513         pci_restore_state(pdev);
 514         pci_save_state(pdev);
 515
 516         if (pci_enable_device_mem(pdev)) {
 517                 dev_err(&pdev->dev,
 518                         "Cannot re-enable PCI device after reset.\n");
 519                 return PCI_ERS_RESULT_DISCONNECT;
 520         }
 521
 522         /* Restore device to previous state */
 523         err = __fbnic_pm_resume(&pdev->dev);
 524
 525         return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
 526 }
 527
 528 static void fbnic_err_resume(struct pci_dev *pdev)
 529 {
 530         __fbnic_pm_attach(&pdev->dev);
 531 }
 532
 533 static const struct pci_error_handlers fbnic_err_handler = {
 534         .error_detected = fbnic_err_error_detected,
 535         .slot_reset     = fbnic_err_slot_reset,
 536         .resume         = fbnic_err_resume,
 537 };
 538
 539 static struct pci_driver fbnic_driver = {
 540         .name           = fbnic_driver_name,
 541         .id_table       = fbnic_pci_tbl,
 542         .probe          = fbnic_probe,
 543         .remove         = fbnic_remove,
 544         .driver.pm      = &fbnic_pm_ops,
 545         .shutdown       = fbnic_shutdown,
 546         .err_handler    = &fbnic_err_handler,
 547 };
 548
 549 /**
 550  * fbnic_init_module - Driver Registration Routine
 551  *
 552  * The first routine called when the driver is loaded.  All it does is
 553  * register with the PCI subsystem.
 554  *
 555  * Return: 0 on success, negative on failure
 556  **/
 557 static int __init fbnic_init_module(void)
 558 {
 559         int err;
 560
 561         fbnic_dbg_init();
 562
 563         err = pci_register_driver(&fbnic_driver);
 564         if (err) {
 565                 fbnic_dbg_exit();
 566                 goto out;
 567         }
 568
 569         pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name);
 570 out:
 571         return err;
 572 }
 573 module_init(fbnic_init_module);
 574
 575 /**
 576  * fbnic_exit_module - Driver Exit Cleanup Routine
 577  *
 578  * Called just before the driver is removed from memory.
 579  **/
 580 static void __exit fbnic_exit_module(void)
 581 {
 582         pci_unregister_driver(&fbnic_driver);
 583
 584         fbnic_dbg_exit();
 585 }
 586 module_exit(fbnic_exit_module);