drivers/infiniband/hw/erdma/erdma_main.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3 /* Authors: Cheng Xu <[email protected]> */
   4 /*          Kai Shen <[email protected]> */
   5 /* Copyright (c) 2020-2022, Alibaba Group. */
   6
   7 #include <linux/module.h>
   8 #include <net/addrconf.h>
   9 #include <rdma/erdma-abi.h>
  10
  11 #include "erdma.h"
  12 #include "erdma_cm.h"
  13 #include "erdma_verbs.h"
  14
  15 MODULE_AUTHOR("Cheng Xu <[email protected]>");
  16 MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver");
  17 MODULE_LICENSE("Dual BSD/GPL");
  18
  19 static int erdma_netdev_event(struct notifier_block *nb, unsigned long event,
  20                               void *arg)
  21 {
  22         struct net_device *netdev = netdev_notifier_info_to_dev(arg);
  23         struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb);
  24
  25         if (dev->netdev == NULL || dev->netdev != netdev)
  26                 goto done;
  27
  28         switch (event) {
  29         case NETDEV_CHANGEMTU:
  30                 if (dev->mtu != netdev->mtu) {
  31                         erdma_set_mtu(dev, netdev->mtu);
  32                         dev->mtu = netdev->mtu;
  33                 }
  34                 break;
  35         case NETDEV_REGISTER:
  36         case NETDEV_UNREGISTER:
  37         case NETDEV_CHANGEADDR:
  38         case NETDEV_GOING_DOWN:
  39         case NETDEV_CHANGE:
  40         default:
  41                 break;
  42         }
  43
  44 done:
  45         return NOTIFY_OK;
  46 }
  47
  48 static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
  49 {
  50         struct net_device *netdev;
  51         int ret = -EPROBE_DEFER;
  52
  53         /* Already binded to a net_device, so we skip. */
  54         if (dev->netdev)
  55                 return 0;
  56
  57         rtnl_lock();
  58         for_each_netdev(&init_net, netdev) {
  59                 /*
  60                  * In erdma, the paired netdev and ibdev should have the same
  61                  * MAC address. erdma can get the value from its PCIe bar
  62                  * registers. Since erdma can not get the paired netdev
  63                  * reference directly, we do a traverse here to get the paired
  64                  * netdev.
  65                  */
  66                 if (ether_addr_equal_unaligned(netdev->perm_addr,
  67                                                dev->attrs.peer_addr)) {
  68                         ret = ib_device_set_netdev(&dev->ibdev, netdev, 1);
  69                         if (ret) {
  70                                 rtnl_unlock();
  71                                 ibdev_warn(&dev->ibdev,
  72                                            "failed (%d) to link netdev", ret);
  73                                 return ret;
  74                         }
  75
  76                         dev->netdev = netdev;
  77                         break;
  78                 }
  79         }
  80
  81         rtnl_unlock();
  82
  83         return ret;
  84 }
  85
  86 static int erdma_device_register(struct erdma_dev *dev)
  87 {
  88         struct ib_device *ibdev = &dev->ibdev;
  89         int ret;
  90
  91         ret = erdma_enum_and_get_netdev(dev);
  92         if (ret)
  93                 return ret;
  94
  95         dev->mtu = dev->netdev->mtu;
  96         addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr);
  97
  98         ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev);
  99         if (ret) {
 100                 dev_err(&dev->pdev->dev,
 101                         "ib_register_device failed: ret = %d\n", ret);
 102                 return ret;
 103         }
 104
 105         dev->netdev_nb.notifier_call = erdma_netdev_event;
 106         ret = register_netdevice_notifier(&dev->netdev_nb);
 107         if (ret) {
 108                 ibdev_err(&dev->ibdev, "failed to register notifier.\n");
 109                 ib_unregister_device(ibdev);
 110         }
 111
 112         return ret;
 113 }
 114
 115 static irqreturn_t erdma_comm_irq_handler(int irq, void *data)
 116 {
 117         struct erdma_dev *dev = data;
 118
 119         erdma_cmdq_completion_handler(&dev->cmdq);
 120         erdma_aeq_event_handler(dev);
 121
 122         return IRQ_HANDLED;
 123 }
 124
 125 static int erdma_request_vectors(struct erdma_dev *dev)
 126 {
 127         int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC);
 128         int ret;
 129
 130         ret = pci_alloc_irq_vectors(dev->pdev, 1, expect_irq_num, PCI_IRQ_MSIX);
 131         if (ret < 0) {
 132                 dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n",
 133                         ret);
 134                 return ret;
 135         }
 136         dev->attrs.irq_num = ret;
 137
 138         return 0;
 139 }
 140
 141 static int erdma_comm_irq_init(struct erdma_dev *dev)
 142 {
 143         snprintf(dev->comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s",
 144                  pci_name(dev->pdev));
 145         dev->comm_irq.msix_vector =
 146                 pci_irq_vector(dev->pdev, ERDMA_MSIX_VECTOR_CMDQ);
 147
 148         cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev->pdev->bus)),
 149                         &dev->comm_irq.affinity_hint_mask);
 150         irq_set_affinity_hint(dev->comm_irq.msix_vector,
 151                               &dev->comm_irq.affinity_hint_mask);
 152
 153         return request_irq(dev->comm_irq.msix_vector, erdma_comm_irq_handler, 0,
 154                            dev->comm_irq.name, dev);
 155 }
 156
 157 static void erdma_comm_irq_uninit(struct erdma_dev *dev)
 158 {
 159         irq_set_affinity_hint(dev->comm_irq.msix_vector, NULL);
 160         free_irq(dev->comm_irq.msix_vector, dev);
 161 }
 162
 163 static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
 164 {
 165         int ret;
 166
 167         dev->proto = erdma_reg_read32(dev, ERDMA_REGS_DEV_PROTO_REG);
 168
 169         dev->resp_pool = dma_pool_create("erdma_resp_pool", &pdev->dev,
 170                                          ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE,
 171                                          0);
 172         if (!dev->resp_pool)
 173                 return -ENOMEM;
 174
 175         dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev,
 176                                        ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0);
 177         if (!dev->db_pool) {
 178                 ret = -ENOMEM;
 179                 goto destroy_resp_pool;
 180         }
 181
 182         ret = dma_set_mask_and_coherent(&pdev->dev,
 183                                         DMA_BIT_MASK(ERDMA_PCI_WIDTH));
 184         if (ret)
 185                 goto destroy_db_pool;
 186
 187         dma_set_max_seg_size(&pdev->dev, UINT_MAX);
 188
 189         return 0;
 190
 191 destroy_db_pool:
 192         dma_pool_destroy(dev->db_pool);
 193
 194 destroy_resp_pool:
 195         dma_pool_destroy(dev->resp_pool);
 196
 197         return ret;
 198 }
 199
 200 static void erdma_device_uninit(struct erdma_dev *dev)
 201 {
 202         dma_pool_destroy(dev->db_pool);
 203         dma_pool_destroy(dev->resp_pool);
 204 }
 205
 206 static void erdma_hw_reset(struct erdma_dev *dev)
 207 {
 208         u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);
 209
 210         erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
 211 }
 212
 213 static int erdma_wait_hw_init_done(struct erdma_dev *dev)
 214 {
 215         int i;
 216
 217         erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,
 218                           FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));
 219
 220         for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
 221                 if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
 222                                            ERDMA_REG_DEV_ST_INIT_DONE_MASK))
 223                         break;
 224
 225                 msleep(ERDMA_REG_ACCESS_WAIT_MS);
 226         }
 227
 228         if (i == ERDMA_WAIT_DEV_DONE_CNT) {
 229                 dev_err(&dev->pdev->dev, "wait init done failed.\n");
 230                 return -ETIMEDOUT;
 231         }
 232
 233         return 0;
 234 }
 235
 236 static const struct pci_device_id erdma_pci_tbl[] = {
 237         { PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },
 238         {}
 239 };
 240
 241 static int erdma_probe_dev(struct pci_dev *pdev)
 242 {
 243         struct erdma_dev *dev;
 244         int bars, err;
 245         u32 version;
 246
 247         err = pci_enable_device(pdev);
 248         if (err) {
 249                 dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err);
 250                 return err;
 251         }
 252
 253         pci_set_master(pdev);
 254
 255         dev = ib_alloc_device(erdma_dev, ibdev);
 256         if (!dev) {
 257                 dev_err(&pdev->dev, "ib_alloc_device failed\n");
 258                 err = -ENOMEM;
 259                 goto err_disable_device;
 260         }
 261
 262         pci_set_drvdata(pdev, dev);
 263         dev->pdev = pdev;
 264         dev->attrs.numa_node = dev_to_node(&pdev->dev);
 265
 266         bars = pci_select_bars(pdev, IORESOURCE_MEM);
 267         err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
 268         if (bars != ERDMA_BAR_MASK || err) {
 269                 err = err ? err : -EINVAL;
 270                 goto err_ib_device_release;
 271         }
 272
 273         dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR);
 274         dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR);
 275
 276         dev->func_bar =
 277                 devm_ioremap(&pdev->dev, dev->func_bar_addr, dev->func_bar_len);
 278         if (!dev->func_bar) {
 279                 dev_err(&pdev->dev, "devm_ioremap failed.\n");
 280                 err = -EFAULT;
 281                 goto err_release_bars;
 282         }
 283
 284         version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG);
 285         if (version == 0) {
 286                 /* we knows that it is a non-functional function. */
 287                 err = -ENODEV;
 288                 goto err_iounmap_func_bar;
 289         }
 290
 291         err = erdma_device_init(dev, pdev);
 292         if (err)
 293                 goto err_iounmap_func_bar;
 294
 295         err = erdma_request_vectors(dev);
 296         if (err)
 297                 goto err_uninit_device;
 298
 299         err = erdma_comm_irq_init(dev);
 300         if (err)
 301                 goto err_free_vectors;
 302
 303         err = erdma_aeq_init(dev);
 304         if (err)
 305                 goto err_uninit_comm_irq;
 306
 307         err = erdma_cmdq_init(dev);
 308         if (err)
 309                 goto err_uninit_aeq;
 310
 311         err = erdma_wait_hw_init_done(dev);
 312         if (err)
 313                 goto err_uninit_cmdq;
 314
 315         err = erdma_ceqs_init(dev);
 316         if (err)
 317                 goto err_reset_hw;
 318
 319         erdma_finish_cmdq_init(dev);
 320
 321         return 0;
 322
 323 err_reset_hw:
 324         erdma_hw_reset(dev);
 325
 326 err_uninit_cmdq:
 327         erdma_cmdq_destroy(dev);
 328
 329 err_uninit_aeq:
 330         erdma_eq_destroy(dev, &dev->aeq);
 331
 332 err_uninit_comm_irq:
 333         erdma_comm_irq_uninit(dev);
 334
 335 err_free_vectors:
 336         pci_free_irq_vectors(dev->pdev);
 337
 338 err_uninit_device:
 339         erdma_device_uninit(dev);
 340
 341 err_iounmap_func_bar:
 342         devm_iounmap(&pdev->dev, dev->func_bar);
 343
 344 err_release_bars:
 345         pci_release_selected_regions(pdev, bars);
 346
 347 err_ib_device_release:
 348         ib_dealloc_device(&dev->ibdev);
 349
 350 err_disable_device:
 351         pci_disable_device(pdev);
 352
 353         return err;
 354 }
 355
 356 static void erdma_remove_dev(struct pci_dev *pdev)
 357 {
 358         struct erdma_dev *dev = pci_get_drvdata(pdev);
 359
 360         erdma_ceqs_uninit(dev);
 361         erdma_hw_reset(dev);
 362         erdma_cmdq_destroy(dev);
 363         erdma_eq_destroy(dev, &dev->aeq);
 364         erdma_comm_irq_uninit(dev);
 365         pci_free_irq_vectors(dev->pdev);
 366         erdma_device_uninit(dev);
 367
 368         devm_iounmap(&pdev->dev, dev->func_bar);
 369         pci_release_selected_regions(pdev, ERDMA_BAR_MASK);
 370
 371         ib_dealloc_device(&dev->ibdev);
 372
 373         pci_disable_device(pdev);
 374 }
 375
 376 #define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap)
 377
 378 static int erdma_dev_attrs_init(struct erdma_dev *dev)
 379 {
 380         int err;
 381         u64 req_hdr, cap0, cap1;
 382
 383         erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_RDMA,
 384                                 CMDQ_OPCODE_QUERY_DEVICE);
 385
 386         err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,
 387                                   &cap1, true);
 388         if (err)
 389                 return err;
 390
 391         dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0);
 392         dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0);
 393         dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1);
 394         dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0);
 395         dev->attrs.max_gid = 1 << ERDMA_GET_CAP(MAX_GID, cap0);
 396         dev->attrs.max_ah = 1 << ERDMA_GET_CAP(MAX_AH, cap0);
 397         dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1);
 398         dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1);
 399         dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);
 400         dev->attrs.max_mr = dev->attrs.max_qp << 1;
 401         dev->attrs.max_cq = dev->attrs.max_qp << 1;
 402         dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);
 403
 404         dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;
 405         dev->attrs.max_ord = ERDMA_MAX_ORD;
 406         dev->attrs.max_ird = ERDMA_MAX_IRD;
 407         dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE;
 408         dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE;
 409         dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD;
 410         dev->attrs.max_pd = ERDMA_MAX_PD;
 411
 412         dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD;
 413         dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr;
 414         dev->res_cb[ERDMA_RES_TYPE_AH].max_cap = dev->attrs.max_ah;
 415
 416         erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_COMMON,
 417                                 CMDQ_OPCODE_QUERY_FW_INFO);
 418
 419         err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,
 420                                   &cap1, true);
 421         if (!err)
 422                 dev->attrs.fw_version =
 423                         FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0);
 424
 425         return err;
 426 }
 427
 428 static int erdma_device_config(struct erdma_dev *dev)
 429 {
 430         struct erdma_cmdq_config_device_req req = {};
 431
 432         if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB))
 433                 return 0;
 434
 435         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
 436                                 CMDQ_OPCODE_CONF_DEVICE);
 437
 438         req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) |
 439                   FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1);
 440
 441         return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
 442                                    true);
 443 }
 444
 445 static int erdma_res_cb_init(struct erdma_dev *dev)
 446 {
 447         int i, j;
 448
 449         for (i = 0; i < ERDMA_RES_CNT; i++) {
 450                 dev->res_cb[i].next_alloc_idx = 1;
 451                 spin_lock_init(&dev->res_cb[i].lock);
 452                 dev->res_cb[i].bitmap =
 453                         bitmap_zalloc(dev->res_cb[i].max_cap, GFP_KERNEL);
 454                 if (!dev->res_cb[i].bitmap)
 455                         goto err;
 456         }
 457
 458         return 0;
 459
 460 err:
 461         for (j = 0; j < i; j++)
 462                 bitmap_free(dev->res_cb[j].bitmap);
 463
 464         return -ENOMEM;
 465 }
 466
 467 static void erdma_res_cb_free(struct erdma_dev *dev)
 468 {
 469         int i;
 470
 471         for (i = 0; i < ERDMA_RES_CNT; i++)
 472                 bitmap_free(dev->res_cb[i].bitmap);
 473 }
 474
 475 static const struct ib_device_ops erdma_device_ops_rocev2 = {
 476         .get_link_layer = erdma_get_link_layer,
 477         .add_gid = erdma_add_gid,
 478         .del_gid = erdma_del_gid,
 479         .query_pkey = erdma_query_pkey,
 480         .create_ah = erdma_create_ah,
 481         .destroy_ah = erdma_destroy_ah,
 482         .query_ah = erdma_query_ah,
 483
 484         INIT_RDMA_OBJ_SIZE(ib_ah, erdma_ah, ibah),
 485 };
 486
 487 static const struct ib_device_ops erdma_device_ops_iwarp = {
 488         .iw_accept = erdma_accept,
 489         .iw_add_ref = erdma_qp_get_ref,
 490         .iw_connect = erdma_connect,
 491         .iw_create_listen = erdma_create_listen,
 492         .iw_destroy_listen = erdma_destroy_listen,
 493         .iw_get_qp = erdma_get_ibqp,
 494         .iw_reject = erdma_reject,
 495         .iw_rem_ref = erdma_qp_put_ref,
 496 };
 497
 498 static const struct ib_device_ops erdma_device_ops = {
 499         .owner = THIS_MODULE,
 500         .driver_id = RDMA_DRIVER_ERDMA,
 501         .uverbs_abi_ver = ERDMA_ABI_VERSION,
 502
 503         .alloc_hw_port_stats = erdma_alloc_hw_port_stats,
 504         .alloc_mr = erdma_ib_alloc_mr,
 505         .alloc_pd = erdma_alloc_pd,
 506         .alloc_ucontext = erdma_alloc_ucontext,
 507         .create_cq = erdma_create_cq,
 508         .create_qp = erdma_create_qp,
 509         .dealloc_pd = erdma_dealloc_pd,
 510         .dealloc_ucontext = erdma_dealloc_ucontext,
 511         .dereg_mr = erdma_dereg_mr,
 512         .destroy_cq = erdma_destroy_cq,
 513         .destroy_qp = erdma_destroy_qp,
 514         .disassociate_ucontext = erdma_disassociate_ucontext,
 515         .get_dma_mr = erdma_get_dma_mr,
 516         .get_hw_stats = erdma_get_hw_stats,
 517         .get_port_immutable = erdma_get_port_immutable,
 518         .map_mr_sg = erdma_map_mr_sg,
 519         .mmap = erdma_mmap,
 520         .mmap_free = erdma_mmap_free,
 521         .post_recv = erdma_post_recv,
 522         .post_send = erdma_post_send,
 523         .poll_cq = erdma_poll_cq,
 524         .query_device = erdma_query_device,
 525         .query_gid = erdma_query_gid,
 526         .query_port = erdma_query_port,
 527         .query_qp = erdma_query_qp,
 528         .req_notify_cq = erdma_req_notify_cq,
 529         .reg_user_mr = erdma_reg_user_mr,
 530         .modify_qp = erdma_modify_qp,
 531
 532         INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),
 533         INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd),
 534         INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext),
 535         INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp),
 536 };
 537
 538 static int erdma_ib_device_add(struct pci_dev *pdev)
 539 {
 540         struct erdma_dev *dev = pci_get_drvdata(pdev);
 541         struct ib_device *ibdev = &dev->ibdev;
 542         u64 mac;
 543         int ret;
 544
 545         ret = erdma_dev_attrs_init(dev);
 546         if (ret)
 547                 return ret;
 548
 549         ret = erdma_device_config(dev);
 550         if (ret)
 551                 return ret;
 552
 553         if (erdma_device_iwarp(dev)) {
 554                 ibdev->node_type = RDMA_NODE_RNIC;
 555                 ib_set_device_ops(ibdev, &erdma_device_ops_iwarp);
 556         } else {
 557                 ibdev->node_type = RDMA_NODE_IB_CA;
 558                 ib_set_device_ops(ibdev, &erdma_device_ops_rocev2);
 559         }
 560
 561         memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC));
 562
 563         /*
 564          * Current model (one-to-one device association):
 565          * One ERDMA device per net_device or, equivalently,
 566          * per physical port.
 567          */
 568         ibdev->phys_port_cnt = 1;
 569         ibdev->num_comp_vectors = dev->attrs.irq_num - 1;
 570
 571         ib_set_device_ops(ibdev, &erdma_device_ops);
 572
 573         INIT_LIST_HEAD(&dev->cep_list);
 574
 575         spin_lock_init(&dev->lock);
 576         xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1);
 577         xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1);
 578         dev->next_alloc_cqn = 1;
 579         dev->next_alloc_qpn = 1;
 580
 581         ret = erdma_res_cb_init(dev);
 582         if (ret)
 583                 return ret;
 584
 585         atomic_set(&dev->num_ctx, 0);
 586
 587         mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG);
 588         mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32;
 589
 590         u64_to_ether_addr(mac, dev->attrs.peer_addr);
 591
 592         dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,
 593                                           WQ_UNBOUND_MAX_ACTIVE);
 594         if (!dev->reflush_wq) {
 595                 ret = -ENOMEM;
 596                 goto err_alloc_workqueue;
 597         }
 598
 599         ret = erdma_device_register(dev);
 600         if (ret)
 601                 goto err_register;
 602
 603         return 0;
 604
 605 err_register:
 606         destroy_workqueue(dev->reflush_wq);
 607 err_alloc_workqueue:
 608         xa_destroy(&dev->qp_xa);
 609         xa_destroy(&dev->cq_xa);
 610
 611         erdma_res_cb_free(dev);
 612
 613         return ret;
 614 }
 615
 616 static void erdma_ib_device_remove(struct pci_dev *pdev)
 617 {
 618         struct erdma_dev *dev = pci_get_drvdata(pdev);
 619
 620         unregister_netdevice_notifier(&dev->netdev_nb);
 621         ib_unregister_device(&dev->ibdev);
 622
 623         destroy_workqueue(dev->reflush_wq);
 624         erdma_res_cb_free(dev);
 625         xa_destroy(&dev->qp_xa);
 626         xa_destroy(&dev->cq_xa);
 627 }
 628
 629 static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 630 {
 631         int ret;
 632
 633         ret = erdma_probe_dev(pdev);
 634         if (ret)
 635                 return ret;
 636
 637         ret = erdma_ib_device_add(pdev);
 638         if (ret) {
 639                 erdma_remove_dev(pdev);
 640                 return ret;
 641         }
 642
 643         return 0;
 644 }
 645
 646 static void erdma_remove(struct pci_dev *pdev)
 647 {
 648         erdma_ib_device_remove(pdev);
 649         erdma_remove_dev(pdev);
 650 }
 651
 652 static struct pci_driver erdma_pci_driver = {
 653         .name = DRV_MODULE_NAME,
 654         .id_table = erdma_pci_tbl,
 655         .probe = erdma_probe,
 656         .remove = erdma_remove
 657 };
 658
 659 MODULE_DEVICE_TABLE(pci, erdma_pci_tbl);
 660
 661 static __init int erdma_init_module(void)
 662 {
 663         int ret;
 664
 665         ret = erdma_cm_init();
 666         if (ret)
 667                 return ret;
 668
 669         ret = pci_register_driver(&erdma_pci_driver);
 670         if (ret)
 671                 erdma_cm_exit();
 672
 673         return ret;
 674 }
 675
 676 static void __exit erdma_exit_module(void)
 677 {
 678         pci_unregister_driver(&erdma_pci_driver);
 679
 680         erdma_cm_exit();
 681 }
 682
 683 module_init(erdma_init_module);
 684 module_exit(erdma_exit_module);