drivers/infiniband/hw/erdma/erdma_verbs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /* Authors: Cheng Xu <[email protected]> */
   4 /*          Kai Shen <[email protected]> */
   5 /* Copyright (c) 2020-2022, Alibaba Group. */
   6
   7 /* Authors: Bernard Metzler <[email protected]> */
   8 /* Copyright (c) 2008-2019, IBM Corporation */
   9
  10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
  11
  12 #include <linux/vmalloc.h>
  13 #include <net/addrconf.h>
  14 #include <rdma/erdma-abi.h>
  15 #include <rdma/ib_umem.h>
  16 #include <rdma/uverbs_ioctl.h>
  17
  18 #include "erdma.h"
  19 #include "erdma_cm.h"
  20 #include "erdma_verbs.h"
  21
  22 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
  23 {
  24         struct erdma_dev *dev = to_edev(qp->ibqp.device);
  25         struct erdma_pd *pd = to_epd(qp->ibqp.pd);
  26         struct erdma_cmdq_create_qp_req req;
  27         struct erdma_uqp *user_qp;
  28         u64 resp0, resp1;
  29         int err;
  30
  31         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
  32                                 CMDQ_OPCODE_CREATE_QP);
  33
  34         req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
  35                               ilog2(qp->attrs.sq_size)) |
  36                    FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
  37         req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
  38                               ilog2(qp->attrs.rq_size)) |
  39                    FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
  40
  41         if (rdma_is_kernel_res(&qp->ibqp.res)) {
  42                 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
  43
  44                 req.sq_cqn_mtt_cfg =
  45                         FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
  46                                    pgsz_range) |
  47                         FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
  48                 req.rq_cqn_mtt_cfg =
  49                         FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
  50                                    pgsz_range) |
  51                         FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
  52
  53                 req.sq_mtt_cfg =
  54                         FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
  55                         FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
  56                         FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
  57                                    ERDMA_MR_INLINE_MTT);
  58                 req.rq_mtt_cfg = req.sq_mtt_cfg;
  59
  60                 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
  61                 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
  62                 req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr +
  63                                           (qp->attrs.sq_size << SQEBB_SHIFT);
  64                 req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr +
  65                                           (qp->attrs.rq_size << RQE_SHIFT);
  66         } else {
  67                 user_qp = &qp->user_qp;
  68                 req.sq_cqn_mtt_cfg = FIELD_PREP(
  69                         ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
  70                         ilog2(user_qp->sq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
  71                 req.sq_cqn_mtt_cfg |=
  72                         FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
  73
  74                 req.rq_cqn_mtt_cfg = FIELD_PREP(
  75                         ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
  76                         ilog2(user_qp->rq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
  77                 req.rq_cqn_mtt_cfg |=
  78                         FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
  79
  80                 req.sq_mtt_cfg = user_qp->sq_mtt.page_offset;
  81                 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
  82                                              user_qp->sq_mtt.mtt_nents) |
  83                                   FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
  84                                              user_qp->sq_mtt.mtt_type);
  85
  86                 req.rq_mtt_cfg = user_qp->rq_mtt.page_offset;
  87                 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
  88                                              user_qp->rq_mtt.mtt_nents) |
  89                                   FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
  90                                              user_qp->rq_mtt.mtt_type);
  91
  92                 req.sq_buf_addr = user_qp->sq_mtt.mtt_entry[0];
  93                 req.rq_buf_addr = user_qp->rq_mtt.mtt_entry[0];
  94
  95                 req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr;
  96                 req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr;
  97
  98                 if (uctx->ext_db.enable) {
  99                         req.sq_cqn_mtt_cfg |=
 100                                 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
 101                         req.db_cfg =
 102                                 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
 103                                            uctx->ext_db.sdb_off) |
 104                                 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
 105                                            uctx->ext_db.rdb_off);
 106                 }
 107         }
 108
 109         err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
 110                                   &resp1);
 111         if (!err)
 112                 qp->attrs.cookie =
 113                         FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
 114
 115         return err;
 116 }
 117
 118 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
 119 {
 120         struct erdma_cmdq_reg_mr_req req;
 121         struct erdma_pd *pd = to_epd(mr->ibmr.pd);
 122         u64 *phy_addr;
 123         int i;
 124
 125         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
 126
 127         req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
 128                    FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
 129                    FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
 130         req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
 131                    FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
 132                    FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
 133         req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
 134                               ilog2(mr->mem.page_size)) |
 135                    FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) |
 136                    FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
 137
 138         if (mr->type == ERDMA_MR_TYPE_DMA)
 139                 goto post_cmd;
 140
 141         if (mr->type == ERDMA_MR_TYPE_NORMAL) {
 142                 req.start_va = mr->mem.va;
 143                 req.size = mr->mem.len;
 144         }
 145
 146         if (mr->type == ERDMA_MR_TYPE_FRMR ||
 147             mr->mem.mtt_type == ERDMA_MR_INDIRECT_MTT) {
 148                 phy_addr = req.phy_addr;
 149                 *phy_addr = mr->mem.mtt_entry[0];
 150         } else {
 151                 phy_addr = req.phy_addr;
 152                 for (i = 0; i < mr->mem.mtt_nents; i++)
 153                         *phy_addr++ = mr->mem.mtt_entry[i];
 154         }
 155
 156 post_cmd:
 157         return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 158 }
 159
 160 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
 161 {
 162         struct erdma_dev *dev = to_edev(cq->ibcq.device);
 163         struct erdma_cmdq_create_cq_req req;
 164         struct erdma_mem *mtt;
 165         u32 page_size;
 166
 167         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
 168                                 CMDQ_OPCODE_CREATE_CQ);
 169
 170         req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
 171                    FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
 172         req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
 173
 174         if (rdma_is_kernel_res(&cq->ibcq.res)) {
 175                 page_size = SZ_32M;
 176                 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
 177                                        ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
 178                 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
 179                 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
 180
 181                 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
 182                             FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK,
 183                                        ERDMA_MR_INLINE_MTT);
 184
 185                 req.first_page_offset = 0;
 186                 req.cq_db_info_addr =
 187                         cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT);
 188         } else {
 189                 mtt = &cq->user_cq.qbuf_mtt;
 190                 req.cfg0 |=
 191                         FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
 192                                    ilog2(mtt->page_size) - ERDMA_HW_PAGE_SHIFT);
 193                 if (mtt->mtt_nents == 1) {
 194                         req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf);
 195                         req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf);
 196                 } else {
 197                         req.qbuf_addr_l = lower_32_bits(mtt->mtt_entry[0]);
 198                         req.qbuf_addr_h = upper_32_bits(mtt->mtt_entry[0]);
 199                 }
 200                 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
 201                                        mtt->mtt_nents);
 202                 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK,
 203                                        mtt->mtt_type);
 204
 205                 req.first_page_offset = mtt->page_offset;
 206                 req.cq_db_info_addr = cq->user_cq.db_info_dma_addr;
 207
 208                 if (uctx->ext_db.enable) {
 209                         req.cfg1 |= FIELD_PREP(
 210                                 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
 211                         req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
 212                                               uctx->ext_db.cdb_off);
 213                 }
 214         }
 215
 216         return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
 217 }
 218
 219 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
 220 {
 221         int idx;
 222         unsigned long flags;
 223
 224         spin_lock_irqsave(&res_cb->lock, flags);
 225         idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
 226                                  res_cb->next_alloc_idx);
 227         if (idx == res_cb->max_cap) {
 228                 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
 229                 if (idx == res_cb->max_cap) {
 230                         res_cb->next_alloc_idx = 1;
 231                         spin_unlock_irqrestore(&res_cb->lock, flags);
 232                         return -ENOSPC;
 233                 }
 234         }
 235
 236         set_bit(idx, res_cb->bitmap);
 237         res_cb->next_alloc_idx = idx + 1;
 238         spin_unlock_irqrestore(&res_cb->lock, flags);
 239
 240         return idx;
 241 }
 242
 243 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
 244 {
 245         unsigned long flags;
 246         u32 used;
 247
 248         spin_lock_irqsave(&res_cb->lock, flags);
 249         used = __test_and_clear_bit(idx, res_cb->bitmap);
 250         spin_unlock_irqrestore(&res_cb->lock, flags);
 251         WARN_ON(!used);
 252 }
 253
 254 static struct rdma_user_mmap_entry *
 255 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
 256                              u32 size, u8 mmap_flag, u64 *mmap_offset)
 257 {
 258         struct erdma_user_mmap_entry *entry =
 259                 kzalloc(sizeof(*entry), GFP_KERNEL);
 260         int ret;
 261
 262         if (!entry)
 263                 return NULL;
 264
 265         entry->address = (u64)address;
 266         entry->mmap_flag = mmap_flag;
 267
 268         size = PAGE_ALIGN(size);
 269
 270         ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
 271                                           size);
 272         if (ret) {
 273                 kfree(entry);
 274                 return NULL;
 275         }
 276
 277         *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 278
 279         return &entry->rdma_entry;
 280 }
 281
 282 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
 283                        struct ib_udata *unused)
 284 {
 285         struct erdma_dev *dev = to_edev(ibdev);
 286
 287         memset(attr, 0, sizeof(*attr));
 288
 289         attr->max_mr_size = dev->attrs.max_mr_size;
 290         attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
 291         attr->vendor_part_id = dev->pdev->device;
 292         attr->hw_ver = dev->pdev->revision;
 293         attr->max_qp = dev->attrs.max_qp - 1;
 294         attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
 295         attr->max_qp_rd_atom = dev->attrs.max_ord;
 296         attr->max_qp_init_rd_atom = dev->attrs.max_ird;
 297         attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
 298         attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
 299         attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
 300         ibdev->local_dma_lkey = dev->attrs.local_dma_key;
 301         attr->max_send_sge = dev->attrs.max_send_sge;
 302         attr->max_recv_sge = dev->attrs.max_recv_sge;
 303         attr->max_sge_rd = dev->attrs.max_sge_rd;
 304         attr->max_cq = dev->attrs.max_cq - 1;
 305         attr->max_cqe = dev->attrs.max_cqe;
 306         attr->max_mr = dev->attrs.max_mr;
 307         attr->max_pd = dev->attrs.max_pd;
 308         attr->max_mw = dev->attrs.max_mw;
 309         attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
 310         attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
 311
 312         if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
 313                 attr->atomic_cap = IB_ATOMIC_GLOB;
 314
 315         attr->fw_ver = dev->attrs.fw_version;
 316
 317         if (dev->netdev)
 318                 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
 319                                     dev->netdev->dev_addr);
 320
 321         return 0;
 322 }
 323
 324 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
 325                     union ib_gid *gid)
 326 {
 327         struct erdma_dev *dev = to_edev(ibdev);
 328
 329         memset(gid, 0, sizeof(*gid));
 330         ether_addr_copy(gid->raw, dev->attrs.peer_addr);
 331
 332         return 0;
 333 }
 334
 335 int erdma_query_port(struct ib_device *ibdev, u32 port,
 336                      struct ib_port_attr *attr)
 337 {
 338         struct erdma_dev *dev = to_edev(ibdev);
 339         struct net_device *ndev = dev->netdev;
 340
 341         memset(attr, 0, sizeof(*attr));
 342
 343         attr->gid_tbl_len = 1;
 344         attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
 345         attr->max_msg_sz = -1;
 346
 347         if (!ndev)
 348                 goto out;
 349
 350         ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
 351         attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
 352         attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
 353         if (netif_running(ndev) && netif_carrier_ok(ndev))
 354                 dev->state = IB_PORT_ACTIVE;
 355         else
 356                 dev->state = IB_PORT_DOWN;
 357         attr->state = dev->state;
 358
 359 out:
 360         if (dev->state == IB_PORT_ACTIVE)
 361                 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 362         else
 363                 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 364
 365         return 0;
 366 }
 367
 368 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
 369                              struct ib_port_immutable *port_immutable)
 370 {
 371         port_immutable->gid_tbl_len = 1;
 372         port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 373
 374         return 0;
 375 }
 376
 377 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 378 {
 379         struct erdma_pd *pd = to_epd(ibpd);
 380         struct erdma_dev *dev = to_edev(ibpd->device);
 381         int pdn;
 382
 383         pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
 384         if (pdn < 0)
 385                 return pdn;
 386
 387         pd->pdn = pdn;
 388
 389         return 0;
 390 }
 391
 392 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 393 {
 394         struct erdma_pd *pd = to_epd(ibpd);
 395         struct erdma_dev *dev = to_edev(ibpd->device);
 396
 397         erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
 398
 399         return 0;
 400 }
 401
 402 static void erdma_flush_worker(struct work_struct *work)
 403 {
 404         struct delayed_work *dwork = to_delayed_work(work);
 405         struct erdma_qp *qp =
 406                 container_of(dwork, struct erdma_qp, reflush_dwork);
 407         struct erdma_cmdq_reflush_req req;
 408
 409         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
 410                                 CMDQ_OPCODE_REFLUSH);
 411         req.qpn = QP_ID(qp);
 412         req.sq_pi = qp->kern_qp.sq_pi;
 413         req.rq_pi = qp->kern_qp.rq_pi;
 414         erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL);
 415 }
 416
 417 static int erdma_qp_validate_cap(struct erdma_dev *dev,
 418                                  struct ib_qp_init_attr *attrs)
 419 {
 420         if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
 421             (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
 422             (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
 423             (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
 424             (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
 425             !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
 426                 return -EINVAL;
 427         }
 428
 429         return 0;
 430 }
 431
 432 static int erdma_qp_validate_attr(struct erdma_dev *dev,
 433                                   struct ib_qp_init_attr *attrs)
 434 {
 435         if (attrs->qp_type != IB_QPT_RC)
 436                 return -EOPNOTSUPP;
 437
 438         if (attrs->srq)
 439                 return -EOPNOTSUPP;
 440
 441         if (!attrs->send_cq || !attrs->recv_cq)
 442                 return -EOPNOTSUPP;
 443
 444         return 0;
 445 }
 446
 447 static void free_kernel_qp(struct erdma_qp *qp)
 448 {
 449         struct erdma_dev *dev = qp->dev;
 450
 451         vfree(qp->kern_qp.swr_tbl);
 452         vfree(qp->kern_qp.rwr_tbl);
 453
 454         if (qp->kern_qp.sq_buf)
 455                 dma_free_coherent(
 456                         &dev->pdev->dev,
 457                         WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
 458                         qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
 459
 460         if (qp->kern_qp.rq_buf)
 461                 dma_free_coherent(
 462                         &dev->pdev->dev,
 463                         WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
 464                         qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
 465 }
 466
 467 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
 468                           struct ib_qp_init_attr *attrs)
 469 {
 470         struct erdma_kqp *kqp = &qp->kern_qp;
 471         int size;
 472
 473         if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
 474                 kqp->sig_all = 1;
 475
 476         kqp->sq_pi = 0;
 477         kqp->sq_ci = 0;
 478         kqp->rq_pi = 0;
 479         kqp->rq_ci = 0;
 480         kqp->hw_sq_db =
 481                 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
 482         kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
 483
 484         kqp->swr_tbl = vmalloc(qp->attrs.sq_size * sizeof(u64));
 485         kqp->rwr_tbl = vmalloc(qp->attrs.rq_size * sizeof(u64));
 486         if (!kqp->swr_tbl || !kqp->rwr_tbl)
 487                 goto err_out;
 488
 489         size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
 490         kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
 491                                          &kqp->sq_buf_dma_addr, GFP_KERNEL);
 492         if (!kqp->sq_buf)
 493                 goto err_out;
 494
 495         size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
 496         kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
 497                                          &kqp->rq_buf_dma_addr, GFP_KERNEL);
 498         if (!kqp->rq_buf)
 499                 goto err_out;
 500
 501         kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT);
 502         kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT);
 503
 504         return 0;
 505
 506 err_out:
 507         free_kernel_qp(qp);
 508         return -ENOMEM;
 509 }
 510
 511 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
 512                            u64 start, u64 len, int access, u64 virt,
 513                            unsigned long req_page_size, u8 force_indirect_mtt)
 514 {
 515         struct ib_block_iter biter;
 516         uint64_t *phy_addr = NULL;
 517         int ret = 0;
 518
 519         mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
 520         if (IS_ERR(mem->umem)) {
 521                 ret = PTR_ERR(mem->umem);
 522                 mem->umem = NULL;
 523                 return ret;
 524         }
 525
 526         mem->va = virt;
 527         mem->len = len;
 528         mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
 529         mem->page_offset = start & (mem->page_size - 1);
 530         mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
 531         mem->page_cnt = mem->mtt_nents;
 532
 533         if (mem->page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES ||
 534             force_indirect_mtt) {
 535                 mem->mtt_type = ERDMA_MR_INDIRECT_MTT;
 536                 mem->mtt_buf =
 537                         alloc_pages_exact(MTT_SIZE(mem->page_cnt), GFP_KERNEL);
 538                 if (!mem->mtt_buf) {
 539                         ret = -ENOMEM;
 540                         goto error_ret;
 541                 }
 542                 phy_addr = mem->mtt_buf;
 543         } else {
 544                 mem->mtt_type = ERDMA_MR_INLINE_MTT;
 545                 phy_addr = mem->mtt_entry;
 546         }
 547
 548         rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) {
 549                 *phy_addr = rdma_block_iter_dma_address(&biter);
 550                 phy_addr++;
 551         }
 552
 553         if (mem->mtt_type == ERDMA_MR_INDIRECT_MTT) {
 554                 mem->mtt_entry[0] =
 555                         dma_map_single(&dev->pdev->dev, mem->mtt_buf,
 556                                        MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE);
 557                 if (dma_mapping_error(&dev->pdev->dev, mem->mtt_entry[0])) {
 558                         free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt));
 559                         mem->mtt_buf = NULL;
 560                         ret = -ENOMEM;
 561                         goto error_ret;
 562                 }
 563         }
 564
 565         return 0;
 566
 567 error_ret:
 568         if (mem->umem) {
 569                 ib_umem_release(mem->umem);
 570                 mem->umem = NULL;
 571         }
 572
 573         return ret;
 574 }
 575
 576 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
 577 {
 578         if (mem->mtt_buf) {
 579                 dma_unmap_single(&dev->pdev->dev, mem->mtt_entry[0],
 580                                  MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE);
 581                 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt));
 582         }
 583
 584         if (mem->umem) {
 585                 ib_umem_release(mem->umem);
 586                 mem->umem = NULL;
 587         }
 588 }
 589
 590 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
 591                                     u64 dbrecords_va,
 592                                     struct erdma_user_dbrecords_page **dbr_page,
 593                                     dma_addr_t *dma_addr)
 594 {
 595         struct erdma_user_dbrecords_page *page = NULL;
 596         int rv = 0;
 597
 598         mutex_lock(&ctx->dbrecords_page_mutex);
 599
 600         list_for_each_entry(page, &ctx->dbrecords_page_list, list)
 601                 if (page->va == (dbrecords_va & PAGE_MASK))
 602                         goto found;
 603
 604         page = kmalloc(sizeof(*page), GFP_KERNEL);
 605         if (!page) {
 606                 rv = -ENOMEM;
 607                 goto out;
 608         }
 609
 610         page->va = (dbrecords_va & PAGE_MASK);
 611         page->refcnt = 0;
 612
 613         page->umem = ib_umem_get(ctx->ibucontext.device,
 614                                  dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
 615         if (IS_ERR(page->umem)) {
 616                 rv = PTR_ERR(page->umem);
 617                 kfree(page);
 618                 goto out;
 619         }
 620
 621         list_add(&page->list, &ctx->dbrecords_page_list);
 622
 623 found:
 624         *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
 625                     (dbrecords_va & ~PAGE_MASK);
 626         *dbr_page = page;
 627         page->refcnt++;
 628
 629 out:
 630         mutex_unlock(&ctx->dbrecords_page_mutex);
 631         return rv;
 632 }
 633
 634 static void
 635 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
 636                            struct erdma_user_dbrecords_page **dbr_page)
 637 {
 638         if (!ctx || !(*dbr_page))
 639                 return;
 640
 641         mutex_lock(&ctx->dbrecords_page_mutex);
 642         if (--(*dbr_page)->refcnt == 0) {
 643                 list_del(&(*dbr_page)->list);
 644                 ib_umem_release((*dbr_page)->umem);
 645                 kfree(*dbr_page);
 646         }
 647
 648         *dbr_page = NULL;
 649         mutex_unlock(&ctx->dbrecords_page_mutex);
 650 }
 651
 652 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 653                         u64 va, u32 len, u64 db_info_va)
 654 {
 655         dma_addr_t db_info_dma_addr;
 656         u32 rq_offset;
 657         int ret;
 658
 659         if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
 660                    qp->attrs.rq_size * RQE_SIZE))
 661                 return -EINVAL;
 662
 663         ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mtt, va,
 664                               qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
 665                               (SZ_1M - SZ_4K), 1);
 666         if (ret)
 667                 return ret;
 668
 669         rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
 670         qp->user_qp.rq_offset = rq_offset;
 671
 672         ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset,
 673                               qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
 674                               (SZ_1M - SZ_4K), 1);
 675         if (ret)
 676                 goto put_sq_mtt;
 677
 678         ret = erdma_map_user_dbrecords(uctx, db_info_va,
 679                                        &qp->user_qp.user_dbr_page,
 680                                        &db_info_dma_addr);
 681         if (ret)
 682                 goto put_rq_mtt;
 683
 684         qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr;
 685         qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE;
 686
 687         return 0;
 688
 689 put_rq_mtt:
 690         put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt);
 691
 692 put_sq_mtt:
 693         put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt);
 694
 695         return ret;
 696 }
 697
 698 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
 699 {
 700         put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt);
 701         put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt);
 702         erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
 703 }
 704
 705 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 706                     struct ib_udata *udata)
 707 {
 708         struct erdma_qp *qp = to_eqp(ibqp);
 709         struct erdma_dev *dev = to_edev(ibqp->device);
 710         struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
 711                 udata, struct erdma_ucontext, ibucontext);
 712         struct erdma_ureq_create_qp ureq;
 713         struct erdma_uresp_create_qp uresp;
 714         int ret;
 715
 716         ret = erdma_qp_validate_cap(dev, attrs);
 717         if (ret)
 718                 goto err_out;
 719
 720         ret = erdma_qp_validate_attr(dev, attrs);
 721         if (ret)
 722                 goto err_out;
 723
 724         qp->scq = to_ecq(attrs->send_cq);
 725         qp->rcq = to_ecq(attrs->recv_cq);
 726         qp->dev = dev;
 727         qp->attrs.cc = dev->attrs.cc;
 728
 729         init_rwsem(&qp->state_lock);
 730         kref_init(&qp->ref);
 731         init_completion(&qp->safe_free);
 732
 733         ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
 734                               XA_LIMIT(1, dev->attrs.max_qp - 1),
 735                               &dev->next_alloc_qpn, GFP_KERNEL);
 736         if (ret < 0) {
 737                 ret = -ENOMEM;
 738                 goto err_out;
 739         }
 740
 741         qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
 742                                                ERDMA_MAX_WQEBB_PER_SQE);
 743         qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
 744
 745         if (uctx) {
 746                 ret = ib_copy_from_udata(&ureq, udata,
 747                                          min(sizeof(ureq), udata->inlen));
 748                 if (ret)
 749                         goto err_out_xa;
 750
 751                 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
 752                                    ureq.db_record_va);
 753                 if (ret)
 754                         goto err_out_xa;
 755
 756                 memset(&uresp, 0, sizeof(uresp));
 757
 758                 uresp.num_sqe = qp->attrs.sq_size;
 759                 uresp.num_rqe = qp->attrs.rq_size;
 760                 uresp.qp_id = QP_ID(qp);
 761                 uresp.rq_offset = qp->user_qp.rq_offset;
 762
 763                 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 764                 if (ret)
 765                         goto err_out_cmd;
 766         } else {
 767                 init_kernel_qp(dev, qp, attrs);
 768         }
 769
 770         qp->attrs.max_send_sge = attrs->cap.max_send_sge;
 771         qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
 772         qp->attrs.state = ERDMA_QP_STATE_IDLE;
 773         INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
 774
 775         ret = create_qp_cmd(uctx, qp);
 776         if (ret)
 777                 goto err_out_cmd;
 778
 779         spin_lock_init(&qp->lock);
 780
 781         return 0;
 782
 783 err_out_cmd:
 784         if (uctx)
 785                 free_user_qp(qp, uctx);
 786         else
 787                 free_kernel_qp(qp);
 788 err_out_xa:
 789         xa_erase(&dev->qp_xa, QP_ID(qp));
 790 err_out:
 791         return ret;
 792 }
 793
 794 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
 795 {
 796         int stag_idx;
 797
 798         stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
 799         if (stag_idx < 0)
 800                 return stag_idx;
 801
 802         /* For now, we always let key field be zero. */
 803         *stag = (stag_idx << 8);
 804
 805         return 0;
 806 }
 807
 808 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
 809 {
 810         struct erdma_dev *dev = to_edev(ibpd->device);
 811         struct erdma_mr *mr;
 812         u32 stag;
 813         int ret;
 814
 815         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 816         if (!mr)
 817                 return ERR_PTR(-ENOMEM);
 818
 819         ret = erdma_create_stag(dev, &stag);
 820         if (ret)
 821                 goto out_free;
 822
 823         mr->type = ERDMA_MR_TYPE_DMA;
 824
 825         mr->ibmr.lkey = stag;
 826         mr->ibmr.rkey = stag;
 827         mr->ibmr.pd = ibpd;
 828         mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
 829         ret = regmr_cmd(dev, mr);
 830         if (ret)
 831                 goto out_remove_stag;
 832
 833         return &mr->ibmr;
 834
 835 out_remove_stag:
 836         erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
 837                        mr->ibmr.lkey >> 8);
 838
 839 out_free:
 840         kfree(mr);
 841
 842         return ERR_PTR(ret);
 843 }
 844
 845 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 846                                 u32 max_num_sg)
 847 {
 848         struct erdma_mr *mr;
 849         struct erdma_dev *dev = to_edev(ibpd->device);
 850         int ret;
 851         u32 stag;
 852
 853         if (mr_type != IB_MR_TYPE_MEM_REG)
 854                 return ERR_PTR(-EOPNOTSUPP);
 855
 856         if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
 857                 return ERR_PTR(-EINVAL);
 858
 859         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 860         if (!mr)
 861                 return ERR_PTR(-ENOMEM);
 862
 863         ret = erdma_create_stag(dev, &stag);
 864         if (ret)
 865                 goto out_free;
 866
 867         mr->type = ERDMA_MR_TYPE_FRMR;
 868
 869         mr->ibmr.lkey = stag;
 870         mr->ibmr.rkey = stag;
 871         mr->ibmr.pd = ibpd;
 872         /* update it in FRMR. */
 873         mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
 874                      ERDMA_MR_ACC_RW;
 875
 876         mr->mem.page_size = PAGE_SIZE; /* update it later. */
 877         mr->mem.page_cnt = max_num_sg;
 878         mr->mem.mtt_type = ERDMA_MR_INDIRECT_MTT;
 879         mr->mem.mtt_buf =
 880                 alloc_pages_exact(MTT_SIZE(mr->mem.page_cnt), GFP_KERNEL);
 881         if (!mr->mem.mtt_buf) {
 882                 ret = -ENOMEM;
 883                 goto out_remove_stag;
 884         }
 885
 886         mr->mem.mtt_entry[0] =
 887                 dma_map_single(&dev->pdev->dev, mr->mem.mtt_buf,
 888                                MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE);
 889         if (dma_mapping_error(&dev->pdev->dev, mr->mem.mtt_entry[0])) {
 890                 ret = -ENOMEM;
 891                 goto out_free_mtt;
 892         }
 893
 894         ret = regmr_cmd(dev, mr);
 895         if (ret)
 896                 goto out_dma_unmap;
 897
 898         return &mr->ibmr;
 899
 900 out_dma_unmap:
 901         dma_unmap_single(&dev->pdev->dev, mr->mem.mtt_entry[0],
 902                          MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE);
 903 out_free_mtt:
 904         free_pages_exact(mr->mem.mtt_buf, MTT_SIZE(mr->mem.page_cnt));
 905
 906 out_remove_stag:
 907         erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
 908                        mr->ibmr.lkey >> 8);
 909
 910 out_free:
 911         kfree(mr);
 912
 913         return ERR_PTR(ret);
 914 }
 915
 916 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
 917 {
 918         struct erdma_mr *mr = to_emr(ibmr);
 919
 920         if (mr->mem.mtt_nents >= mr->mem.page_cnt)
 921                 return -1;
 922
 923         *((u64 *)mr->mem.mtt_buf + mr->mem.mtt_nents) = addr;
 924         mr->mem.mtt_nents++;
 925
 926         return 0;
 927 }
 928
 929 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 930                     unsigned int *sg_offset)
 931 {
 932         struct erdma_mr *mr = to_emr(ibmr);
 933         int num;
 934
 935         mr->mem.mtt_nents = 0;
 936
 937         num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
 938                              erdma_set_page);
 939
 940         return num;
 941 }
 942
 943 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 944                                 u64 virt, int access, struct ib_udata *udata)
 945 {
 946         struct erdma_mr *mr = NULL;
 947         struct erdma_dev *dev = to_edev(ibpd->device);
 948         u32 stag;
 949         int ret;
 950
 951         if (!len || len > dev->attrs.max_mr_size)
 952                 return ERR_PTR(-EINVAL);
 953
 954         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 955         if (!mr)
 956                 return ERR_PTR(-ENOMEM);
 957
 958         ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
 959                               SZ_2G - SZ_4K, 0);
 960         if (ret)
 961                 goto err_out_free;
 962
 963         ret = erdma_create_stag(dev, &stag);
 964         if (ret)
 965                 goto err_out_put_mtt;
 966
 967         mr->ibmr.lkey = mr->ibmr.rkey = stag;
 968         mr->ibmr.pd = ibpd;
 969         mr->mem.va = virt;
 970         mr->mem.len = len;
 971         mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
 972         mr->valid = 1;
 973         mr->type = ERDMA_MR_TYPE_NORMAL;
 974
 975         ret = regmr_cmd(dev, mr);
 976         if (ret)
 977                 goto err_out_mr;
 978
 979         return &mr->ibmr;
 980
 981 err_out_mr:
 982         erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
 983                        mr->ibmr.lkey >> 8);
 984
 985 err_out_put_mtt:
 986         put_mtt_entries(dev, &mr->mem);
 987
 988 err_out_free:
 989         kfree(mr);
 990
 991         return ERR_PTR(ret);
 992 }
 993
 994 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 995 {
 996         struct erdma_mr *mr;
 997         struct erdma_dev *dev = to_edev(ibmr->device);
 998         struct erdma_cmdq_dereg_mr_req req;
 999         int ret;
1000
1001         mr = to_emr(ibmr);
1002
1003         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1004                                 CMDQ_OPCODE_DEREG_MR);
1005
1006         req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
1007                   FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
1008
1009         ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1010         if (ret)
1011                 return ret;
1012
1013         erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
1014
1015         put_mtt_entries(dev, &mr->mem);
1016
1017         kfree(mr);
1018         return 0;
1019 }
1020
1021 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1022 {
1023         struct erdma_cq *cq = to_ecq(ibcq);
1024         struct erdma_dev *dev = to_edev(ibcq->device);
1025         struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1026                 udata, struct erdma_ucontext, ibucontext);
1027         int err;
1028         struct erdma_cmdq_destroy_cq_req req;
1029
1030         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1031                                 CMDQ_OPCODE_DESTROY_CQ);
1032         req.cqn = cq->cqn;
1033
1034         err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1035         if (err)
1036                 return err;
1037
1038         if (rdma_is_kernel_res(&cq->ibcq.res)) {
1039                 dma_free_coherent(&dev->pdev->dev,
1040                                   WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1041                                   cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1042         } else {
1043                 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1044                 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1045         }
1046
1047         xa_erase(&dev->cq_xa, cq->cqn);
1048
1049         return 0;
1050 }
1051
1052 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1053 {
1054         struct erdma_qp *qp = to_eqp(ibqp);
1055         struct erdma_dev *dev = to_edev(ibqp->device);
1056         struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1057                 udata, struct erdma_ucontext, ibucontext);
1058         struct erdma_qp_attrs qp_attrs;
1059         int err;
1060         struct erdma_cmdq_destroy_qp_req req;
1061
1062         down_write(&qp->state_lock);
1063         qp_attrs.state = ERDMA_QP_STATE_ERROR;
1064         erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
1065         up_write(&qp->state_lock);
1066
1067         cancel_delayed_work_sync(&qp->reflush_dwork);
1068
1069         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1070                                 CMDQ_OPCODE_DESTROY_QP);
1071         req.qpn = QP_ID(qp);
1072
1073         err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1074         if (err)
1075                 return err;
1076
1077         erdma_qp_put(qp);
1078         wait_for_completion(&qp->safe_free);
1079
1080         if (rdma_is_kernel_res(&qp->ibqp.res)) {
1081                 vfree(qp->kern_qp.swr_tbl);
1082                 vfree(qp->kern_qp.rwr_tbl);
1083                 dma_free_coherent(
1084                         &dev->pdev->dev,
1085                         WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
1086                         qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
1087                 dma_free_coherent(
1088                         &dev->pdev->dev,
1089                         WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
1090                         qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
1091         } else {
1092                 put_mtt_entries(dev, &qp->user_qp.sq_mtt);
1093                 put_mtt_entries(dev, &qp->user_qp.rq_mtt);
1094                 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1095         }
1096
1097         if (qp->cep)
1098                 erdma_cep_put(qp->cep);
1099         xa_erase(&dev->qp_xa, QP_ID(qp));
1100
1101         return 0;
1102 }
1103
1104 void erdma_qp_get_ref(struct ib_qp *ibqp)
1105 {
1106         erdma_qp_get(to_eqp(ibqp));
1107 }
1108
1109 void erdma_qp_put_ref(struct ib_qp *ibqp)
1110 {
1111         erdma_qp_put(to_eqp(ibqp));
1112 }
1113
1114 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1115 {
1116         struct rdma_user_mmap_entry *rdma_entry;
1117         struct erdma_user_mmap_entry *entry;
1118         pgprot_t prot;
1119         int err;
1120
1121         rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1122         if (!rdma_entry)
1123                 return -EINVAL;
1124
1125         entry = to_emmap(rdma_entry);
1126
1127         switch (entry->mmap_flag) {
1128         case ERDMA_MMAP_IO_NC:
1129                 /* map doorbell. */
1130                 prot = pgprot_device(vma->vm_page_prot);
1131                 break;
1132         default:
1133                 err = -EINVAL;
1134                 goto put_entry;
1135         }
1136
1137         err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1138                                 prot, rdma_entry);
1139
1140 put_entry:
1141         rdma_user_mmap_entry_put(rdma_entry);
1142         return err;
1143 }
1144
1145 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1146 {
1147         struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1148
1149         kfree(entry);
1150 }
1151
1152 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
1153                               bool ext_db_en)
1154 {
1155         struct erdma_cmdq_ext_db_req req = {};
1156         u64 val0, val1;
1157         int ret;
1158
1159         /*
1160          * CAP_SYS_RAWIO is required if hardware does not support extend
1161          * doorbell mechanism.
1162          */
1163         if (!ext_db_en && !capable(CAP_SYS_RAWIO))
1164                 return -EPERM;
1165
1166         if (!ext_db_en) {
1167                 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;
1168                 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1169                 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1170                 return 0;
1171         }
1172
1173         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1174                                 CMDQ_OPCODE_ALLOC_DB);
1175
1176         req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1177                   FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1178                   FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1179
1180         ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1);
1181         if (ret)
1182                 return ret;
1183
1184         ctx->ext_db.enable = true;
1185         ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);
1186         ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);
1187         ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);
1188
1189         ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);
1190         ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);
1191         ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);
1192
1193         return 0;
1194 }
1195
1196 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
1197 {
1198         struct erdma_cmdq_ext_db_req req = {};
1199         int ret;
1200
1201         if (!ctx->ext_db.enable)
1202                 return;
1203
1204         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1205                                 CMDQ_OPCODE_FREE_DB);
1206
1207         req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1208                   FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1209                   FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1210
1211         req.sdb_off = ctx->ext_db.sdb_off;
1212         req.rdb_off = ctx->ext_db.rdb_off;
1213         req.cdb_off = ctx->ext_db.cdb_off;
1214
1215         ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1216         if (ret)
1217                 ibdev_err_ratelimited(&dev->ibdev,
1218                                       "free db resources failed %d", ret);
1219 }
1220
1221 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1222 {
1223         rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1224         rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1225         rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1226 }
1227
1228 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1229 {
1230         struct erdma_ucontext *ctx = to_ectx(ibctx);
1231         struct erdma_dev *dev = to_edev(ibctx->device);
1232         int ret;
1233         struct erdma_uresp_alloc_ctx uresp = {};
1234
1235         if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1236                 ret = -ENOMEM;
1237                 goto err_out;
1238         }
1239
1240         if (udata->outlen < sizeof(uresp)) {
1241                 ret = -EINVAL;
1242                 goto err_out;
1243         }
1244
1245         INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1246         mutex_init(&ctx->dbrecords_page_mutex);
1247
1248         ret = alloc_db_resources(dev, ctx,
1249                                  !!(dev->attrs.cap_flags &
1250                                     ERDMA_DEV_CAP_FLAGS_EXTEND_DB));
1251         if (ret)
1252                 goto err_out;
1253
1254         ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1255                 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1256         if (!ctx->sq_db_mmap_entry) {
1257                 ret = -ENOMEM;
1258                 goto err_free_ext_db;
1259         }
1260
1261         ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1262                 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1263         if (!ctx->rq_db_mmap_entry) {
1264                 ret = -EINVAL;
1265                 goto err_put_mmap_entries;
1266         }
1267
1268         ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1269                 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1270         if (!ctx->cq_db_mmap_entry) {
1271                 ret = -EINVAL;
1272                 goto err_put_mmap_entries;
1273         }
1274
1275         uresp.dev_id = dev->pdev->device;
1276
1277         ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1278         if (ret)
1279                 goto err_put_mmap_entries;
1280
1281         return 0;
1282
1283 err_put_mmap_entries:
1284         erdma_uctx_user_mmap_entries_remove(ctx);
1285
1286 err_free_ext_db:
1287         free_db_resources(dev, ctx);
1288
1289 err_out:
1290         atomic_dec(&dev->num_ctx);
1291         return ret;
1292 }
1293
1294 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1295 {
1296         struct erdma_dev *dev = to_edev(ibctx->device);
1297         struct erdma_ucontext *ctx = to_ectx(ibctx);
1298
1299         erdma_uctx_user_mmap_entries_remove(ctx);
1300         free_db_resources(dev, ctx);
1301         atomic_dec(&dev->num_ctx);
1302 }
1303
1304 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = {
1305         [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE,
1306         [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE,
1307         [IB_QPS_RTR] = ERDMA_QP_STATE_RTR,
1308         [IB_QPS_RTS] = ERDMA_QP_STATE_RTS,
1309         [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING,
1310         [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE,
1311         [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR
1312 };
1313
1314 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1315                     struct ib_udata *udata)
1316 {
1317         struct erdma_qp_attrs new_attrs;
1318         enum erdma_qp_attr_mask erdma_attr_mask = 0;
1319         struct erdma_qp *qp = to_eqp(ibqp);
1320         int ret = 0;
1321
1322         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1323                 return -EOPNOTSUPP;
1324
1325         memset(&new_attrs, 0, sizeof(new_attrs));
1326
1327         if (attr_mask & IB_QP_STATE) {
1328                 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state];
1329
1330                 erdma_attr_mask |= ERDMA_QP_ATTR_STATE;
1331         }
1332
1333         down_write(&qp->state_lock);
1334
1335         ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask);
1336
1337         up_write(&qp->state_lock);
1338
1339         return ret;
1340 }
1341
1342 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1343                    int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1344 {
1345         struct erdma_qp *qp;
1346         struct erdma_dev *dev;
1347
1348         if (ibqp && qp_attr && qp_init_attr) {
1349                 qp = to_eqp(ibqp);
1350                 dev = to_edev(ibqp->device);
1351         } else {
1352                 return -EINVAL;
1353         }
1354
1355         qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1356         qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1357
1358         qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1359         qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1360         qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1361         qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1362
1363         qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1364         qp_attr->max_rd_atomic = qp->attrs.irq_size;
1365         qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1366
1367         qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1368                                    IB_ACCESS_REMOTE_WRITE |
1369                                    IB_ACCESS_REMOTE_READ;
1370
1371         qp_init_attr->cap = qp_attr->cap;
1372
1373         return 0;
1374 }
1375
1376 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1377                               struct erdma_ureq_create_cq *ureq)
1378 {
1379         int ret;
1380         struct erdma_dev *dev = to_edev(cq->ibcq.device);
1381
1382         ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mtt, ureq->qbuf_va,
1383                               ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1384                               1);
1385         if (ret)
1386                 return ret;
1387
1388         ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1389                                        &cq->user_cq.user_dbr_page,
1390                                        &cq->user_cq.db_info_dma_addr);
1391         if (ret)
1392                 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1393
1394         return ret;
1395 }
1396
1397 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1398 {
1399         struct erdma_dev *dev = to_edev(cq->ibcq.device);
1400
1401         cq->kern_cq.qbuf =
1402                 dma_alloc_coherent(&dev->pdev->dev,
1403                                    WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1404                                    &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1405         if (!cq->kern_cq.qbuf)
1406                 return -ENOMEM;
1407
1408         cq->kern_cq.db_record =
1409                 (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT));
1410         spin_lock_init(&cq->kern_cq.lock);
1411         /* use default cqdb addr */
1412         cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1413
1414         return 0;
1415 }
1416
1417 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1418                     struct ib_udata *udata)
1419 {
1420         struct erdma_cq *cq = to_ecq(ibcq);
1421         struct erdma_dev *dev = to_edev(ibcq->device);
1422         unsigned int depth = attr->cqe;
1423         int ret;
1424         struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1425                 udata, struct erdma_ucontext, ibucontext);
1426
1427         if (depth > dev->attrs.max_cqe)
1428                 return -EINVAL;
1429
1430         depth = roundup_pow_of_two(depth);
1431         cq->ibcq.cqe = depth;
1432         cq->depth = depth;
1433         cq->assoc_eqn = attr->comp_vector + 1;
1434
1435         ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1436                               XA_LIMIT(1, dev->attrs.max_cq - 1),
1437                               &dev->next_alloc_cqn, GFP_KERNEL);
1438         if (ret < 0)
1439                 return ret;
1440
1441         if (!rdma_is_kernel_res(&ibcq->res)) {
1442                 struct erdma_ureq_create_cq ureq;
1443                 struct erdma_uresp_create_cq uresp;
1444
1445                 ret = ib_copy_from_udata(&ureq, udata,
1446                                          min(udata->inlen, sizeof(ureq)));
1447                 if (ret)
1448                         goto err_out_xa;
1449
1450                 ret = erdma_init_user_cq(ctx, cq, &ureq);
1451                 if (ret)
1452                         goto err_out_xa;
1453
1454                 uresp.cq_id = cq->cqn;
1455                 uresp.num_cqe = depth;
1456
1457                 ret = ib_copy_to_udata(udata, &uresp,
1458                                        min(sizeof(uresp), udata->outlen));
1459                 if (ret)
1460                         goto err_free_res;
1461         } else {
1462                 ret = erdma_init_kernel_cq(cq);
1463                 if (ret)
1464                         goto err_out_xa;
1465         }
1466
1467         ret = create_cq_cmd(ctx, cq);
1468         if (ret)
1469                 goto err_free_res;
1470
1471         return 0;
1472
1473 err_free_res:
1474         if (!rdma_is_kernel_res(&ibcq->res)) {
1475                 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1476                 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1477         } else {
1478                 dma_free_coherent(&dev->pdev->dev,
1479                                   WARPPED_BUFSIZE(depth << CQE_SHIFT),
1480                                   cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1481         }
1482
1483 err_out_xa:
1484         xa_erase(&dev->cq_xa, cq->cqn);
1485
1486         return ret;
1487 }
1488
1489 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
1490 {
1491         struct erdma_cmdq_config_mtu_req req;
1492
1493         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1494                                 CMDQ_OPCODE_CONF_MTU);
1495         req.mtu = mtu;
1496
1497         erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1498 }
1499
1500 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
1501 {
1502         struct ib_event event;
1503
1504         event.device = &dev->ibdev;
1505         event.element.port_num = 1;
1506         event.event = reason;
1507
1508         ib_dispatch_event(&event);
1509 }