drivers/infiniband/hw/bnxt_re/ib_verbs.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: IB Verbs interpreter
  37  */
  38
  39 #include <linux/interrupt.h>
  40 #include <linux/types.h>
  41 #include <linux/pci.h>
  42 #include <linux/netdevice.h>
  43 #include <linux/if_ether.h>
  44 #include <net/addrconf.h>
  45
  46 #include <rdma/ib_verbs.h>
  47 #include <rdma/ib_user_verbs.h>
  48 #include <rdma/ib_umem.h>
  49 #include <rdma/ib_addr.h>
  50 #include <rdma/ib_mad.h>
  51 #include <rdma/ib_cache.h>
  52 #include <rdma/uverbs_ioctl.h>
  53 #include <linux/hashtable.h>
  54
  55 #include "roce_hsi.h"
  56 #include "qplib_res.h"
  57 #include "qplib_sp.h"
  58 #include "qplib_fp.h"
  59 #include "qplib_rcfw.h"
  60
  61 #include "bnxt_re.h"
  62 #include "ib_verbs.h"
  63 #include "debugfs.h"
  64
  65 #include <rdma/uverbs_types.h>
  66 #include <rdma/uverbs_std_types.h>
  67
  68 #include <rdma/ib_user_ioctl_cmds.h>
  69
  70 #define UVERBS_MODULE_NAME bnxt_re
  71 #include <rdma/uverbs_named_ioctl.h>
  72
  73 #include <rdma/bnxt_re-abi.h>
  74
  75 static int __from_ib_access_flags(int iflags)
  76 {
  77         int qflags = 0;
  78
  79         if (iflags & IB_ACCESS_LOCAL_WRITE)
  80                 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
  81         if (iflags & IB_ACCESS_REMOTE_READ)
  82                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
  83         if (iflags & IB_ACCESS_REMOTE_WRITE)
  84                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
  85         if (iflags & IB_ACCESS_REMOTE_ATOMIC)
  86                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
  87         if (iflags & IB_ACCESS_MW_BIND)
  88                 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
  89         if (iflags & IB_ZERO_BASED)
  90                 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
  91         if (iflags & IB_ACCESS_ON_DEMAND)
  92                 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
  93         return qflags;
  94 };
  95
  96 static int __to_ib_access_flags(int qflags)
  97 {
  98         int iflags = 0;
  99
 100         if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
 101                 iflags |= IB_ACCESS_LOCAL_WRITE;
 102         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
 103                 iflags |= IB_ACCESS_REMOTE_WRITE;
 104         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
 105                 iflags |= IB_ACCESS_REMOTE_READ;
 106         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
 107                 iflags |= IB_ACCESS_REMOTE_ATOMIC;
 108         if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
 109                 iflags |= IB_ACCESS_MW_BIND;
 110         if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
 111                 iflags |= IB_ZERO_BASED;
 112         if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
 113                 iflags |= IB_ACCESS_ON_DEMAND;
 114         return iflags;
 115 }
 116
 117 static u8 __qp_access_flags_from_ib(struct bnxt_qplib_chip_ctx *cctx, int iflags)
 118 {
 119         u8 qflags = 0;
 120
 121         if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
 122                 /* For Wh+ */
 123                 return (u8)__from_ib_access_flags(iflags);
 124
 125         /* For P5, P7 and later chips */
 126         if (iflags & IB_ACCESS_LOCAL_WRITE)
 127                 qflags |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
 128         if (iflags & IB_ACCESS_REMOTE_WRITE)
 129                 qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
 130         if (iflags & IB_ACCESS_REMOTE_READ)
 131                 qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
 132         if (iflags & IB_ACCESS_REMOTE_ATOMIC)
 133                 qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC;
 134
 135         return qflags;
 136 }
 137
 138 static int __qp_access_flags_to_ib(struct bnxt_qplib_chip_ctx *cctx, u8 qflags)
 139 {
 140         int iflags = 0;
 141
 142         if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
 143                 /* For Wh+ */
 144                 return __to_ib_access_flags(qflags);
 145
 146         /* For P5, P7 and later chips */
 147         if (qflags & CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE)
 148                 iflags |= IB_ACCESS_LOCAL_WRITE;
 149         if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE)
 150                 iflags |= IB_ACCESS_REMOTE_WRITE;
 151         if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_READ)
 152                 iflags |= IB_ACCESS_REMOTE_READ;
 153         if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC)
 154                 iflags |= IB_ACCESS_REMOTE_ATOMIC;
 155
 156         return iflags;
 157 }
 158
 159 static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,
 160                                                    struct bnxt_qplib_mrw *qplib_mr)
 161 {
 162         if (_is_relaxed_ordering_supported(rdev->dev_attr->dev_cap_flags2) &&
 163             pcie_relaxed_ordering_enabled(rdev->en_dev->pdev))
 164                 qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
 165 }
 166
 167 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 168                              struct bnxt_qplib_sge *sg_list, int num)
 169 {
 170         int i, total = 0;
 171
 172         for (i = 0; i < num; i++) {
 173                 sg_list[i].addr = ib_sg_list[i].addr;
 174                 sg_list[i].lkey = ib_sg_list[i].lkey;
 175                 sg_list[i].size = ib_sg_list[i].length;
 176                 total += sg_list[i].size;
 177         }
 178         return total;
 179 }
 180
 181 /* Device */
 182 int bnxt_re_query_device(struct ib_device *ibdev,
 183                          struct ib_device_attr *ib_attr,
 184                          struct ib_udata *udata)
 185 {
 186         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 187         struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
 188
 189         memset(ib_attr, 0, sizeof(*ib_attr));
 190         memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
 191                min(sizeof(dev_attr->fw_ver),
 192                    sizeof(ib_attr->fw_ver)));
 193         addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid,
 194                             rdev->netdev->dev_addr);
 195         ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
 196         ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED;
 197
 198         ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
 199         ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
 200         ib_attr->hw_ver = rdev->en_dev->pdev->revision;
 201         ib_attr->max_qp = dev_attr->max_qp;
 202         ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
 203         ib_attr->device_cap_flags =
 204                                     IB_DEVICE_CURR_QP_STATE_MOD
 205                                     | IB_DEVICE_RC_RNR_NAK_GEN
 206                                     | IB_DEVICE_SHUTDOWN_PORT
 207                                     | IB_DEVICE_SYS_IMAGE_GUID
 208                                     | IB_DEVICE_RESIZE_MAX_WR
 209                                     | IB_DEVICE_PORT_ACTIVE_EVENT
 210                                     | IB_DEVICE_N_NOTIFY_CQ
 211                                     | IB_DEVICE_MEM_WINDOW
 212                                     | IB_DEVICE_MEM_WINDOW_TYPE_2B
 213                                     | IB_DEVICE_MEM_MGT_EXTENSIONS;
 214         ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
 215         ib_attr->max_send_sge = dev_attr->max_qp_sges;
 216         ib_attr->max_recv_sge = dev_attr->max_qp_sges;
 217         ib_attr->max_sge_rd = dev_attr->max_qp_sges;
 218         ib_attr->max_cq = dev_attr->max_cq;
 219         ib_attr->max_cqe = dev_attr->max_cq_wqes;
 220         ib_attr->max_mr = dev_attr->max_mr;
 221         ib_attr->max_pd = dev_attr->max_pd;
 222         ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
 223         ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
 224         ib_attr->atomic_cap = IB_ATOMIC_NONE;
 225         ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
 226         if (dev_attr->is_atomic) {
 227                 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
 228                 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
 229         }
 230
 231         ib_attr->max_ee_rd_atom = 0;
 232         ib_attr->max_res_rd_atom = 0;
 233         ib_attr->max_ee_init_rd_atom = 0;
 234         ib_attr->max_ee = 0;
 235         ib_attr->max_rdd = 0;
 236         ib_attr->max_mw = dev_attr->max_mw;
 237         ib_attr->max_raw_ipv6_qp = 0;
 238         ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
 239         ib_attr->max_mcast_grp = 0;
 240         ib_attr->max_mcast_qp_attach = 0;
 241         ib_attr->max_total_mcast_qp_attach = 0;
 242         ib_attr->max_ah = dev_attr->max_ah;
 243
 244         ib_attr->max_srq = dev_attr->max_srq;
 245         ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
 246         ib_attr->max_srq_sge = dev_attr->max_srq_sges;
 247
 248         ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
 249
 250         ib_attr->max_pkeys = 1;
 251         ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
 252         return 0;
 253 }
 254
 255 int bnxt_re_modify_device(struct ib_device *ibdev,
 256                           int device_modify_mask,
 257                           struct ib_device_modify *device_modify)
 258 {
 259         ibdev_dbg(ibdev, "Modify device with mask 0x%x", device_modify_mask);
 260
 261         if (device_modify_mask & ~IB_DEVICE_MODIFY_NODE_DESC)
 262                 return -EOPNOTSUPP;
 263
 264         if (!(device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC))
 265                 return 0;
 266
 267         memcpy(ibdev->node_desc, device_modify->node_desc, IB_DEVICE_NODE_DESC_MAX);
 268         return 0;
 269 }
 270
 271 /* Port */
 272 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
 273                        struct ib_port_attr *port_attr)
 274 {
 275         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 276         struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
 277         int rc;
 278
 279         memset(port_attr, 0, sizeof(*port_attr));
 280
 281         if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
 282                 port_attr->state = IB_PORT_ACTIVE;
 283                 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 284         } else {
 285                 port_attr->state = IB_PORT_DOWN;
 286                 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 287         }
 288         port_attr->max_mtu = IB_MTU_4096;
 289         port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
 290         port_attr->gid_tbl_len = dev_attr->max_sgid;
 291         port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 292                                     IB_PORT_DEVICE_MGMT_SUP |
 293                                     IB_PORT_VENDOR_CLASS_SUP;
 294         port_attr->ip_gids = true;
 295
 296         port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
 297         port_attr->bad_pkey_cntr = 0;
 298         port_attr->qkey_viol_cntr = 0;
 299         port_attr->pkey_tbl_len = dev_attr->max_pkey;
 300         port_attr->lid = 0;
 301         port_attr->sm_lid = 0;
 302         port_attr->lmc = 0;
 303         port_attr->max_vl_num = 4;
 304         port_attr->sm_sl = 0;
 305         port_attr->subnet_timeout = 0;
 306         port_attr->init_type_reply = 0;
 307         rc = ib_get_eth_speed(&rdev->ibdev, port_num, &port_attr->active_speed,
 308                               &port_attr->active_width);
 309
 310         return rc;
 311 }
 312
 313 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 314                                struct ib_port_immutable *immutable)
 315 {
 316         struct ib_port_attr port_attr;
 317
 318         if (bnxt_re_query_port(ibdev, port_num, &port_attr))
 319                 return -EINVAL;
 320
 321         immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
 322         immutable->gid_tbl_len = port_attr.gid_tbl_len;
 323         immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
 324         immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
 325         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 326         return 0;
 327 }
 328
 329 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
 330 {
 331         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 332
 333         snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
 334                  rdev->dev_attr->fw_ver[0], rdev->dev_attr->fw_ver[1],
 335                  rdev->dev_attr->fw_ver[2], rdev->dev_attr->fw_ver[3]);
 336 }
 337
 338 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
 339                        u16 index, u16 *pkey)
 340 {
 341         if (index > 0)
 342                 return -EINVAL;
 343
 344         *pkey = IB_DEFAULT_PKEY_FULL;
 345
 346         return 0;
 347 }
 348
 349 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
 350                       int index, union ib_gid *gid)
 351 {
 352         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 353         int rc;
 354
 355         /* Ignore port_num */
 356         memset(gid, 0, sizeof(*gid));
 357         rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
 358                                  &rdev->qplib_res.sgid_tbl, index,
 359                                  (struct bnxt_qplib_gid *)gid);
 360         return rc;
 361 }
 362
 363 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
 364 {
 365         int rc = 0;
 366         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 367         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 368         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 369         struct bnxt_qplib_gid *gid_to_del;
 370         u16 vlan_id = 0xFFFF;
 371
 372         /* Delete the entry from the hardware */
 373         ctx = *context;
 374         if (!ctx)
 375                 return -EINVAL;
 376
 377         if (sgid_tbl && sgid_tbl->active) {
 378                 if (ctx->idx >= sgid_tbl->max)
 379                         return -EINVAL;
 380                 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
 381                 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
 382                 /* DEL_GID is called in WQ context(netdevice_event_work_handler)
 383                  * or via the ib_unregister_device path. In the former case QP1
 384                  * may not be destroyed yet, in which case just return as FW
 385                  * needs that entry to be present and will fail it's deletion.
 386                  * We could get invoked again after QP1 is destroyed OR get an
 387                  * ADD_GID call with a different GID value for the same index
 388                  * where we issue MODIFY_GID cmd to update the GID entry -- TBD
 389                  */
 390                 if (ctx->idx == 0 &&
 391                     rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
 392                     ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
 393                         ibdev_dbg(&rdev->ibdev,
 394                                   "Trying to delete GID0 while QP1 is alive\n");
 395                         return -EFAULT;
 396                 }
 397                 ctx->refcnt--;
 398                 if (!ctx->refcnt) {
 399                         rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
 400                                                  vlan_id,  true);
 401                         if (rc) {
 402                                 ibdev_err(&rdev->ibdev,
 403                                           "Failed to remove GID: %#x", rc);
 404                         } else {
 405                                 ctx_tbl = sgid_tbl->ctx;
 406                                 ctx_tbl[ctx->idx] = NULL;
 407                                 kfree(ctx);
 408                         }
 409                 }
 410         } else {
 411                 return -EINVAL;
 412         }
 413         return rc;
 414 }
 415
 416 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
 417 {
 418         int rc;
 419         u32 tbl_idx = 0;
 420         u16 vlan_id = 0xFFFF;
 421         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 422         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 423         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 424
 425         rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
 426         if (rc)
 427                 return rc;
 428
 429         rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
 430                                  rdev->qplib_res.netdev->dev_addr,
 431                                  vlan_id, true, &tbl_idx);
 432         if (rc == -EALREADY) {
 433                 ctx_tbl = sgid_tbl->ctx;
 434                 ctx_tbl[tbl_idx]->refcnt++;
 435                 *context = ctx_tbl[tbl_idx];
 436                 return 0;
 437         }
 438
 439         if (rc < 0) {
 440                 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
 441                 return rc;
 442         }
 443
 444         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 445         if (!ctx)
 446                 return -ENOMEM;
 447         ctx_tbl = sgid_tbl->ctx;
 448         ctx->idx = tbl_idx;
 449         ctx->refcnt = 1;
 450         ctx_tbl[tbl_idx] = ctx;
 451         *context = ctx;
 452
 453         return rc;
 454 }
 455
 456 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
 457                                             u32 port_num)
 458 {
 459         return IB_LINK_LAYER_ETHERNET;
 460 }
 461
 462 #define BNXT_RE_FENCE_PBL_SIZE  DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
 463
 464 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
 465 {
 466         struct bnxt_re_fence_data *fence = &pd->fence;
 467         struct ib_mr *ib_mr = &fence->mr->ib_mr;
 468         struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
 469         struct bnxt_re_dev *rdev = pd->rdev;
 470
 471         if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
 472                 return;
 473
 474         memset(wqe, 0, sizeof(*wqe));
 475         wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
 476         wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
 477         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
 478         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
 479         wqe->bind.zero_based = false;
 480         wqe->bind.parent_l_key = ib_mr->lkey;
 481         wqe->bind.va = (u64)(unsigned long)fence->va;
 482         wqe->bind.length = fence->size;
 483         wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
 484         wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
 485
 486         /* Save the initial rkey in fence structure for now;
 487          * wqe->bind.r_key will be set at (re)bind time.
 488          */
 489         fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
 490 }
 491
 492 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
 493 {
 494         struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
 495                                              qplib_qp);
 496         struct ib_pd *ib_pd = qp->ib_qp.pd;
 497         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 498         struct bnxt_re_fence_data *fence = &pd->fence;
 499         struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
 500         struct bnxt_qplib_swqe wqe;
 501         int rc;
 502
 503         memcpy(&wqe, fence_wqe, sizeof(wqe));
 504         wqe.bind.r_key = fence->bind_rkey;
 505         fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
 506
 507         ibdev_dbg(&qp->rdev->ibdev,
 508                   "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
 509                 wqe.bind.r_key, qp->qplib_qp.id, pd);
 510         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
 511         if (rc) {
 512                 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
 513                 return rc;
 514         }
 515         bnxt_qplib_post_send_db(&qp->qplib_qp);
 516
 517         return rc;
 518 }
 519
 520 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
 521 {
 522         struct bnxt_re_fence_data *fence = &pd->fence;
 523         struct bnxt_re_dev *rdev = pd->rdev;
 524         struct device *dev = &rdev->en_dev->pdev->dev;
 525         struct bnxt_re_mr *mr = fence->mr;
 526
 527         if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
 528                 return;
 529
 530         if (fence->mw) {
 531                 bnxt_re_dealloc_mw(fence->mw);
 532                 fence->mw = NULL;
 533         }
 534         if (mr) {
 535                 if (mr->ib_mr.rkey)
 536                         bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
 537                                              true);
 538                 if (mr->ib_mr.lkey)
 539                         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
 540                 kfree(mr);
 541                 fence->mr = NULL;
 542         }
 543         if (fence->dma_addr) {
 544                 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
 545                                  DMA_BIDIRECTIONAL);
 546                 fence->dma_addr = 0;
 547         }
 548 }
 549
 550 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 551 {
 552         int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
 553         struct bnxt_re_fence_data *fence = &pd->fence;
 554         struct bnxt_re_dev *rdev = pd->rdev;
 555         struct device *dev = &rdev->en_dev->pdev->dev;
 556         struct bnxt_re_mr *mr = NULL;
 557         dma_addr_t dma_addr = 0;
 558         struct ib_mw *mw;
 559         int rc;
 560
 561         if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
 562                 return 0;
 563
 564         dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
 565                                   DMA_BIDIRECTIONAL);
 566         rc = dma_mapping_error(dev, dma_addr);
 567         if (rc) {
 568                 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
 569                 rc = -EIO;
 570                 fence->dma_addr = 0;
 571                 goto fail;
 572         }
 573         fence->dma_addr = dma_addr;
 574
 575         /* Allocate a MR */
 576         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 577         if (!mr) {
 578                 rc = -ENOMEM;
 579                 goto fail;
 580         }
 581         fence->mr = mr;
 582         mr->rdev = rdev;
 583         mr->qplib_mr.pd = &pd->qplib_pd;
 584         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
 585         mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
 586         if (!_is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)) {
 587                 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
 588                 if (rc) {
 589                         ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
 590                         goto fail;
 591                 }
 592
 593                 /* Register MR */
 594                 mr->ib_mr.lkey = mr->qplib_mr.lkey;
 595         } else {
 596                 mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
 597         }
 598         mr->qplib_mr.va = (u64)(unsigned long)fence->va;
 599         mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
 600         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
 601                                BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
 602         if (rc) {
 603                 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
 604                 goto fail;
 605         }
 606         mr->ib_mr.rkey = mr->qplib_mr.rkey;
 607
 608         /* Create a fence MW only for kernel consumers */
 609         mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
 610         if (IS_ERR(mw)) {
 611                 ibdev_err(&rdev->ibdev,
 612                           "Failed to create fence-MW for PD: %p\n", pd);
 613                 rc = PTR_ERR(mw);
 614                 goto fail;
 615         }
 616         fence->mw = mw;
 617
 618         bnxt_re_create_fence_wqe(pd);
 619         return 0;
 620
 621 fail:
 622         bnxt_re_destroy_fence_mr(pd);
 623         return rc;
 624 }
 625
 626 static struct bnxt_re_user_mmap_entry*
 627 bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
 628                           enum bnxt_re_mmap_flag mmap_flag, u64 *offset)
 629 {
 630         struct bnxt_re_user_mmap_entry *entry;
 631         int ret;
 632
 633         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 634         if (!entry)
 635                 return NULL;
 636
 637         entry->mem_offset = mem_offset;
 638         entry->mmap_flag = mmap_flag;
 639         entry->uctx = uctx;
 640
 641         switch (mmap_flag) {
 642         case BNXT_RE_MMAP_SH_PAGE:
 643                 ret = rdma_user_mmap_entry_insert_exact(&uctx->ib_uctx,
 644                                                         &entry->rdma_entry, PAGE_SIZE, 0);
 645                 break;
 646         case BNXT_RE_MMAP_UC_DB:
 647         case BNXT_RE_MMAP_WC_DB:
 648         case BNXT_RE_MMAP_DBR_BAR:
 649         case BNXT_RE_MMAP_DBR_PAGE:
 650         case BNXT_RE_MMAP_TOGGLE_PAGE:
 651                 ret = rdma_user_mmap_entry_insert(&uctx->ib_uctx,
 652                                                   &entry->rdma_entry, PAGE_SIZE);
 653                 break;
 654         default:
 655                 ret = -EINVAL;
 656                 break;
 657         }
 658
 659         if (ret) {
 660                 kfree(entry);
 661                 return NULL;
 662         }
 663         if (offset)
 664                 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 665
 666         return entry;
 667 }
 668
 669 /* Protection Domains */
 670 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 671 {
 672         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 673         struct bnxt_re_dev *rdev = pd->rdev;
 674
 675         if (udata) {
 676                 rdma_user_mmap_entry_remove(pd->pd_db_mmap);
 677                 pd->pd_db_mmap = NULL;
 678         }
 679
 680         bnxt_re_destroy_fence_mr(pd);
 681
 682         if (pd->qplib_pd.id) {
 683                 if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res,
 684                                            &rdev->qplib_res.pd_tbl,
 685                                            &pd->qplib_pd))
 686                         atomic_dec(&rdev->stats.res.pd_count);
 687         }
 688         return 0;
 689 }
 690
 691 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 692 {
 693         struct ib_device *ibdev = ibpd->device;
 694         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 695         struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
 696                 udata, struct bnxt_re_ucontext, ib_uctx);
 697         struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
 698         struct bnxt_re_user_mmap_entry *entry = NULL;
 699         u32 active_pds;
 700         int rc = 0;
 701
 702         pd->rdev = rdev;
 703         if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
 704                 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
 705                 rc = -ENOMEM;
 706                 goto fail;
 707         }
 708
 709         if (udata) {
 710                 struct bnxt_re_pd_resp resp = {};
 711
 712                 if (!ucntx->dpi.dbr) {
 713                         /* Allocate DPI in alloc_pd to avoid failing of
 714                          * ibv_devinfo and family of application when DPIs
 715                          * are depleted.
 716                          */
 717                         if (bnxt_qplib_alloc_dpi(&rdev->qplib_res,
 718                                                  &ucntx->dpi, ucntx, BNXT_QPLIB_DPI_TYPE_UC)) {
 719                                 rc = -ENOMEM;
 720                                 goto dbfail;
 721                         }
 722                 }
 723
 724                 resp.pdid = pd->qplib_pd.id;
 725                 /* Still allow mapping this DBR to the new user PD. */
 726                 resp.dpi = ucntx->dpi.dpi;
 727
 728                 entry = bnxt_re_mmap_entry_insert(ucntx, (u64)ucntx->dpi.umdbr,
 729                                                   BNXT_RE_MMAP_UC_DB, &resp.dbr);
 730
 731                 if (!entry) {
 732                         rc = -ENOMEM;
 733                         goto dbfail;
 734                 }
 735
 736                 pd->pd_db_mmap = &entry->rdma_entry;
 737
 738                 rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
 739                 if (rc) {
 740                         rdma_user_mmap_entry_remove(pd->pd_db_mmap);
 741                         rc = -EFAULT;
 742                         goto dbfail;
 743                 }
 744         }
 745
 746         if (!udata)
 747                 if (bnxt_re_create_fence_mr(pd))
 748                         ibdev_warn(&rdev->ibdev,
 749                                    "Failed to create Fence-MR\n");
 750         active_pds = atomic_inc_return(&rdev->stats.res.pd_count);
 751         if (active_pds > rdev->stats.res.pd_watermark)
 752                 rdev->stats.res.pd_watermark = active_pds;
 753
 754         return 0;
 755 dbfail:
 756         bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 757                               &pd->qplib_pd);
 758 fail:
 759         return rc;
 760 }
 761
 762 /* Address Handles */
 763 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
 764 {
 765         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 766         struct bnxt_re_dev *rdev = ah->rdev;
 767         bool block = true;
 768         int rc;
 769
 770         block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
 771         rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block);
 772         if (BNXT_RE_CHECK_RC(rc)) {
 773                 if (rc == -ETIMEDOUT)
 774                         rc = 0;
 775                 else
 776                         goto fail;
 777         }
 778         atomic_dec(&rdev->stats.res.ah_count);
 779 fail:
 780         return rc;
 781 }
 782
 783 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
 784 {
 785         u8 nw_type;
 786
 787         switch (ntype) {
 788         case RDMA_NETWORK_IPV4:
 789                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
 790                 break;
 791         case RDMA_NETWORK_IPV6:
 792                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
 793                 break;
 794         default:
 795                 nw_type = CMDQ_CREATE_AH_TYPE_V1;
 796                 break;
 797         }
 798         return nw_type;
 799 }
 800
 801 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 802                       struct ib_udata *udata)
 803 {
 804         struct ib_pd *ib_pd = ib_ah->pd;
 805         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 806         struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 807         const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 808         struct bnxt_re_dev *rdev = pd->rdev;
 809         const struct ib_gid_attr *sgid_attr;
 810         struct bnxt_re_gid_ctx *ctx;
 811         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 812         u32 active_ahs;
 813         u8 nw_type;
 814         int rc;
 815
 816         if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 817                 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
 818                 return -EINVAL;
 819         }
 820
 821         ah->rdev = rdev;
 822         ah->qplib_ah.pd = &pd->qplib_pd;
 823
 824         /* Supply the configuration for the HW */
 825         memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
 826                sizeof(union ib_gid));
 827         sgid_attr = grh->sgid_attr;
 828         /* Get the HW context of the GID. The reference
 829          * of GID table entry is already taken by the caller.
 830          */
 831         ctx = rdma_read_gid_hw_context(sgid_attr);
 832         ah->qplib_ah.sgid_index = ctx->idx;
 833         ah->qplib_ah.host_sgid_index = grh->sgid_index;
 834         ah->qplib_ah.traffic_class = grh->traffic_class;
 835         ah->qplib_ah.flow_label = grh->flow_label;
 836         ah->qplib_ah.hop_limit = grh->hop_limit;
 837         ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
 838
 839         /* Get network header type for this GID */
 840         nw_type = rdma_gid_attr_network_type(sgid_attr);
 841         ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
 842
 843         memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
 844         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
 845                                   !(init_attr->flags &
 846                                     RDMA_CREATE_AH_SLEEPABLE));
 847         if (rc) {
 848                 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
 849                 return rc;
 850         }
 851
 852         /* Write AVID to shared page. */
 853         if (udata) {
 854                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
 855                         udata, struct bnxt_re_ucontext, ib_uctx);
 856                 unsigned long flag;
 857                 u32 *wrptr;
 858
 859                 spin_lock_irqsave(&uctx->sh_lock, flag);
 860                 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
 861                 *wrptr = ah->qplib_ah.id;
 862                 wmb(); /* make sure cache is updated. */
 863                 spin_unlock_irqrestore(&uctx->sh_lock, flag);
 864         }
 865         active_ahs = atomic_inc_return(&rdev->stats.res.ah_count);
 866         if (active_ahs > rdev->stats.res.ah_watermark)
 867                 rdev->stats.res.ah_watermark = active_ahs;
 868
 869         return 0;
 870 }
 871
 872 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 873 {
 874         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 875
 876         ah_attr->type = ib_ah->type;
 877         rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
 878         memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
 879         rdma_ah_set_grh(ah_attr, NULL, 0,
 880                         ah->qplib_ah.host_sgid_index,
 881                         0, ah->qplib_ah.traffic_class);
 882         rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
 883         rdma_ah_set_port_num(ah_attr, 1);
 884         rdma_ah_set_static_rate(ah_attr, 0);
 885         return 0;
 886 }
 887
 888 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
 889         __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
 890 {
 891         unsigned long flags;
 892
 893         spin_lock_irqsave(&qp->scq->cq_lock, flags);
 894         if (qp->rcq != qp->scq)
 895                 spin_lock(&qp->rcq->cq_lock);
 896         else
 897                 __acquire(&qp->rcq->cq_lock);
 898
 899         return flags;
 900 }
 901
 902 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
 903                         unsigned long flags)
 904         __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
 905 {
 906         if (qp->rcq != qp->scq)
 907                 spin_unlock(&qp->rcq->cq_lock);
 908         else
 909                 __release(&qp->rcq->cq_lock);
 910         spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
 911 }
 912
 913 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
 914 {
 915         struct bnxt_re_qp *gsi_sqp;
 916         struct bnxt_re_ah *gsi_sah;
 917         struct bnxt_re_dev *rdev;
 918         int rc;
 919
 920         rdev = qp->rdev;
 921         gsi_sqp = rdev->gsi_ctx.gsi_sqp;
 922         gsi_sah = rdev->gsi_ctx.gsi_sah;
 923
 924         ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
 925         bnxt_qplib_destroy_ah(&rdev->qplib_res,
 926                               &gsi_sah->qplib_ah,
 927                               true);
 928         atomic_dec(&rdev->stats.res.ah_count);
 929         bnxt_qplib_clean_qp(&qp->qplib_qp);
 930
 931         ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
 932         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 933         if (rc) {
 934                 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
 935                 goto fail;
 936         }
 937         bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 938
 939         /* remove from active qp list */
 940         mutex_lock(&rdev->qp_lock);
 941         list_del(&gsi_sqp->list);
 942         mutex_unlock(&rdev->qp_lock);
 943         atomic_dec(&rdev->stats.res.qp_count);
 944
 945         kfree(rdev->gsi_ctx.sqp_tbl);
 946         kfree(gsi_sah);
 947         kfree(gsi_sqp);
 948         rdev->gsi_ctx.gsi_sqp = NULL;
 949         rdev->gsi_ctx.gsi_sah = NULL;
 950         rdev->gsi_ctx.sqp_tbl = NULL;
 951
 952         return 0;
 953 fail:
 954         return rc;
 955 }
 956
 957 /* Queue Pairs */
 958 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 959 {
 960         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 961         struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
 962         struct bnxt_re_dev *rdev = qp->rdev;
 963         struct bnxt_qplib_nq *scq_nq = NULL;
 964         struct bnxt_qplib_nq *rcq_nq = NULL;
 965         unsigned int flags;
 966         int rc;
 967
 968         bnxt_re_debug_rem_qpinfo(rdev, qp);
 969
 970         bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
 971
 972         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 973         if (rc)
 974                 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
 975
 976         if (rdma_is_kernel_res(&qp->ib_qp.res)) {
 977                 flags = bnxt_re_lock_cqs(qp);
 978                 bnxt_qplib_clean_qp(&qp->qplib_qp);
 979                 bnxt_re_unlock_cqs(qp, flags);
 980         }
 981
 982         bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
 983
 984         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
 985                 bnxt_re_destroy_gsi_sqp(qp);
 986
 987         mutex_lock(&rdev->qp_lock);
 988         list_del(&qp->list);
 989         mutex_unlock(&rdev->qp_lock);
 990         atomic_dec(&rdev->stats.res.qp_count);
 991         if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
 992                 atomic_dec(&rdev->stats.res.rc_qp_count);
 993         else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
 994                 atomic_dec(&rdev->stats.res.ud_qp_count);
 995
 996         ib_umem_release(qp->rumem);
 997         ib_umem_release(qp->sumem);
 998
 999         /* Flush all the entries of notification queue associated with
1000          * given qp.
1001          */
1002         scq_nq = qplib_qp->scq->nq;
1003         rcq_nq = qplib_qp->rcq->nq;
1004         bnxt_re_synchronize_nq(scq_nq);
1005         if (scq_nq != rcq_nq)
1006                 bnxt_re_synchronize_nq(rcq_nq);
1007
1008         return 0;
1009 }
1010
1011 static u8 __from_ib_qp_type(enum ib_qp_type type)
1012 {
1013         switch (type) {
1014         case IB_QPT_GSI:
1015                 return CMDQ_CREATE_QP1_TYPE_GSI;
1016         case IB_QPT_RC:
1017                 return CMDQ_CREATE_QP_TYPE_RC;
1018         case IB_QPT_UD:
1019                 return CMDQ_CREATE_QP_TYPE_UD;
1020         default:
1021                 return IB_QPT_MAX;
1022         }
1023 }
1024
1025 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
1026                                    int rsge, int max)
1027 {
1028         if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1029                 rsge = max;
1030         return bnxt_re_get_rwqe_size(rsge);
1031 }
1032
1033 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
1034 {
1035         u16 wqe_size, calc_ils;
1036
1037         wqe_size = bnxt_re_get_swqe_size(nsge);
1038         if (ilsize) {
1039                 calc_ils = sizeof(struct sq_send_hdr) + ilsize;
1040                 wqe_size = max_t(u16, calc_ils, wqe_size);
1041                 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
1042         }
1043         return wqe_size;
1044 }
1045
1046 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
1047                                    struct ib_qp_init_attr *init_attr)
1048 {
1049         struct bnxt_qplib_dev_attr *dev_attr;
1050         struct bnxt_qplib_qp *qplqp;
1051         struct bnxt_re_dev *rdev;
1052         struct bnxt_qplib_q *sq;
1053         int align, ilsize;
1054
1055         rdev = qp->rdev;
1056         qplqp = &qp->qplib_qp;
1057         sq = &qplqp->sq;
1058         dev_attr = rdev->dev_attr;
1059
1060         align = sizeof(struct sq_send_hdr);
1061         ilsize = ALIGN(init_attr->cap.max_inline_data, align);
1062
1063         /* For gen p4 and gen p5 fixed wqe compatibility mode
1064          * wqe size is fixed to 128 bytes - ie 6 SGEs
1065          */
1066         if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) {
1067                 sq->wqe_size = bnxt_re_get_swqe_size(BNXT_STATIC_MAX_SGE);
1068                 sq->max_sge = BNXT_STATIC_MAX_SGE;
1069         } else {
1070                 sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
1071                 if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
1072                         return -EINVAL;
1073         }
1074
1075         if (init_attr->cap.max_inline_data) {
1076                 qplqp->max_inline_data = sq->wqe_size -
1077                         sizeof(struct sq_send_hdr);
1078                 init_attr->cap.max_inline_data = qplqp->max_inline_data;
1079         }
1080
1081         return 0;
1082 }
1083
1084 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
1085                                 struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,
1086                                 struct bnxt_re_qp_req *ureq)
1087 {
1088         struct bnxt_qplib_qp *qplib_qp;
1089         int bytes = 0, psn_sz;
1090         struct ib_umem *umem;
1091         int psn_nume;
1092
1093         qplib_qp = &qp->qplib_qp;
1094
1095         bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
1096         /* Consider mapping PSN search memory only for RC QPs. */
1097         if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
1098                 psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
1099                                                    sizeof(struct sq_psn_search_ext) :
1100                                                    sizeof(struct sq_psn_search);
1101                 if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) {
1102                         psn_nume = ureq->sq_slots;
1103                 } else {
1104                         psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1105                         qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
1106                                  sizeof(struct bnxt_qplib_sge));
1107                 }
1108                 if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2))
1109                         psn_nume = roundup_pow_of_two(psn_nume);
1110                 bytes += (psn_nume * psn_sz);
1111         }
1112
1113         bytes = PAGE_ALIGN(bytes);
1114         umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes,
1115                            IB_ACCESS_LOCAL_WRITE);
1116         if (IS_ERR(umem))
1117                 return PTR_ERR(umem);
1118
1119         qp->sumem = umem;
1120         qplib_qp->sq.sg_info.umem = umem;
1121         qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
1122         qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
1123         qplib_qp->qp_handle = ureq->qp_handle;
1124
1125         if (!qp->qplib_qp.srq) {
1126                 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
1127                 bytes = PAGE_ALIGN(bytes);
1128                 umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes,
1129                                    IB_ACCESS_LOCAL_WRITE);
1130                 if (IS_ERR(umem))
1131                         goto rqfail;
1132                 qp->rumem = umem;
1133                 qplib_qp->rq.sg_info.umem = umem;
1134                 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
1135                 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
1136         }
1137
1138         qplib_qp->dpi = &cntx->dpi;
1139         return 0;
1140 rqfail:
1141         ib_umem_release(qp->sumem);
1142         qp->sumem = NULL;
1143         memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
1144
1145         return PTR_ERR(umem);
1146 }
1147
1148 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
1149                                 (struct bnxt_re_pd *pd,
1150                                  struct bnxt_qplib_res *qp1_res,
1151                                  struct bnxt_qplib_qp *qp1_qp)
1152 {
1153         struct bnxt_re_dev *rdev = pd->rdev;
1154         struct bnxt_re_ah *ah;
1155         union ib_gid sgid;
1156         int rc;
1157
1158         ah = kzalloc(sizeof(*ah), GFP_KERNEL);
1159         if (!ah)
1160                 return NULL;
1161
1162         ah->rdev = rdev;
1163         ah->qplib_ah.pd = &pd->qplib_pd;
1164
1165         rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
1166         if (rc)
1167                 goto fail;
1168
1169         /* supply the dgid data same as sgid */
1170         memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
1171                sizeof(union ib_gid));
1172         ah->qplib_ah.sgid_index = 0;
1173
1174         ah->qplib_ah.traffic_class = 0;
1175         ah->qplib_ah.flow_label = 0;
1176         ah->qplib_ah.hop_limit = 1;
1177         ah->qplib_ah.sl = 0;
1178         /* Have DMAC same as SMAC */
1179         ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1180
1181         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1182         if (rc) {
1183                 ibdev_err(&rdev->ibdev,
1184                           "Failed to allocate HW AH for Shadow QP");
1185                 goto fail;
1186         }
1187         atomic_inc(&rdev->stats.res.ah_count);
1188
1189         return ah;
1190
1191 fail:
1192         kfree(ah);
1193         return NULL;
1194 }
1195
1196 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1197                                 (struct bnxt_re_pd *pd,
1198                                  struct bnxt_qplib_res *qp1_res,
1199                                  struct bnxt_qplib_qp *qp1_qp)
1200 {
1201         struct bnxt_re_dev *rdev = pd->rdev;
1202         struct bnxt_re_qp *qp;
1203         int rc;
1204
1205         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1206         if (!qp)
1207                 return NULL;
1208
1209         qp->rdev = rdev;
1210
1211         /* Initialize the shadow QP structure from the QP1 values */
1212         ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1213
1214         qp->qplib_qp.pd = &pd->qplib_pd;
1215         qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1216         qp->qplib_qp.type = IB_QPT_UD;
1217
1218         qp->qplib_qp.max_inline_data = 0;
1219         qp->qplib_qp.sig_type = true;
1220
1221         /* Shadow QP SQ depth should be same as QP1 RQ depth */
1222         qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1223         qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1224         qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe;
1225         qp->qplib_qp.sq.max_sge = 2;
1226         /* Q full delta can be 1 since it is internal QP */
1227         qp->qplib_qp.sq.q_full_delta = 1;
1228         qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1229         qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1230
1231         qp->qplib_qp.scq = qp1_qp->scq;
1232         qp->qplib_qp.rcq = qp1_qp->rcq;
1233
1234         qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1235         qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1236         qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe;
1237         qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1238         /* Q full delta can be 1 since it is internal QP */
1239         qp->qplib_qp.rq.q_full_delta = 1;
1240         qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1241         qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1242
1243         qp->qplib_qp.mtu = qp1_qp->mtu;
1244
1245         qp->qplib_qp.sq_hdr_buf_size = 0;
1246         qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1247         qp->qplib_qp.dpi = &rdev->dpi_privileged;
1248
1249         rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1250         if (rc)
1251                 goto fail;
1252
1253         spin_lock_init(&qp->sq_lock);
1254         INIT_LIST_HEAD(&qp->list);
1255         mutex_lock(&rdev->qp_lock);
1256         list_add_tail(&qp->list, &rdev->qp_list);
1257         atomic_inc(&rdev->stats.res.qp_count);
1258         mutex_unlock(&rdev->qp_lock);
1259         return qp;
1260 fail:
1261         kfree(qp);
1262         return NULL;
1263 }
1264
1265 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1266                                 struct ib_qp_init_attr *init_attr,
1267                                 struct bnxt_re_ucontext *uctx)
1268 {
1269         struct bnxt_qplib_dev_attr *dev_attr;
1270         struct bnxt_qplib_qp *qplqp;
1271         struct bnxt_re_dev *rdev;
1272         struct bnxt_qplib_q *rq;
1273         int entries;
1274
1275         rdev = qp->rdev;
1276         qplqp = &qp->qplib_qp;
1277         rq = &qplqp->rq;
1278         dev_attr = rdev->dev_attr;
1279
1280         if (init_attr->srq) {
1281                 struct bnxt_re_srq *srq;
1282
1283                 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1284                 qplqp->srq = &srq->qplib_srq;
1285                 rq->max_wqe = 0;
1286         } else {
1287                 rq->max_sge = init_attr->cap.max_recv_sge;
1288                 if (rq->max_sge > dev_attr->max_qp_sges)
1289                         rq->max_sge = dev_attr->max_qp_sges;
1290                 init_attr->cap.max_recv_sge = rq->max_sge;
1291                 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1292                                                        dev_attr->max_qp_sges);
1293                 /* Allocate 1 more than what's provided so posting max doesn't
1294                  * mean empty.
1295                  */
1296                 entries = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, uctx);
1297                 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1298                 rq->max_sw_wqe = rq->max_wqe;
1299                 rq->q_full_delta = 0;
1300                 rq->sg_info.pgsize = PAGE_SIZE;
1301                 rq->sg_info.pgshft = PAGE_SHIFT;
1302         }
1303
1304         return 0;
1305 }
1306
1307 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1308 {
1309         struct bnxt_qplib_dev_attr *dev_attr;
1310         struct bnxt_qplib_qp *qplqp;
1311         struct bnxt_re_dev *rdev;
1312
1313         rdev = qp->rdev;
1314         qplqp = &qp->qplib_qp;
1315         dev_attr = rdev->dev_attr;
1316
1317         if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) {
1318                 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1319                 if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1320                         qplqp->rq.max_sge = dev_attr->max_qp_sges;
1321                 qplqp->rq.max_sge = 6;
1322         }
1323 }
1324
1325 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1326                                 struct ib_qp_init_attr *init_attr,
1327                                 struct bnxt_re_ucontext *uctx,
1328                                 struct bnxt_re_qp_req *ureq)
1329 {
1330         struct bnxt_qplib_dev_attr *dev_attr;
1331         struct bnxt_qplib_qp *qplqp;
1332         struct bnxt_re_dev *rdev;
1333         struct bnxt_qplib_q *sq;
1334         int diff = 0;
1335         int entries;
1336         int rc;
1337
1338         rdev = qp->rdev;
1339         qplqp = &qp->qplib_qp;
1340         sq = &qplqp->sq;
1341         dev_attr = rdev->dev_attr;
1342
1343         sq->max_sge = init_attr->cap.max_send_sge;
1344         entries = init_attr->cap.max_send_wr;
1345         if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
1346                 sq->max_wqe = ureq->sq_slots;
1347                 sq->max_sw_wqe = ureq->sq_slots;
1348                 sq->wqe_size = sizeof(struct sq_sge);
1349         } else {
1350                 if (sq->max_sge > dev_attr->max_qp_sges) {
1351                         sq->max_sge = dev_attr->max_qp_sges;
1352                         init_attr->cap.max_send_sge = sq->max_sge;
1353                 }
1354
1355                 rc = bnxt_re_setup_swqe_size(qp, init_attr);
1356                 if (rc)
1357                         return rc;
1358
1359                 /* Allocate 128 + 1 more than what's provided */
1360                 diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1361                         0 : BNXT_QPLIB_RESERVED_QP_WRS;
1362                 entries = bnxt_re_init_depth(entries + diff + 1, uctx);
1363                 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1364                 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
1365                         sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp->wqe_mode, true);
1366                 else
1367                         sq->max_sw_wqe = sq->max_wqe;
1368
1369         }
1370         sq->q_full_delta = diff + 1;
1371         /*
1372          * Reserving one slot for Phantom WQE. Application can
1373          * post one extra entry in this case. But allowing this to avoid
1374          * unexpected Queue full condition
1375          */
1376         qplqp->sq.q_full_delta -= 1;
1377         qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1378         qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1379
1380         return 0;
1381 }
1382
1383 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1384                                        struct ib_qp_init_attr *init_attr,
1385                                        struct bnxt_re_ucontext *uctx)
1386 {
1387         struct bnxt_qplib_dev_attr *dev_attr;
1388         struct bnxt_qplib_qp *qplqp;
1389         struct bnxt_re_dev *rdev;
1390         int entries;
1391
1392         rdev = qp->rdev;
1393         qplqp = &qp->qplib_qp;
1394         dev_attr = rdev->dev_attr;
1395
1396         if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) {
1397                 entries = bnxt_re_init_depth(init_attr->cap.max_send_wr + 1, uctx);
1398                 qplqp->sq.max_wqe = min_t(u32, entries,
1399                                           dev_attr->max_qp_wqes + 1);
1400                 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1401                         init_attr->cap.max_send_wr;
1402                 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1403                 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1404                         qplqp->sq.max_sge = dev_attr->max_qp_sges;
1405         }
1406 }
1407
1408 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1409                                 struct ib_qp_init_attr *init_attr)
1410 {
1411         struct bnxt_qplib_chip_ctx *chip_ctx;
1412         int qptype;
1413
1414         chip_ctx = rdev->chip_ctx;
1415
1416         qptype = __from_ib_qp_type(init_attr->qp_type);
1417         if (qptype == IB_QPT_MAX) {
1418                 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
1419                 qptype = -EOPNOTSUPP;
1420                 goto out;
1421         }
1422
1423         if (bnxt_qplib_is_chip_gen_p5_p7(chip_ctx) &&
1424             init_attr->qp_type == IB_QPT_GSI)
1425                 qptype = CMDQ_CREATE_QP_TYPE_GSI;
1426 out:
1427         return qptype;
1428 }
1429
1430 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1431                                 struct ib_qp_init_attr *init_attr,
1432                                 struct bnxt_re_ucontext *uctx,
1433                                 struct bnxt_re_qp_req *ureq)
1434 {
1435         struct bnxt_qplib_dev_attr *dev_attr;
1436         struct bnxt_qplib_qp *qplqp;
1437         struct bnxt_re_dev *rdev;
1438         struct bnxt_re_cq *cq;
1439         int rc = 0, qptype;
1440
1441         rdev = qp->rdev;
1442         qplqp = &qp->qplib_qp;
1443         dev_attr = rdev->dev_attr;
1444
1445         /* Setup misc params */
1446         ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1447         qplqp->pd = &pd->qplib_pd;
1448         qplqp->qp_handle = (u64)qplqp;
1449         qplqp->max_inline_data = init_attr->cap.max_inline_data;
1450         qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1451         qptype = bnxt_re_init_qp_type(rdev, init_attr);
1452         if (qptype < 0) {
1453                 rc = qptype;
1454                 goto out;
1455         }
1456         qplqp->type = (u8)qptype;
1457         qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx);
1458         if (init_attr->qp_type == IB_QPT_RC) {
1459                 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1460                 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1461         }
1462         qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1463         qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1464         if (init_attr->create_flags) {
1465                 ibdev_dbg(&rdev->ibdev,
1466                           "QP create flags 0x%x not supported",
1467                           init_attr->create_flags);
1468                 return -EOPNOTSUPP;
1469         }
1470
1471         /* Setup CQs */
1472         if (init_attr->send_cq) {
1473                 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1474                 qplqp->scq = &cq->qplib_cq;
1475                 qp->scq = cq;
1476         }
1477
1478         if (init_attr->recv_cq) {
1479                 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1480                 qplqp->rcq = &cq->qplib_cq;
1481                 qp->rcq = cq;
1482         }
1483
1484         /* Setup RQ/SRQ */
1485         rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);
1486         if (rc)
1487                 goto out;
1488         if (init_attr->qp_type == IB_QPT_GSI)
1489                 bnxt_re_adjust_gsi_rq_attr(qp);
1490
1491         /* Setup SQ */
1492         rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq);
1493         if (rc)
1494                 goto out;
1495         if (init_attr->qp_type == IB_QPT_GSI)
1496                 bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
1497
1498         if (uctx) /* This will update DPI and qp_handle */
1499                 rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq);
1500 out:
1501         return rc;
1502 }
1503
1504 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1505                                      struct bnxt_re_pd *pd)
1506 {
1507         struct bnxt_re_sqp_entries *sqp_tbl;
1508         struct bnxt_re_dev *rdev;
1509         struct bnxt_re_qp *sqp;
1510         struct bnxt_re_ah *sah;
1511         int rc = 0;
1512
1513         rdev = qp->rdev;
1514         /* Create a shadow QP to handle the QP1 traffic */
1515         sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl),
1516                           GFP_KERNEL);
1517         if (!sqp_tbl)
1518                 return -ENOMEM;
1519         rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1520
1521         sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1522         if (!sqp) {
1523                 rc = -ENODEV;
1524                 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
1525                 goto out;
1526         }
1527         rdev->gsi_ctx.gsi_sqp = sqp;
1528
1529         sqp->rcq = qp->rcq;
1530         sqp->scq = qp->scq;
1531         sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1532                                           &qp->qplib_qp);
1533         if (!sah) {
1534                 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1535                                       &sqp->qplib_qp);
1536                 rc = -ENODEV;
1537                 ibdev_err(&rdev->ibdev,
1538                           "Failed to create AH entry for ShadowQP");
1539                 goto out;
1540         }
1541         rdev->gsi_ctx.gsi_sah = sah;
1542
1543         return 0;
1544 out:
1545         kfree(sqp_tbl);
1546         return rc;
1547 }
1548
1549 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1550                                  struct ib_qp_init_attr *init_attr)
1551 {
1552         struct bnxt_re_dev *rdev;
1553         struct bnxt_qplib_qp *qplqp;
1554         int rc;
1555
1556         rdev = qp->rdev;
1557         qplqp = &qp->qplib_qp;
1558
1559         qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1560         qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1561
1562         rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1563         if (rc) {
1564                 ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
1565                 goto out;
1566         }
1567
1568         rc = bnxt_re_create_shadow_gsi(qp, pd);
1569 out:
1570         return rc;
1571 }
1572
1573 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1574                                    struct ib_qp_init_attr *init_attr,
1575                                    struct bnxt_qplib_dev_attr *dev_attr)
1576 {
1577         bool rc = true;
1578
1579         if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1580             init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1581             init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1582             init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1583             init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1584                 ibdev_err(&rdev->ibdev,
1585                           "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1586                           init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1587                           init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1588                           init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1589                           init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1590                           init_attr->cap.max_inline_data,
1591                           dev_attr->max_inline_data);
1592                 rc = false;
1593         }
1594         return rc;
1595 }
1596
1597 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1598                       struct ib_udata *udata)
1599 {
1600         struct bnxt_qplib_dev_attr *dev_attr;
1601         struct bnxt_re_ucontext *uctx;
1602         struct bnxt_re_qp_req ureq;
1603         struct bnxt_re_dev *rdev;
1604         struct bnxt_re_pd *pd;
1605         struct bnxt_re_qp *qp;
1606         struct ib_pd *ib_pd;
1607         u32 active_qps;
1608         int rc;
1609
1610         ib_pd = ib_qp->pd;
1611         pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1612         rdev = pd->rdev;
1613         dev_attr = rdev->dev_attr;
1614         qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1615
1616         uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
1617         if (udata)
1618                 if (ib_copy_from_udata(&ureq, udata,  min(udata->inlen, sizeof(ureq))))
1619                         return -EFAULT;
1620
1621         rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1622         if (!rc) {
1623                 rc = -EINVAL;
1624                 goto fail;
1625         }
1626
1627         qp->rdev = rdev;
1628         rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq);
1629         if (rc)
1630                 goto fail;
1631
1632         if (qp_init_attr->qp_type == IB_QPT_GSI &&
1633             !(bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))) {
1634                 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1635                 if (rc == -ENODEV)
1636                         goto qp_destroy;
1637                 if (rc)
1638                         goto fail;
1639         } else {
1640                 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1641                 if (rc) {
1642                         ibdev_err(&rdev->ibdev, "Failed to create HW QP");
1643                         goto free_umem;
1644                 }
1645                 if (udata) {
1646                         struct bnxt_re_qp_resp resp;
1647
1648                         resp.qpid = qp->qplib_qp.id;
1649                         resp.rsvd = 0;
1650                         rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1651                         if (rc) {
1652                                 ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
1653                                 goto qp_destroy;
1654                         }
1655                 }
1656         }
1657
1658         qp->ib_qp.qp_num = qp->qplib_qp.id;
1659         if (qp_init_attr->qp_type == IB_QPT_GSI)
1660                 rdev->gsi_ctx.gsi_qp = qp;
1661         spin_lock_init(&qp->sq_lock);
1662         spin_lock_init(&qp->rq_lock);
1663         INIT_LIST_HEAD(&qp->list);
1664         mutex_lock(&rdev->qp_lock);
1665         list_add_tail(&qp->list, &rdev->qp_list);
1666         mutex_unlock(&rdev->qp_lock);
1667         active_qps = atomic_inc_return(&rdev->stats.res.qp_count);
1668         if (active_qps > rdev->stats.res.qp_watermark)
1669                 rdev->stats.res.qp_watermark = active_qps;
1670         if (qp_init_attr->qp_type == IB_QPT_RC) {
1671                 active_qps = atomic_inc_return(&rdev->stats.res.rc_qp_count);
1672                 if (active_qps > rdev->stats.res.rc_qp_watermark)
1673                         rdev->stats.res.rc_qp_watermark = active_qps;
1674         } else if (qp_init_attr->qp_type == IB_QPT_UD) {
1675                 active_qps = atomic_inc_return(&rdev->stats.res.ud_qp_count);
1676                 if (active_qps > rdev->stats.res.ud_qp_watermark)
1677                         rdev->stats.res.ud_qp_watermark = active_qps;
1678         }
1679         bnxt_re_debug_add_qpinfo(rdev, qp);
1680
1681         return 0;
1682 qp_destroy:
1683         bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1684 free_umem:
1685         ib_umem_release(qp->rumem);
1686         ib_umem_release(qp->sumem);
1687 fail:
1688         return rc;
1689 }
1690
1691 static u8 __from_ib_qp_state(enum ib_qp_state state)
1692 {
1693         switch (state) {
1694         case IB_QPS_RESET:
1695                 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1696         case IB_QPS_INIT:
1697                 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1698         case IB_QPS_RTR:
1699                 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1700         case IB_QPS_RTS:
1701                 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1702         case IB_QPS_SQD:
1703                 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1704         case IB_QPS_SQE:
1705                 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1706         case IB_QPS_ERR:
1707         default:
1708                 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1709         }
1710 }
1711
1712 static enum ib_qp_state __to_ib_qp_state(u8 state)
1713 {
1714         switch (state) {
1715         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1716                 return IB_QPS_RESET;
1717         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1718                 return IB_QPS_INIT;
1719         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1720                 return IB_QPS_RTR;
1721         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1722                 return IB_QPS_RTS;
1723         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1724                 return IB_QPS_SQD;
1725         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1726                 return IB_QPS_SQE;
1727         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1728         default:
1729                 return IB_QPS_ERR;
1730         }
1731 }
1732
1733 static u32 __from_ib_mtu(enum ib_mtu mtu)
1734 {
1735         switch (mtu) {
1736         case IB_MTU_256:
1737                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1738         case IB_MTU_512:
1739                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1740         case IB_MTU_1024:
1741                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1742         case IB_MTU_2048:
1743                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1744         case IB_MTU_4096:
1745                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1746         default:
1747                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1748         }
1749 }
1750
1751 static enum ib_mtu __to_ib_mtu(u32 mtu)
1752 {
1753         switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1754         case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1755                 return IB_MTU_256;
1756         case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1757                 return IB_MTU_512;
1758         case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1759                 return IB_MTU_1024;
1760         case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1761                 return IB_MTU_2048;
1762         case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1763                 return IB_MTU_4096;
1764         default:
1765                 return IB_MTU_2048;
1766         }
1767 }
1768
1769 /* Shared Receive Queues */
1770 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
1771 {
1772         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1773                                                ib_srq);
1774         struct bnxt_re_dev *rdev = srq->rdev;
1775         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1776         struct bnxt_qplib_nq *nq = NULL;
1777
1778         if (qplib_srq->cq)
1779                 nq = qplib_srq->cq->nq;
1780         if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
1781                 free_page((unsigned long)srq->uctx_srq_page);
1782                 hash_del(&srq->hash_entry);
1783         }
1784         bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
1785         ib_umem_release(srq->umem);
1786         atomic_dec(&rdev->stats.res.srq_count);
1787         if (nq)
1788                 nq->budget--;
1789         return 0;
1790 }
1791
1792 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1793                                  struct bnxt_re_pd *pd,
1794                                  struct bnxt_re_srq *srq,
1795                                  struct ib_udata *udata)
1796 {
1797         struct bnxt_re_srq_req ureq;
1798         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1799         struct ib_umem *umem;
1800         int bytes = 0;
1801         struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1802                 udata, struct bnxt_re_ucontext, ib_uctx);
1803
1804         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1805                 return -EFAULT;
1806
1807         bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1808         bytes = PAGE_ALIGN(bytes);
1809         umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
1810                            IB_ACCESS_LOCAL_WRITE);
1811         if (IS_ERR(umem))
1812                 return PTR_ERR(umem);
1813
1814         srq->umem = umem;
1815         qplib_srq->sg_info.umem = umem;
1816         qplib_srq->sg_info.pgsize = PAGE_SIZE;
1817         qplib_srq->sg_info.pgshft = PAGE_SHIFT;
1818         qplib_srq->srq_handle = ureq.srq_handle;
1819         qplib_srq->dpi = &cntx->dpi;
1820
1821         return 0;
1822 }
1823
1824 int bnxt_re_create_srq(struct ib_srq *ib_srq,
1825                        struct ib_srq_init_attr *srq_init_attr,
1826                        struct ib_udata *udata)
1827 {
1828         struct bnxt_qplib_dev_attr *dev_attr;
1829         struct bnxt_qplib_nq *nq = NULL;
1830         struct bnxt_re_ucontext *uctx;
1831         struct bnxt_re_dev *rdev;
1832         struct bnxt_re_srq *srq;
1833         struct bnxt_re_pd *pd;
1834         struct ib_pd *ib_pd;
1835         u32 active_srqs;
1836         int rc, entries;
1837
1838         ib_pd = ib_srq->pd;
1839         pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1840         rdev = pd->rdev;
1841         dev_attr = rdev->dev_attr;
1842         srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1843
1844         if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1845                 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
1846                 rc = -EINVAL;
1847                 goto exit;
1848         }
1849
1850         if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1851                 rc = -EOPNOTSUPP;
1852                 goto exit;
1853         }
1854
1855         uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
1856         srq->rdev = rdev;
1857         srq->qplib_srq.pd = &pd->qplib_pd;
1858         srq->qplib_srq.dpi = &rdev->dpi_privileged;
1859         /* Allocate 1 more than what's provided so posting max doesn't
1860          * mean empty
1861          */
1862         entries = bnxt_re_init_depth(srq_init_attr->attr.max_wr + 1, uctx);
1863         if (entries > dev_attr->max_srq_wqes + 1)
1864                 entries = dev_attr->max_srq_wqes + 1;
1865         srq->qplib_srq.max_wqe = entries;
1866
1867         srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1868          /* 128 byte wqe size for SRQ . So use max sges */
1869         srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
1870         srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1871         srq->srq_limit = srq_init_attr->attr.srq_limit;
1872         srq->qplib_srq.eventq_hw_ring_id = rdev->nqr->nq[0].ring_id;
1873         nq = &rdev->nqr->nq[0];
1874
1875         if (udata) {
1876                 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1877                 if (rc)
1878                         goto fail;
1879         }
1880
1881         rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1882         if (rc) {
1883                 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
1884                 goto fail;
1885         }
1886
1887         if (udata) {
1888                 struct bnxt_re_srq_resp resp = {};
1889
1890                 resp.srqid = srq->qplib_srq.id;
1891                 if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
1892                         hash_add(rdev->srq_hash, &srq->hash_entry, srq->qplib_srq.id);
1893                         srq->uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);
1894                         if (!srq->uctx_srq_page) {
1895                                 rc = -ENOMEM;
1896                                 goto fail;
1897                         }
1898                         resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;
1899                 }
1900                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1901                 if (rc) {
1902                         ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
1903                         bnxt_qplib_destroy_srq(&rdev->qplib_res,
1904                                                &srq->qplib_srq);
1905                         goto fail;
1906                 }
1907         }
1908         if (nq)
1909                 nq->budget++;
1910         active_srqs = atomic_inc_return(&rdev->stats.res.srq_count);
1911         if (active_srqs > rdev->stats.res.srq_watermark)
1912                 rdev->stats.res.srq_watermark = active_srqs;
1913         spin_lock_init(&srq->lock);
1914
1915         return 0;
1916
1917 fail:
1918         ib_umem_release(srq->umem);
1919 exit:
1920         return rc;
1921 }
1922
1923 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1924                        enum ib_srq_attr_mask srq_attr_mask,
1925                        struct ib_udata *udata)
1926 {
1927         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1928                                                ib_srq);
1929         struct bnxt_re_dev *rdev = srq->rdev;
1930         int rc;
1931
1932         switch (srq_attr_mask) {
1933         case IB_SRQ_MAX_WR:
1934                 /* SRQ resize is not supported */
1935                 return -EINVAL;
1936         case IB_SRQ_LIMIT:
1937                 /* Change the SRQ threshold */
1938                 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1939                         return -EINVAL;
1940
1941                 srq->qplib_srq.threshold = srq_attr->srq_limit;
1942                 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
1943                 if (rc) {
1944                         ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!");
1945                         return rc;
1946                 }
1947                 /* On success, update the shadow */
1948                 srq->srq_limit = srq_attr->srq_limit;
1949                 /* No need to Build and send response back to udata */
1950                 return 0;
1951         default:
1952                 ibdev_err(&rdev->ibdev,
1953                           "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
1954                 return -EINVAL;
1955         }
1956 }
1957
1958 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
1959 {
1960         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1961                                                ib_srq);
1962         struct bnxt_re_srq tsrq;
1963         struct bnxt_re_dev *rdev = srq->rdev;
1964         int rc;
1965
1966         /* Get live SRQ attr */
1967         tsrq.qplib_srq.id = srq->qplib_srq.id;
1968         rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
1969         if (rc) {
1970                 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
1971                 return rc;
1972         }
1973         srq_attr->max_wr = srq->qplib_srq.max_wqe;
1974         srq_attr->max_sge = srq->qplib_srq.max_sge;
1975         srq_attr->srq_limit = tsrq.qplib_srq.threshold;
1976
1977         return 0;
1978 }
1979
1980 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
1981                           const struct ib_recv_wr **bad_wr)
1982 {
1983         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1984                                                ib_srq);
1985         struct bnxt_qplib_swqe wqe;
1986         unsigned long flags;
1987         int rc = 0;
1988
1989         spin_lock_irqsave(&srq->lock, flags);
1990         while (wr) {
1991                 /* Transcribe each ib_recv_wr to qplib_swqe */
1992                 wqe.num_sge = wr->num_sge;
1993                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
1994                 wqe.wr_id = wr->wr_id;
1995                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
1996
1997                 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
1998                 if (rc) {
1999                         *bad_wr = wr;
2000                         break;
2001                 }
2002                 wr = wr->next;
2003         }
2004         spin_unlock_irqrestore(&srq->lock, flags);
2005
2006         return rc;
2007 }
2008 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
2009                                     struct bnxt_re_qp *qp1_qp,
2010                                     int qp_attr_mask)
2011 {
2012         struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
2013         int rc;
2014
2015         if (qp_attr_mask & IB_QP_STATE) {
2016                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2017                 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
2018         }
2019         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2020                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2021                 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
2022         }
2023
2024         if (qp_attr_mask & IB_QP_QKEY) {
2025                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2026                 /* Using a Random  QKEY */
2027                 qp->qplib_qp.qkey = 0x81818181;
2028         }
2029         if (qp_attr_mask & IB_QP_SQ_PSN) {
2030                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2031                 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
2032         }
2033
2034         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2035         if (rc)
2036                 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
2037         return rc;
2038 }
2039
2040 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2041                       int qp_attr_mask, struct ib_udata *udata)
2042 {
2043         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2044         struct bnxt_re_dev *rdev = qp->rdev;
2045         struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
2046         enum ib_qp_state curr_qp_state, new_qp_state;
2047         int rc, entries;
2048         unsigned int flags;
2049         u8 nw_type;
2050
2051         if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2052                 return -EOPNOTSUPP;
2053
2054         qp->qplib_qp.modify_flags = 0;
2055         if (qp_attr_mask & IB_QP_STATE) {
2056                 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
2057                 new_qp_state = qp_attr->qp_state;
2058                 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
2059                                         ib_qp->qp_type, qp_attr_mask)) {
2060                         ibdev_err(&rdev->ibdev,
2061                                   "Invalid attribute mask: %#x specified ",
2062                                   qp_attr_mask);
2063                         ibdev_err(&rdev->ibdev,
2064                                   "for qpn: %#x type: %#x",
2065                                   ib_qp->qp_num, ib_qp->qp_type);
2066                         ibdev_err(&rdev->ibdev,
2067                                   "curr_qp_state=0x%x, new_qp_state=0x%x\n",
2068                                   curr_qp_state, new_qp_state);
2069                         return -EINVAL;
2070                 }
2071                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2072                 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
2073
2074                 if (!qp->sumem &&
2075                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
2076                         ibdev_dbg(&rdev->ibdev,
2077                                   "Move QP = %p to flush list\n", qp);
2078                         flags = bnxt_re_lock_cqs(qp);
2079                         bnxt_qplib_add_flush_qp(&qp->qplib_qp);
2080                         bnxt_re_unlock_cqs(qp, flags);
2081                 }
2082                 if (!qp->sumem &&
2083                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
2084                         ibdev_dbg(&rdev->ibdev,
2085                                   "Move QP = %p out of flush list\n", qp);
2086                         flags = bnxt_re_lock_cqs(qp);
2087                         bnxt_qplib_clean_qp(&qp->qplib_qp);
2088                         bnxt_re_unlock_cqs(qp, flags);
2089                 }
2090         }
2091         if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
2092                 qp->qplib_qp.modify_flags |=
2093                                 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
2094                 qp->qplib_qp.en_sqd_async_notify = true;
2095         }
2096         if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
2097                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
2098                 qp->qplib_qp.access =
2099                         __qp_access_flags_from_ib(qp->qplib_qp.cctx,
2100                                                   qp_attr->qp_access_flags);
2101                 /* LOCAL_WRITE access must be set to allow RC receive */
2102                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
2103         }
2104         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2105                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2106                 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
2107         }
2108         if (qp_attr_mask & IB_QP_QKEY) {
2109                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2110                 qp->qplib_qp.qkey = qp_attr->qkey;
2111         }
2112         if (qp_attr_mask & IB_QP_AV) {
2113                 const struct ib_global_route *grh =
2114                         rdma_ah_read_grh(&qp_attr->ah_attr);
2115                 const struct ib_gid_attr *sgid_attr;
2116                 struct bnxt_re_gid_ctx *ctx;
2117
2118                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
2119                                      CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
2120                                      CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
2121                                      CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
2122                                      CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
2123                                      CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
2124                                      CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
2125                 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
2126                        sizeof(qp->qplib_qp.ah.dgid.data));
2127                 qp->qplib_qp.ah.flow_label = grh->flow_label;
2128                 sgid_attr = grh->sgid_attr;
2129                 /* Get the HW context of the GID. The reference
2130                  * of GID table entry is already taken by the caller.
2131                  */
2132                 ctx = rdma_read_gid_hw_context(sgid_attr);
2133                 qp->qplib_qp.ah.sgid_index = ctx->idx;
2134                 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
2135                 qp->qplib_qp.ah.hop_limit = grh->hop_limit;
2136                 qp->qplib_qp.ah.traffic_class = grh->traffic_class >> 2;
2137                 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
2138                 ether_addr_copy(qp->qplib_qp.ah.dmac,
2139                                 qp_attr->ah_attr.roce.dmac);
2140
2141                 rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
2142                                              &qp->qplib_qp.smac[0]);
2143                 if (rc)
2144                         return rc;
2145
2146                 nw_type = rdma_gid_attr_network_type(sgid_attr);
2147                 switch (nw_type) {
2148                 case RDMA_NETWORK_IPV4:
2149                         qp->qplib_qp.nw_type =
2150                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
2151                         break;
2152                 case RDMA_NETWORK_IPV6:
2153                         qp->qplib_qp.nw_type =
2154                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
2155                         break;
2156                 default:
2157                         qp->qplib_qp.nw_type =
2158                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
2159                         break;
2160                 }
2161         }
2162
2163         if (qp_attr->qp_state == IB_QPS_RTR) {
2164                 enum ib_mtu qpmtu;
2165
2166                 qpmtu = iboe_get_mtu(rdev->netdev->mtu);
2167                 if (qp_attr_mask & IB_QP_PATH_MTU) {
2168                         if (ib_mtu_enum_to_int(qp_attr->path_mtu) >
2169                             ib_mtu_enum_to_int(qpmtu))
2170                                 return -EINVAL;
2171                         qpmtu = qp_attr->path_mtu;
2172                 }
2173
2174                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
2175                 qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu);
2176                 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qpmtu);
2177         }
2178
2179         if (qp_attr_mask & IB_QP_TIMEOUT) {
2180                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
2181                 qp->qplib_qp.timeout = qp_attr->timeout;
2182         }
2183         if (qp_attr_mask & IB_QP_RETRY_CNT) {
2184                 qp->qplib_qp.modify_flags |=
2185                                 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
2186                 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
2187         }
2188         if (qp_attr_mask & IB_QP_RNR_RETRY) {
2189                 qp->qplib_qp.modify_flags |=
2190                                 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
2191                 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
2192         }
2193         if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
2194                 qp->qplib_qp.modify_flags |=
2195                                 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
2196                 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
2197         }
2198         if (qp_attr_mask & IB_QP_RQ_PSN) {
2199                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
2200                 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
2201         }
2202         if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2203                 qp->qplib_qp.modify_flags |=
2204                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
2205                 /* Cap the max_rd_atomic to device max */
2206                 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
2207                                                    dev_attr->max_qp_rd_atom);
2208         }
2209         if (qp_attr_mask & IB_QP_SQ_PSN) {
2210                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2211                 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
2212         }
2213         if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2214                 if (qp_attr->max_dest_rd_atomic >
2215                     dev_attr->max_qp_init_rd_atom) {
2216                         ibdev_err(&rdev->ibdev,
2217                                   "max_dest_rd_atomic requested%d is > dev_max%d",
2218                                   qp_attr->max_dest_rd_atomic,
2219                                   dev_attr->max_qp_init_rd_atom);
2220                         return -EINVAL;
2221                 }
2222
2223                 qp->qplib_qp.modify_flags |=
2224                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
2225                 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
2226         }
2227         if (qp_attr_mask & IB_QP_CAP) {
2228                 struct bnxt_re_ucontext *uctx =
2229                         rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
2230
2231                 qp->qplib_qp.modify_flags |=
2232                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
2233                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
2234                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
2235                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
2236                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
2237                 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
2238                     (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
2239                     (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
2240                     (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2241                     (qp_attr->cap.max_inline_data >=
2242                                                 dev_attr->max_inline_data)) {
2243                         ibdev_err(&rdev->ibdev,
2244                                   "Create QP failed - max exceeded");
2245                         return -EINVAL;
2246                 }
2247                 entries = bnxt_re_init_depth(qp_attr->cap.max_send_wr, uctx);
2248                 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2249                                                 dev_attr->max_qp_wqes + 1);
2250                 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2251                                                 qp_attr->cap.max_send_wr;
2252                 /*
2253                  * Reserving one slot for Phantom WQE. Some application can
2254                  * post one extra entry in this case. Allowing this to avoid
2255                  * unexpected Queue full condition
2256                  */
2257                 qp->qplib_qp.sq.q_full_delta -= 1;
2258                 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2259                 if (qp->qplib_qp.rq.max_wqe) {
2260                         entries = bnxt_re_init_depth(qp_attr->cap.max_recv_wr, uctx);
2261                         qp->qplib_qp.rq.max_wqe =
2262                                 min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2263                         qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe;
2264                         qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2265                                                        qp_attr->cap.max_recv_wr;
2266                         qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2267                 } else {
2268                         /* SRQ was used prior, just ignore the RQ caps */
2269                 }
2270         }
2271         if (qp_attr_mask & IB_QP_DEST_QPN) {
2272                 qp->qplib_qp.modify_flags |=
2273                                 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2274                 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2275         }
2276         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2277         if (rc) {
2278                 ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
2279                 return rc;
2280         }
2281         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
2282                 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2283         return rc;
2284 }
2285
2286 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2287                      int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2288 {
2289         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2290         struct bnxt_re_dev *rdev = qp->rdev;
2291         struct bnxt_qplib_qp *qplib_qp;
2292         int rc;
2293
2294         qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2295         if (!qplib_qp)
2296                 return -ENOMEM;
2297
2298         qplib_qp->id = qp->qplib_qp.id;
2299         qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2300
2301         rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2302         if (rc) {
2303                 ibdev_err(&rdev->ibdev, "Failed to query HW QP");
2304                 goto out;
2305         }
2306         qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2307         qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2308         qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2309         qp_attr->qp_access_flags = __qp_access_flags_to_ib(qp->qplib_qp.cctx,
2310                                                            qplib_qp->access);
2311         qp_attr->pkey_index = qplib_qp->pkey_index;
2312         qp_attr->qkey = qplib_qp->qkey;
2313         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2314         rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label,
2315                         qplib_qp->ah.host_sgid_index,
2316                         qplib_qp->ah.hop_limit,
2317                         qplib_qp->ah.traffic_class);
2318         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2319         rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2320         ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2321         qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2322         qp_attr->timeout = qplib_qp->timeout;
2323         qp_attr->retry_cnt = qplib_qp->retry_cnt;
2324         qp_attr->rnr_retry = qplib_qp->rnr_retry;
2325         qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2326         qp_attr->port_num = __to_ib_port_num(qplib_qp->port_id);
2327         qp_attr->rq_psn = qplib_qp->rq.psn;
2328         qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2329         qp_attr->sq_psn = qplib_qp->sq.psn;
2330         qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2331         qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2332                                                          IB_SIGNAL_REQ_WR;
2333         qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2334
2335         qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2336         qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2337         qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2338         qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2339         qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2340         qp_init_attr->cap = qp_attr->cap;
2341
2342 out:
2343         kfree(qplib_qp);
2344         return rc;
2345 }
2346
2347 /* Routine for sending QP1 packets for RoCE V1 an V2
2348  */
2349 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2350                                      const struct ib_send_wr *wr,
2351                                      struct bnxt_qplib_swqe *wqe,
2352                                      int payload_size)
2353 {
2354         struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2355                                              ib_ah);
2356         struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2357         const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2358         struct bnxt_qplib_sge sge;
2359         u8 nw_type;
2360         u16 ether_type;
2361         union ib_gid dgid;
2362         bool is_eth = false;
2363         bool is_vlan = false;
2364         bool is_grh = false;
2365         bool is_udp = false;
2366         u8 ip_version = 0;
2367         u16 vlan_id = 0xFFFF;
2368         void *buf;
2369         int i, rc;
2370
2371         memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2372
2373         rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2374         if (rc)
2375                 return rc;
2376
2377         /* Get network header type for this GID */
2378         nw_type = rdma_gid_attr_network_type(sgid_attr);
2379         switch (nw_type) {
2380         case RDMA_NETWORK_IPV4:
2381                 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2382                 break;
2383         case RDMA_NETWORK_IPV6:
2384                 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2385                 break;
2386         default:
2387                 nw_type = BNXT_RE_ROCE_V1_PACKET;
2388                 break;
2389         }
2390         memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2391         is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2392         if (is_udp) {
2393                 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
2394                         ip_version = 4;
2395                         ether_type = ETH_P_IP;
2396                 } else {
2397                         ip_version = 6;
2398                         ether_type = ETH_P_IPV6;
2399                 }
2400                 is_grh = false;
2401         } else {
2402                 ether_type = ETH_P_IBOE;
2403                 is_grh = true;
2404         }
2405
2406         is_eth = true;
2407         is_vlan = vlan_id && (vlan_id < 0x1000);
2408
2409         ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2410                           ip_version, is_udp, 0, &qp->qp1_hdr);
2411
2412         /* ETH */
2413         ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2414         ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2415
2416         /* For vlan, check the sgid for vlan existence */
2417
2418         if (!is_vlan) {
2419                 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2420         } else {
2421                 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2422                 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2423         }
2424
2425         if (is_grh || (ip_version == 6)) {
2426                 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2427                        sizeof(sgid_attr->gid));
2428                 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2429                        sizeof(sgid_attr->gid));
2430                 qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
2431         }
2432
2433         if (ip_version == 4) {
2434                 qp->qp1_hdr.ip4.tos = 0;
2435                 qp->qp1_hdr.ip4.id = 0;
2436                 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2437                 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2438
2439                 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2440                 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2441                 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2442         }
2443
2444         if (is_udp) {
2445                 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2446                 qp->qp1_hdr.udp.sport = htons(0x8CD1);
2447                 qp->qp1_hdr.udp.csum = 0;
2448         }
2449
2450         /* BTH */
2451         if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2452                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2453                 qp->qp1_hdr.immediate_present = 1;
2454         } else {
2455                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2456         }
2457         if (wr->send_flags & IB_SEND_SOLICITED)
2458                 qp->qp1_hdr.bth.solicited_event = 1;
2459         /* pad_count */
2460         qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2461
2462         /* P_key for QP1 is for all members */
2463         qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2464         qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2465         qp->qp1_hdr.bth.ack_req = 0;
2466         qp->send_psn++;
2467         qp->send_psn &= BTH_PSN_MASK;
2468         qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2469         /* DETH */
2470         /* Use the priviledged Q_Key for QP1 */
2471         qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2472         qp->qp1_hdr.deth.source_qpn = IB_QP1;
2473
2474         /* Pack the QP1 to the transmit buffer */
2475         buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2476         if (buf) {
2477                 ib_ud_header_pack(&qp->qp1_hdr, buf);
2478                 for (i = wqe->num_sge; i; i--) {
2479                         wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2480                         wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2481                         wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2482                 }
2483
2484                 /*
2485                  * Max Header buf size for IPV6 RoCE V2 is 86,
2486                  * which is same as the QP1 SQ header buffer.
2487                  * Header buf size for IPV4 RoCE V2 can be 66.
2488                  * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2489                  * Subtract 20 bytes from QP1 SQ header buf size
2490                  */
2491                 if (is_udp && ip_version == 4)
2492                         sge.size -= 20;
2493                 /*
2494                  * Max Header buf size for RoCE V1 is 78.
2495                  * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2496                  * Subtract 8 bytes from QP1 SQ header buf size
2497                  */
2498                 if (!is_udp)
2499                         sge.size -= 8;
2500
2501                 /* Subtract 4 bytes for non vlan packets */
2502                 if (!is_vlan)
2503                         sge.size -= 4;
2504
2505                 wqe->sg_list[0].addr = sge.addr;
2506                 wqe->sg_list[0].lkey = sge.lkey;
2507                 wqe->sg_list[0].size = sge.size;
2508                 wqe->num_sge++;
2509
2510         } else {
2511                 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
2512                 rc = -ENOMEM;
2513         }
2514         return rc;
2515 }
2516
2517 /* For the MAD layer, it only provides the recv SGE the size of
2518  * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
2519  * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
2520  * receive packet (334 bytes) with no VLAN and then copy the GRH
2521  * and the MAD datagram out to the provided SGE.
2522  */
2523 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2524                                             const struct ib_recv_wr *wr,
2525                                             struct bnxt_qplib_swqe *wqe,
2526                                             int payload_size)
2527 {
2528         struct bnxt_re_sqp_entries *sqp_entry;
2529         struct bnxt_qplib_sge ref, sge;
2530         struct bnxt_re_dev *rdev;
2531         u32 rq_prod_index;
2532
2533         rdev = qp->rdev;
2534
2535         rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2536
2537         if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2538                 return -ENOMEM;
2539
2540         /* Create 1 SGE to receive the entire
2541          * ethernet packet
2542          */
2543         /* Save the reference from ULP */
2544         ref.addr = wqe->sg_list[0].addr;
2545         ref.lkey = wqe->sg_list[0].lkey;
2546         ref.size = wqe->sg_list[0].size;
2547
2548         sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2549
2550         /* SGE 1 */
2551         wqe->sg_list[0].addr = sge.addr;
2552         wqe->sg_list[0].lkey = sge.lkey;
2553         wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2554         sge.size -= wqe->sg_list[0].size;
2555
2556         sqp_entry->sge.addr = ref.addr;
2557         sqp_entry->sge.lkey = ref.lkey;
2558         sqp_entry->sge.size = ref.size;
2559         /* Store the wrid for reporting completion */
2560         sqp_entry->wrid = wqe->wr_id;
2561         /* change the wqe->wrid to table index */
2562         wqe->wr_id = rq_prod_index;
2563         return 0;
2564 }
2565
2566 static int is_ud_qp(struct bnxt_re_qp *qp)
2567 {
2568         return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2569                 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2570 }
2571
2572 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2573                                   const struct ib_send_wr *wr,
2574                                   struct bnxt_qplib_swqe *wqe)
2575 {
2576         struct bnxt_re_ah *ah = NULL;
2577
2578         if (is_ud_qp(qp)) {
2579                 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2580                 wqe->send.q_key = ud_wr(wr)->remote_qkey;
2581                 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2582                 wqe->send.avid = ah->qplib_ah.id;
2583         }
2584         switch (wr->opcode) {
2585         case IB_WR_SEND:
2586                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2587                 break;
2588         case IB_WR_SEND_WITH_IMM:
2589                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2590                 wqe->send.imm_data = be32_to_cpu(wr->ex.imm_data);
2591                 break;
2592         case IB_WR_SEND_WITH_INV:
2593                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2594                 wqe->send.inv_key = wr->ex.invalidate_rkey;
2595                 break;
2596         default:
2597                 return -EINVAL;
2598         }
2599         if (wr->send_flags & IB_SEND_SIGNALED)
2600                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2601         if (wr->send_flags & IB_SEND_FENCE)
2602                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2603         if (wr->send_flags & IB_SEND_SOLICITED)
2604                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2605         if (wr->send_flags & IB_SEND_INLINE)
2606                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2607
2608         return 0;
2609 }
2610
2611 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2612                                   struct bnxt_qplib_swqe *wqe)
2613 {
2614         switch (wr->opcode) {
2615         case IB_WR_RDMA_WRITE:
2616                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2617                 break;
2618         case IB_WR_RDMA_WRITE_WITH_IMM:
2619                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2620                 wqe->rdma.imm_data = be32_to_cpu(wr->ex.imm_data);
2621                 break;
2622         case IB_WR_RDMA_READ:
2623                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2624                 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2625                 break;
2626         default:
2627                 return -EINVAL;
2628         }
2629         wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2630         wqe->rdma.r_key = rdma_wr(wr)->rkey;
2631         if (wr->send_flags & IB_SEND_SIGNALED)
2632                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2633         if (wr->send_flags & IB_SEND_FENCE)
2634                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2635         if (wr->send_flags & IB_SEND_SOLICITED)
2636                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2637         if (wr->send_flags & IB_SEND_INLINE)
2638                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2639
2640         return 0;
2641 }
2642
2643 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2644                                     struct bnxt_qplib_swqe *wqe)
2645 {
2646         switch (wr->opcode) {
2647         case IB_WR_ATOMIC_CMP_AND_SWP:
2648                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2649                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2650                 wqe->atomic.swap_data = atomic_wr(wr)->swap;
2651                 break;
2652         case IB_WR_ATOMIC_FETCH_AND_ADD:
2653                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2654                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2655                 break;
2656         default:
2657                 return -EINVAL;
2658         }
2659         wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2660         wqe->atomic.r_key = atomic_wr(wr)->rkey;
2661         if (wr->send_flags & IB_SEND_SIGNALED)
2662                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2663         if (wr->send_flags & IB_SEND_FENCE)
2664                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2665         if (wr->send_flags & IB_SEND_SOLICITED)
2666                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2667         return 0;
2668 }
2669
2670 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2671                                  struct bnxt_qplib_swqe *wqe)
2672 {
2673         wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2674         wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2675
2676         if (wr->send_flags & IB_SEND_SIGNALED)
2677                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2678         if (wr->send_flags & IB_SEND_SOLICITED)
2679                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2680
2681         return 0;
2682 }
2683
2684 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2685                                  struct bnxt_qplib_swqe *wqe)
2686 {
2687         struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2688         struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2689         int access = wr->access;
2690
2691         wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2692         wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2693         wqe->frmr.page_list = mr->pages;
2694         wqe->frmr.page_list_len = mr->npages;
2695         wqe->frmr.levels = qplib_frpl->hwq.level;
2696         wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2697
2698         if (wr->wr.send_flags & IB_SEND_SIGNALED)
2699                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2700
2701         if (access & IB_ACCESS_LOCAL_WRITE)
2702                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2703         if (access & IB_ACCESS_REMOTE_READ)
2704                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2705         if (access & IB_ACCESS_REMOTE_WRITE)
2706                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2707         if (access & IB_ACCESS_REMOTE_ATOMIC)
2708                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2709         if (access & IB_ACCESS_MW_BIND)
2710                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2711
2712         wqe->frmr.l_key = wr->key;
2713         wqe->frmr.length = wr->mr->length;
2714         wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
2715         wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
2716         wqe->frmr.va = wr->mr->iova;
2717         return 0;
2718 }
2719
2720 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2721                                     const struct ib_send_wr *wr,
2722                                     struct bnxt_qplib_swqe *wqe)
2723 {
2724         /*  Copy the inline data to the data  field */
2725         u8 *in_data;
2726         u32 i, sge_len;
2727         void *sge_addr;
2728
2729         in_data = wqe->inline_data;
2730         for (i = 0; i < wr->num_sge; i++) {
2731                 sge_addr = (void *)(unsigned long)
2732                                 wr->sg_list[i].addr;
2733                 sge_len = wr->sg_list[i].length;
2734
2735                 if ((sge_len + wqe->inline_len) >
2736                     BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2737                         ibdev_err(&rdev->ibdev,
2738                                   "Inline data size requested > supported value");
2739                         return -EINVAL;
2740                 }
2741                 sge_len = wr->sg_list[i].length;
2742
2743                 memcpy(in_data, sge_addr, sge_len);
2744                 in_data += wr->sg_list[i].length;
2745                 wqe->inline_len += wr->sg_list[i].length;
2746         }
2747         return wqe->inline_len;
2748 }
2749
2750 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2751                                    const struct ib_send_wr *wr,
2752                                    struct bnxt_qplib_swqe *wqe)
2753 {
2754         int payload_sz = 0;
2755
2756         if (wr->send_flags & IB_SEND_INLINE)
2757                 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2758         else
2759                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2760                                                wqe->num_sge);
2761
2762         return payload_sz;
2763 }
2764
2765 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2766 {
2767         if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2768              qp->ib_qp.qp_type == IB_QPT_GSI ||
2769              qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2770              qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2771                 int qp_attr_mask;
2772                 struct ib_qp_attr qp_attr;
2773
2774                 qp_attr_mask = IB_QP_STATE;
2775                 qp_attr.qp_state = IB_QPS_RTS;
2776                 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2777                 qp->qplib_qp.wqe_cnt = 0;
2778         }
2779 }
2780
2781 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2782                                        struct bnxt_re_qp *qp,
2783                                        const struct ib_send_wr *wr)
2784 {
2785         int rc = 0, payload_sz = 0;
2786         unsigned long flags;
2787
2788         spin_lock_irqsave(&qp->sq_lock, flags);
2789         while (wr) {
2790                 struct bnxt_qplib_swqe wqe = {};
2791
2792                 /* Common */
2793                 wqe.num_sge = wr->num_sge;
2794                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2795                         ibdev_err(&rdev->ibdev,
2796                                   "Limit exceeded for Send SGEs");
2797                         rc = -EINVAL;
2798                         goto bad;
2799                 }
2800
2801                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2802                 if (payload_sz < 0) {
2803                         rc = -EINVAL;
2804                         goto bad;
2805                 }
2806                 wqe.wr_id = wr->wr_id;
2807
2808                 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2809
2810                 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2811                 if (!rc)
2812                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2813 bad:
2814                 if (rc) {
2815                         ibdev_err(&rdev->ibdev,
2816                                   "Post send failed opcode = %#x rc = %d",
2817                                   wr->opcode, rc);
2818                         break;
2819                 }
2820                 wr = wr->next;
2821         }
2822         bnxt_qplib_post_send_db(&qp->qplib_qp);
2823         if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
2824                 bnxt_ud_qp_hw_stall_workaround(qp);
2825         spin_unlock_irqrestore(&qp->sq_lock, flags);
2826         return rc;
2827 }
2828
2829 static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)
2830 {
2831         /* Need unconditional fence for non-wire memory opcode
2832          * to work as expected.
2833          */
2834         if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||
2835             wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||
2836             wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||
2837             wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)
2838                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2839 }
2840
2841 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2842                       const struct ib_send_wr **bad_wr)
2843 {
2844         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2845         struct bnxt_qplib_swqe wqe;
2846         int rc = 0, payload_sz = 0;
2847         unsigned long flags;
2848
2849         spin_lock_irqsave(&qp->sq_lock, flags);
2850         while (wr) {
2851                 /* House keeping */
2852                 memset(&wqe, 0, sizeof(wqe));
2853
2854                 /* Common */
2855                 wqe.num_sge = wr->num_sge;
2856                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2857                         ibdev_err(&qp->rdev->ibdev,
2858                                   "Limit exceeded for Send SGEs");
2859                         rc = -EINVAL;
2860                         goto bad;
2861                 }
2862
2863                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2864                 if (payload_sz < 0) {
2865                         rc = -EINVAL;
2866                         goto bad;
2867                 }
2868                 wqe.wr_id = wr->wr_id;
2869
2870                 switch (wr->opcode) {
2871                 case IB_WR_SEND:
2872                 case IB_WR_SEND_WITH_IMM:
2873                         if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
2874                                 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2875                                                                payload_sz);
2876                                 if (rc)
2877                                         goto bad;
2878                                 wqe.rawqp1.lflags |=
2879                                         SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2880                         }
2881                         switch (wr->send_flags) {
2882                         case IB_SEND_IP_CSUM:
2883                                 wqe.rawqp1.lflags |=
2884                                         SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2885                                 break;
2886                         default:
2887                                 break;
2888                         }
2889                         fallthrough;
2890                 case IB_WR_SEND_WITH_INV:
2891                         rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2892                         break;
2893                 case IB_WR_RDMA_WRITE:
2894                 case IB_WR_RDMA_WRITE_WITH_IMM:
2895                 case IB_WR_RDMA_READ:
2896                         rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2897                         break;
2898                 case IB_WR_ATOMIC_CMP_AND_SWP:
2899                 case IB_WR_ATOMIC_FETCH_AND_ADD:
2900                         rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2901                         break;
2902                 case IB_WR_RDMA_READ_WITH_INV:
2903                         ibdev_err(&qp->rdev->ibdev,
2904                                   "RDMA Read with Invalidate is not supported");
2905                         rc = -EINVAL;
2906                         goto bad;
2907                 case IB_WR_LOCAL_INV:
2908                         rc = bnxt_re_build_inv_wqe(wr, &wqe);
2909                         break;
2910                 case IB_WR_REG_MR:
2911                         rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2912                         break;
2913                 default:
2914                         /* Unsupported WRs */
2915                         ibdev_err(&qp->rdev->ibdev,
2916                                   "WR (%#x) is not supported", wr->opcode);
2917                         rc = -EINVAL;
2918                         goto bad;
2919                 }
2920                 if (!rc) {
2921                         if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
2922                                 bnxt_re_legacy_set_uc_fence(&wqe);
2923                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2924                 }
2925 bad:
2926                 if (rc) {
2927                         ibdev_err(&qp->rdev->ibdev,
2928                                   "post_send failed op:%#x qps = %#x rc = %d\n",
2929                                   wr->opcode, qp->qplib_qp.state, rc);
2930                         *bad_wr = wr;
2931                         break;
2932                 }
2933                 wr = wr->next;
2934         }
2935         bnxt_qplib_post_send_db(&qp->qplib_qp);
2936         if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
2937                 bnxt_ud_qp_hw_stall_workaround(qp);
2938         spin_unlock_irqrestore(&qp->sq_lock, flags);
2939
2940         return rc;
2941 }
2942
2943 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2944                                        struct bnxt_re_qp *qp,
2945                                        const struct ib_recv_wr *wr)
2946 {
2947         struct bnxt_qplib_swqe wqe;
2948         int rc = 0;
2949
2950         while (wr) {
2951                 /* House keeping */
2952                 memset(&wqe, 0, sizeof(wqe));
2953
2954                 /* Common */
2955                 wqe.num_sge = wr->num_sge;
2956                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2957                         ibdev_err(&rdev->ibdev,
2958                                   "Limit exceeded for Receive SGEs");
2959                         rc = -EINVAL;
2960                         break;
2961                 }
2962                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2963                 wqe.wr_id = wr->wr_id;
2964                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2965
2966                 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2967                 if (rc)
2968                         break;
2969
2970                 wr = wr->next;
2971         }
2972         if (!rc)
2973                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2974         return rc;
2975 }
2976
2977 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
2978                       const struct ib_recv_wr **bad_wr)
2979 {
2980         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2981         struct bnxt_qplib_swqe wqe;
2982         int rc = 0, payload_sz = 0;
2983         unsigned long flags;
2984         u32 count = 0;
2985
2986         spin_lock_irqsave(&qp->rq_lock, flags);
2987         while (wr) {
2988                 /* House keeping */
2989                 memset(&wqe, 0, sizeof(wqe));
2990
2991                 /* Common */
2992                 wqe.num_sge = wr->num_sge;
2993                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2994                         ibdev_err(&qp->rdev->ibdev,
2995                                   "Limit exceeded for Receive SGEs");
2996                         rc = -EINVAL;
2997                         *bad_wr = wr;
2998                         break;
2999                 }
3000
3001                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
3002                                                wr->num_sge);
3003                 wqe.wr_id = wr->wr_id;
3004                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3005
3006                 if (ib_qp->qp_type == IB_QPT_GSI &&
3007                     qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
3008                         rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
3009                                                               payload_sz);
3010                 if (!rc)
3011                         rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
3012                 if (rc) {
3013                         *bad_wr = wr;
3014                         break;
3015                 }
3016
3017                 /* Ring DB if the RQEs posted reaches a threshold value */
3018                 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
3019                         bnxt_qplib_post_recv_db(&qp->qplib_qp);
3020                         count = 0;
3021                 }
3022
3023                 wr = wr->next;
3024         }
3025
3026         if (count)
3027                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
3028
3029         spin_unlock_irqrestore(&qp->rq_lock, flags);
3030
3031         return rc;
3032 }
3033
3034 static struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)
3035 {
3036         int min, indx;
3037
3038         mutex_lock(&rdev->nqr->load_lock);
3039         for (indx = 0, min = 0; indx < (rdev->nqr->num_msix - 1); indx++) {
3040                 if (rdev->nqr->nq[min].load > rdev->nqr->nq[indx].load)
3041                         min = indx;
3042         }
3043         rdev->nqr->nq[min].load++;
3044         mutex_unlock(&rdev->nqr->load_lock);
3045
3046         return &rdev->nqr->nq[min];
3047 }
3048
3049 static void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)
3050 {
3051         mutex_lock(&rdev->nqr->load_lock);
3052         nq->load--;
3053         mutex_unlock(&rdev->nqr->load_lock);
3054 }
3055
3056 /* Completion Queues */
3057 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
3058 {
3059         struct bnxt_qplib_chip_ctx *cctx;
3060         struct bnxt_qplib_nq *nq;
3061         struct bnxt_re_dev *rdev;
3062         struct bnxt_re_cq *cq;
3063
3064         cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3065         rdev = cq->rdev;
3066         nq = cq->qplib_cq.nq;
3067         cctx = rdev->chip_ctx;
3068
3069         if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3070                 free_page((unsigned long)cq->uctx_cq_page);
3071                 hash_del(&cq->hash_entry);
3072         }
3073         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3074
3075         bnxt_re_put_nq(rdev, nq);
3076         ib_umem_release(cq->umem);
3077
3078         atomic_dec(&rdev->stats.res.cq_count);
3079         nq->budget--;
3080         kfree(cq->cql);
3081         return 0;
3082 }
3083
3084 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
3085                       struct uverbs_attr_bundle *attrs)
3086 {
3087         struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3088         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
3089         struct ib_udata *udata = &attrs->driver_udata;
3090         struct bnxt_re_ucontext *uctx =
3091                 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3092         struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
3093         struct bnxt_qplib_chip_ctx *cctx;
3094         int cqe = attr->cqe;
3095         int rc, entries;
3096         u32 active_cqs;
3097
3098         if (attr->flags)
3099                 return -EOPNOTSUPP;
3100
3101         /* Validate CQ fields */
3102         if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3103                 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
3104                 return -EINVAL;
3105         }
3106
3107         cq->rdev = rdev;
3108         cctx = rdev->chip_ctx;
3109         cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
3110
3111         entries = bnxt_re_init_depth(cqe + 1, uctx);
3112         if (entries > dev_attr->max_cq_wqes + 1)
3113                 entries = dev_attr->max_cq_wqes + 1;
3114
3115         cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3116         cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3117         if (udata) {
3118                 struct bnxt_re_cq_req req;
3119                 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
3120                         rc = -EFAULT;
3121                         goto fail;
3122                 }
3123
3124                 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
3125                                        entries * sizeof(struct cq_base),
3126                                        IB_ACCESS_LOCAL_WRITE);
3127                 if (IS_ERR(cq->umem)) {
3128                         rc = PTR_ERR(cq->umem);
3129                         goto fail;
3130                 }
3131                 cq->qplib_cq.sg_info.umem = cq->umem;
3132                 cq->qplib_cq.dpi = &uctx->dpi;
3133         } else {
3134                 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
3135                 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
3136                                   GFP_KERNEL);
3137                 if (!cq->cql) {
3138                         rc = -ENOMEM;
3139                         goto fail;
3140                 }
3141
3142                 cq->qplib_cq.dpi = &rdev->dpi_privileged;
3143         }
3144         cq->qplib_cq.max_wqe = entries;
3145         cq->qplib_cq.coalescing = &rdev->cq_coalescing;
3146         cq->qplib_cq.nq = bnxt_re_get_nq(rdev);
3147         cq->qplib_cq.cnq_hw_ring_id = cq->qplib_cq.nq->ring_id;
3148
3149         rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
3150         if (rc) {
3151                 ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
3152                 goto fail;
3153         }
3154
3155         cq->ib_cq.cqe = entries;
3156         cq->cq_period = cq->qplib_cq.period;
3157
3158         active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);
3159         if (active_cqs > rdev->stats.res.cq_watermark)
3160                 rdev->stats.res.cq_watermark = active_cqs;
3161         spin_lock_init(&cq->cq_lock);
3162
3163         if (udata) {
3164                 struct bnxt_re_cq_resp resp = {};
3165
3166                 if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3167                         hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id);
3168                         /* Allocate a page */
3169                         cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
3170                         if (!cq->uctx_cq_page) {
3171                                 rc = -ENOMEM;
3172                                 goto c2fail;
3173                         }
3174                         resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;
3175                 }
3176                 resp.cqid = cq->qplib_cq.id;
3177                 resp.tail = cq->qplib_cq.hwq.cons;
3178                 resp.phase = cq->qplib_cq.period;
3179                 resp.rsvd = 0;
3180                 rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
3181                 if (rc) {
3182                         ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
3183                         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3184                         goto free_mem;
3185                 }
3186         }
3187
3188         return 0;
3189
3190 free_mem:
3191         free_page((unsigned long)cq->uctx_cq_page);
3192 c2fail:
3193         ib_umem_release(cq->umem);
3194 fail:
3195         kfree(cq->cql);
3196         return rc;
3197 }
3198
3199 static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
3200 {
3201         struct bnxt_re_dev *rdev = cq->rdev;
3202
3203         bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
3204
3205         cq->qplib_cq.max_wqe = cq->resize_cqe;
3206         if (cq->resize_umem) {
3207                 ib_umem_release(cq->umem);
3208                 cq->umem = cq->resize_umem;
3209                 cq->resize_umem = NULL;
3210                 cq->resize_cqe = 0;
3211         }
3212 }
3213
3214 int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
3215 {
3216         struct bnxt_qplib_sg_info sg_info = {};
3217         struct bnxt_qplib_dpi *orig_dpi = NULL;
3218         struct bnxt_qplib_dev_attr *dev_attr;
3219         struct bnxt_re_ucontext *uctx = NULL;
3220         struct bnxt_re_resize_cq_req req;
3221         struct bnxt_re_dev *rdev;
3222         struct bnxt_re_cq *cq;
3223         int rc, entries;
3224
3225         cq =  container_of(ibcq, struct bnxt_re_cq, ib_cq);
3226         rdev = cq->rdev;
3227         dev_attr = rdev->dev_attr;
3228         if (!ibcq->uobject) {
3229                 ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported");
3230                 return -EOPNOTSUPP;
3231         }
3232
3233         if (cq->resize_umem) {
3234                 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy",
3235                           cq->qplib_cq.id);
3236                 return -EBUSY;
3237         }
3238
3239         /* Check the requested cq depth out of supported depth */
3240         if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3241                 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - out of range cqe %d",
3242                           cq->qplib_cq.id, cqe);
3243                 return -EINVAL;
3244         }
3245
3246         uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3247         entries = bnxt_re_init_depth(cqe + 1, uctx);
3248         if (entries > dev_attr->max_cq_wqes + 1)
3249                 entries = dev_attr->max_cq_wqes + 1;
3250
3251         /* uverbs consumer */
3252         if (ib_copy_from_udata(&req, udata, sizeof(req))) {
3253                 rc = -EFAULT;
3254                 goto fail;
3255         }
3256
3257         cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va,
3258                                       entries * sizeof(struct cq_base),
3259                                       IB_ACCESS_LOCAL_WRITE);
3260         if (IS_ERR(cq->resize_umem)) {
3261                 rc = PTR_ERR(cq->resize_umem);
3262                 cq->resize_umem = NULL;
3263                 ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %d\n",
3264                           __func__, rc);
3265                 goto fail;
3266         }
3267         cq->resize_cqe = entries;
3268         memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info));
3269         orig_dpi = cq->qplib_cq.dpi;
3270
3271         cq->qplib_cq.sg_info.umem = cq->resize_umem;
3272         cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3273         cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3274         cq->qplib_cq.dpi = &uctx->dpi;
3275
3276         rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
3277         if (rc) {
3278                 ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!",
3279                           cq->qplib_cq.id);
3280                 goto fail;
3281         }
3282
3283         cq->ib_cq.cqe = cq->resize_cqe;
3284         atomic_inc(&rdev->stats.res.resize_count);
3285
3286         return 0;
3287
3288 fail:
3289         if (cq->resize_umem) {
3290                 ib_umem_release(cq->resize_umem);
3291                 cq->resize_umem = NULL;
3292                 cq->resize_cqe = 0;
3293                 memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info));
3294                 cq->qplib_cq.dpi = orig_dpi;
3295         }
3296         return rc;
3297 }
3298
3299 static u8 __req_to_ib_wc_status(u8 qstatus)
3300 {
3301         switch (qstatus) {
3302         case CQ_REQ_STATUS_OK:
3303                 return IB_WC_SUCCESS;
3304         case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
3305                 return IB_WC_BAD_RESP_ERR;
3306         case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
3307                 return IB_WC_LOC_LEN_ERR;
3308         case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
3309                 return IB_WC_LOC_QP_OP_ERR;
3310         case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
3311                 return IB_WC_LOC_PROT_ERR;
3312         case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
3313                 return IB_WC_GENERAL_ERR;
3314         case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
3315                 return IB_WC_REM_INV_REQ_ERR;
3316         case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
3317                 return IB_WC_REM_ACCESS_ERR;
3318         case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
3319                 return IB_WC_REM_OP_ERR;
3320         case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
3321                 return IB_WC_RNR_RETRY_EXC_ERR;
3322         case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
3323                 return IB_WC_RETRY_EXC_ERR;
3324         case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
3325                 return IB_WC_WR_FLUSH_ERR;
3326         default:
3327                 return IB_WC_GENERAL_ERR;
3328         }
3329         return 0;
3330 }
3331
3332 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
3333 {
3334         switch (qstatus) {
3335         case CQ_RES_RAWETH_QP1_STATUS_OK:
3336                 return IB_WC_SUCCESS;
3337         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
3338                 return IB_WC_LOC_ACCESS_ERR;
3339         case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
3340                 return IB_WC_LOC_LEN_ERR;
3341         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
3342                 return IB_WC_LOC_PROT_ERR;
3343         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
3344                 return IB_WC_LOC_QP_OP_ERR;
3345         case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
3346                 return IB_WC_GENERAL_ERR;
3347         case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
3348                 return IB_WC_WR_FLUSH_ERR;
3349         case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
3350                 return IB_WC_WR_FLUSH_ERR;
3351         default:
3352                 return IB_WC_GENERAL_ERR;
3353         }
3354 }
3355
3356 static u8 __rc_to_ib_wc_status(u8 qstatus)
3357 {
3358         switch (qstatus) {
3359         case CQ_RES_RC_STATUS_OK:
3360                 return IB_WC_SUCCESS;
3361         case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
3362                 return IB_WC_LOC_ACCESS_ERR;
3363         case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
3364                 return IB_WC_LOC_LEN_ERR;
3365         case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
3366                 return IB_WC_LOC_PROT_ERR;
3367         case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
3368                 return IB_WC_LOC_QP_OP_ERR;
3369         case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
3370                 return IB_WC_GENERAL_ERR;
3371         case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
3372                 return IB_WC_REM_INV_REQ_ERR;
3373         case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
3374                 return IB_WC_WR_FLUSH_ERR;
3375         case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
3376                 return IB_WC_WR_FLUSH_ERR;
3377         default:
3378                 return IB_WC_GENERAL_ERR;
3379         }
3380 }
3381
3382 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
3383 {
3384         switch (cqe->type) {
3385         case BNXT_QPLIB_SWQE_TYPE_SEND:
3386                 wc->opcode = IB_WC_SEND;
3387                 break;
3388         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
3389                 wc->opcode = IB_WC_SEND;
3390                 wc->wc_flags |= IB_WC_WITH_IMM;
3391                 break;
3392         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3393                 wc->opcode = IB_WC_SEND;
3394                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3395                 break;
3396         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3397                 wc->opcode = IB_WC_RDMA_WRITE;
3398                 break;
3399         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3400                 wc->opcode = IB_WC_RDMA_WRITE;
3401                 wc->wc_flags |= IB_WC_WITH_IMM;
3402                 break;
3403         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3404                 wc->opcode = IB_WC_RDMA_READ;
3405                 break;
3406         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3407                 wc->opcode = IB_WC_COMP_SWAP;
3408                 break;
3409         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3410                 wc->opcode = IB_WC_FETCH_ADD;
3411                 break;
3412         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3413                 wc->opcode = IB_WC_LOCAL_INV;
3414                 break;
3415         case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3416                 wc->opcode = IB_WC_REG_MR;
3417                 break;
3418         default:
3419                 wc->opcode = IB_WC_SEND;
3420                 break;
3421         }
3422
3423         wc->status = __req_to_ib_wc_status(cqe->status);
3424 }
3425
3426 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3427                                      u16 raweth_qp1_flags2)
3428 {
3429         bool is_ipv6 = false, is_ipv4 = false;
3430
3431         /* raweth_qp1_flags Bit 9-6 indicates itype */
3432         if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3433             != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3434                 return -1;
3435
3436         if (raweth_qp1_flags2 &
3437             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3438             raweth_qp1_flags2 &
3439             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
3440                 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3441                 (raweth_qp1_flags2 &
3442                  CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3443                         (is_ipv6 = true) : (is_ipv4 = true);
3444                 return ((is_ipv6) ?
3445                          BNXT_RE_ROCEV2_IPV6_PACKET :
3446                          BNXT_RE_ROCEV2_IPV4_PACKET);
3447         } else {
3448                 return BNXT_RE_ROCE_V1_PACKET;
3449         }
3450 }
3451
3452 static int bnxt_re_to_ib_nw_type(int nw_type)
3453 {
3454         u8 nw_hdr_type = 0xFF;
3455
3456         switch (nw_type) {
3457         case BNXT_RE_ROCE_V1_PACKET:
3458                 nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3459                 break;
3460         case BNXT_RE_ROCEV2_IPV4_PACKET:
3461                 nw_hdr_type = RDMA_NETWORK_IPV4;
3462                 break;
3463         case BNXT_RE_ROCEV2_IPV6_PACKET:
3464                 nw_hdr_type = RDMA_NETWORK_IPV6;
3465                 break;
3466         }
3467         return nw_hdr_type;
3468 }
3469
3470 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3471                                        void *rq_hdr_buf)
3472 {
3473         u8 *tmp_buf = NULL;
3474         struct ethhdr *eth_hdr;
3475         u16 eth_type;
3476         bool rc = false;
3477
3478         tmp_buf = (u8 *)rq_hdr_buf;
3479         /*
3480          * If dest mac is not same as I/F mac, this could be a
3481          * loopback address or multicast address, check whether
3482          * it is a loopback packet
3483          */
3484         if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
3485                 tmp_buf += 4;
3486                 /* Check the  ether type */
3487                 eth_hdr = (struct ethhdr *)tmp_buf;
3488                 eth_type = ntohs(eth_hdr->h_proto);
3489                 switch (eth_type) {
3490                 case ETH_P_IBOE:
3491                         rc = true;
3492                         break;
3493                 case ETH_P_IP:
3494                 case ETH_P_IPV6: {
3495                         u32 len;
3496                         struct udphdr *udp_hdr;
3497
3498                         len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3499                                                       sizeof(struct ipv6hdr));
3500                         tmp_buf += sizeof(struct ethhdr) + len;
3501                         udp_hdr = (struct udphdr *)tmp_buf;
3502                         if (ntohs(udp_hdr->dest) ==
3503                                     ROCE_V2_UDP_DPORT)
3504                                 rc = true;
3505                         break;
3506                         }
3507                 default:
3508                         break;
3509                 }
3510         }
3511
3512         return rc;
3513 }
3514
3515 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
3516                                          struct bnxt_qplib_cqe *cqe)
3517 {
3518         struct bnxt_re_dev *rdev = gsi_qp->rdev;
3519         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3520         struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
3521         dma_addr_t shrq_hdr_buf_map;
3522         struct ib_sge s_sge[2] = {};
3523         struct ib_sge r_sge[2] = {};
3524         struct bnxt_re_ah *gsi_sah;
3525         struct ib_recv_wr rwr = {};
3526         dma_addr_t rq_hdr_buf_map;
3527         struct ib_ud_wr udwr = {};
3528         struct ib_send_wr *swr;
3529         u32 skip_bytes = 0;
3530         int pkt_type = 0;
3531         void *rq_hdr_buf;
3532         u32 offset = 0;
3533         u32 tbl_idx;
3534         int rc;
3535
3536         swr = &udwr.wr;
3537         tbl_idx = cqe->wr_id;
3538
3539         rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3540                         (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3541         rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3542                                                           tbl_idx);
3543
3544         /* Shadow QP header buffer */
3545         shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3546                                                             tbl_idx);
3547         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3548
3549         /* Store this cqe */
3550         memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
3551         sqp_entry->qp1_qp = gsi_qp;
3552
3553         /* Find packet type from the cqe */
3554
3555         pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
3556                                              cqe->raweth_qp1_flags2);
3557         if (pkt_type < 0) {
3558                 ibdev_err(&rdev->ibdev, "Invalid packet\n");
3559                 return -EINVAL;
3560         }
3561
3562         /* Adjust the offset for the user buffer and post in the rq */
3563
3564         if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3565                 offset = 20;
3566
3567         /*
3568          * QP1 loopback packet has 4 bytes of internal header before
3569          * ether header. Skip these four bytes.
3570          */
3571         if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3572                 skip_bytes = 4;
3573
3574         /* First send SGE . Skip the ether header*/
3575         s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3576                         + skip_bytes;
3577         s_sge[0].lkey = 0xFFFFFFFF;
3578         s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3579                                 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3580
3581         /* Second Send SGE */
3582         s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3583                         BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3584         if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3585                 s_sge[1].addr += 8;
3586         s_sge[1].lkey = 0xFFFFFFFF;
3587         s_sge[1].length = 256;
3588
3589         /* First recv SGE */
3590
3591         r_sge[0].addr = shrq_hdr_buf_map;
3592         r_sge[0].lkey = 0xFFFFFFFF;
3593         r_sge[0].length = 40;
3594
3595         r_sge[1].addr = sqp_entry->sge.addr + offset;
3596         r_sge[1].lkey = sqp_entry->sge.lkey;
3597         r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3598
3599         /* Create receive work request */
3600         rwr.num_sge = 2;
3601         rwr.sg_list = r_sge;
3602         rwr.wr_id = tbl_idx;
3603         rwr.next = NULL;
3604
3605         rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
3606         if (rc) {
3607                 ibdev_err(&rdev->ibdev,
3608                           "Failed to post Rx buffers to shadow QP");
3609                 return -ENOMEM;
3610         }
3611
3612         swr->num_sge = 2;
3613         swr->sg_list = s_sge;
3614         swr->wr_id = tbl_idx;
3615         swr->opcode = IB_WR_SEND;
3616         swr->next = NULL;
3617         gsi_sah = rdev->gsi_ctx.gsi_sah;
3618         udwr.ah = &gsi_sah->ib_ah;
3619         udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3620         udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
3621
3622         /* post data received  in the send queue */
3623         return bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
3624 }
3625
3626 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3627                                           struct bnxt_qplib_cqe *cqe)
3628 {
3629         wc->opcode = IB_WC_RECV;
3630         wc->status = __rawqp1_to_ib_wc_status(cqe->status);
3631         wc->wc_flags |= IB_WC_GRH;
3632 }
3633
3634 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3635                                         u16 vlan_id)
3636 {
3637         /*
3638          * Check if the vlan is configured in the host.  If not configured, it
3639          * can be a transparent VLAN. So dont report the vlan id.
3640          */
3641         if (!__vlan_find_dev_deep_rcu(rdev->netdev,
3642                                       htons(ETH_P_8021Q), vlan_id))
3643                 return false;
3644         return true;
3645 }
3646
3647 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3648                                 u16 *vid, u8 *sl)
3649 {
3650         bool ret = false;
3651         u32 metadata;
3652         u16 tpid;
3653
3654         metadata = orig_cqe->raweth_qp1_metadata;
3655         if (orig_cqe->raweth_qp1_flags2 &
3656                 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3657                 tpid = ((metadata &
3658                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3659                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3660                 if (tpid == ETH_P_8021Q) {
3661                         *vid = metadata &
3662                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3663                         *sl = (metadata &
3664                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3665                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3666                         ret = true;
3667                 }
3668         }
3669
3670         return ret;
3671 }
3672
3673 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3674                                       struct bnxt_qplib_cqe *cqe)
3675 {
3676         wc->opcode = IB_WC_RECV;
3677         wc->status = __rc_to_ib_wc_status(cqe->status);
3678
3679         if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3680                 wc->wc_flags |= IB_WC_WITH_IMM;
3681         if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3682                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3683         if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3684             (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3685                 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3686 }
3687
3688 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
3689                                              struct ib_wc *wc,
3690                                              struct bnxt_qplib_cqe *cqe)
3691 {
3692         struct bnxt_re_dev *rdev = gsi_sqp->rdev;
3693         struct bnxt_re_qp *gsi_qp = NULL;
3694         struct bnxt_qplib_cqe *orig_cqe = NULL;
3695         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3696         int nw_type;
3697         u32 tbl_idx;
3698         u16 vlan_id;
3699         u8 sl;
3700
3701         tbl_idx = cqe->wr_id;
3702
3703         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3704         gsi_qp = sqp_entry->qp1_qp;
3705         orig_cqe = &sqp_entry->cqe;
3706
3707         wc->wr_id = sqp_entry->wrid;
3708         wc->byte_len = orig_cqe->length;
3709         wc->qp = &gsi_qp->ib_qp;
3710
3711         wc->ex.imm_data = cpu_to_be32(orig_cqe->immdata);
3712         wc->src_qp = orig_cqe->src_qp;
3713         memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
3714         if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) {
3715                 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3716                         wc->vlan_id = vlan_id;
3717                         wc->sl = sl;
3718                         wc->wc_flags |= IB_WC_WITH_VLAN;
3719                 }
3720         }
3721         wc->port_num = 1;
3722         wc->vendor_err = orig_cqe->status;
3723
3724         wc->opcode = IB_WC_RECV;
3725         wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
3726         wc->wc_flags |= IB_WC_GRH;
3727
3728         nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
3729                                             orig_cqe->raweth_qp1_flags2);
3730         if (nw_type >= 0) {
3731                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3732                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3733         }
3734 }
3735
3736 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
3737                                       struct ib_wc *wc,
3738                                       struct bnxt_qplib_cqe *cqe)
3739 {
3740         struct bnxt_re_dev *rdev;
3741         u16 vlan_id = 0;
3742         u8 nw_type;
3743
3744         rdev = qp->rdev;
3745         wc->opcode = IB_WC_RECV;
3746         wc->status = __rc_to_ib_wc_status(cqe->status);
3747
3748         if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
3749                 wc->wc_flags |= IB_WC_WITH_IMM;
3750         /* report only on GSI QP for Thor */
3751         if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
3752                 wc->wc_flags |= IB_WC_GRH;
3753                 memcpy(wc->smac, cqe->smac, ETH_ALEN);
3754                 wc->wc_flags |= IB_WC_WITH_SMAC;
3755                 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
3756                         vlan_id = (cqe->cfa_meta & 0xFFF);
3757                 }
3758                 /* Mark only if vlan_id is non zero */
3759                 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3760                         wc->vlan_id = vlan_id;
3761                         wc->wc_flags |= IB_WC_WITH_VLAN;
3762                 }
3763                 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
3764                            CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
3765                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3766                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3767         }
3768
3769 }
3770
3771 static int send_phantom_wqe(struct bnxt_re_qp *qp)
3772 {
3773         struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
3774         unsigned long flags;
3775         int rc;
3776
3777         spin_lock_irqsave(&qp->sq_lock, flags);
3778
3779         rc = bnxt_re_bind_fence_mw(lib_qp);
3780         if (!rc) {
3781                 lib_qp->sq.phantom_wqe_cnt++;
3782                 ibdev_dbg(&qp->rdev->ibdev,
3783                           "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
3784                           lib_qp->id, lib_qp->sq.hwq.prod,
3785                           HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
3786                           lib_qp->sq.phantom_wqe_cnt);
3787         }
3788
3789         spin_unlock_irqrestore(&qp->sq_lock, flags);
3790         return rc;
3791 }
3792
3793 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
3794 {
3795         struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3796         struct bnxt_re_qp *qp, *sh_qp;
3797         struct bnxt_qplib_cqe *cqe;
3798         int i, ncqe, budget;
3799         struct bnxt_qplib_q *sq;
3800         struct bnxt_qplib_qp *lib_qp;
3801         u32 tbl_idx;
3802         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3803         unsigned long flags;
3804
3805         /* User CQ; the only processing we do is to
3806          * complete any pending CQ resize operation.
3807          */
3808         if (cq->umem) {
3809                 if (cq->resize_umem)
3810                         bnxt_re_resize_cq_complete(cq);
3811                 return 0;
3812         }
3813
3814         spin_lock_irqsave(&cq->cq_lock, flags);
3815         budget = min_t(u32, num_entries, cq->max_cql);
3816         num_entries = budget;
3817         if (!cq->cql) {
3818                 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use");
3819                 goto exit;
3820         }
3821         cqe = &cq->cql[0];
3822         while (budget) {
3823                 lib_qp = NULL;
3824                 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
3825                 if (lib_qp) {
3826                         sq = &lib_qp->sq;
3827                         if (sq->send_phantom) {
3828                                 qp = container_of(lib_qp,
3829                                                   struct bnxt_re_qp, qplib_qp);
3830                                 if (send_phantom_wqe(qp) == -ENOMEM)
3831                                         ibdev_err(&cq->rdev->ibdev,
3832                                                   "Phantom failed! Scheduled to send again\n");
3833                                 else
3834                                         sq->send_phantom = false;
3835                         }
3836                 }
3837                 if (ncqe < budget)
3838                         ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
3839                                                               cqe + ncqe,
3840                                                               budget - ncqe);
3841
3842                 if (!ncqe)
3843                         break;
3844
3845                 for (i = 0; i < ncqe; i++, cqe++) {
3846                         /* Transcribe each qplib_wqe back to ib_wc */
3847                         memset(wc, 0, sizeof(*wc));
3848
3849                         wc->wr_id = cqe->wr_id;
3850                         wc->byte_len = cqe->length;
3851                         qp = container_of
3852                                 ((struct bnxt_qplib_qp *)
3853                                  (unsigned long)(cqe->qp_handle),
3854                                  struct bnxt_re_qp, qplib_qp);
3855                         wc->qp = &qp->ib_qp;
3856                         if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3857                                 wc->ex.imm_data = cpu_to_be32(cqe->immdata);
3858                         else
3859                                 wc->ex.invalidate_rkey = cqe->invrkey;
3860                         wc->src_qp = cqe->src_qp;
3861                         memcpy(wc->smac, cqe->smac, ETH_ALEN);
3862                         wc->port_num = 1;
3863                         wc->vendor_err = cqe->status;
3864
3865                         switch (cqe->opcode) {
3866                         case CQ_BASE_CQE_TYPE_REQ:
3867                                 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3868                                 if (sh_qp &&
3869                                     qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3870                                         /* Handle this completion with
3871                                          * the stored completion
3872                                          */
3873                                         memset(wc, 0, sizeof(*wc));
3874                                         continue;
3875                                 }
3876                                 bnxt_re_process_req_wc(wc, cqe);
3877                                 break;
3878                         case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3879                                 if (!cqe->status) {
3880                                         int rc = 0;
3881
3882                                         rc = bnxt_re_process_raw_qp_pkt_rx
3883                                                                 (qp, cqe);
3884                                         if (!rc) {
3885                                                 memset(wc, 0, sizeof(*wc));
3886                                                 continue;
3887                                         }
3888                                         cqe->status = -1;
3889                                 }
3890                                 /* Errors need not be looped back.
3891                                  * But change the wr_id to the one
3892                                  * stored in the table
3893                                  */
3894                                 tbl_idx = cqe->wr_id;
3895                                 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx];
3896                                 wc->wr_id = sqp_entry->wrid;
3897                                 bnxt_re_process_res_rawqp1_wc(wc, cqe);
3898                                 break;
3899                         case CQ_BASE_CQE_TYPE_RES_RC:
3900                                 bnxt_re_process_res_rc_wc(wc, cqe);
3901                                 break;
3902                         case CQ_BASE_CQE_TYPE_RES_UD:
3903                                 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3904                                 if (sh_qp &&
3905                                     qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3906                                         /* Handle this completion with
3907                                          * the stored completion
3908                                          */
3909                                         if (cqe->status) {
3910                                                 continue;
3911                                         } else {
3912                                                 bnxt_re_process_res_shadow_qp_wc
3913                                                                 (qp, wc, cqe);
3914                                                 break;
3915                                         }
3916                                 }
3917                                 bnxt_re_process_res_ud_wc(qp, wc, cqe);
3918                                 break;
3919                         default:
3920                                 ibdev_err(&cq->rdev->ibdev,
3921                                           "POLL CQ : type 0x%x not handled",
3922                                           cqe->opcode);
3923                                 continue;
3924                         }
3925                         wc++;
3926                         budget--;
3927                 }
3928         }
3929 exit:
3930         spin_unlock_irqrestore(&cq->cq_lock, flags);
3931         return num_entries - budget;
3932 }
3933
3934 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
3935                           enum ib_cq_notify_flags ib_cqn_flags)
3936 {
3937         struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3938         int type = 0, rc = 0;
3939         unsigned long flags;
3940
3941         spin_lock_irqsave(&cq->cq_lock, flags);
3942         /* Trigger on the very next completion */
3943         if (ib_cqn_flags & IB_CQ_NEXT_COMP)
3944                 type = DBC_DBC_TYPE_CQ_ARMALL;
3945         /* Trigger on the next solicited completion */
3946         else if (ib_cqn_flags & IB_CQ_SOLICITED)
3947                 type = DBC_DBC_TYPE_CQ_ARMSE;
3948
3949         /* Poll to see if there are missed events */
3950         if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3951             !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) {
3952                 rc = 1;
3953                 goto exit;
3954         }
3955         bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
3956
3957 exit:
3958         spin_unlock_irqrestore(&cq->cq_lock, flags);
3959         return rc;
3960 }
3961
3962 /* Memory Regions */
3963 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
3964 {
3965         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3966         struct bnxt_re_dev *rdev = pd->rdev;
3967         struct bnxt_re_mr *mr;
3968         u32 active_mrs;
3969         int rc;
3970
3971         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3972         if (!mr)
3973                 return ERR_PTR(-ENOMEM);
3974
3975         mr->rdev = rdev;
3976         mr->qplib_mr.pd = &pd->qplib_pd;
3977         mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
3978         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3979
3980         if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
3981                 bnxt_re_check_and_set_relaxed_ordering(rdev, &mr->qplib_mr);
3982
3983         /* Allocate and register 0 as the address */
3984         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3985         if (rc)
3986                 goto fail;
3987
3988         mr->qplib_mr.hwq.level = PBL_LVL_MAX;
3989         mr->qplib_mr.total_size = -1; /* Infinte length */
3990         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0,
3991                                PAGE_SIZE);
3992         if (rc)
3993                 goto fail_mr;
3994
3995         mr->ib_mr.lkey = mr->qplib_mr.lkey;
3996         if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
3997                                IB_ACCESS_REMOTE_ATOMIC))
3998                 mr->ib_mr.rkey = mr->ib_mr.lkey;
3999         active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4000         if (active_mrs > rdev->stats.res.mr_watermark)
4001                 rdev->stats.res.mr_watermark = active_mrs;
4002
4003         return &mr->ib_mr;
4004
4005 fail_mr:
4006         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4007 fail:
4008         kfree(mr);
4009         return ERR_PTR(rc);
4010 }
4011
4012 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
4013 {
4014         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4015         struct bnxt_re_dev *rdev = mr->rdev;
4016         int rc;
4017
4018         rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4019         if (rc) {
4020                 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc);
4021                 return rc;
4022         }
4023
4024         if (mr->pages) {
4025                 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
4026                                                         &mr->qplib_frpl);
4027                 kfree(mr->pages);
4028                 mr->npages = 0;
4029                 mr->pages = NULL;
4030         }
4031         ib_umem_release(mr->ib_umem);
4032
4033         kfree(mr);
4034         atomic_dec(&rdev->stats.res.mr_count);
4035         return rc;
4036 }
4037
4038 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
4039 {
4040         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4041
4042         if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
4043                 return -ENOMEM;
4044
4045         mr->pages[mr->npages++] = addr;
4046         return 0;
4047 }
4048
4049 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
4050                       unsigned int *sg_offset)
4051 {
4052         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4053
4054         mr->npages = 0;
4055         return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
4056 }
4057
4058 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
4059                                u32 max_num_sg)
4060 {
4061         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4062         struct bnxt_re_dev *rdev = pd->rdev;
4063         struct bnxt_re_mr *mr = NULL;
4064         u32 active_mrs;
4065         int rc;
4066
4067         if (type != IB_MR_TYPE_MEM_REG) {
4068                 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type);
4069                 return ERR_PTR(-EINVAL);
4070         }
4071         if (max_num_sg > MAX_PBL_LVL_1_PGS)
4072                 return ERR_PTR(-EINVAL);
4073
4074         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4075         if (!mr)
4076                 return ERR_PTR(-ENOMEM);
4077
4078         mr->rdev = rdev;
4079         mr->qplib_mr.pd = &pd->qplib_pd;
4080         mr->qplib_mr.access_flags = BNXT_QPLIB_FR_PMR;
4081         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4082
4083         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4084         if (rc)
4085                 goto bail;
4086
4087         mr->ib_mr.lkey = mr->qplib_mr.lkey;
4088         mr->ib_mr.rkey = mr->ib_mr.lkey;
4089
4090         mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
4091         if (!mr->pages) {
4092                 rc = -ENOMEM;
4093                 goto fail;
4094         }
4095         rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
4096                                                  &mr->qplib_frpl, max_num_sg);
4097         if (rc) {
4098                 ibdev_err(&rdev->ibdev,
4099                           "Failed to allocate HW FR page list");
4100                 goto fail_mr;
4101         }
4102
4103         active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4104         if (active_mrs > rdev->stats.res.mr_watermark)
4105                 rdev->stats.res.mr_watermark = active_mrs;
4106         return &mr->ib_mr;
4107
4108 fail_mr:
4109         kfree(mr->pages);
4110 fail:
4111         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4112 bail:
4113         kfree(mr);
4114         return ERR_PTR(rc);
4115 }
4116
4117 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
4118                                struct ib_udata *udata)
4119 {
4120         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4121         struct bnxt_re_dev *rdev = pd->rdev;
4122         struct bnxt_re_mw *mw;
4123         u32 active_mws;
4124         int rc;
4125
4126         mw = kzalloc(sizeof(*mw), GFP_KERNEL);
4127         if (!mw)
4128                 return ERR_PTR(-ENOMEM);
4129         mw->rdev = rdev;
4130         mw->qplib_mw.pd = &pd->qplib_pd;
4131
4132         mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
4133                                CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
4134                                CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
4135         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
4136         if (rc) {
4137                 ibdev_err(&rdev->ibdev, "Allocate MW failed!");
4138                 goto fail;
4139         }
4140         mw->ib_mw.rkey = mw->qplib_mw.rkey;
4141
4142         active_mws = atomic_inc_return(&rdev->stats.res.mw_count);
4143         if (active_mws > rdev->stats.res.mw_watermark)
4144                 rdev->stats.res.mw_watermark = active_mws;
4145         return &mw->ib_mw;
4146
4147 fail:
4148         kfree(mw);
4149         return ERR_PTR(rc);
4150 }
4151
4152 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
4153 {
4154         struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
4155         struct bnxt_re_dev *rdev = mw->rdev;
4156         int rc;
4157
4158         rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
4159         if (rc) {
4160                 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc);
4161                 return rc;
4162         }
4163
4164         kfree(mw);
4165         atomic_dec(&rdev->stats.res.mw_count);
4166         return rc;
4167 }
4168
4169 static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr,
4170                                            int mr_access_flags, struct ib_umem *umem)
4171 {
4172         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4173         struct bnxt_re_dev *rdev = pd->rdev;
4174         unsigned long page_size;
4175         struct bnxt_re_mr *mr;
4176         int umem_pgs, rc;
4177         u32 active_mrs;
4178
4179         if (length > BNXT_RE_MAX_MR_SIZE) {
4180                 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n",
4181                           length, BNXT_RE_MAX_MR_SIZE);
4182                 return ERR_PTR(-ENOMEM);
4183         }
4184
4185         page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr);
4186         if (!page_size) {
4187                 ibdev_err(&rdev->ibdev, "umem page size unsupported!");
4188                 return ERR_PTR(-EINVAL);
4189         }
4190
4191         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4192         if (!mr)
4193                 return ERR_PTR(-ENOMEM);
4194
4195         mr->rdev = rdev;
4196         mr->qplib_mr.pd = &pd->qplib_pd;
4197         mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
4198         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
4199
4200         if (!_is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)) {
4201                 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4202                 if (rc) {
4203                         ibdev_err(&rdev->ibdev, "Failed to allocate MR rc = %d", rc);
4204                         rc = -EIO;
4205                         goto free_mr;
4206                 }
4207                 /* The fixed portion of the rkey is the same as the lkey */
4208                 mr->ib_mr.rkey = mr->qplib_mr.rkey;
4209         } else {
4210                 mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
4211         }
4212         mr->ib_umem = umem;
4213         mr->qplib_mr.va = virt_addr;
4214         mr->qplib_mr.total_size = length;
4215
4216         if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
4217                 bnxt_re_check_and_set_relaxed_ordering(rdev, &mr->qplib_mr);
4218
4219         umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
4220         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem,
4221                                umem_pgs, page_size);
4222         if (rc) {
4223                 ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc);
4224                 rc = -EIO;
4225                 goto free_mrw;
4226         }
4227
4228         mr->ib_mr.lkey = mr->qplib_mr.lkey;
4229         mr->ib_mr.rkey = mr->qplib_mr.lkey;
4230         active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4231         if (active_mrs > rdev->stats.res.mr_watermark)
4232                 rdev->stats.res.mr_watermark = active_mrs;
4233
4234         return &mr->ib_mr;
4235
4236 free_mrw:
4237         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4238 free_mr:
4239         kfree(mr);
4240         return ERR_PTR(rc);
4241 }
4242
4243 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
4244                                   u64 virt_addr, int mr_access_flags,
4245                                   struct ib_udata *udata)
4246 {
4247         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4248         struct bnxt_re_dev *rdev = pd->rdev;
4249         struct ib_umem *umem;
4250         struct ib_mr *ib_mr;
4251
4252         umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
4253         if (IS_ERR(umem))
4254                 return ERR_CAST(umem);
4255
4256         ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4257         if (IS_ERR(ib_mr))
4258                 ib_umem_release(umem);
4259         return ib_mr;
4260 }
4261
4262 struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
4263                                          u64 length, u64 virt_addr, int fd,
4264                                          int mr_access_flags,
4265                                          struct uverbs_attr_bundle *attrs)
4266 {
4267         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4268         struct bnxt_re_dev *rdev = pd->rdev;
4269         struct ib_umem_dmabuf *umem_dmabuf;
4270         struct ib_umem *umem;
4271         struct ib_mr *ib_mr;
4272
4273         umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length,
4274                                                 fd, mr_access_flags);
4275         if (IS_ERR(umem_dmabuf))
4276                 return ERR_CAST(umem_dmabuf);
4277
4278         umem = &umem_dmabuf->umem;
4279
4280         ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4281         if (IS_ERR(ib_mr))
4282                 ib_umem_release(umem);
4283         return ib_mr;
4284 }
4285
4286 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
4287 {
4288         struct ib_device *ibdev = ctx->device;
4289         struct bnxt_re_ucontext *uctx =
4290                 container_of(ctx, struct bnxt_re_ucontext, ib_uctx);
4291         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
4292         struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
4293         struct bnxt_re_user_mmap_entry *entry;
4294         struct bnxt_re_uctx_resp resp = {};
4295         struct bnxt_re_uctx_req ureq = {};
4296         u32 chip_met_rev_num = 0;
4297         int rc;
4298
4299         ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
4300
4301         if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
4302                 ibdev_dbg(ibdev, " is different from the device %d ",
4303                           BNXT_RE_ABI_VERSION);
4304                 return -EPERM;
4305         }
4306
4307         uctx->rdev = rdev;
4308
4309         uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
4310         if (!uctx->shpg) {
4311                 rc = -ENOMEM;
4312                 goto fail;
4313         }
4314         spin_lock_init(&uctx->sh_lock);
4315
4316         resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
4317         chip_met_rev_num = rdev->chip_ctx->chip_num;
4318         chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
4319                              BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
4320         chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
4321                              BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
4322         resp.chip_id0 = chip_met_rev_num;
4323         /*Temp, Use xa_alloc instead */
4324         resp.dev_id = rdev->en_dev->pdev->devfn;
4325         resp.max_qp = rdev->qplib_ctx.qpc_count;
4326         resp.pg_size = PAGE_SIZE;
4327         resp.cqe_sz = sizeof(struct cq_base);
4328         resp.max_cqd = dev_attr->max_cq_wqes;
4329
4330         if (rdev->chip_ctx->modes.db_push)
4331                 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED;
4332
4333         entry = bnxt_re_mmap_entry_insert(uctx, 0, BNXT_RE_MMAP_SH_PAGE, NULL);
4334         if (!entry) {
4335                 rc = -ENOMEM;
4336                 goto cfail;
4337         }
4338         uctx->shpage_mmap = &entry->rdma_entry;
4339         if (rdev->pacing.dbr_pacing)
4340                 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED;
4341
4342         if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2))
4343                 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED;
4344
4345         if (udata->inlen >= sizeof(ureq)) {
4346                 rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq)));
4347                 if (rc)
4348                         goto cfail;
4349                 if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) {
4350                         resp.comp_mask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED;
4351                         uctx->cmask |= BNXT_RE_UCNTX_CAP_POW2_DISABLED;
4352                 }
4353                 if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) {
4354                         resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;
4355                         resp.mode = rdev->chip_ctx->modes.wqe_mode;
4356                         if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
4357                                 uctx->cmask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
4358                 }
4359         }
4360
4361         rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
4362         if (rc) {
4363                 ibdev_err(ibdev, "Failed to copy user context");
4364                 rc = -EFAULT;
4365                 goto cfail;
4366         }
4367
4368         return 0;
4369 cfail:
4370         free_page((unsigned long)uctx->shpg);
4371         uctx->shpg = NULL;
4372 fail:
4373         return rc;
4374 }
4375
4376 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
4377 {
4378         struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4379                                                    struct bnxt_re_ucontext,
4380                                                    ib_uctx);
4381
4382         struct bnxt_re_dev *rdev = uctx->rdev;
4383
4384         rdma_user_mmap_entry_remove(uctx->shpage_mmap);
4385         uctx->shpage_mmap = NULL;
4386         if (uctx->shpg)
4387                 free_page((unsigned long)uctx->shpg);
4388
4389         if (uctx->dpi.dbr) {
4390                 /* Free DPI only if this is the first PD allocated by the
4391                  * application and mark the context dpi as NULL
4392                  */
4393                 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->dpi);
4394                 uctx->dpi.dbr = NULL;
4395         }
4396 }
4397
4398 static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq_id)
4399 {
4400         struct bnxt_re_cq *cq = NULL, *tmp_cq;
4401
4402         hash_for_each_possible(rdev->cq_hash, tmp_cq, hash_entry, cq_id) {
4403                 if (tmp_cq->qplib_cq.id == cq_id) {
4404                         cq = tmp_cq;
4405                         break;
4406                 }
4407         }
4408         return cq;
4409 }
4410
4411 static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)
4412 {
4413         struct bnxt_re_srq *srq = NULL, *tmp_srq;
4414
4415         hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) {
4416                 if (tmp_srq->qplib_srq.id == srq_id) {
4417                         srq = tmp_srq;
4418                         break;
4419                 }
4420         }
4421         return srq;
4422 }
4423
4424 /* Helper function to mmap the virtual memory from user app */
4425 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
4426 {
4427         struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4428                                                    struct bnxt_re_ucontext,
4429                                                    ib_uctx);
4430         struct bnxt_re_user_mmap_entry *bnxt_entry;
4431         struct rdma_user_mmap_entry *rdma_entry;
4432         int ret = 0;
4433         u64 pfn;
4434
4435         rdma_entry = rdma_user_mmap_entry_get(&uctx->ib_uctx, vma);
4436         if (!rdma_entry)
4437                 return -EINVAL;
4438
4439         bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4440                                   rdma_entry);
4441
4442         switch (bnxt_entry->mmap_flag) {
4443         case BNXT_RE_MMAP_WC_DB:
4444                 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4445                 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4446                                         pgprot_writecombine(vma->vm_page_prot),
4447                                         rdma_entry);
4448                 break;
4449         case BNXT_RE_MMAP_UC_DB:
4450                 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4451                 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4452                                         pgprot_noncached(vma->vm_page_prot),
4453                                 rdma_entry);
4454                 break;
4455         case BNXT_RE_MMAP_SH_PAGE:
4456                 ret = vm_insert_page(vma, vma->vm_start, virt_to_page(uctx->shpg));
4457                 break;
4458         case BNXT_RE_MMAP_DBR_BAR:
4459                 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4460                 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4461                                         pgprot_noncached(vma->vm_page_prot),
4462                                         rdma_entry);
4463                 break;
4464         case BNXT_RE_MMAP_DBR_PAGE:
4465         case BNXT_RE_MMAP_TOGGLE_PAGE:
4466                 /* Driver doesn't expect write access for user space */
4467                 if (vma->vm_flags & VM_WRITE)
4468                         ret = -EFAULT;
4469                 else
4470                         ret = vm_insert_page(vma, vma->vm_start,
4471                                              virt_to_page((void *)bnxt_entry->mem_offset));
4472                 break;
4473         default:
4474                 ret = -EINVAL;
4475                 break;
4476         }
4477
4478         rdma_user_mmap_entry_put(rdma_entry);
4479         return ret;
4480 }
4481
4482 void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
4483 {
4484         struct bnxt_re_user_mmap_entry *bnxt_entry;
4485
4486         bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4487                                   rdma_entry);
4488
4489         kfree(bnxt_entry);
4490 }
4491
4492 static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs)
4493 {
4494         struct bnxt_re_ucontext *uctx;
4495
4496         uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4497         bnxt_re_pacing_alert(uctx->rdev);
4498         return 0;
4499 }
4500
4501 static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs)
4502 {
4503         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
4504         enum bnxt_re_alloc_page_type alloc_type;
4505         struct bnxt_re_user_mmap_entry *entry;
4506         enum bnxt_re_mmap_flag mmap_flag;
4507         struct bnxt_qplib_chip_ctx *cctx;
4508         struct bnxt_re_ucontext *uctx;
4509         struct bnxt_re_dev *rdev;
4510         u64 mmap_offset;
4511         u32 length;
4512         u32 dpi;
4513         u64 addr;
4514         int err;
4515
4516         uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4517         if (IS_ERR(uctx))
4518                 return PTR_ERR(uctx);
4519
4520         err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE);
4521         if (err)
4522                 return err;
4523
4524         rdev = uctx->rdev;
4525         cctx = rdev->chip_ctx;
4526
4527         switch (alloc_type) {
4528         case BNXT_RE_ALLOC_WC_PAGE:
4529                 if (cctx->modes.db_push)  {
4530                         if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi,
4531                                                  uctx, BNXT_QPLIB_DPI_TYPE_WC))
4532                                 return -ENOMEM;
4533                         length = PAGE_SIZE;
4534                         dpi = uctx->wcdpi.dpi;
4535                         addr = (u64)uctx->wcdpi.umdbr;
4536                         mmap_flag = BNXT_RE_MMAP_WC_DB;
4537                 } else {
4538                         return -EINVAL;
4539                 }
4540
4541                 break;
4542         case BNXT_RE_ALLOC_DBR_BAR_PAGE:
4543                 length = PAGE_SIZE;
4544                 addr = (u64)rdev->pacing.dbr_bar_addr;
4545                 mmap_flag = BNXT_RE_MMAP_DBR_BAR;
4546                 break;
4547
4548         case BNXT_RE_ALLOC_DBR_PAGE:
4549                 length = PAGE_SIZE;
4550                 addr = (u64)rdev->pacing.dbr_page;
4551                 mmap_flag = BNXT_RE_MMAP_DBR_PAGE;
4552                 break;
4553
4554         default:
4555                 return -EOPNOTSUPP;
4556         }
4557
4558         entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset);
4559         if (!entry)
4560                 return -ENOMEM;
4561
4562         uobj->object = entry;
4563         uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
4564         err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4565                              &mmap_offset, sizeof(mmap_offset));
4566         if (err)
4567                 return err;
4568
4569         err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4570                              &length, sizeof(length));
4571         if (err)
4572                 return err;
4573
4574         err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI,
4575                              &dpi, sizeof(length));
4576         if (err)
4577                 return err;
4578
4579         return 0;
4580 }
4581
4582 static int alloc_page_obj_cleanup(struct ib_uobject *uobject,
4583                                   enum rdma_remove_reason why,
4584                             struct uverbs_attr_bundle *attrs)
4585 {
4586         struct  bnxt_re_user_mmap_entry *entry = uobject->object;
4587         struct bnxt_re_ucontext *uctx = entry->uctx;
4588
4589         switch (entry->mmap_flag) {
4590         case BNXT_RE_MMAP_WC_DB:
4591                 if (uctx && uctx->wcdpi.dbr) {
4592                         struct bnxt_re_dev *rdev = uctx->rdev;
4593
4594                         bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi);
4595                         uctx->wcdpi.dbr = NULL;
4596                 }
4597                 break;
4598         case BNXT_RE_MMAP_DBR_BAR:
4599         case BNXT_RE_MMAP_DBR_PAGE:
4600                 break;
4601         default:
4602                 goto exit;
4603         }
4604         rdma_user_mmap_entry_remove(&entry->rdma_entry);
4605 exit:
4606         return 0;
4607 }
4608
4609 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE,
4610                             UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE,
4611                                             BNXT_RE_OBJECT_ALLOC_PAGE,
4612                                             UVERBS_ACCESS_NEW,
4613                                             UA_MANDATORY),
4614                             UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE,
4615                                                  enum bnxt_re_alloc_page_type,
4616                                                  UA_MANDATORY),
4617                             UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4618                                                 UVERBS_ATTR_TYPE(u64),
4619                                                 UA_MANDATORY),
4620                             UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4621                                                 UVERBS_ATTR_TYPE(u32),
4622                                                 UA_MANDATORY),
4623                             UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI,
4624                                                 UVERBS_ATTR_TYPE(u32),
4625                                                 UA_MANDATORY));
4626
4627 DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE,
4628                                     UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE,
4629                                                     BNXT_RE_OBJECT_ALLOC_PAGE,
4630                                                     UVERBS_ACCESS_DESTROY,
4631                                                     UA_MANDATORY));
4632
4633 DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE,
4634                             UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup),
4635                             &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE),
4636                             &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE));
4637
4638 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV);
4639
4640 DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV,
4641                               &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV));
4642
4643 /* Toggle MEM */
4644 static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs)
4645 {
4646         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
4647         enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE;
4648         enum bnxt_re_get_toggle_mem_type res_type;
4649         struct bnxt_re_user_mmap_entry *entry;
4650         struct bnxt_re_ucontext *uctx;
4651         struct ib_ucontext *ib_uctx;
4652         struct bnxt_re_dev *rdev;
4653         struct bnxt_re_srq *srq;
4654         u32 length = PAGE_SIZE;
4655         struct bnxt_re_cq *cq;
4656         u64 mem_offset;
4657         u32 offset = 0;
4658         u64 addr = 0;
4659         u32 res_id;
4660         int err;
4661
4662         ib_uctx = ib_uverbs_get_ucontext(attrs);
4663         if (IS_ERR(ib_uctx))
4664                 return PTR_ERR(ib_uctx);
4665
4666         err = uverbs_get_const(&res_type, attrs, BNXT_RE_TOGGLE_MEM_TYPE);
4667         if (err)
4668                 return err;
4669
4670         uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
4671         rdev = uctx->rdev;
4672         err = uverbs_copy_from(&res_id, attrs, BNXT_RE_TOGGLE_MEM_RES_ID);
4673         if (err)
4674                 return err;
4675
4676         switch (res_type) {
4677         case BNXT_RE_CQ_TOGGLE_MEM:
4678                 cq = bnxt_re_search_for_cq(rdev, res_id);
4679                 if (!cq)
4680                         return -EINVAL;
4681
4682                 addr = (u64)cq->uctx_cq_page;
4683                 break;
4684         case BNXT_RE_SRQ_TOGGLE_MEM:
4685                 srq = bnxt_re_search_for_srq(rdev, res_id);
4686                 if (!srq)
4687                         return -EINVAL;
4688
4689                 addr = (u64)srq->uctx_srq_page;
4690                 break;
4691
4692         default:
4693                 return -EOPNOTSUPP;
4694         }
4695
4696         entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mem_offset);
4697         if (!entry)
4698                 return -ENOMEM;
4699
4700         uobj->object = entry;
4701         uverbs_finalize_uobj_create(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
4702         err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
4703                              &mem_offset, sizeof(mem_offset));
4704         if (err)
4705                 return err;
4706
4707         err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
4708                              &length, sizeof(length));
4709         if (err)
4710                 return err;
4711
4712         err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
4713                              &offset, sizeof(length));
4714         if (err)
4715                 return err;
4716
4717         return 0;
4718 }
4719
4720 static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject,
4721                                       enum rdma_remove_reason why,
4722                                       struct uverbs_attr_bundle *attrs)
4723 {
4724         struct  bnxt_re_user_mmap_entry *entry = uobject->object;
4725
4726         rdma_user_mmap_entry_remove(&entry->rdma_entry);
4727         return 0;
4728 }
4729
4730 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM,
4731                             UVERBS_ATTR_IDR(BNXT_RE_TOGGLE_MEM_HANDLE,
4732                                             BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4733                                             UVERBS_ACCESS_NEW,
4734                                             UA_MANDATORY),
4735                             UVERBS_ATTR_CONST_IN(BNXT_RE_TOGGLE_MEM_TYPE,
4736                                                  enum bnxt_re_get_toggle_mem_type,
4737                                                  UA_MANDATORY),
4738                             UVERBS_ATTR_PTR_IN(BNXT_RE_TOGGLE_MEM_RES_ID,
4739                                                UVERBS_ATTR_TYPE(u32),
4740                                                UA_MANDATORY),
4741                             UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
4742                                                 UVERBS_ATTR_TYPE(u64),
4743                                                 UA_MANDATORY),
4744                             UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
4745                                                 UVERBS_ATTR_TYPE(u32),
4746                                                 UA_MANDATORY),
4747                             UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
4748                                                 UVERBS_ATTR_TYPE(u32),
4749                                                 UA_MANDATORY));
4750
4751 DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,
4752                                     UVERBS_ATTR_IDR(BNXT_RE_RELEASE_TOGGLE_MEM_HANDLE,
4753                                                     BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4754                                                     UVERBS_ACCESS_DESTROY,
4755                                                     UA_MANDATORY));
4756
4757 DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4758                             UVERBS_TYPE_ALLOC_IDR(get_toggle_mem_obj_cleanup),
4759                             &UVERBS_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM),
4760                             &UVERBS_METHOD(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM));
4761
4762 const struct uapi_definition bnxt_re_uapi_defs[] = {
4763         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
4764         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
4765         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),
4766         {}
4767 };