drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/uverbs_ioctl.h>
  46
  47 #include <linux/qed/common_hsi.h>
  48 #include "qedr_hsi_rdma.h"
  49 #include <linux/qed/qed_if.h>
  50 #include "qedr.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_roce_cm.h"
  54
  55 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  56 #define RDMA_MAX_SGE_PER_SRQ    (4)
  57 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  58
  59 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  60
  61 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  62                                         size_t len)
  63 {
  64         size_t min_len = min_t(size_t, len, udata->outlen);
  65
  66         return ib_copy_to_udata(udata, src, min_len);
  67 }
  68
  69 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  70 {
  71         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  72                 return -EINVAL;
  73
  74         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  75         return 0;
  76 }
  77
  78 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
  79                       int index, union ib_gid *sgid)
  80 {
  81         struct qedr_dev *dev = get_qedr_dev(ibdev);
  82
  83         memset(sgid->raw, 0, sizeof(sgid->raw));
  84         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  85
  86         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  87                  sgid->global.interface_id, sgid->global.subnet_prefix);
  88
  89         return 0;
  90 }
  91
  92 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  93 {
  94         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
  95         struct qedr_device_attr *qattr = &dev->attr;
  96         struct qedr_srq *srq = get_qedr_srq(ibsrq);
  97
  98         srq_attr->srq_limit = srq->srq_limit;
  99         srq_attr->max_wr = qattr->max_srq_wr;
 100         srq_attr->max_sge = qattr->max_sge;
 101
 102         return 0;
 103 }
 104
 105 int qedr_query_device(struct ib_device *ibdev,
 106                       struct ib_device_attr *attr, struct ib_udata *udata)
 107 {
 108         struct qedr_dev *dev = get_qedr_dev(ibdev);
 109         struct qedr_device_attr *qattr = &dev->attr;
 110
 111         if (!dev->rdma_ctx) {
 112                 DP_ERR(dev,
 113                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 114                        dev->rdma_ctx);
 115                 return -EINVAL;
 116         }
 117
 118         memset(attr, 0, sizeof(*attr));
 119
 120         attr->fw_ver = qattr->fw_ver;
 121         attr->sys_image_guid = qattr->sys_image_guid;
 122         attr->max_mr_size = qattr->max_mr_size;
 123         attr->page_size_cap = qattr->page_size_caps;
 124         attr->vendor_id = qattr->vendor_id;
 125         attr->vendor_part_id = qattr->vendor_part_id;
 126         attr->hw_ver = qattr->hw_ver;
 127         attr->max_qp = qattr->max_qp;
 128         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 129         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 130             IB_DEVICE_RC_RNR_NAK_GEN |
 131             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 132
 133         attr->max_send_sge = qattr->max_sge;
 134         attr->max_recv_sge = qattr->max_sge;
 135         attr->max_sge_rd = qattr->max_sge;
 136         attr->max_cq = qattr->max_cq;
 137         attr->max_cqe = qattr->max_cqe;
 138         attr->max_mr = qattr->max_mr;
 139         attr->max_mw = qattr->max_mw;
 140         attr->max_pd = qattr->max_pd;
 141         attr->atomic_cap = dev->atomic_cap;
 142         attr->max_fmr = qattr->max_fmr;
 143         attr->max_map_per_fmr = 16;
 144         attr->max_qp_init_rd_atom =
 145             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 146         attr->max_qp_rd_atom =
 147             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 148                 attr->max_qp_init_rd_atom);
 149
 150         attr->max_srq = qattr->max_srq;
 151         attr->max_srq_sge = qattr->max_srq_sge;
 152         attr->max_srq_wr = qattr->max_srq_wr;
 153
 154         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 155         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 156         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
 157         attr->max_ah = qattr->max_ah;
 158
 159         return 0;
 160 }
 161
 162 #define QEDR_SPEED_SDR          (1)
 163 #define QEDR_SPEED_DDR          (2)
 164 #define QEDR_SPEED_QDR          (4)
 165 #define QEDR_SPEED_FDR10        (8)
 166 #define QEDR_SPEED_FDR          (16)
 167 #define QEDR_SPEED_EDR          (32)
 168
 169 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
 170                                             u8 *ib_width)
 171 {
 172         switch (speed) {
 173         case 1000:
 174                 *ib_speed = QEDR_SPEED_SDR;
 175                 *ib_width = IB_WIDTH_1X;
 176                 break;
 177         case 10000:
 178                 *ib_speed = QEDR_SPEED_QDR;
 179                 *ib_width = IB_WIDTH_1X;
 180                 break;
 181
 182         case 20000:
 183                 *ib_speed = QEDR_SPEED_DDR;
 184                 *ib_width = IB_WIDTH_4X;
 185                 break;
 186
 187         case 25000:
 188                 *ib_speed = QEDR_SPEED_EDR;
 189                 *ib_width = IB_WIDTH_1X;
 190                 break;
 191
 192         case 40000:
 193                 *ib_speed = QEDR_SPEED_QDR;
 194                 *ib_width = IB_WIDTH_4X;
 195                 break;
 196
 197         case 50000:
 198                 *ib_speed = QEDR_SPEED_QDR;
 199                 *ib_width = IB_WIDTH_4X;
 200                 break;
 201
 202         case 100000:
 203                 *ib_speed = QEDR_SPEED_EDR;
 204                 *ib_width = IB_WIDTH_4X;
 205                 break;
 206
 207         default:
 208                 /* Unsupported */
 209                 *ib_speed = QEDR_SPEED_SDR;
 210                 *ib_width = IB_WIDTH_1X;
 211         }
 212 }
 213
 214 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 215 {
 216         struct qedr_dev *dev;
 217         struct qed_rdma_port *rdma_port;
 218
 219         dev = get_qedr_dev(ibdev);
 220
 221         if (!dev->rdma_ctx) {
 222                 DP_ERR(dev, "rdma_ctx is NULL\n");
 223                 return -EINVAL;
 224         }
 225
 226         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 227
 228         /* *attr being zeroed by the caller, avoid zeroing it here */
 229         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 230                 attr->state = IB_PORT_ACTIVE;
 231                 attr->phys_state = 5;
 232         } else {
 233                 attr->state = IB_PORT_DOWN;
 234                 attr->phys_state = 3;
 235         }
 236         attr->max_mtu = IB_MTU_4096;
 237         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 238         attr->lid = 0;
 239         attr->lmc = 0;
 240         attr->sm_lid = 0;
 241         attr->sm_sl = 0;
 242         attr->ip_gids = true;
 243         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 244                 attr->gid_tbl_len = 1;
 245                 attr->pkey_tbl_len = 1;
 246         } else {
 247                 attr->gid_tbl_len = QEDR_MAX_SGID;
 248                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 249         }
 250         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 251         attr->qkey_viol_cntr = 0;
 252         get_link_speed_and_width(rdma_port->link_speed,
 253                                  &attr->active_speed, &attr->active_width);
 254         attr->max_msg_sz = rdma_port->max_msg_size;
 255         attr->max_vl_num = 4;
 256
 257         return 0;
 258 }
 259
 260 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
 261                      struct ib_port_modify *props)
 262 {
 263         return 0;
 264 }
 265
 266 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 267                          unsigned long len)
 268 {
 269         struct qedr_mm *mm;
 270
 271         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 272         if (!mm)
 273                 return -ENOMEM;
 274
 275         mm->key.phy_addr = phy_addr;
 276         /* This function might be called with a length which is not a multiple
 277          * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
 278          * forces this granularity by increasing the requested size if needed.
 279          * When qedr_mmap is called, it will search the list with the updated
 280          * length as a key. To prevent search failures, the length is rounded up
 281          * in advance to PAGE_SIZE.
 282          */
 283         mm->key.len = roundup(len, PAGE_SIZE);
 284         INIT_LIST_HEAD(&mm->entry);
 285
 286         mutex_lock(&uctx->mm_list_lock);
 287         list_add(&mm->entry, &uctx->mm_head);
 288         mutex_unlock(&uctx->mm_list_lock);
 289
 290         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 291                  "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 292                  (unsigned long long)mm->key.phy_addr,
 293                  (unsigned long)mm->key.len, uctx);
 294
 295         return 0;
 296 }
 297
 298 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 299                              unsigned long len)
 300 {
 301         bool found = false;
 302         struct qedr_mm *mm;
 303
 304         mutex_lock(&uctx->mm_list_lock);
 305         list_for_each_entry(mm, &uctx->mm_head, entry) {
 306                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
 307                         continue;
 308
 309                 found = true;
 310                 break;
 311         }
 312         mutex_unlock(&uctx->mm_list_lock);
 313         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 314                  "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
 315                  mm->key.phy_addr, mm->key.len, uctx, found);
 316
 317         return found;
 318 }
 319
 320 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 321 {
 322         struct ib_device *ibdev = uctx->device;
 323         int rc;
 324         struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 325         struct qedr_alloc_ucontext_resp uresp = {};
 326         struct qedr_dev *dev = get_qedr_dev(ibdev);
 327         struct qed_rdma_add_user_out_params oparams;
 328
 329         if (!udata)
 330                 return -EFAULT;
 331
 332         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 333         if (rc) {
 334                 DP_ERR(dev,
 335                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 336                        rc);
 337                 return rc;
 338         }
 339
 340         ctx->dpi = oparams.dpi;
 341         ctx->dpi_addr = oparams.dpi_addr;
 342         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 343         ctx->dpi_size = oparams.dpi_size;
 344         INIT_LIST_HEAD(&ctx->mm_head);
 345         mutex_init(&ctx->mm_list_lock);
 346
 347         uresp.dpm_enabled = dev->user_dpm_enabled;
 348         uresp.wids_enabled = 1;
 349         uresp.wid_count = oparams.wid_count;
 350         uresp.db_pa = ctx->dpi_phys_addr;
 351         uresp.db_size = ctx->dpi_size;
 352         uresp.max_send_wr = dev->attr.max_sqe;
 353         uresp.max_recv_wr = dev->attr.max_rqe;
 354         uresp.max_srq_wr = dev->attr.max_srq_wr;
 355         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 356         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 357         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 358         uresp.max_cqes = QEDR_MAX_CQES;
 359
 360         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 361         if (rc)
 362                 return rc;
 363
 364         ctx->dev = dev;
 365
 366         rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
 367         if (rc)
 368                 return rc;
 369
 370         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 371                  &ctx->ibucontext);
 372         return 0;
 373 }
 374
 375 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 376 {
 377         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 378         struct qedr_mm *mm, *tmp;
 379
 380         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 381                  uctx);
 382         uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
 383
 384         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 385                 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 386                          "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 387                          mm->key.phy_addr, mm->key.len, uctx);
 388                 list_del(&mm->entry);
 389                 kfree(mm);
 390         }
 391 }
 392
 393 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 394 {
 395         struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
 396         struct qedr_dev *dev = get_qedr_dev(context->device);
 397         unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
 398         unsigned long len = (vma->vm_end - vma->vm_start);
 399         unsigned long dpi_start;
 400
 401         dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
 402
 403         DP_DEBUG(dev, QEDR_MSG_INIT,
 404                  "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
 405                  (void *)vma->vm_start, (void *)vma->vm_end,
 406                  (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
 407
 408         if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
 409                 DP_ERR(dev,
 410                        "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
 411                        (void *)vma->vm_start, (void *)vma->vm_end);
 412                 return -EINVAL;
 413         }
 414
 415         if (!qedr_search_mmap(ucontext, phys_addr, len)) {
 416                 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
 417                        vma->vm_pgoff);
 418                 return -EINVAL;
 419         }
 420
 421         if (phys_addr < dpi_start ||
 422             ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
 423                 DP_ERR(dev,
 424                        "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
 425                        (void *)phys_addr, (void *)dpi_start,
 426                        ucontext->dpi_size);
 427                 return -EINVAL;
 428         }
 429
 430         if (vma->vm_flags & VM_READ) {
 431                 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
 432                 return -EINVAL;
 433         }
 434
 435         vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 436         return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
 437                                   vma->vm_page_prot);
 438 }
 439
 440 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 441 {
 442         struct ib_device *ibdev = ibpd->device;
 443         struct qedr_dev *dev = get_qedr_dev(ibdev);
 444         struct qedr_pd *pd = get_qedr_pd(ibpd);
 445         u16 pd_id;
 446         int rc;
 447
 448         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 449                  udata ? "User Lib" : "Kernel");
 450
 451         if (!dev->rdma_ctx) {
 452                 DP_ERR(dev, "invalid RDMA context\n");
 453                 return -EINVAL;
 454         }
 455
 456         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 457         if (rc)
 458                 return rc;
 459
 460         pd->pd_id = pd_id;
 461
 462         if (udata) {
 463                 struct qedr_alloc_pd_uresp uresp = {
 464                         .pd_id = pd_id,
 465                 };
 466                 struct qedr_ucontext *context = rdma_udata_to_drv_context(
 467                         udata, struct qedr_ucontext, ibucontext);
 468
 469                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 470                 if (rc) {
 471                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 472                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 473                         return rc;
 474                 }
 475
 476                 pd->uctx = context;
 477                 pd->uctx->pd = pd;
 478         }
 479
 480         return 0;
 481 }
 482
 483 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 484 {
 485         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 486         struct qedr_pd *pd = get_qedr_pd(ibpd);
 487
 488         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 489         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 490 }
 491
 492 static void qedr_free_pbl(struct qedr_dev *dev,
 493                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 494 {
 495         struct pci_dev *pdev = dev->pdev;
 496         int i;
 497
 498         for (i = 0; i < pbl_info->num_pbls; i++) {
 499                 if (!pbl[i].va)
 500                         continue;
 501                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 502                                   pbl[i].va, pbl[i].pa);
 503         }
 504
 505         kfree(pbl);
 506 }
 507
 508 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 509 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 510
 511 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 512 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 513 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 514
 515 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 516                                            struct qedr_pbl_info *pbl_info,
 517                                            gfp_t flags)
 518 {
 519         struct pci_dev *pdev = dev->pdev;
 520         struct qedr_pbl *pbl_table;
 521         dma_addr_t *pbl_main_tbl;
 522         dma_addr_t pa;
 523         void *va;
 524         int i;
 525
 526         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 527         if (!pbl_table)
 528                 return ERR_PTR(-ENOMEM);
 529
 530         for (i = 0; i < pbl_info->num_pbls; i++) {
 531                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 532                                         flags);
 533                 if (!va)
 534                         goto err;
 535
 536                 pbl_table[i].va = va;
 537                 pbl_table[i].pa = pa;
 538         }
 539
 540         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 541          * the first one with physical pointers to all of the rest
 542          */
 543         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 544         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 545                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 546
 547         return pbl_table;
 548
 549 err:
 550         for (i--; i >= 0; i--)
 551                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 552                                   pbl_table[i].va, pbl_table[i].pa);
 553
 554         qedr_free_pbl(dev, pbl_info, pbl_table);
 555
 556         return ERR_PTR(-ENOMEM);
 557 }
 558
 559 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 560                                 struct qedr_pbl_info *pbl_info,
 561                                 u32 num_pbes, int two_layer_capable)
 562 {
 563         u32 pbl_capacity;
 564         u32 pbl_size;
 565         u32 num_pbls;
 566
 567         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 568                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 569                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 570                                num_pbes);
 571                         return -EINVAL;
 572                 }
 573
 574                 /* calculate required pbl page size */
 575                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 576                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 577                                NUM_PBES_ON_PAGE(pbl_size);
 578
 579                 while (pbl_capacity < num_pbes) {
 580                         pbl_size *= 2;
 581                         pbl_capacity = pbl_size / sizeof(u64);
 582                         pbl_capacity = pbl_capacity * pbl_capacity;
 583                 }
 584
 585                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 586                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 587                 pbl_info->two_layered = true;
 588         } else {
 589                 /* One layered PBL */
 590                 num_pbls = 1;
 591                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 592                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 593                 pbl_info->two_layered = false;
 594         }
 595
 596         pbl_info->num_pbls = num_pbls;
 597         pbl_info->pbl_size = pbl_size;
 598         pbl_info->num_pbes = num_pbes;
 599
 600         DP_DEBUG(dev, QEDR_MSG_MR,
 601                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 602                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 603
 604         return 0;
 605 }
 606
 607 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 608                                struct qedr_pbl *pbl,
 609                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 610 {
 611         int pbe_cnt, total_num_pbes = 0;
 612         u32 fw_pg_cnt, fw_pg_per_umem_pg;
 613         struct qedr_pbl *pbl_tbl;
 614         struct sg_dma_page_iter sg_iter;
 615         struct regpair *pbe;
 616         u64 pg_addr;
 617
 618         if (!pbl_info->num_pbes)
 619                 return;
 620
 621         /* If we have a two layered pbl, the first pbl points to the rest
 622          * of the pbls and the first entry lays on the second pbl in the table
 623          */
 624         if (pbl_info->two_layered)
 625                 pbl_tbl = &pbl[1];
 626         else
 627                 pbl_tbl = pbl;
 628
 629         pbe = (struct regpair *)pbl_tbl->va;
 630         if (!pbe) {
 631                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 632                 return;
 633         }
 634
 635         pbe_cnt = 0;
 636
 637         fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
 638
 639         for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
 640                 pg_addr = sg_page_iter_dma_address(&sg_iter);
 641                 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
 642                         pbe->lo = cpu_to_le32(pg_addr);
 643                         pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 644
 645                         pg_addr += BIT(pg_shift);
 646                         pbe_cnt++;
 647                         total_num_pbes++;
 648                         pbe++;
 649
 650                         if (total_num_pbes == pbl_info->num_pbes)
 651                                 return;
 652
 653                         /* If the given pbl is full storing the pbes,
 654                          * move to next pbl.
 655                          */
 656                         if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 657                                 pbl_tbl++;
 658                                 pbe = (struct regpair *)pbl_tbl->va;
 659                                 pbe_cnt = 0;
 660                         }
 661
 662                         fw_pg_cnt++;
 663                 }
 664         }
 665 }
 666
 667 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 668                               struct qedr_cq *cq, struct ib_udata *udata)
 669 {
 670         struct qedr_create_cq_uresp uresp;
 671         int rc;
 672
 673         memset(&uresp, 0, sizeof(uresp));
 674
 675         uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 676         uresp.icid = cq->icid;
 677
 678         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 679         if (rc)
 680                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 681
 682         return rc;
 683 }
 684
 685 static void consume_cqe(struct qedr_cq *cq)
 686 {
 687         if (cq->latest_cqe == cq->toggle_cqe)
 688                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 689
 690         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 691 }
 692
 693 static inline int qedr_align_cq_entries(int entries)
 694 {
 695         u64 size, aligned_size;
 696
 697         /* We allocate an extra entry that we don't report to the FW. */
 698         size = (entries + 1) * QEDR_CQE_SIZE;
 699         aligned_size = ALIGN(size, PAGE_SIZE);
 700
 701         return aligned_size / QEDR_CQE_SIZE;
 702 }
 703
 704 static inline int qedr_init_user_queue(struct ib_udata *udata,
 705                                        struct qedr_dev *dev,
 706                                        struct qedr_userq *q, u64 buf_addr,
 707                                        size_t buf_len, int access, int dmasync,
 708                                        int alloc_and_init)
 709 {
 710         u32 fw_pages;
 711         int rc;
 712
 713         q->buf_addr = buf_addr;
 714         q->buf_len = buf_len;
 715         q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync);
 716         if (IS_ERR(q->umem)) {
 717                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 718                        PTR_ERR(q->umem));
 719                 return PTR_ERR(q->umem);
 720         }
 721
 722         fw_pages = ib_umem_page_count(q->umem) <<
 723             (PAGE_SHIFT - FW_PAGE_SHIFT);
 724
 725         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 726         if (rc)
 727                 goto err0;
 728
 729         if (alloc_and_init) {
 730                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 731                 if (IS_ERR(q->pbl_tbl)) {
 732                         rc = PTR_ERR(q->pbl_tbl);
 733                         goto err0;
 734                 }
 735                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 736                                    FW_PAGE_SHIFT);
 737         } else {
 738                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 739                 if (!q->pbl_tbl) {
 740                         rc = -ENOMEM;
 741                         goto err0;
 742                 }
 743         }
 744
 745         return 0;
 746
 747 err0:
 748         ib_umem_release(q->umem);
 749         q->umem = NULL;
 750
 751         return rc;
 752 }
 753
 754 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 755                                        struct qedr_ucontext *ctx,
 756                                        struct qedr_dev *dev, int vector,
 757                                        int chain_entries, int page_cnt,
 758                                        u64 pbl_ptr,
 759                                        struct qed_rdma_create_cq_in_params
 760                                        *params)
 761 {
 762         memset(params, 0, sizeof(*params));
 763         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 764         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 765         params->cnq_id = vector;
 766         params->cq_size = chain_entries - 1;
 767         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 768         params->pbl_num_pages = page_cnt;
 769         params->pbl_ptr = pbl_ptr;
 770         params->pbl_two_level = 0;
 771 }
 772
 773 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 774 {
 775         cq->db.data.agg_flags = flags;
 776         cq->db.data.value = cpu_to_le32(cons);
 777         writeq(cq->db.raw, cq->db_addr);
 778
 779         /* Make sure write would stick */
 780         mmiowb();
 781 }
 782
 783 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 784 {
 785         struct qedr_cq *cq = get_qedr_cq(ibcq);
 786         unsigned long sflags;
 787         struct qedr_dev *dev;
 788
 789         dev = get_qedr_dev(ibcq->device);
 790
 791         if (cq->destroyed) {
 792                 DP_ERR(dev,
 793                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 794                        cq, cq->icid);
 795                 return -EINVAL;
 796         }
 797
 798
 799         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 800                 return 0;
 801
 802         spin_lock_irqsave(&cq->cq_lock, sflags);
 803
 804         cq->arm_flags = 0;
 805
 806         if (flags & IB_CQ_SOLICITED)
 807                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 808
 809         if (flags & IB_CQ_NEXT_COMP)
 810                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 811
 812         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 813
 814         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 815
 816         return 0;
 817 }
 818
 819 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 820                              const struct ib_cq_init_attr *attr,
 821                              struct ib_udata *udata)
 822 {
 823         struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 824                 udata, struct qedr_ucontext, ibucontext);
 825         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 826         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 827         struct qedr_dev *dev = get_qedr_dev(ibdev);
 828         struct qed_rdma_create_cq_in_params params;
 829         struct qedr_create_cq_ureq ureq;
 830         int vector = attr->comp_vector;
 831         int entries = attr->cqe;
 832         struct qedr_cq *cq;
 833         int chain_entries;
 834         int page_cnt;
 835         u64 pbl_ptr;
 836         u16 icid;
 837         int rc;
 838
 839         DP_DEBUG(dev, QEDR_MSG_INIT,
 840                  "create_cq: called from %s. entries=%d, vector=%d\n",
 841                  udata ? "User Lib" : "Kernel", entries, vector);
 842
 843         if (entries > QEDR_MAX_CQES) {
 844                 DP_ERR(dev,
 845                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 846                        entries, QEDR_MAX_CQES);
 847                 return ERR_PTR(-EINVAL);
 848         }
 849
 850         chain_entries = qedr_align_cq_entries(entries);
 851         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 852
 853         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 854         if (!cq)
 855                 return ERR_PTR(-ENOMEM);
 856
 857         if (udata) {
 858                 memset(&ureq, 0, sizeof(ureq));
 859                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
 860                         DP_ERR(dev,
 861                                "create cq: problem copying data from user space\n");
 862                         goto err0;
 863                 }
 864
 865                 if (!ureq.len) {
 866                         DP_ERR(dev,
 867                                "create cq: cannot create a cq with 0 entries\n");
 868                         goto err0;
 869                 }
 870
 871                 cq->cq_type = QEDR_CQ_TYPE_USER;
 872
 873                 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 874                                           ureq.len, IB_ACCESS_LOCAL_WRITE, 1,
 875                                           1);
 876                 if (rc)
 877                         goto err0;
 878
 879                 pbl_ptr = cq->q.pbl_tbl->pa;
 880                 page_cnt = cq->q.pbl_info.num_pbes;
 881
 882                 cq->ibcq.cqe = chain_entries;
 883         } else {
 884                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 885
 886                 rc = dev->ops->common->chain_alloc(dev->cdev,
 887                                                    QED_CHAIN_USE_TO_CONSUME,
 888                                                    QED_CHAIN_MODE_PBL,
 889                                                    QED_CHAIN_CNT_TYPE_U32,
 890                                                    chain_entries,
 891                                                    sizeof(union rdma_cqe),
 892                                                    &cq->pbl, NULL);
 893                 if (rc)
 894                         goto err1;
 895
 896                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 897                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 898                 cq->ibcq.cqe = cq->pbl.capacity;
 899         }
 900
 901         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 902                             pbl_ptr, &params);
 903
 904         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 905         if (rc)
 906                 goto err2;
 907
 908         cq->icid = icid;
 909         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 910         spin_lock_init(&cq->cq_lock);
 911
 912         if (udata) {
 913                 rc = qedr_copy_cq_uresp(dev, cq, udata);
 914                 if (rc)
 915                         goto err3;
 916         } else {
 917                 /* Generate doorbell address. */
 918                 cq->db_addr = dev->db_addr +
 919                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 920                 cq->db.data.icid = cq->icid;
 921                 cq->db.data.params = DB_AGG_CMD_SET <<
 922                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
 923
 924                 /* point to the very last element, passing it we will toggle */
 925                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
 926                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 927                 cq->latest_cqe = NULL;
 928                 consume_cqe(cq);
 929                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
 930         }
 931
 932         DP_DEBUG(dev, QEDR_MSG_CQ,
 933                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
 934                  cq->icid, cq, params.cq_size);
 935
 936         return &cq->ibcq;
 937
 938 err3:
 939         destroy_iparams.icid = cq->icid;
 940         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
 941                                   &destroy_oparams);
 942 err2:
 943         if (udata)
 944                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
 945         else
 946                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 947 err1:
 948         if (udata)
 949                 ib_umem_release(cq->q.umem);
 950 err0:
 951         kfree(cq);
 952         return ERR_PTR(-EINVAL);
 953 }
 954
 955 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
 956 {
 957         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
 958         struct qedr_cq *cq = get_qedr_cq(ibcq);
 959
 960         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
 961
 962         return 0;
 963 }
 964
 965 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
 966 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
 967
 968 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 969 {
 970         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
 971         struct qed_rdma_destroy_cq_out_params oparams;
 972         struct qed_rdma_destroy_cq_in_params iparams;
 973         struct qedr_cq *cq = get_qedr_cq(ibcq);
 974         int iter;
 975         int rc;
 976
 977         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
 978
 979         cq->destroyed = 1;
 980
 981         /* GSIs CQs are handled by driver, so they don't exist in the FW */
 982         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 983                 goto done;
 984
 985         iparams.icid = cq->icid;
 986         rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
 987         if (rc)
 988                 return rc;
 989
 990         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 991
 992         if (udata) {
 993                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
 994                 ib_umem_release(cq->q.umem);
 995         }
 996
 997         /* We don't want the IRQ handler to handle a non-existing CQ so we
 998          * wait until all CNQ interrupts, if any, are received. This will always
 999          * happen and will always happen very fast. If not, then a serious error
1000          * has occured. That is why we can use a long delay.
1001          * We spin for a short time so we don’t lose time on context switching
1002          * in case all the completions are handled in that span. Otherwise
1003          * we sleep for a while and check again. Since the CNQ may be
1004          * associated with (only) the current CPU we use msleep to allow the
1005          * current CPU to be freed.
1006          * The CNQ notification is increased in qedr_irq_handler().
1007          */
1008         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1009         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1010                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1011                 iter--;
1012         }
1013
1014         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1015         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1016                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1017                 iter--;
1018         }
1019
1020         if (oparams.num_cq_notif != cq->cnq_notif)
1021                 goto err;
1022
1023         /* Note that we don't need to have explicit code to wait for the
1024          * completion of the event handler because it is invoked from the EQ.
1025          * Since the destroy CQ ramrod has also been received on the EQ we can
1026          * be certain that there's no event handler in process.
1027          */
1028 done:
1029         cq->sig = ~cq->sig;
1030
1031         kfree(cq);
1032
1033         return 0;
1034
1035 err:
1036         DP_ERR(dev,
1037                "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1038                cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1039
1040         return -EINVAL;
1041 }
1042
1043 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1044                                           struct ib_qp_attr *attr,
1045                                           int attr_mask,
1046                                           struct qed_rdma_modify_qp_in_params
1047                                           *qp_params)
1048 {
1049         const struct ib_gid_attr *gid_attr;
1050         enum rdma_network_type nw_type;
1051         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1052         u32 ipv4_addr;
1053         int ret;
1054         int i;
1055
1056         gid_attr = grh->sgid_attr;
1057         ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1058         if (ret)
1059                 return ret;
1060
1061         nw_type = rdma_gid_attr_network_type(gid_attr);
1062         switch (nw_type) {
1063         case RDMA_NETWORK_IPV6:
1064                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1065                        sizeof(qp_params->sgid));
1066                 memcpy(&qp_params->dgid.bytes[0],
1067                        &grh->dgid,
1068                        sizeof(qp_params->dgid));
1069                 qp_params->roce_mode = ROCE_V2_IPV6;
1070                 SET_FIELD(qp_params->modify_flags,
1071                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1072                 break;
1073         case RDMA_NETWORK_IB:
1074                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1075                        sizeof(qp_params->sgid));
1076                 memcpy(&qp_params->dgid.bytes[0],
1077                        &grh->dgid,
1078                        sizeof(qp_params->dgid));
1079                 qp_params->roce_mode = ROCE_V1;
1080                 break;
1081         case RDMA_NETWORK_IPV4:
1082                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1083                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1084                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1085                 qp_params->sgid.ipv4_addr = ipv4_addr;
1086                 ipv4_addr =
1087                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1088                 qp_params->dgid.ipv4_addr = ipv4_addr;
1089                 SET_FIELD(qp_params->modify_flags,
1090                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1091                 qp_params->roce_mode = ROCE_V2_IPV4;
1092                 break;
1093         }
1094
1095         for (i = 0; i < 4; i++) {
1096                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1097                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1098         }
1099
1100         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1101                 qp_params->vlan_id = 0;
1102
1103         return 0;
1104 }
1105
1106 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1107                                struct ib_qp_init_attr *attrs,
1108                                struct ib_udata *udata)
1109 {
1110         struct qedr_device_attr *qattr = &dev->attr;
1111
1112         /* QP0... attrs->qp_type == IB_QPT_GSI */
1113         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1114                 DP_DEBUG(dev, QEDR_MSG_QP,
1115                          "create qp: unsupported qp type=0x%x requested\n",
1116                          attrs->qp_type);
1117                 return -EINVAL;
1118         }
1119
1120         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1121                 DP_ERR(dev,
1122                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1123                        attrs->cap.max_send_wr, qattr->max_sqe);
1124                 return -EINVAL;
1125         }
1126
1127         if (attrs->cap.max_inline_data > qattr->max_inline) {
1128                 DP_ERR(dev,
1129                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1130                        attrs->cap.max_inline_data, qattr->max_inline);
1131                 return -EINVAL;
1132         }
1133
1134         if (attrs->cap.max_send_sge > qattr->max_sge) {
1135                 DP_ERR(dev,
1136                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1137                        attrs->cap.max_send_sge, qattr->max_sge);
1138                 return -EINVAL;
1139         }
1140
1141         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1142                 DP_ERR(dev,
1143                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1144                        attrs->cap.max_recv_sge, qattr->max_sge);
1145                 return -EINVAL;
1146         }
1147
1148         /* Unprivileged user space cannot create special QP */
1149         if (udata && attrs->qp_type == IB_QPT_GSI) {
1150                 DP_ERR(dev,
1151                        "create qp: userspace can't create special QPs of type=0x%x\n",
1152                        attrs->qp_type);
1153                 return -EINVAL;
1154         }
1155
1156         return 0;
1157 }
1158
1159 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1160                                struct qedr_srq *srq, struct ib_udata *udata)
1161 {
1162         struct qedr_create_srq_uresp uresp = {};
1163         int rc;
1164
1165         uresp.srq_id = srq->srq_id;
1166
1167         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1168         if (rc)
1169                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1170
1171         return rc;
1172 }
1173
1174 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1175                                struct qedr_create_qp_uresp *uresp,
1176                                struct qedr_qp *qp)
1177 {
1178         /* iWARP requires two doorbells per RQ. */
1179         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1180                 uresp->rq_db_offset =
1181                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1182                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1183         } else {
1184                 uresp->rq_db_offset =
1185                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1186         }
1187
1188         uresp->rq_icid = qp->icid;
1189 }
1190
1191 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1192                                struct qedr_create_qp_uresp *uresp,
1193                                struct qedr_qp *qp)
1194 {
1195         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1196
1197         /* iWARP uses the same cid for rq and sq */
1198         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1199                 uresp->sq_icid = qp->icid;
1200         else
1201                 uresp->sq_icid = qp->icid + 1;
1202 }
1203
1204 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1205                               struct qedr_qp *qp, struct ib_udata *udata)
1206 {
1207         struct qedr_create_qp_uresp uresp;
1208         int rc;
1209
1210         memset(&uresp, 0, sizeof(uresp));
1211         qedr_copy_sq_uresp(dev, &uresp, qp);
1212         qedr_copy_rq_uresp(dev, &uresp, qp);
1213
1214         uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1215         uresp.qp_id = qp->qp_id;
1216
1217         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1218         if (rc)
1219                 DP_ERR(dev,
1220                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1221                        qp->icid);
1222
1223         return rc;
1224 }
1225
1226 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1227                                       struct qedr_qp *qp,
1228                                       struct qedr_pd *pd,
1229                                       struct ib_qp_init_attr *attrs)
1230 {
1231         spin_lock_init(&qp->q_lock);
1232         atomic_set(&qp->refcnt, 1);
1233         qp->pd = pd;
1234         qp->qp_type = attrs->qp_type;
1235         qp->max_inline_data = attrs->cap.max_inline_data;
1236         qp->sq.max_sges = attrs->cap.max_send_sge;
1237         qp->state = QED_ROCE_QP_STATE_RESET;
1238         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1239         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1240         qp->dev = dev;
1241
1242         if (attrs->srq) {
1243                 qp->srq = get_qedr_srq(attrs->srq);
1244         } else {
1245                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1246                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1247                 DP_DEBUG(dev, QEDR_MSG_QP,
1248                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1249                          qp->rq.max_sges, qp->rq_cq->icid);
1250         }
1251
1252         DP_DEBUG(dev, QEDR_MSG_QP,
1253                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1254                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1255                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1256         DP_DEBUG(dev, QEDR_MSG_QP,
1257                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1258                  qp->sq.max_sges, qp->sq_cq->icid);
1259 }
1260
1261 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1262 {
1263         qp->sq.db = dev->db_addr +
1264                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1265         qp->sq.db_data.data.icid = qp->icid + 1;
1266         if (!qp->srq) {
1267                 qp->rq.db = dev->db_addr +
1268                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1269                 qp->rq.db_data.data.icid = qp->icid;
1270         }
1271 }
1272
1273 static int qedr_check_srq_params(struct qedr_dev *dev,
1274                                  struct ib_srq_init_attr *attrs,
1275                                  struct ib_udata *udata)
1276 {
1277         struct qedr_device_attr *qattr = &dev->attr;
1278
1279         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1280                 DP_ERR(dev,
1281                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1282                        attrs->attr.max_wr, qattr->max_srq_wr);
1283                 return -EINVAL;
1284         }
1285
1286         if (attrs->attr.max_sge > qattr->max_sge) {
1287                 DP_ERR(dev,
1288                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1289                        attrs->attr.max_sge, qattr->max_sge);
1290                 return -EINVAL;
1291         }
1292
1293         return 0;
1294 }
1295
1296 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1297 {
1298         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1299         ib_umem_release(srq->usrq.umem);
1300         ib_umem_release(srq->prod_umem);
1301 }
1302
1303 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1304 {
1305         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1306         struct qedr_dev *dev = srq->dev;
1307
1308         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1309
1310         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1311                           hw_srq->virt_prod_pair_addr,
1312                           hw_srq->phy_prod_pair_addr);
1313 }
1314
1315 static int qedr_init_srq_user_params(struct ib_udata *udata,
1316                                      struct qedr_srq *srq,
1317                                      struct qedr_create_srq_ureq *ureq,
1318                                      int access, int dmasync)
1319 {
1320         struct scatterlist *sg;
1321         int rc;
1322
1323         rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1324                                   ureq->srq_len, access, dmasync, 1);
1325         if (rc)
1326                 return rc;
1327
1328         srq->prod_umem =
1329                 ib_umem_get(udata, ureq->prod_pair_addr,
1330                             sizeof(struct rdma_srq_producers), access, dmasync);
1331         if (IS_ERR(srq->prod_umem)) {
1332                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1333                 ib_umem_release(srq->usrq.umem);
1334                 DP_ERR(srq->dev,
1335                        "create srq: failed ib_umem_get for producer, got %ld\n",
1336                        PTR_ERR(srq->prod_umem));
1337                 return PTR_ERR(srq->prod_umem);
1338         }
1339
1340         sg = srq->prod_umem->sg_head.sgl;
1341         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1342
1343         return 0;
1344 }
1345
1346 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1347                                         struct qedr_dev *dev,
1348                                         struct ib_srq_init_attr *init_attr)
1349 {
1350         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1351         dma_addr_t phy_prod_pair_addr;
1352         u32 num_elems;
1353         void *va;
1354         int rc;
1355
1356         va = dma_alloc_coherent(&dev->pdev->dev,
1357                                 sizeof(struct rdma_srq_producers),
1358                                 &phy_prod_pair_addr, GFP_KERNEL);
1359         if (!va) {
1360                 DP_ERR(dev,
1361                        "create srq: failed to allocate dma memory for producer\n");
1362                 return -ENOMEM;
1363         }
1364
1365         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1366         hw_srq->virt_prod_pair_addr = va;
1367
1368         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1369         rc = dev->ops->common->chain_alloc(dev->cdev,
1370                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1371                                            QED_CHAIN_MODE_PBL,
1372                                            QED_CHAIN_CNT_TYPE_U32,
1373                                            num_elems,
1374                                            QEDR_SRQ_WQE_ELEM_SIZE,
1375                                            &hw_srq->pbl, NULL);
1376         if (rc)
1377                 goto err0;
1378
1379         hw_srq->num_elems = num_elems;
1380
1381         return 0;
1382
1383 err0:
1384         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1385                           va, phy_prod_pair_addr);
1386         return rc;
1387 }
1388
1389 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1390                     struct ib_udata *udata)
1391 {
1392         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1393         struct qed_rdma_create_srq_in_params in_params = {};
1394         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1395         struct qed_rdma_create_srq_out_params out_params;
1396         struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1397         struct qedr_create_srq_ureq ureq = {};
1398         u64 pbl_base_addr, phy_prod_pair_addr;
1399         struct qedr_srq_hwq_info *hw_srq;
1400         u32 page_cnt, page_size;
1401         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1402         int rc = 0;
1403
1404         DP_DEBUG(dev, QEDR_MSG_QP,
1405                  "create SRQ called from %s (pd %p)\n",
1406                  (udata) ? "User lib" : "kernel", pd);
1407
1408         rc = qedr_check_srq_params(dev, init_attr, udata);
1409         if (rc)
1410                 return -EINVAL;
1411
1412         srq->dev = dev;
1413         hw_srq = &srq->hw_srq;
1414         spin_lock_init(&srq->lock);
1415
1416         hw_srq->max_wr = init_attr->attr.max_wr;
1417         hw_srq->max_sges = init_attr->attr.max_sge;
1418
1419         if (udata) {
1420                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1421                         DP_ERR(dev,
1422                                "create srq: problem copying data from user space\n");
1423                         goto err0;
1424                 }
1425
1426                 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0);
1427                 if (rc)
1428                         goto err0;
1429
1430                 page_cnt = srq->usrq.pbl_info.num_pbes;
1431                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1432                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1433                 page_size = PAGE_SIZE;
1434         } else {
1435                 struct qed_chain *pbl;
1436
1437                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1438                 if (rc)
1439                         goto err0;
1440
1441                 pbl = &hw_srq->pbl;
1442                 page_cnt = qed_chain_get_page_cnt(pbl);
1443                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1444                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1445                 page_size = QED_CHAIN_PAGE_SIZE;
1446         }
1447
1448         in_params.pd_id = pd->pd_id;
1449         in_params.pbl_base_addr = pbl_base_addr;
1450         in_params.prod_pair_addr = phy_prod_pair_addr;
1451         in_params.num_pages = page_cnt;
1452         in_params.page_size = page_size;
1453
1454         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1455         if (rc)
1456                 goto err1;
1457
1458         srq->srq_id = out_params.srq_id;
1459
1460         if (udata) {
1461                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1462                 if (rc)
1463                         goto err2;
1464         }
1465
1466         rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1467         if (rc)
1468                 goto err2;
1469
1470         DP_DEBUG(dev, QEDR_MSG_SRQ,
1471                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1472         return 0;
1473
1474 err2:
1475         destroy_in_params.srq_id = srq->srq_id;
1476
1477         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1478 err1:
1479         if (udata)
1480                 qedr_free_srq_user_params(srq);
1481         else
1482                 qedr_free_srq_kernel_params(srq);
1483 err0:
1484         return -EFAULT;
1485 }
1486
1487 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1488 {
1489         struct qed_rdma_destroy_srq_in_params in_params = {};
1490         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1491         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1492
1493         xa_erase_irq(&dev->srqs, srq->srq_id);
1494         in_params.srq_id = srq->srq_id;
1495         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1496
1497         if (ibsrq->uobject)
1498                 qedr_free_srq_user_params(srq);
1499         else
1500                 qedr_free_srq_kernel_params(srq);
1501
1502         DP_DEBUG(dev, QEDR_MSG_SRQ,
1503                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1504                  srq->srq_id);
1505 }
1506
1507 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1508                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1509 {
1510         struct qed_rdma_modify_srq_in_params in_params = {};
1511         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1512         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1513         int rc;
1514
1515         if (attr_mask & IB_SRQ_MAX_WR) {
1516                 DP_ERR(dev,
1517                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1518                        attr_mask, srq);
1519                 return -EINVAL;
1520         }
1521
1522         if (attr_mask & IB_SRQ_LIMIT) {
1523                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1524                         DP_ERR(dev,
1525                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1526                                attr->srq_limit, srq->hw_srq.max_wr);
1527                         return -EINVAL;
1528                 }
1529
1530                 in_params.srq_id = srq->srq_id;
1531                 in_params.wqe_limit = attr->srq_limit;
1532                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1533                 if (rc)
1534                         return rc;
1535         }
1536
1537         srq->srq_limit = attr->srq_limit;
1538
1539         DP_DEBUG(dev, QEDR_MSG_SRQ,
1540                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1541
1542         return 0;
1543 }
1544
1545 static inline void
1546 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1547                               struct qedr_pd *pd,
1548                               struct qedr_qp *qp,
1549                               struct ib_qp_init_attr *attrs,
1550                               bool fmr_and_reserved_lkey,
1551                               struct qed_rdma_create_qp_in_params *params)
1552 {
1553         /* QP handle to be written in an async event */
1554         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1555         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1556
1557         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1558         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1559         params->pd = pd->pd_id;
1560         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1561         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1562         params->stats_queue = 0;
1563         params->srq_id = 0;
1564         params->use_srq = false;
1565
1566         if (!qp->srq) {
1567                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1568
1569         } else {
1570                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1571                 params->srq_id = qp->srq->srq_id;
1572                 params->use_srq = true;
1573         }
1574 }
1575
1576 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1577 {
1578         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1579                  "qp=%p. "
1580                  "sq_addr=0x%llx, "
1581                  "sq_len=%zd, "
1582                  "rq_addr=0x%llx, "
1583                  "rq_len=%zd"
1584                  "\n",
1585                  qp,
1586                  qp->usq.buf_addr,
1587                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1588 }
1589
1590 static inline void
1591 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1592                             struct qedr_qp *qp,
1593                             struct qed_rdma_create_qp_out_params *out_params)
1594 {
1595         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1596         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1597
1598         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1599                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1600         if (!qp->srq) {
1601                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1602                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1603         }
1604
1605         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1606                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1607 }
1608
1609 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1610 {
1611         if (qp->usq.umem)
1612                 ib_umem_release(qp->usq.umem);
1613         qp->usq.umem = NULL;
1614
1615         if (qp->urq.umem)
1616                 ib_umem_release(qp->urq.umem);
1617         qp->urq.umem = NULL;
1618 }
1619
1620 static int qedr_create_user_qp(struct qedr_dev *dev,
1621                                struct qedr_qp *qp,
1622                                struct ib_pd *ibpd,
1623                                struct ib_udata *udata,
1624                                struct ib_qp_init_attr *attrs)
1625 {
1626         struct qed_rdma_create_qp_in_params in_params;
1627         struct qed_rdma_create_qp_out_params out_params;
1628         struct qedr_pd *pd = get_qedr_pd(ibpd);
1629         struct qedr_create_qp_ureq ureq;
1630         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1631         int rc = -EINVAL;
1632
1633         memset(&ureq, 0, sizeof(ureq));
1634         rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1635         if (rc) {
1636                 DP_ERR(dev, "Problem copying data from user space\n");
1637                 return rc;
1638         }
1639
1640         /* SQ - read access only (0), dma sync not required (0) */
1641         rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1642                                   ureq.sq_len, 0, 0, alloc_and_init);
1643         if (rc)
1644                 return rc;
1645
1646         if (!qp->srq) {
1647                 /* RQ - read access only (0), dma sync not required (0) */
1648                 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1649                                           ureq.rq_len, 0, 0, alloc_and_init);
1650                 if (rc)
1651                         return rc;
1652         }
1653
1654         memset(&in_params, 0, sizeof(in_params));
1655         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1656         in_params.qp_handle_lo = ureq.qp_handle_lo;
1657         in_params.qp_handle_hi = ureq.qp_handle_hi;
1658         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1659         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1660         if (!qp->srq) {
1661                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1662                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1663         }
1664
1665         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1666                                               &in_params, &out_params);
1667
1668         if (!qp->qed_qp) {
1669                 rc = -ENOMEM;
1670                 goto err1;
1671         }
1672
1673         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1674                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1675
1676         qp->qp_id = out_params.qp_id;
1677         qp->icid = out_params.icid;
1678
1679         rc = qedr_copy_qp_uresp(dev, qp, udata);
1680         if (rc)
1681                 goto err;
1682
1683         qedr_qp_user_print(dev, qp);
1684
1685         return 0;
1686 err:
1687         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1688         if (rc)
1689                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1690
1691 err1:
1692         qedr_cleanup_user(dev, qp);
1693         return rc;
1694 }
1695
1696 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1697 {
1698         qp->sq.db = dev->db_addr +
1699             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1700         qp->sq.db_data.data.icid = qp->icid;
1701
1702         qp->rq.db = dev->db_addr +
1703                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1704         qp->rq.db_data.data.icid = qp->icid;
1705         qp->rq.iwarp_db2 = dev->db_addr +
1706                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1707         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1708         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1709 }
1710
1711 static int
1712 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1713                            struct qedr_qp *qp,
1714                            struct qed_rdma_create_qp_in_params *in_params,
1715                            u32 n_sq_elems, u32 n_rq_elems)
1716 {
1717         struct qed_rdma_create_qp_out_params out_params;
1718         int rc;
1719
1720         rc = dev->ops->common->chain_alloc(dev->cdev,
1721                                            QED_CHAIN_USE_TO_PRODUCE,
1722                                            QED_CHAIN_MODE_PBL,
1723                                            QED_CHAIN_CNT_TYPE_U32,
1724                                            n_sq_elems,
1725                                            QEDR_SQE_ELEMENT_SIZE,
1726                                            &qp->sq.pbl, NULL);
1727
1728         if (rc)
1729                 return rc;
1730
1731         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1732         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1733
1734         rc = dev->ops->common->chain_alloc(dev->cdev,
1735                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1736                                            QED_CHAIN_MODE_PBL,
1737                                            QED_CHAIN_CNT_TYPE_U32,
1738                                            n_rq_elems,
1739                                            QEDR_RQE_ELEMENT_SIZE,
1740                                            &qp->rq.pbl, NULL);
1741         if (rc)
1742                 return rc;
1743
1744         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1745         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1746
1747         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1748                                               in_params, &out_params);
1749
1750         if (!qp->qed_qp)
1751                 return -EINVAL;
1752
1753         qp->qp_id = out_params.qp_id;
1754         qp->icid = out_params.icid;
1755
1756         qedr_set_roce_db_info(dev, qp);
1757         return rc;
1758 }
1759
1760 static int
1761 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1762                             struct qedr_qp *qp,
1763                             struct qed_rdma_create_qp_in_params *in_params,
1764                             u32 n_sq_elems, u32 n_rq_elems)
1765 {
1766         struct qed_rdma_create_qp_out_params out_params;
1767         struct qed_chain_ext_pbl ext_pbl;
1768         int rc;
1769
1770         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1771                                                      QEDR_SQE_ELEMENT_SIZE,
1772                                                      QED_CHAIN_MODE_PBL);
1773         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1774                                                      QEDR_RQE_ELEMENT_SIZE,
1775                                                      QED_CHAIN_MODE_PBL);
1776
1777         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1778                                               in_params, &out_params);
1779
1780         if (!qp->qed_qp)
1781                 return -EINVAL;
1782
1783         /* Now we allocate the chain */
1784         ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1785         ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1786
1787         rc = dev->ops->common->chain_alloc(dev->cdev,
1788                                            QED_CHAIN_USE_TO_PRODUCE,
1789                                            QED_CHAIN_MODE_PBL,
1790                                            QED_CHAIN_CNT_TYPE_U32,
1791                                            n_sq_elems,
1792                                            QEDR_SQE_ELEMENT_SIZE,
1793                                            &qp->sq.pbl, &ext_pbl);
1794
1795         if (rc)
1796                 goto err;
1797
1798         ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1799         ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1800
1801         rc = dev->ops->common->chain_alloc(dev->cdev,
1802                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1803                                            QED_CHAIN_MODE_PBL,
1804                                            QED_CHAIN_CNT_TYPE_U32,
1805                                            n_rq_elems,
1806                                            QEDR_RQE_ELEMENT_SIZE,
1807                                            &qp->rq.pbl, &ext_pbl);
1808
1809         if (rc)
1810                 goto err;
1811
1812         qp->qp_id = out_params.qp_id;
1813         qp->icid = out_params.icid;
1814
1815         qedr_set_iwarp_db_info(dev, qp);
1816         return rc;
1817
1818 err:
1819         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1820
1821         return rc;
1822 }
1823
1824 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1825 {
1826         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1827         kfree(qp->wqe_wr_id);
1828
1829         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1830         kfree(qp->rqe_wr_id);
1831 }
1832
1833 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1834                                  struct qedr_qp *qp,
1835                                  struct ib_pd *ibpd,
1836                                  struct ib_qp_init_attr *attrs)
1837 {
1838         struct qed_rdma_create_qp_in_params in_params;
1839         struct qedr_pd *pd = get_qedr_pd(ibpd);
1840         int rc = -EINVAL;
1841         u32 n_rq_elems;
1842         u32 n_sq_elems;
1843         u32 n_sq_entries;
1844
1845         memset(&in_params, 0, sizeof(in_params));
1846
1847         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1848          * the ring. The ring should allow at least a single WR, even if the
1849          * user requested none, due to allocation issues.
1850          * We should add an extra WR since the prod and cons indices of
1851          * wqe_wr_id are managed in such a way that the WQ is considered full
1852          * when (prod+1)%max_wr==cons. We currently don't do that because we
1853          * double the number of entries due an iSER issue that pushes far more
1854          * WRs than indicated. If we decline its ib_post_send() then we get
1855          * error prints in the dmesg we'd like to avoid.
1856          */
1857         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1858                               dev->attr.max_sqe);
1859
1860         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1861                                 GFP_KERNEL);
1862         if (!qp->wqe_wr_id) {
1863                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1864                 return -ENOMEM;
1865         }
1866
1867         /* QP handle to be written in CQE */
1868         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1869         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1870
1871         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1872          * the ring. There ring should allow at least a single WR, even if the
1873          * user requested none, due to allocation issues.
1874          */
1875         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1876
1877         /* Allocate driver internal RQ array */
1878         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1879                                 GFP_KERNEL);
1880         if (!qp->rqe_wr_id) {
1881                 DP_ERR(dev,
1882                        "create qp: failed RQ shadow memory allocation\n");
1883                 kfree(qp->wqe_wr_id);
1884                 return -ENOMEM;
1885         }
1886
1887         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1888
1889         n_sq_entries = attrs->cap.max_send_wr;
1890         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1891         n_sq_entries = max_t(u32, n_sq_entries, 1);
1892         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1893
1894         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1895
1896         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1897                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1898                                                  n_sq_elems, n_rq_elems);
1899         else
1900                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1901                                                 n_sq_elems, n_rq_elems);
1902         if (rc)
1903                 qedr_cleanup_kernel(dev, qp);
1904
1905         return rc;
1906 }
1907
1908 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1909                              struct ib_qp_init_attr *attrs,
1910                              struct ib_udata *udata)
1911 {
1912         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1913         struct qedr_pd *pd = get_qedr_pd(ibpd);
1914         struct qedr_qp *qp;
1915         struct ib_qp *ibqp;
1916         int rc = 0;
1917
1918         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1919                  udata ? "user library" : "kernel", pd);
1920
1921         rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
1922         if (rc)
1923                 return ERR_PTR(rc);
1924
1925         DP_DEBUG(dev, QEDR_MSG_QP,
1926                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1927                  udata ? "user library" : "kernel", attrs->event_handler, pd,
1928                  get_qedr_cq(attrs->send_cq),
1929                  get_qedr_cq(attrs->send_cq)->icid,
1930                  get_qedr_cq(attrs->recv_cq),
1931                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
1932
1933         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1934         if (!qp) {
1935                 DP_ERR(dev, "create qp: failed allocating memory\n");
1936                 return ERR_PTR(-ENOMEM);
1937         }
1938
1939         qedr_set_common_qp_params(dev, qp, pd, attrs);
1940
1941         if (attrs->qp_type == IB_QPT_GSI) {
1942                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1943                 if (IS_ERR(ibqp))
1944                         kfree(qp);
1945                 return ibqp;
1946         }
1947
1948         if (udata)
1949                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1950         else
1951                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1952
1953         if (rc)
1954                 goto err;
1955
1956         qp->ibqp.qp_num = qp->qp_id;
1957
1958         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1959                 rc = xa_insert_irq(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1960                 if (rc)
1961                         goto err;
1962         }
1963
1964         return &qp->ibqp;
1965
1966 err:
1967         kfree(qp);
1968
1969         return ERR_PTR(-EFAULT);
1970 }
1971
1972 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1973 {
1974         switch (qp_state) {
1975         case QED_ROCE_QP_STATE_RESET:
1976                 return IB_QPS_RESET;
1977         case QED_ROCE_QP_STATE_INIT:
1978                 return IB_QPS_INIT;
1979         case QED_ROCE_QP_STATE_RTR:
1980                 return IB_QPS_RTR;
1981         case QED_ROCE_QP_STATE_RTS:
1982                 return IB_QPS_RTS;
1983         case QED_ROCE_QP_STATE_SQD:
1984                 return IB_QPS_SQD;
1985         case QED_ROCE_QP_STATE_ERR:
1986                 return IB_QPS_ERR;
1987         case QED_ROCE_QP_STATE_SQE:
1988                 return IB_QPS_SQE;
1989         }
1990         return IB_QPS_ERR;
1991 }
1992
1993 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1994                                         enum ib_qp_state qp_state)
1995 {
1996         switch (qp_state) {
1997         case IB_QPS_RESET:
1998                 return QED_ROCE_QP_STATE_RESET;
1999         case IB_QPS_INIT:
2000                 return QED_ROCE_QP_STATE_INIT;
2001         case IB_QPS_RTR:
2002                 return QED_ROCE_QP_STATE_RTR;
2003         case IB_QPS_RTS:
2004                 return QED_ROCE_QP_STATE_RTS;
2005         case IB_QPS_SQD:
2006                 return QED_ROCE_QP_STATE_SQD;
2007         case IB_QPS_ERR:
2008                 return QED_ROCE_QP_STATE_ERR;
2009         default:
2010                 return QED_ROCE_QP_STATE_ERR;
2011         }
2012 }
2013
2014 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2015 {
2016         qed_chain_reset(&qph->pbl);
2017         qph->prod = 0;
2018         qph->cons = 0;
2019         qph->wqe_cons = 0;
2020         qph->db_data.data.value = cpu_to_le16(0);
2021 }
2022
2023 static int qedr_update_qp_state(struct qedr_dev *dev,
2024                                 struct qedr_qp *qp,
2025                                 enum qed_roce_qp_state cur_state,
2026                                 enum qed_roce_qp_state new_state)
2027 {
2028         int status = 0;
2029
2030         if (new_state == cur_state)
2031                 return 0;
2032
2033         switch (cur_state) {
2034         case QED_ROCE_QP_STATE_RESET:
2035                 switch (new_state) {
2036                 case QED_ROCE_QP_STATE_INIT:
2037                         qp->prev_wqe_size = 0;
2038                         qedr_reset_qp_hwq_info(&qp->sq);
2039                         qedr_reset_qp_hwq_info(&qp->rq);
2040                         break;
2041                 default:
2042                         status = -EINVAL;
2043                         break;
2044                 }
2045                 break;
2046         case QED_ROCE_QP_STATE_INIT:
2047                 switch (new_state) {
2048                 case QED_ROCE_QP_STATE_RTR:
2049                         /* Update doorbell (in case post_recv was
2050                          * done before move to RTR)
2051                          */
2052
2053                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2054                                 writel(qp->rq.db_data.raw, qp->rq.db);
2055                                 /* Make sure write takes effect */
2056                                 mmiowb();
2057                         }
2058                         break;
2059                 case QED_ROCE_QP_STATE_ERR:
2060                         break;
2061                 default:
2062                         /* Invalid state change. */
2063                         status = -EINVAL;
2064                         break;
2065                 }
2066                 break;
2067         case QED_ROCE_QP_STATE_RTR:
2068                 /* RTR->XXX */
2069                 switch (new_state) {
2070                 case QED_ROCE_QP_STATE_RTS:
2071                         break;
2072                 case QED_ROCE_QP_STATE_ERR:
2073                         break;
2074                 default:
2075                         /* Invalid state change. */
2076                         status = -EINVAL;
2077                         break;
2078                 }
2079                 break;
2080         case QED_ROCE_QP_STATE_RTS:
2081                 /* RTS->XXX */
2082                 switch (new_state) {
2083                 case QED_ROCE_QP_STATE_SQD:
2084                         break;
2085                 case QED_ROCE_QP_STATE_ERR:
2086                         break;
2087                 default:
2088                         /* Invalid state change. */
2089                         status = -EINVAL;
2090                         break;
2091                 }
2092                 break;
2093         case QED_ROCE_QP_STATE_SQD:
2094                 /* SQD->XXX */
2095                 switch (new_state) {
2096                 case QED_ROCE_QP_STATE_RTS:
2097                 case QED_ROCE_QP_STATE_ERR:
2098                         break;
2099                 default:
2100                         /* Invalid state change. */
2101                         status = -EINVAL;
2102                         break;
2103                 }
2104                 break;
2105         case QED_ROCE_QP_STATE_ERR:
2106                 /* ERR->XXX */
2107                 switch (new_state) {
2108                 case QED_ROCE_QP_STATE_RESET:
2109                         if ((qp->rq.prod != qp->rq.cons) ||
2110                             (qp->sq.prod != qp->sq.cons)) {
2111                                 DP_NOTICE(dev,
2112                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2113                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2114                                           qp->sq.cons);
2115                                 status = -EINVAL;
2116                         }
2117                         break;
2118                 default:
2119                         status = -EINVAL;
2120                         break;
2121                 }
2122                 break;
2123         default:
2124                 status = -EINVAL;
2125                 break;
2126         }
2127
2128         return status;
2129 }
2130
2131 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2132                    int attr_mask, struct ib_udata *udata)
2133 {
2134         struct qedr_qp *qp = get_qedr_qp(ibqp);
2135         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2136         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2137         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2138         enum ib_qp_state old_qp_state, new_qp_state;
2139         enum qed_roce_qp_state cur_state;
2140         int rc = 0;
2141
2142         DP_DEBUG(dev, QEDR_MSG_QP,
2143                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2144                  attr->qp_state);
2145
2146         old_qp_state = qedr_get_ibqp_state(qp->state);
2147         if (attr_mask & IB_QP_STATE)
2148                 new_qp_state = attr->qp_state;
2149         else
2150                 new_qp_state = old_qp_state;
2151
2152         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2153                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2154                                         ibqp->qp_type, attr_mask)) {
2155                         DP_ERR(dev,
2156                                "modify qp: invalid attribute mask=0x%x specified for\n"
2157                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2158                                attr_mask, qp->qp_id, ibqp->qp_type,
2159                                old_qp_state, new_qp_state);
2160                         rc = -EINVAL;
2161                         goto err;
2162                 }
2163         }
2164
2165         /* Translate the masks... */
2166         if (attr_mask & IB_QP_STATE) {
2167                 SET_FIELD(qp_params.modify_flags,
2168                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2169                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2170         }
2171
2172         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2173                 qp_params.sqd_async = true;
2174
2175         if (attr_mask & IB_QP_PKEY_INDEX) {
2176                 SET_FIELD(qp_params.modify_flags,
2177                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2178                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2179                         rc = -EINVAL;
2180                         goto err;
2181                 }
2182
2183                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2184         }
2185
2186         if (attr_mask & IB_QP_QKEY)
2187                 qp->qkey = attr->qkey;
2188
2189         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2190                 SET_FIELD(qp_params.modify_flags,
2191                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2192                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2193                                                   IB_ACCESS_REMOTE_READ;
2194                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2195                                                    IB_ACCESS_REMOTE_WRITE;
2196                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2197                                                IB_ACCESS_REMOTE_ATOMIC;
2198         }
2199
2200         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2201                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2202                         return -EINVAL;
2203
2204                 if (attr_mask & IB_QP_PATH_MTU) {
2205                         if (attr->path_mtu < IB_MTU_256 ||
2206                             attr->path_mtu > IB_MTU_4096) {
2207                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2208                                 rc = -EINVAL;
2209                                 goto err;
2210                         }
2211                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2212                                       ib_mtu_enum_to_int(iboe_get_mtu
2213                                                          (dev->ndev->mtu)));
2214                 }
2215
2216                 if (!qp->mtu) {
2217                         qp->mtu =
2218                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2219                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2220                 }
2221
2222                 SET_FIELD(qp_params.modify_flags,
2223                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2224
2225                 qp_params.traffic_class_tos = grh->traffic_class;
2226                 qp_params.flow_label = grh->flow_label;
2227                 qp_params.hop_limit_ttl = grh->hop_limit;
2228
2229                 qp->sgid_idx = grh->sgid_index;
2230
2231                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2232                 if (rc) {
2233                         DP_ERR(dev,
2234                                "modify qp: problems with GID index %d (rc=%d)\n",
2235                                grh->sgid_index, rc);
2236                         return rc;
2237                 }
2238
2239                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2240                                    qp_params.remote_mac_addr);
2241                 if (rc)
2242                         return rc;
2243
2244                 qp_params.use_local_mac = true;
2245                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2246
2247                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2248                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2249                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2250                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2251                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2252                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2253                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2254                          qp_params.remote_mac_addr);
2255
2256                 qp_params.mtu = qp->mtu;
2257                 qp_params.lb_indication = false;
2258         }
2259
2260         if (!qp_params.mtu) {
2261                 /* Stay with current MTU */
2262                 if (qp->mtu)
2263                         qp_params.mtu = qp->mtu;
2264                 else
2265                         qp_params.mtu =
2266                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2267         }
2268
2269         if (attr_mask & IB_QP_TIMEOUT) {
2270                 SET_FIELD(qp_params.modify_flags,
2271                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2272
2273                 /* The received timeout value is an exponent used like this:
2274                  *    "12.7.34 LOCAL ACK TIMEOUT
2275                  *    Value representing the transport (ACK) timeout for use by
2276                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2277                  * The FW expects timeout in msec so we need to divide the usec
2278                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2279                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2280                  * The value of zero means infinite so we use a 'max_t' to make
2281                  * sure that sub 1 msec values will be configured as 1 msec.
2282                  */
2283                 if (attr->timeout)
2284                         qp_params.ack_timeout =
2285                                         1 << max_t(int, attr->timeout - 8, 0);
2286                 else
2287                         qp_params.ack_timeout = 0;
2288         }
2289
2290         if (attr_mask & IB_QP_RETRY_CNT) {
2291                 SET_FIELD(qp_params.modify_flags,
2292                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2293                 qp_params.retry_cnt = attr->retry_cnt;
2294         }
2295
2296         if (attr_mask & IB_QP_RNR_RETRY) {
2297                 SET_FIELD(qp_params.modify_flags,
2298                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2299                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2300         }
2301
2302         if (attr_mask & IB_QP_RQ_PSN) {
2303                 SET_FIELD(qp_params.modify_flags,
2304                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2305                 qp_params.rq_psn = attr->rq_psn;
2306                 qp->rq_psn = attr->rq_psn;
2307         }
2308
2309         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2310                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2311                         rc = -EINVAL;
2312                         DP_ERR(dev,
2313                                "unsupported max_rd_atomic=%d, supported=%d\n",
2314                                attr->max_rd_atomic,
2315                                dev->attr.max_qp_req_rd_atomic_resc);
2316                         goto err;
2317                 }
2318
2319                 SET_FIELD(qp_params.modify_flags,
2320                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2321                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2322         }
2323
2324         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2325                 SET_FIELD(qp_params.modify_flags,
2326                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2327                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2328         }
2329
2330         if (attr_mask & IB_QP_SQ_PSN) {
2331                 SET_FIELD(qp_params.modify_flags,
2332                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2333                 qp_params.sq_psn = attr->sq_psn;
2334                 qp->sq_psn = attr->sq_psn;
2335         }
2336
2337         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2338                 if (attr->max_dest_rd_atomic >
2339                     dev->attr.max_qp_resp_rd_atomic_resc) {
2340                         DP_ERR(dev,
2341                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2342                                attr->max_dest_rd_atomic,
2343                                dev->attr.max_qp_resp_rd_atomic_resc);
2344
2345                         rc = -EINVAL;
2346                         goto err;
2347                 }
2348
2349                 SET_FIELD(qp_params.modify_flags,
2350                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2351                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2352         }
2353
2354         if (attr_mask & IB_QP_DEST_QPN) {
2355                 SET_FIELD(qp_params.modify_flags,
2356                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2357
2358                 qp_params.dest_qp = attr->dest_qp_num;
2359                 qp->dest_qp_num = attr->dest_qp_num;
2360         }
2361
2362         cur_state = qp->state;
2363
2364         /* Update the QP state before the actual ramrod to prevent a race with
2365          * fast path. Modifying the QP state to error will cause the device to
2366          * flush the CQEs and while polling the flushed CQEs will considered as
2367          * a potential issue if the QP isn't in error state.
2368          */
2369         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2370             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2371                 qp->state = QED_ROCE_QP_STATE_ERR;
2372
2373         if (qp->qp_type != IB_QPT_GSI)
2374                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2375                                               qp->qed_qp, &qp_params);
2376
2377         if (attr_mask & IB_QP_STATE) {
2378                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2379                         rc = qedr_update_qp_state(dev, qp, cur_state,
2380                                                   qp_params.new_state);
2381                 qp->state = qp_params.new_state;
2382         }
2383
2384 err:
2385         return rc;
2386 }
2387
2388 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2389 {
2390         int ib_qp_acc_flags = 0;
2391
2392         if (params->incoming_rdma_write_en)
2393                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2394         if (params->incoming_rdma_read_en)
2395                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2396         if (params->incoming_atomic_en)
2397                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2398         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2399         return ib_qp_acc_flags;
2400 }
2401
2402 int qedr_query_qp(struct ib_qp *ibqp,
2403                   struct ib_qp_attr *qp_attr,
2404                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2405 {
2406         struct qed_rdma_query_qp_out_params params;
2407         struct qedr_qp *qp = get_qedr_qp(ibqp);
2408         struct qedr_dev *dev = qp->dev;
2409         int rc = 0;
2410
2411         memset(&params, 0, sizeof(params));
2412
2413         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2414         if (rc)
2415                 goto err;
2416
2417         memset(qp_attr, 0, sizeof(*qp_attr));
2418         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2419
2420         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2421         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2422         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2423         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2424         qp_attr->rq_psn = params.rq_psn;
2425         qp_attr->sq_psn = params.sq_psn;
2426         qp_attr->dest_qp_num = params.dest_qp;
2427
2428         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2429
2430         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2431         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2432         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2433         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2434         qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2435         qp_init_attr->cap = qp_attr->cap;
2436
2437         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2438         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2439                         params.flow_label, qp->sgid_idx,
2440                         params.hop_limit_ttl, params.traffic_class_tos);
2441         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2442         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2443         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2444         qp_attr->timeout = params.timeout;
2445         qp_attr->rnr_retry = params.rnr_retry;
2446         qp_attr->retry_cnt = params.retry_cnt;
2447         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2448         qp_attr->pkey_index = params.pkey_index;
2449         qp_attr->port_num = 1;
2450         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2451         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2452         qp_attr->alt_pkey_index = 0;
2453         qp_attr->alt_port_num = 0;
2454         qp_attr->alt_timeout = 0;
2455         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2456
2457         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2458         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2459         qp_attr->max_rd_atomic = params.max_rd_atomic;
2460         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2461
2462         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2463                  qp_attr->cap.max_inline_data);
2464
2465 err:
2466         return rc;
2467 }
2468
2469 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2470                                   struct ib_udata *udata)
2471 {
2472         int rc = 0;
2473
2474         if (qp->qp_type != IB_QPT_GSI) {
2475                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2476                 if (rc)
2477                         return rc;
2478         }
2479
2480         if (udata)
2481                 qedr_cleanup_user(dev, qp);
2482         else
2483                 qedr_cleanup_kernel(dev, qp);
2484
2485         return 0;
2486 }
2487
2488 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2489 {
2490         struct qedr_qp *qp = get_qedr_qp(ibqp);
2491         struct qedr_dev *dev = qp->dev;
2492         struct ib_qp_attr attr;
2493         int attr_mask = 0;
2494         int rc = 0;
2495
2496         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2497                  qp, qp->qp_type);
2498
2499         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2500                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2501                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2502                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2503
2504                         attr.qp_state = IB_QPS_ERR;
2505                         attr_mask |= IB_QP_STATE;
2506
2507                         /* Change the QP state to ERROR */
2508                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2509                 }
2510         } else {
2511                 /* Wait for the connect/accept to complete */
2512                 if (qp->ep) {
2513                         int wait_count = 1;
2514
2515                         while (qp->ep->during_connect) {
2516                                 DP_DEBUG(dev, QEDR_MSG_QP,
2517                                          "Still in during connect/accept\n");
2518
2519                                 msleep(100);
2520                                 if (wait_count++ > 200) {
2521                                         DP_NOTICE(dev,
2522                                                   "during connect timeout\n");
2523                                         break;
2524                                 }
2525                         }
2526                 }
2527         }
2528
2529         if (qp->qp_type == IB_QPT_GSI)
2530                 qedr_destroy_gsi_qp(dev);
2531
2532         qedr_free_qp_resources(dev, qp, udata);
2533
2534         if (atomic_dec_and_test(&qp->refcnt) &&
2535             rdma_protocol_iwarp(&dev->ibdev, 1)) {
2536                 xa_erase_irq(&dev->qps, qp->qp_id);
2537                 kfree(qp);
2538         }
2539         return rc;
2540 }
2541
2542 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
2543                    struct ib_udata *udata)
2544 {
2545         struct qedr_ah *ah = get_qedr_ah(ibah);
2546
2547         rdma_copy_ah_attr(&ah->attr, attr);
2548
2549         return 0;
2550 }
2551
2552 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2553 {
2554         struct qedr_ah *ah = get_qedr_ah(ibah);
2555
2556         rdma_destroy_ah_attr(&ah->attr);
2557 }
2558
2559 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2560 {
2561         struct qedr_pbl *pbl, *tmp;
2562
2563         if (info->pbl_table)
2564                 list_add_tail(&info->pbl_table->list_entry,
2565                               &info->free_pbl_list);
2566
2567         if (!list_empty(&info->inuse_pbl_list))
2568                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2569
2570         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2571                 list_del(&pbl->list_entry);
2572                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2573         }
2574 }
2575
2576 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2577                         size_t page_list_len, bool two_layered)
2578 {
2579         struct qedr_pbl *tmp;
2580         int rc;
2581
2582         INIT_LIST_HEAD(&info->free_pbl_list);
2583         INIT_LIST_HEAD(&info->inuse_pbl_list);
2584
2585         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2586                                   page_list_len, two_layered);
2587         if (rc)
2588                 goto done;
2589
2590         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2591         if (IS_ERR(info->pbl_table)) {
2592                 rc = PTR_ERR(info->pbl_table);
2593                 goto done;
2594         }
2595
2596         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2597                  &info->pbl_table->pa);
2598
2599         /* in usual case we use 2 PBLs, so we add one to free
2600          * list and allocating another one
2601          */
2602         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2603         if (IS_ERR(tmp)) {
2604                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2605                 goto done;
2606         }
2607
2608         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2609
2610         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2611
2612 done:
2613         if (rc)
2614                 free_mr_info(dev, info);
2615
2616         return rc;
2617 }
2618
2619 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2620                                u64 usr_addr, int acc, struct ib_udata *udata)
2621 {
2622         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2623         struct qedr_mr *mr;
2624         struct qedr_pd *pd;
2625         int rc = -ENOMEM;
2626
2627         pd = get_qedr_pd(ibpd);
2628         DP_DEBUG(dev, QEDR_MSG_MR,
2629                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2630                  pd->pd_id, start, len, usr_addr, acc);
2631
2632         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2633                 return ERR_PTR(-EINVAL);
2634
2635         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2636         if (!mr)
2637                 return ERR_PTR(rc);
2638
2639         mr->type = QEDR_MR_USER;
2640
2641         mr->umem = ib_umem_get(udata, start, len, acc, 0);
2642         if (IS_ERR(mr->umem)) {
2643                 rc = -EFAULT;
2644                 goto err0;
2645         }
2646
2647         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2648         if (rc)
2649                 goto err1;
2650
2651         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2652                            &mr->info.pbl_info, PAGE_SHIFT);
2653
2654         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2655         if (rc) {
2656                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2657                 goto err1;
2658         }
2659
2660         /* Index only, 18 bit long, lkey = itid << 8 | key */
2661         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2662         mr->hw_mr.key = 0;
2663         mr->hw_mr.pd = pd->pd_id;
2664         mr->hw_mr.local_read = 1;
2665         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2666         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2667         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2668         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2669         mr->hw_mr.mw_bind = false;
2670         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2671         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2672         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2673         mr->hw_mr.page_size_log = PAGE_SHIFT;
2674         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2675         mr->hw_mr.length = len;
2676         mr->hw_mr.vaddr = usr_addr;
2677         mr->hw_mr.zbva = false;
2678         mr->hw_mr.phy_mr = false;
2679         mr->hw_mr.dma_mr = false;
2680
2681         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2682         if (rc) {
2683                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2684                 goto err2;
2685         }
2686
2687         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2688         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2689             mr->hw_mr.remote_atomic)
2690                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2691
2692         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2693                  mr->ibmr.lkey);
2694         return &mr->ibmr;
2695
2696 err2:
2697         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2698 err1:
2699         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2700 err0:
2701         kfree(mr);
2702         return ERR_PTR(rc);
2703 }
2704
2705 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
2706 {
2707         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2708         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2709         int rc = 0;
2710
2711         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2712         if (rc)
2713                 return rc;
2714
2715         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2716
2717         if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2718                 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2719
2720         /* it could be user registered memory. */
2721         if (mr->umem)
2722                 ib_umem_release(mr->umem);
2723
2724         kfree(mr);
2725
2726         return rc;
2727 }
2728
2729 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2730                                        int max_page_list_len)
2731 {
2732         struct qedr_pd *pd = get_qedr_pd(ibpd);
2733         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2734         struct qedr_mr *mr;
2735         int rc = -ENOMEM;
2736
2737         DP_DEBUG(dev, QEDR_MSG_MR,
2738                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2739                  max_page_list_len);
2740
2741         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2742         if (!mr)
2743                 return ERR_PTR(rc);
2744
2745         mr->dev = dev;
2746         mr->type = QEDR_MR_FRMR;
2747
2748         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2749         if (rc)
2750                 goto err0;
2751
2752         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2753         if (rc) {
2754                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2755                 goto err0;
2756         }
2757
2758         /* Index only, 18 bit long, lkey = itid << 8 | key */
2759         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2760         mr->hw_mr.key = 0;
2761         mr->hw_mr.pd = pd->pd_id;
2762         mr->hw_mr.local_read = 1;
2763         mr->hw_mr.local_write = 0;
2764         mr->hw_mr.remote_read = 0;
2765         mr->hw_mr.remote_write = 0;
2766         mr->hw_mr.remote_atomic = 0;
2767         mr->hw_mr.mw_bind = false;
2768         mr->hw_mr.pbl_ptr = 0;
2769         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2770         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2771         mr->hw_mr.fbo = 0;
2772         mr->hw_mr.length = 0;
2773         mr->hw_mr.vaddr = 0;
2774         mr->hw_mr.zbva = false;
2775         mr->hw_mr.phy_mr = true;
2776         mr->hw_mr.dma_mr = false;
2777
2778         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2779         if (rc) {
2780                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2781                 goto err1;
2782         }
2783
2784         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2785         mr->ibmr.rkey = mr->ibmr.lkey;
2786
2787         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2788         return mr;
2789
2790 err1:
2791         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2792 err0:
2793         kfree(mr);
2794         return ERR_PTR(rc);
2795 }
2796
2797 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2798                             u32 max_num_sg, struct ib_udata *udata)
2799 {
2800         struct qedr_mr *mr;
2801
2802         if (mr_type != IB_MR_TYPE_MEM_REG)
2803                 return ERR_PTR(-EINVAL);
2804
2805         mr = __qedr_alloc_mr(ibpd, max_num_sg);
2806
2807         if (IS_ERR(mr))
2808                 return ERR_PTR(-EINVAL);
2809
2810         return &mr->ibmr;
2811 }
2812
2813 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2814 {
2815         struct qedr_mr *mr = get_qedr_mr(ibmr);
2816         struct qedr_pbl *pbl_table;
2817         struct regpair *pbe;
2818         u32 pbes_in_page;
2819
2820         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2821                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2822                 return -ENOMEM;
2823         }
2824
2825         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2826                  mr->npages, addr);
2827
2828         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2829         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2830         pbe = (struct regpair *)pbl_table->va;
2831         pbe +=  mr->npages % pbes_in_page;
2832         pbe->lo = cpu_to_le32((u32)addr);
2833         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2834
2835         mr->npages++;
2836
2837         return 0;
2838 }
2839
2840 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2841 {
2842         int work = info->completed - info->completed_handled - 1;
2843
2844         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2845         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2846                 struct qedr_pbl *pbl;
2847
2848                 /* Free all the page list that are possible to be freed
2849                  * (all the ones that were invalidated), under the assumption
2850                  * that if an FMR was completed successfully that means that
2851                  * if there was an invalidate operation before it also ended
2852                  */
2853                 pbl = list_first_entry(&info->inuse_pbl_list,
2854                                        struct qedr_pbl, list_entry);
2855                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2856                 info->completed_handled++;
2857         }
2858 }
2859
2860 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2861                    int sg_nents, unsigned int *sg_offset)
2862 {
2863         struct qedr_mr *mr = get_qedr_mr(ibmr);
2864
2865         mr->npages = 0;
2866
2867         handle_completed_mrs(mr->dev, &mr->info);
2868         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2869 }
2870
2871 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2872 {
2873         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2874         struct qedr_pd *pd = get_qedr_pd(ibpd);
2875         struct qedr_mr *mr;
2876         int rc;
2877
2878         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2879         if (!mr)
2880                 return ERR_PTR(-ENOMEM);
2881
2882         mr->type = QEDR_MR_DMA;
2883
2884         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2885         if (rc) {
2886                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2887                 goto err1;
2888         }
2889
2890         /* index only, 18 bit long, lkey = itid << 8 | key */
2891         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2892         mr->hw_mr.pd = pd->pd_id;
2893         mr->hw_mr.local_read = 1;
2894         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2895         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2896         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2897         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2898         mr->hw_mr.dma_mr = true;
2899
2900         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2901         if (rc) {
2902                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2903                 goto err2;
2904         }
2905
2906         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2907         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2908             mr->hw_mr.remote_atomic)
2909                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2910
2911         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2912         return &mr->ibmr;
2913
2914 err2:
2915         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2916 err1:
2917         kfree(mr);
2918         return ERR_PTR(rc);
2919 }
2920
2921 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2922 {
2923         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2924 }
2925
2926 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2927 {
2928         int i, len = 0;
2929
2930         for (i = 0; i < num_sge; i++)
2931                 len += sg_list[i].length;
2932
2933         return len;
2934 }
2935
2936 static void swap_wqe_data64(u64 *p)
2937 {
2938         int i;
2939
2940         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2941                 *p = cpu_to_be64(cpu_to_le64(*p));
2942 }
2943
2944 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2945                                        struct qedr_qp *qp, u8 *wqe_size,
2946                                        const struct ib_send_wr *wr,
2947                                        const struct ib_send_wr **bad_wr,
2948                                        u8 *bits, u8 bit)
2949 {
2950         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2951         char *seg_prt, *wqe;
2952         int i, seg_siz;
2953
2954         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2955                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2956                 *bad_wr = wr;
2957                 return 0;
2958         }
2959
2960         if (!data_size)
2961                 return data_size;
2962
2963         *bits |= bit;
2964
2965         seg_prt = NULL;
2966         wqe = NULL;
2967         seg_siz = 0;
2968
2969         /* Copy data inline */
2970         for (i = 0; i < wr->num_sge; i++) {
2971                 u32 len = wr->sg_list[i].length;
2972                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2973
2974                 while (len > 0) {
2975                         u32 cur;
2976
2977                         /* New segment required */
2978                         if (!seg_siz) {
2979                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2980                                 seg_prt = wqe;
2981                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
2982                                 (*wqe_size)++;
2983                         }
2984
2985                         /* Calculate currently allowed length */
2986                         cur = min_t(u32, len, seg_siz);
2987                         memcpy(seg_prt, src, cur);
2988
2989                         /* Update segment variables */
2990                         seg_prt += cur;
2991                         seg_siz -= cur;
2992
2993                         /* Update sge variables */
2994                         src += cur;
2995                         len -= cur;
2996
2997                         /* Swap fully-completed segments */
2998                         if (!seg_siz)
2999                                 swap_wqe_data64((u64 *)wqe);
3000                 }
3001         }
3002
3003         /* swap last not completed segment */
3004         if (seg_siz)
3005                 swap_wqe_data64((u64 *)wqe);
3006
3007         return data_size;
3008 }
3009
3010 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3011         do {                                                    \
3012                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3013                 (sge)->length = cpu_to_le32(vlength);           \
3014                 (sge)->flags = cpu_to_le32(vflags);             \
3015         } while (0)
3016
3017 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3018         do {                                                    \
3019                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3020                 (hdr)->num_sges = num_sge;                      \
3021         } while (0)
3022
3023 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3024         do {                                                    \
3025                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3026                 (sge)->length = cpu_to_le32(vlength);           \
3027                 (sge)->l_key = cpu_to_le32(vlkey);              \
3028         } while (0)
3029
3030 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3031                                 const struct ib_send_wr *wr)
3032 {
3033         u32 data_size = 0;
3034         int i;
3035
3036         for (i = 0; i < wr->num_sge; i++) {
3037                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3038
3039                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3040                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3041                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3042                 data_size += wr->sg_list[i].length;
3043         }
3044
3045         if (wqe_size)
3046                 *wqe_size += wr->num_sge;
3047
3048         return data_size;
3049 }
3050
3051 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3052                                      struct qedr_qp *qp,
3053                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3054                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3055                                      const struct ib_send_wr *wr,
3056                                      const struct ib_send_wr **bad_wr)
3057 {
3058         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3059         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3060
3061         if (wr->send_flags & IB_SEND_INLINE &&
3062             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3063              wr->opcode == IB_WR_RDMA_WRITE)) {
3064                 u8 flags = 0;
3065
3066                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3067                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3068                                                    bad_wr, &rwqe->flags, flags);
3069         }
3070
3071         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3072 }
3073
3074 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3075                                      struct qedr_qp *qp,
3076                                      struct rdma_sq_send_wqe_1st *swqe,
3077                                      struct rdma_sq_send_wqe_2st *swqe2,
3078                                      const struct ib_send_wr *wr,
3079                                      const struct ib_send_wr **bad_wr)
3080 {
3081         memset(swqe2, 0, sizeof(*swqe2));
3082         if (wr->send_flags & IB_SEND_INLINE) {
3083                 u8 flags = 0;
3084
3085                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3086                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3087                                                    bad_wr, &swqe->flags, flags);
3088         }
3089
3090         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3091 }
3092
3093 static int qedr_prepare_reg(struct qedr_qp *qp,
3094                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3095                             const struct ib_reg_wr *wr)
3096 {
3097         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3098         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3099
3100         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3101         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3102         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3103         fwqe1->l_key = wr->key;
3104
3105         fwqe2->access_ctrl = 0;
3106
3107         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3108                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3109         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3110                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3111         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3112                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3113         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3114         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3115                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3116         fwqe2->fmr_ctrl = 0;
3117
3118         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3119                    ilog2(mr->ibmr.page_size) - 12);
3120
3121         fwqe2->length_hi = 0;
3122         fwqe2->length_lo = mr->ibmr.length;
3123         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3124         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3125
3126         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3127
3128         return 0;
3129 }
3130
3131 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3132 {
3133         switch (opcode) {
3134         case IB_WR_RDMA_WRITE:
3135         case IB_WR_RDMA_WRITE_WITH_IMM:
3136                 return IB_WC_RDMA_WRITE;
3137         case IB_WR_SEND_WITH_IMM:
3138         case IB_WR_SEND:
3139         case IB_WR_SEND_WITH_INV:
3140                 return IB_WC_SEND;
3141         case IB_WR_RDMA_READ:
3142         case IB_WR_RDMA_READ_WITH_INV:
3143                 return IB_WC_RDMA_READ;
3144         case IB_WR_ATOMIC_CMP_AND_SWP:
3145                 return IB_WC_COMP_SWAP;
3146         case IB_WR_ATOMIC_FETCH_AND_ADD:
3147                 return IB_WC_FETCH_ADD;
3148         case IB_WR_REG_MR:
3149                 return IB_WC_REG_MR;
3150         case IB_WR_LOCAL_INV:
3151                 return IB_WC_LOCAL_INV;
3152         default:
3153                 return IB_WC_SEND;
3154         }
3155 }
3156
3157 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3158                                       const struct ib_send_wr *wr)
3159 {
3160         int wq_is_full, err_wr, pbl_is_full;
3161         struct qedr_dev *dev = qp->dev;
3162
3163         /* prevent SQ overflow and/or processing of a bad WR */
3164         err_wr = wr->num_sge > qp->sq.max_sges;
3165         wq_is_full = qedr_wq_is_full(&qp->sq);
3166         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3167                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3168         if (wq_is_full || err_wr || pbl_is_full) {
3169                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3170                         DP_ERR(dev,
3171                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3172                                qp);
3173                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3174                 }
3175
3176                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3177                         DP_ERR(dev,
3178                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3179                                qp);
3180                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3181                 }
3182
3183                 if (pbl_is_full &&
3184                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3185                         DP_ERR(dev,
3186                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3187                                qp);
3188                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3189                 }
3190                 return false;
3191         }
3192         return true;
3193 }
3194
3195 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3196                             const struct ib_send_wr **bad_wr)
3197 {
3198         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3199         struct qedr_qp *qp = get_qedr_qp(ibqp);
3200         struct rdma_sq_atomic_wqe_1st *awqe1;
3201         struct rdma_sq_atomic_wqe_2nd *awqe2;
3202         struct rdma_sq_atomic_wqe_3rd *awqe3;
3203         struct rdma_sq_send_wqe_2st *swqe2;
3204         struct rdma_sq_local_inv_wqe *iwqe;
3205         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3206         struct rdma_sq_send_wqe_1st *swqe;
3207         struct rdma_sq_rdma_wqe_1st *rwqe;
3208         struct rdma_sq_fmr_wqe_1st *fwqe1;
3209         struct rdma_sq_common_wqe *wqe;
3210         u32 length;
3211         int rc = 0;
3212         bool comp;
3213
3214         if (!qedr_can_post_send(qp, wr)) {
3215                 *bad_wr = wr;
3216                 return -ENOMEM;
3217         }
3218
3219         wqe = qed_chain_produce(&qp->sq.pbl);
3220         qp->wqe_wr_id[qp->sq.prod].signaled =
3221                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3222
3223         wqe->flags = 0;
3224         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3225                    !!(wr->send_flags & IB_SEND_SOLICITED));
3226         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3227         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3228         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3229                    !!(wr->send_flags & IB_SEND_FENCE));
3230         wqe->prev_wqe_size = qp->prev_wqe_size;
3231
3232         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3233
3234         switch (wr->opcode) {
3235         case IB_WR_SEND_WITH_IMM:
3236                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3237                         rc = -EINVAL;
3238                         *bad_wr = wr;
3239                         break;
3240                 }
3241                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3242                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3243                 swqe->wqe_size = 2;
3244                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3245
3246                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3247                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3248                                                    wr, bad_wr);
3249                 swqe->length = cpu_to_le32(length);
3250                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3251                 qp->prev_wqe_size = swqe->wqe_size;
3252                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3253                 break;
3254         case IB_WR_SEND:
3255                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3256                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3257
3258                 swqe->wqe_size = 2;
3259                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3260                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3261                                                    wr, bad_wr);
3262                 swqe->length = cpu_to_le32(length);
3263                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3264                 qp->prev_wqe_size = swqe->wqe_size;
3265                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3266                 break;
3267         case IB_WR_SEND_WITH_INV:
3268                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3269                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3270                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3271                 swqe->wqe_size = 2;
3272                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3273                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3274                                                    wr, bad_wr);
3275                 swqe->length = cpu_to_le32(length);
3276                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3277                 qp->prev_wqe_size = swqe->wqe_size;
3278                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3279                 break;
3280
3281         case IB_WR_RDMA_WRITE_WITH_IMM:
3282                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3283                         rc = -EINVAL;
3284                         *bad_wr = wr;
3285                         break;
3286                 }
3287                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3288                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3289
3290                 rwqe->wqe_size = 2;
3291                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3292                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3293                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3294                                                    wr, bad_wr);
3295                 rwqe->length = cpu_to_le32(length);
3296                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3297                 qp->prev_wqe_size = rwqe->wqe_size;
3298                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3299                 break;
3300         case IB_WR_RDMA_WRITE:
3301                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3302                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3303
3304                 rwqe->wqe_size = 2;
3305                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3306                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3307                                                    wr, bad_wr);
3308                 rwqe->length = cpu_to_le32(length);
3309                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3310                 qp->prev_wqe_size = rwqe->wqe_size;
3311                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3312                 break;
3313         case IB_WR_RDMA_READ_WITH_INV:
3314                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3315                 /* fallthrough -- same is identical to RDMA READ */
3316
3317         case IB_WR_RDMA_READ:
3318                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3319                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3320
3321                 rwqe->wqe_size = 2;
3322                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3323                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3324                                                    wr, bad_wr);
3325                 rwqe->length = cpu_to_le32(length);
3326                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3327                 qp->prev_wqe_size = rwqe->wqe_size;
3328                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3329                 break;
3330
3331         case IB_WR_ATOMIC_CMP_AND_SWP:
3332         case IB_WR_ATOMIC_FETCH_AND_ADD:
3333                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3334                 awqe1->wqe_size = 4;
3335
3336                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3337                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3338                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3339
3340                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3341
3342                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3343                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3344                         DMA_REGPAIR_LE(awqe3->swap_data,
3345                                        atomic_wr(wr)->compare_add);
3346                 } else {
3347                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3348                         DMA_REGPAIR_LE(awqe3->swap_data,
3349                                        atomic_wr(wr)->swap);
3350                         DMA_REGPAIR_LE(awqe3->cmp_data,
3351                                        atomic_wr(wr)->compare_add);
3352                 }
3353
3354                 qedr_prepare_sq_sges(qp, NULL, wr);
3355
3356                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3357                 qp->prev_wqe_size = awqe1->wqe_size;
3358                 break;
3359
3360         case IB_WR_LOCAL_INV:
3361                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3362                 iwqe->wqe_size = 1;
3363
3364                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3365                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3366                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3367                 qp->prev_wqe_size = iwqe->wqe_size;
3368                 break;
3369         case IB_WR_REG_MR:
3370                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3371                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3372                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3373                 fwqe1->wqe_size = 2;
3374
3375                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3376                 if (rc) {
3377                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3378                         *bad_wr = wr;
3379                         break;
3380                 }
3381
3382                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3383                 qp->prev_wqe_size = fwqe1->wqe_size;
3384                 break;
3385         default:
3386                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3387                 rc = -EINVAL;
3388                 *bad_wr = wr;
3389                 break;
3390         }
3391
3392         if (*bad_wr) {
3393                 u16 value;
3394
3395                 /* Restore prod to its position before
3396                  * this WR was processed
3397                  */
3398                 value = le16_to_cpu(qp->sq.db_data.data.value);
3399                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3400
3401                 /* Restore prev_wqe_size */
3402                 qp->prev_wqe_size = wqe->prev_wqe_size;
3403                 rc = -EINVAL;
3404                 DP_ERR(dev, "POST SEND FAILED\n");
3405         }
3406
3407         return rc;
3408 }
3409
3410 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3411                    const struct ib_send_wr **bad_wr)
3412 {
3413         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3414         struct qedr_qp *qp = get_qedr_qp(ibqp);
3415         unsigned long flags;
3416         int rc = 0;
3417
3418         *bad_wr = NULL;
3419
3420         if (qp->qp_type == IB_QPT_GSI)
3421                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3422
3423         spin_lock_irqsave(&qp->q_lock, flags);
3424
3425         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3426                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3427                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3428                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3429                         spin_unlock_irqrestore(&qp->q_lock, flags);
3430                         *bad_wr = wr;
3431                         DP_DEBUG(dev, QEDR_MSG_CQ,
3432                                  "QP in wrong state! QP icid=0x%x state %d\n",
3433                                  qp->icid, qp->state);
3434                         return -EINVAL;
3435                 }
3436         }
3437
3438         while (wr) {
3439                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3440                 if (rc)
3441                         break;
3442
3443                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3444
3445                 qedr_inc_sw_prod(&qp->sq);
3446
3447                 qp->sq.db_data.data.value++;
3448
3449                 wr = wr->next;
3450         }
3451
3452         /* Trigger doorbell
3453          * If there was a failure in the first WR then it will be triggered in
3454          * vane. However this is not harmful (as long as the producer value is
3455          * unchanged). For performance reasons we avoid checking for this
3456          * redundant doorbell.
3457          *
3458          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3459          * soon as we give the doorbell, we could get a completion
3460          * for this wr, therefore we need to make sure that the
3461          * memory is updated before giving the doorbell.
3462          * During qedr_poll_cq, rmb is called before accessing the
3463          * cqe. This covers for the smp_rmb as well.
3464          */
3465         smp_wmb();
3466         writel(qp->sq.db_data.raw, qp->sq.db);
3467
3468         /* Make sure write sticks */
3469         mmiowb();
3470
3471         spin_unlock_irqrestore(&qp->q_lock, flags);
3472
3473         return rc;
3474 }
3475
3476 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3477 {
3478         u32 used;
3479
3480         /* Calculate number of elements used based on producer
3481          * count and consumer count and subtract it from max
3482          * work request supported so that we get elements left.
3483          */
3484         used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3485
3486         return hw_srq->max_wr - used;
3487 }
3488
3489 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3490                        const struct ib_recv_wr **bad_wr)
3491 {
3492         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3493         struct qedr_srq_hwq_info *hw_srq;
3494         struct qedr_dev *dev = srq->dev;
3495         struct qed_chain *pbl;
3496         unsigned long flags;
3497         int status = 0;
3498         u32 num_sge;
3499         u32 offset;
3500
3501         spin_lock_irqsave(&srq->lock, flags);
3502
3503         hw_srq = &srq->hw_srq;
3504         pbl = &srq->hw_srq.pbl;
3505         while (wr) {
3506                 struct rdma_srq_wqe_header *hdr;
3507                 int i;
3508
3509                 if (!qedr_srq_elem_left(hw_srq) ||
3510                     wr->num_sge > srq->hw_srq.max_sges) {
3511                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3512                                hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3513                                wr->num_sge, srq->hw_srq.max_sges);
3514                         status = -ENOMEM;
3515                         *bad_wr = wr;
3516                         break;
3517                 }
3518
3519                 hdr = qed_chain_produce(pbl);
3520                 num_sge = wr->num_sge;
3521                 /* Set number of sge and work request id in header */
3522                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3523
3524                 srq->hw_srq.wr_prod_cnt++;
3525                 hw_srq->wqe_prod++;
3526                 hw_srq->sge_prod++;
3527
3528                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3529                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3530                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3531
3532                 for (i = 0; i < wr->num_sge; i++) {
3533                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3534
3535                         /* Set SGE length, lkey and address */
3536                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3537                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3538
3539                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3540                                  "[%d]: len %d key %x addr %x:%x\n",
3541                                  i, srq_sge->length, srq_sge->l_key,
3542                                  srq_sge->addr.hi, srq_sge->addr.lo);
3543                         hw_srq->sge_prod++;
3544                 }
3545
3546                 /* Flush WQE and SGE information before
3547                  * updating producer.
3548                  */
3549                 wmb();
3550
3551                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3552                  * in first 4 bytes and need to update WQE producer in
3553                  * next 4 bytes.
3554                  */
3555                 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3556                 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3557                 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3558                         hw_srq->wqe_prod;
3559
3560                 /* Flush producer after updating it. */
3561                 wmb();
3562                 wr = wr->next;
3563         }
3564
3565         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3566                  qed_chain_get_elem_left(pbl));
3567         spin_unlock_irqrestore(&srq->lock, flags);
3568
3569         return status;
3570 }
3571
3572 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3573                    const struct ib_recv_wr **bad_wr)
3574 {
3575         struct qedr_qp *qp = get_qedr_qp(ibqp);
3576         struct qedr_dev *dev = qp->dev;
3577         unsigned long flags;
3578         int status = 0;
3579
3580         if (qp->qp_type == IB_QPT_GSI)
3581                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3582
3583         spin_lock_irqsave(&qp->q_lock, flags);
3584
3585         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3586                 spin_unlock_irqrestore(&qp->q_lock, flags);
3587                 *bad_wr = wr;
3588                 return -EINVAL;
3589         }
3590
3591         while (wr) {
3592                 int i;
3593
3594                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3595                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3596                     wr->num_sge > qp->rq.max_sges) {
3597                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3598                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3599                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3600                                qp->rq.max_sges);
3601                         status = -ENOMEM;
3602                         *bad_wr = wr;
3603                         break;
3604                 }
3605                 for (i = 0; i < wr->num_sge; i++) {
3606                         u32 flags = 0;
3607                         struct rdma_rq_sge *rqe =
3608                             qed_chain_produce(&qp->rq.pbl);
3609
3610                         /* First one must include the number
3611                          * of SGE in the list
3612                          */
3613                         if (!i)
3614                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3615                                           wr->num_sge);
3616
3617                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3618                                   wr->sg_list[i].lkey);
3619
3620                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3621                                    wr->sg_list[i].length, flags);
3622                 }
3623
3624                 /* Special case of no sges. FW requires between 1-4 sges...
3625                  * in this case we need to post 1 sge with length zero. this is
3626                  * because rdma write with immediate consumes an RQ.
3627                  */
3628                 if (!wr->num_sge) {
3629                         u32 flags = 0;
3630                         struct rdma_rq_sge *rqe =
3631                             qed_chain_produce(&qp->rq.pbl);
3632
3633                         /* First one must include the number
3634                          * of SGE in the list
3635                          */
3636                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3637                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3638
3639                         RQ_SGE_SET(rqe, 0, 0, flags);
3640                         i = 1;
3641                 }
3642
3643                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3644                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3645
3646                 qedr_inc_sw_prod(&qp->rq);
3647
3648                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3649                  * soon as we give the doorbell, we could get a completion
3650                  * for this wr, therefore we need to make sure that the
3651                  * memory is update before giving the doorbell.
3652                  * During qedr_poll_cq, rmb is called before accessing the
3653                  * cqe. This covers for the smp_rmb as well.
3654                  */
3655                 smp_wmb();
3656
3657                 qp->rq.db_data.data.value++;
3658
3659                 writel(qp->rq.db_data.raw, qp->rq.db);
3660
3661                 /* Make sure write sticks */
3662                 mmiowb();
3663
3664                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3665                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3666                         mmiowb();       /* for second doorbell */
3667                 }
3668
3669                 wr = wr->next;
3670         }
3671
3672         spin_unlock_irqrestore(&qp->q_lock, flags);
3673
3674         return status;
3675 }
3676
3677 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3678 {
3679         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3680
3681         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3682                 cq->pbl_toggle;
3683 }
3684
3685 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3686 {
3687         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3688         struct qedr_qp *qp;
3689
3690         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3691                                                    resp_cqe->qp_handle.lo,
3692                                                    u64);
3693         return qp;
3694 }
3695
3696 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3697 {
3698         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3699
3700         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3701 }
3702
3703 /* Return latest CQE (needs processing) */
3704 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3705 {
3706         return cq->latest_cqe;
3707 }
3708
3709 /* In fmr we need to increase the number of fmr completed counter for the fmr
3710  * algorithm determining whether we can free a pbl or not.
3711  * we need to perform this whether the work request was signaled or not. for
3712  * this purpose we call this function from the condition that checks if a wr
3713  * should be skipped, to make sure we don't miss it ( possibly this fmr
3714  * operation was not signalted)
3715  */
3716 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3717 {
3718         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3719                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3720 }
3721
3722 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3723                        struct qedr_cq *cq, int num_entries,
3724                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3725                        int force)
3726 {
3727         u16 cnt = 0;
3728
3729         while (num_entries && qp->sq.wqe_cons != hw_cons) {
3730                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3731                         qedr_chk_if_fmr(qp);
3732                         /* skip WC */
3733                         goto next_cqe;
3734                 }
3735
3736                 /* fill WC */
3737                 wc->status = status;
3738                 wc->vendor_err = 0;
3739                 wc->wc_flags = 0;
3740                 wc->src_qp = qp->id;
3741                 wc->qp = &qp->ibqp;
3742
3743                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3744                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3745
3746                 switch (wc->opcode) {
3747                 case IB_WC_RDMA_WRITE:
3748                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3749                         break;
3750                 case IB_WC_COMP_SWAP:
3751                 case IB_WC_FETCH_ADD:
3752                         wc->byte_len = 8;
3753                         break;
3754                 case IB_WC_REG_MR:
3755                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3756                         break;
3757                 case IB_WC_RDMA_READ:
3758                 case IB_WC_SEND:
3759                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3760                         break;
3761                 default:
3762                         break;
3763                 }
3764
3765                 num_entries--;
3766                 wc++;
3767                 cnt++;
3768 next_cqe:
3769                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3770                         qed_chain_consume(&qp->sq.pbl);
3771                 qedr_inc_sw_cons(&qp->sq);
3772         }
3773
3774         return cnt;
3775 }
3776
3777 static int qedr_poll_cq_req(struct qedr_dev *dev,
3778                             struct qedr_qp *qp, struct qedr_cq *cq,
3779                             int num_entries, struct ib_wc *wc,
3780                             struct rdma_cqe_requester *req)
3781 {
3782         int cnt = 0;
3783
3784         switch (req->status) {
3785         case RDMA_CQE_REQ_STS_OK:
3786                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3787                                   IB_WC_SUCCESS, 0);
3788                 break;
3789         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3790                 if (qp->state != QED_ROCE_QP_STATE_ERR)
3791                         DP_DEBUG(dev, QEDR_MSG_CQ,
3792                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3793                                  cq->icid, qp->icid);
3794                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3795                                   IB_WC_WR_FLUSH_ERR, 1);
3796                 break;
3797         default:
3798                 /* process all WQE before the cosumer */
3799                 qp->state = QED_ROCE_QP_STATE_ERR;
3800                 cnt = process_req(dev, qp, cq, num_entries, wc,
3801                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
3802                 wc += cnt;
3803                 /* if we have extra WC fill it with actual error info */
3804                 if (cnt < num_entries) {
3805                         enum ib_wc_status wc_status;
3806
3807                         switch (req->status) {
3808                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3809                                 DP_ERR(dev,
3810                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3811                                        cq->icid, qp->icid);
3812                                 wc_status = IB_WC_BAD_RESP_ERR;
3813                                 break;
3814                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3815                                 DP_ERR(dev,
3816                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3817                                        cq->icid, qp->icid);
3818                                 wc_status = IB_WC_LOC_LEN_ERR;
3819                                 break;
3820                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3821                                 DP_ERR(dev,
3822                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3823                                        cq->icid, qp->icid);
3824                                 wc_status = IB_WC_LOC_QP_OP_ERR;
3825                                 break;
3826                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3827                                 DP_ERR(dev,
3828                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3829                                        cq->icid, qp->icid);
3830                                 wc_status = IB_WC_LOC_PROT_ERR;
3831                                 break;
3832                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3833                                 DP_ERR(dev,
3834                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3835                                        cq->icid, qp->icid);
3836                                 wc_status = IB_WC_MW_BIND_ERR;
3837                                 break;
3838                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3839                                 DP_ERR(dev,
3840                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3841                                        cq->icid, qp->icid);
3842                                 wc_status = IB_WC_REM_INV_REQ_ERR;
3843                                 break;
3844                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3845                                 DP_ERR(dev,
3846                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3847                                        cq->icid, qp->icid);
3848                                 wc_status = IB_WC_REM_ACCESS_ERR;
3849                                 break;
3850                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3851                                 DP_ERR(dev,
3852                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3853                                        cq->icid, qp->icid);
3854                                 wc_status = IB_WC_REM_OP_ERR;
3855                                 break;
3856                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3857                                 DP_ERR(dev,
3858                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3859                                        cq->icid, qp->icid);
3860                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3861                                 break;
3862                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3863                                 DP_ERR(dev,
3864                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3865                                        cq->icid, qp->icid);
3866                                 wc_status = IB_WC_RETRY_EXC_ERR;
3867                                 break;
3868                         default:
3869                                 DP_ERR(dev,
3870                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3871                                        cq->icid, qp->icid);
3872                                 wc_status = IB_WC_GENERAL_ERR;
3873                         }
3874                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3875                                            wc_status, 1);
3876                 }
3877         }
3878
3879         return cnt;
3880 }
3881
3882 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3883 {
3884         switch (status) {
3885         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3886                 return IB_WC_LOC_ACCESS_ERR;
3887         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3888                 return IB_WC_LOC_LEN_ERR;
3889         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3890                 return IB_WC_LOC_QP_OP_ERR;
3891         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3892                 return IB_WC_LOC_PROT_ERR;
3893         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3894                 return IB_WC_MW_BIND_ERR;
3895         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3896                 return IB_WC_REM_INV_RD_REQ_ERR;
3897         case RDMA_CQE_RESP_STS_OK:
3898                 return IB_WC_SUCCESS;
3899         default:
3900                 return IB_WC_GENERAL_ERR;
3901         }
3902 }
3903
3904 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3905                                           struct ib_wc *wc)
3906 {
3907         wc->status = IB_WC_SUCCESS;
3908         wc->byte_len = le32_to_cpu(resp->length);
3909
3910         if (resp->flags & QEDR_RESP_IMM) {
3911                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
3912                 wc->wc_flags |= IB_WC_WITH_IMM;
3913
3914                 if (resp->flags & QEDR_RESP_RDMA)
3915                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3916
3917                 if (resp->flags & QEDR_RESP_INV)
3918                         return -EINVAL;
3919
3920         } else if (resp->flags & QEDR_RESP_INV) {
3921                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3922                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3923
3924                 if (resp->flags & QEDR_RESP_RDMA)
3925                         return -EINVAL;
3926
3927         } else if (resp->flags & QEDR_RESP_RDMA) {
3928                 return -EINVAL;
3929         }
3930
3931         return 0;
3932 }
3933
3934 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3935                                struct qedr_cq *cq, struct ib_wc *wc,
3936                                struct rdma_cqe_responder *resp, u64 wr_id)
3937 {
3938         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3939         wc->opcode = IB_WC_RECV;
3940         wc->wc_flags = 0;
3941
3942         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3943                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3944                         DP_ERR(dev,
3945                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3946                                cq, cq->icid, resp->flags);
3947
3948         } else {
3949                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3950                 if (wc->status == IB_WC_GENERAL_ERR)
3951                         DP_ERR(dev,
3952                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3953                                cq, cq->icid, resp->status);
3954         }
3955
3956         /* Fill the rest of the WC */
3957         wc->vendor_err = 0;
3958         wc->src_qp = qp->id;
3959         wc->qp = &qp->ibqp;
3960         wc->wr_id = wr_id;
3961 }
3962
3963 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3964                                 struct qedr_cq *cq, struct ib_wc *wc,
3965                                 struct rdma_cqe_responder *resp)
3966 {
3967         struct qedr_srq *srq = qp->srq;
3968         u64 wr_id;
3969
3970         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
3971                          le32_to_cpu(resp->srq_wr_id.lo), u64);
3972
3973         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3974                 wc->status = IB_WC_WR_FLUSH_ERR;
3975                 wc->vendor_err = 0;
3976                 wc->wr_id = wr_id;
3977                 wc->byte_len = 0;
3978                 wc->src_qp = qp->id;
3979                 wc->qp = &qp->ibqp;
3980                 wc->wr_id = wr_id;
3981         } else {
3982                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3983         }
3984         srq->hw_srq.wr_cons_cnt++;
3985
3986         return 1;
3987 }
3988 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3989                             struct qedr_cq *cq, struct ib_wc *wc,
3990                             struct rdma_cqe_responder *resp)
3991 {
3992         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3993
3994         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3995
3996         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3997                 qed_chain_consume(&qp->rq.pbl);
3998         qedr_inc_sw_cons(&qp->rq);
3999
4000         return 1;
4001 }
4002
4003 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4004                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4005 {
4006         u16 cnt = 0;
4007
4008         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4009                 /* fill WC */
4010                 wc->status = IB_WC_WR_FLUSH_ERR;
4011                 wc->vendor_err = 0;
4012                 wc->wc_flags = 0;
4013                 wc->src_qp = qp->id;
4014                 wc->byte_len = 0;
4015                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4016                 wc->qp = &qp->ibqp;
4017                 num_entries--;
4018                 wc++;
4019                 cnt++;
4020                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4021                         qed_chain_consume(&qp->rq.pbl);
4022                 qedr_inc_sw_cons(&qp->rq);
4023         }
4024
4025         return cnt;
4026 }
4027
4028 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4029                                  struct rdma_cqe_responder *resp, int *update)
4030 {
4031         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4032                 consume_cqe(cq);
4033                 *update |= 1;
4034         }
4035 }
4036
4037 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4038                                  struct qedr_cq *cq, int num_entries,
4039                                  struct ib_wc *wc,
4040                                  struct rdma_cqe_responder *resp)
4041 {
4042         int cnt;
4043
4044         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4045         consume_cqe(cq);
4046
4047         return cnt;
4048 }
4049
4050 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4051                              struct qedr_cq *cq, int num_entries,
4052                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4053                              int *update)
4054 {
4055         int cnt;
4056
4057         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4058                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4059                                          resp->rq_cons_or_srq_id);
4060                 try_consume_resp_cqe(cq, qp, resp, update);
4061         } else {
4062                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4063                 consume_cqe(cq);
4064                 *update |= 1;
4065         }
4066
4067         return cnt;
4068 }
4069
4070 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4071                                 struct rdma_cqe_requester *req, int *update)
4072 {
4073         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4074                 consume_cqe(cq);
4075                 *update |= 1;
4076         }
4077 }
4078
4079 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4080 {
4081         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4082         struct qedr_cq *cq = get_qedr_cq(ibcq);
4083         union rdma_cqe *cqe;
4084         u32 old_cons, new_cons;
4085         unsigned long flags;
4086         int update = 0;
4087         int done = 0;
4088
4089         if (cq->destroyed) {
4090                 DP_ERR(dev,
4091                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4092                        cq, cq->icid);
4093                 return 0;
4094         }
4095
4096         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4097                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4098
4099         spin_lock_irqsave(&cq->cq_lock, flags);
4100         cqe = cq->latest_cqe;
4101         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4102         while (num_entries && is_valid_cqe(cq, cqe)) {
4103                 struct qedr_qp *qp;
4104                 int cnt = 0;
4105
4106                 /* prevent speculative reads of any field of CQE */
4107                 rmb();
4108
4109                 qp = cqe_get_qp(cqe);
4110                 if (!qp) {
4111                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4112                         break;
4113                 }
4114
4115                 wc->qp = &qp->ibqp;
4116
4117                 switch (cqe_get_type(cqe)) {
4118                 case RDMA_CQE_TYPE_REQUESTER:
4119                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4120                                                &cqe->req);
4121                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4122                         break;
4123                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4124                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4125                                                 &cqe->resp, &update);
4126                         break;
4127                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4128                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4129                                                     wc, &cqe->resp);
4130                         update = 1;
4131                         break;
4132                 case RDMA_CQE_TYPE_INVALID:
4133                 default:
4134                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4135                                cqe_get_type(cqe));
4136                 }
4137                 num_entries -= cnt;
4138                 wc += cnt;
4139                 done += cnt;
4140
4141                 cqe = get_cqe(cq);
4142         }
4143         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4144
4145         cq->cq_cons += new_cons - old_cons;
4146
4147         if (update)
4148                 /* doorbell notifies abount latest VALID entry,
4149                  * but chain already point to the next INVALID one
4150                  */
4151                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4152
4153         spin_unlock_irqrestore(&cq->cq_lock, flags);
4154         return done;
4155 }
4156
4157 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4158                      u8 port_num,
4159                      const struct ib_wc *in_wc,
4160                      const struct ib_grh *in_grh,
4161                      const struct ib_mad_hdr *mad_hdr,
4162                      size_t in_mad_size, struct ib_mad_hdr *out_mad,
4163                      size_t *out_mad_size, u16 *out_mad_pkey_index)
4164 {
4165         struct qedr_dev *dev = get_qedr_dev(ibdev);
4166
4167         DP_DEBUG(dev, QEDR_MSG_GSI,
4168                  "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4169                  mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4170                  mad_hdr->class_specific, mad_hdr->class_version,
4171                  mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4172         return IB_MAD_RESULT_SUCCESS;
4173 }