drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/uverbs_ioctl.h>
  46
  47 #include <linux/qed/common_hsi.h>
  48 #include "qedr_hsi_rdma.h"
  49 #include <linux/qed/qed_if.h>
  50 #include "qedr.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_roce_cm.h"
  54 #include "qedr_iw_cm.h"
  55
  56 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  57 #define RDMA_MAX_SGE_PER_SRQ    (4)
  58 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62 enum {
  63         QEDR_USER_MMAP_IO_WC = 0,
  64         QEDR_USER_MMAP_PHYS_PAGE,
  65 };
  66
  67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68                                         size_t len)
  69 {
  70         size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72         return ib_copy_to_udata(udata, src, min_len);
  73 }
  74
  75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
  76 {
  77         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78                 return -EINVAL;
  79
  80         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  81         return 0;
  82 }
  83
  84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
  85                       int index, union ib_gid *sgid)
  86 {
  87         struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89         memset(sgid->raw, 0, sizeof(sgid->raw));
  90         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93                  sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95         return 0;
  96 }
  97
  98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99 {
 100         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101         struct qedr_device_attr *qattr = &dev->attr;
 102         struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104         srq_attr->srq_limit = srq->srq_limit;
 105         srq_attr->max_wr = qattr->max_srq_wr;
 106         srq_attr->max_sge = qattr->max_sge;
 107
 108         return 0;
 109 }
 110
 111 int qedr_query_device(struct ib_device *ibdev,
 112                       struct ib_device_attr *attr, struct ib_udata *udata)
 113 {
 114         struct qedr_dev *dev = get_qedr_dev(ibdev);
 115         struct qedr_device_attr *qattr = &dev->attr;
 116
 117         if (!dev->rdma_ctx) {
 118                 DP_ERR(dev,
 119                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120                        dev->rdma_ctx);
 121                 return -EINVAL;
 122         }
 123
 124         memset(attr, 0, sizeof(*attr));
 125
 126         attr->fw_ver = qattr->fw_ver;
 127         attr->sys_image_guid = qattr->sys_image_guid;
 128         attr->max_mr_size = qattr->max_mr_size;
 129         attr->page_size_cap = qattr->page_size_caps;
 130         attr->vendor_id = qattr->vendor_id;
 131         attr->vendor_part_id = qattr->vendor_part_id;
 132         attr->hw_ver = qattr->hw_ver;
 133         attr->max_qp = qattr->max_qp;
 134         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136             IB_DEVICE_RC_RNR_NAK_GEN |
 137             IB_DEVICE_MEM_MGT_EXTENSIONS;
 138         attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
 139
 140         if (!rdma_protocol_iwarp(&dev->ibdev, 1))
 141                 attr->device_cap_flags |= IB_DEVICE_XRC;
 142         attr->max_send_sge = qattr->max_sge;
 143         attr->max_recv_sge = qattr->max_sge;
 144         attr->max_sge_rd = qattr->max_sge;
 145         attr->max_cq = qattr->max_cq;
 146         attr->max_cqe = qattr->max_cqe;
 147         attr->max_mr = qattr->max_mr;
 148         attr->max_mw = qattr->max_mw;
 149         attr->max_pd = qattr->max_pd;
 150         attr->atomic_cap = dev->atomic_cap;
 151         attr->max_qp_init_rd_atom =
 152             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 153         attr->max_qp_rd_atom =
 154             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 155                 attr->max_qp_init_rd_atom);
 156
 157         attr->max_srq = qattr->max_srq;
 158         attr->max_srq_sge = qattr->max_srq_sge;
 159         attr->max_srq_wr = qattr->max_srq_wr;
 160
 161         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 162         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 163         attr->max_pkeys = qattr->max_pkey;
 164         attr->max_ah = qattr->max_ah;
 165
 166         return 0;
 167 }
 168
 169 static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
 170                                             u8 *ib_width)
 171 {
 172         switch (speed) {
 173         case 1000:
 174                 *ib_speed = IB_SPEED_SDR;
 175                 *ib_width = IB_WIDTH_1X;
 176                 break;
 177         case 10000:
 178                 *ib_speed = IB_SPEED_QDR;
 179                 *ib_width = IB_WIDTH_1X;
 180                 break;
 181
 182         case 20000:
 183                 *ib_speed = IB_SPEED_DDR;
 184                 *ib_width = IB_WIDTH_4X;
 185                 break;
 186
 187         case 25000:
 188                 *ib_speed = IB_SPEED_EDR;
 189                 *ib_width = IB_WIDTH_1X;
 190                 break;
 191
 192         case 40000:
 193                 *ib_speed = IB_SPEED_QDR;
 194                 *ib_width = IB_WIDTH_4X;
 195                 break;
 196
 197         case 50000:
 198                 *ib_speed = IB_SPEED_HDR;
 199                 *ib_width = IB_WIDTH_1X;
 200                 break;
 201
 202         case 100000:
 203                 *ib_speed = IB_SPEED_EDR;
 204                 *ib_width = IB_WIDTH_4X;
 205                 break;
 206
 207         default:
 208                 /* Unsupported */
 209                 *ib_speed = IB_SPEED_SDR;
 210                 *ib_width = IB_WIDTH_1X;
 211         }
 212 }
 213
 214 int qedr_query_port(struct ib_device *ibdev, u32 port,
 215                     struct ib_port_attr *attr)
 216 {
 217         struct qedr_dev *dev;
 218         struct qed_rdma_port *rdma_port;
 219
 220         dev = get_qedr_dev(ibdev);
 221
 222         if (!dev->rdma_ctx) {
 223                 DP_ERR(dev, "rdma_ctx is NULL\n");
 224                 return -EINVAL;
 225         }
 226
 227         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 228
 229         /* *attr being zeroed by the caller, avoid zeroing it here */
 230         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 231                 attr->state = IB_PORT_ACTIVE;
 232                 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 233         } else {
 234                 attr->state = IB_PORT_DOWN;
 235                 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 236         }
 237         attr->max_mtu = IB_MTU_4096;
 238         attr->lid = 0;
 239         attr->lmc = 0;
 240         attr->sm_lid = 0;
 241         attr->sm_sl = 0;
 242         attr->ip_gids = true;
 243         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 244                 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
 245                 attr->gid_tbl_len = 1;
 246         } else {
 247                 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 248                 attr->gid_tbl_len = QEDR_MAX_SGID;
 249                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 250         }
 251         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 252         attr->qkey_viol_cntr = 0;
 253         get_link_speed_and_width(rdma_port->link_speed,
 254                                  &attr->active_speed, &attr->active_width);
 255         attr->max_msg_sz = rdma_port->max_msg_size;
 256         attr->max_vl_num = 4;
 257
 258         return 0;
 259 }
 260
 261 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 262 {
 263         struct ib_device *ibdev = uctx->device;
 264         int rc;
 265         struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 266         struct qedr_alloc_ucontext_resp uresp = {};
 267         struct qedr_alloc_ucontext_req ureq = {};
 268         struct qedr_dev *dev = get_qedr_dev(ibdev);
 269         struct qed_rdma_add_user_out_params oparams;
 270         struct qedr_user_mmap_entry *entry;
 271
 272         if (!udata)
 273                 return -EFAULT;
 274
 275         if (udata->inlen) {
 276                 rc = ib_copy_from_udata(&ureq, udata,
 277                                         min(sizeof(ureq), udata->inlen));
 278                 if (rc) {
 279                         DP_ERR(dev, "Problem copying data from user space\n");
 280                         return -EFAULT;
 281                 }
 282                 ctx->edpm_mode = !!(ureq.context_flags &
 283                                     QEDR_ALLOC_UCTX_EDPM_MODE);
 284                 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 285         }
 286
 287         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 288         if (rc) {
 289                 DP_ERR(dev,
 290                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 291                        rc);
 292                 return rc;
 293         }
 294
 295         ctx->dpi = oparams.dpi;
 296         ctx->dpi_addr = oparams.dpi_addr;
 297         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 298         ctx->dpi_size = oparams.dpi_size;
 299         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 300         if (!entry) {
 301                 rc = -ENOMEM;
 302                 goto err;
 303         }
 304
 305         entry->io_address = ctx->dpi_phys_addr;
 306         entry->length = ctx->dpi_size;
 307         entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 308         entry->dpi = ctx->dpi;
 309         entry->dev = dev;
 310         rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 311                                          ctx->dpi_size);
 312         if (rc) {
 313                 kfree(entry);
 314                 goto err;
 315         }
 316         ctx->db_mmap_entry = &entry->rdma_entry;
 317
 318         if (!dev->user_dpm_enabled)
 319                 uresp.dpm_flags = 0;
 320         else if (rdma_protocol_iwarp(&dev->ibdev, 1))
 321                 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
 322         else
 323                 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
 324                                   QEDR_DPM_TYPE_ROCE_LEGACY |
 325                                   QEDR_DPM_TYPE_ROCE_EDPM_MODE;
 326
 327         if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
 328                 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
 329                 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
 330                 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
 331                 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
 332         }
 333
 334         uresp.wids_enabled = 1;
 335         uresp.wid_count = oparams.wid_count;
 336         uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 337         uresp.db_size = ctx->dpi_size;
 338         uresp.max_send_wr = dev->attr.max_sqe;
 339         uresp.max_recv_wr = dev->attr.max_rqe;
 340         uresp.max_srq_wr = dev->attr.max_srq_wr;
 341         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 342         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 343         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 344         uresp.max_cqes = QEDR_MAX_CQES;
 345
 346         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 347         if (rc)
 348                 goto err;
 349
 350         ctx->dev = dev;
 351
 352         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 353                  &ctx->ibucontext);
 354         return 0;
 355
 356 err:
 357         if (!ctx->db_mmap_entry)
 358                 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 359         else
 360                 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 361
 362         return rc;
 363 }
 364
 365 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 366 {
 367         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 368
 369         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 370                  uctx);
 371
 372         rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 373 }
 374
 375 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 376 {
 377         struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 378         struct qedr_dev *dev = entry->dev;
 379
 380         if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 381                 free_page((unsigned long)entry->address);
 382         else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 383                 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 384
 385         kfree(entry);
 386 }
 387
 388 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 389 {
 390         struct ib_device *dev = ucontext->device;
 391         size_t length = vma->vm_end - vma->vm_start;
 392         struct rdma_user_mmap_entry *rdma_entry;
 393         struct qedr_user_mmap_entry *entry;
 394         int rc = 0;
 395         u64 pfn;
 396
 397         ibdev_dbg(dev,
 398                   "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 399                   vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 400
 401         rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 402         if (!rdma_entry) {
 403                 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 404                           vma->vm_pgoff);
 405                 return -EINVAL;
 406         }
 407         entry = get_qedr_mmap_entry(rdma_entry);
 408         ibdev_dbg(dev,
 409                   "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 410                   entry->io_address, length, entry->mmap_flag);
 411
 412         switch (entry->mmap_flag) {
 413         case QEDR_USER_MMAP_IO_WC:
 414                 pfn = entry->io_address >> PAGE_SHIFT;
 415                 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 416                                        pgprot_writecombine(vma->vm_page_prot),
 417                                        rdma_entry);
 418                 break;
 419         case QEDR_USER_MMAP_PHYS_PAGE:
 420                 rc = vm_insert_page(vma, vma->vm_start,
 421                                     virt_to_page(entry->address));
 422                 break;
 423         default:
 424                 rc = -EINVAL;
 425         }
 426
 427         if (rc)
 428                 ibdev_dbg(dev,
 429                           "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 430                           entry->io_address, length, entry->mmap_flag, rc);
 431
 432         rdma_user_mmap_entry_put(rdma_entry);
 433         return rc;
 434 }
 435
 436 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 437 {
 438         struct ib_device *ibdev = ibpd->device;
 439         struct qedr_dev *dev = get_qedr_dev(ibdev);
 440         struct qedr_pd *pd = get_qedr_pd(ibpd);
 441         u16 pd_id;
 442         int rc;
 443
 444         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 445                  udata ? "User Lib" : "Kernel");
 446
 447         if (!dev->rdma_ctx) {
 448                 DP_ERR(dev, "invalid RDMA context\n");
 449                 return -EINVAL;
 450         }
 451
 452         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 453         if (rc)
 454                 return rc;
 455
 456         pd->pd_id = pd_id;
 457
 458         if (udata) {
 459                 struct qedr_alloc_pd_uresp uresp = {
 460                         .pd_id = pd_id,
 461                 };
 462                 struct qedr_ucontext *context = rdma_udata_to_drv_context(
 463                         udata, struct qedr_ucontext, ibucontext);
 464
 465                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 466                 if (rc) {
 467                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 468                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 469                         return rc;
 470                 }
 471
 472                 pd->uctx = context;
 473                 pd->uctx->pd = pd;
 474         }
 475
 476         return 0;
 477 }
 478
 479 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 480 {
 481         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 482         struct qedr_pd *pd = get_qedr_pd(ibpd);
 483
 484         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 485         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 486         return 0;
 487 }
 488
 489
 490 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 491 {
 492         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 493         struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
 494
 495         return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
 496 }
 497
 498 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 499 {
 500         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 501         u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
 502
 503         dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
 504         return 0;
 505 }
 506 static void qedr_free_pbl(struct qedr_dev *dev,
 507                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 508 {
 509         struct pci_dev *pdev = dev->pdev;
 510         int i;
 511
 512         for (i = 0; i < pbl_info->num_pbls; i++) {
 513                 if (!pbl[i].va)
 514                         continue;
 515                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 516                                   pbl[i].va, pbl[i].pa);
 517         }
 518
 519         kfree(pbl);
 520 }
 521
 522 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 523 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 524
 525 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 526 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 527 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 528
 529 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 530                                            struct qedr_pbl_info *pbl_info,
 531                                            gfp_t flags)
 532 {
 533         struct pci_dev *pdev = dev->pdev;
 534         struct qedr_pbl *pbl_table;
 535         dma_addr_t *pbl_main_tbl;
 536         dma_addr_t pa;
 537         void *va;
 538         int i;
 539
 540         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 541         if (!pbl_table)
 542                 return ERR_PTR(-ENOMEM);
 543
 544         for (i = 0; i < pbl_info->num_pbls; i++) {
 545                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 546                                         flags);
 547                 if (!va)
 548                         goto err;
 549
 550                 pbl_table[i].va = va;
 551                 pbl_table[i].pa = pa;
 552         }
 553
 554         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 555          * the first one with physical pointers to all of the rest
 556          */
 557         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 558         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 559                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 560
 561         return pbl_table;
 562
 563 err:
 564         for (i--; i >= 0; i--)
 565                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 566                                   pbl_table[i].va, pbl_table[i].pa);
 567
 568         qedr_free_pbl(dev, pbl_info, pbl_table);
 569
 570         return ERR_PTR(-ENOMEM);
 571 }
 572
 573 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 574                                 struct qedr_pbl_info *pbl_info,
 575                                 u32 num_pbes, int two_layer_capable)
 576 {
 577         u32 pbl_capacity;
 578         u32 pbl_size;
 579         u32 num_pbls;
 580
 581         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 582                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 583                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 584                                num_pbes);
 585                         return -EINVAL;
 586                 }
 587
 588                 /* calculate required pbl page size */
 589                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 590                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 591                                NUM_PBES_ON_PAGE(pbl_size);
 592
 593                 while (pbl_capacity < num_pbes) {
 594                         pbl_size *= 2;
 595                         pbl_capacity = pbl_size / sizeof(u64);
 596                         pbl_capacity = pbl_capacity * pbl_capacity;
 597                 }
 598
 599                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 600                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 601                 pbl_info->two_layered = true;
 602         } else {
 603                 /* One layered PBL */
 604                 num_pbls = 1;
 605                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 606                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 607                 pbl_info->two_layered = false;
 608         }
 609
 610         pbl_info->num_pbls = num_pbls;
 611         pbl_info->pbl_size = pbl_size;
 612         pbl_info->num_pbes = num_pbes;
 613
 614         DP_DEBUG(dev, QEDR_MSG_MR,
 615                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 616                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 617
 618         return 0;
 619 }
 620
 621 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 622                                struct qedr_pbl *pbl,
 623                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 624 {
 625         int pbe_cnt, total_num_pbes = 0;
 626         struct qedr_pbl *pbl_tbl;
 627         struct ib_block_iter biter;
 628         struct regpair *pbe;
 629
 630         if (!pbl_info->num_pbes)
 631                 return;
 632
 633         /* If we have a two layered pbl, the first pbl points to the rest
 634          * of the pbls and the first entry lays on the second pbl in the table
 635          */
 636         if (pbl_info->two_layered)
 637                 pbl_tbl = &pbl[1];
 638         else
 639                 pbl_tbl = pbl;
 640
 641         pbe = (struct regpair *)pbl_tbl->va;
 642         if (!pbe) {
 643                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 644                 return;
 645         }
 646
 647         pbe_cnt = 0;
 648
 649         rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
 650                 u64 pg_addr = rdma_block_iter_dma_address(&biter);
 651
 652                 pbe->lo = cpu_to_le32(pg_addr);
 653                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 654
 655                 pbe_cnt++;
 656                 total_num_pbes++;
 657                 pbe++;
 658
 659                 if (total_num_pbes == pbl_info->num_pbes)
 660                         return;
 661
 662                 /* If the given pbl is full storing the pbes, move to next pbl.
 663                  */
 664                 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 665                         pbl_tbl++;
 666                         pbe = (struct regpair *)pbl_tbl->va;
 667                         pbe_cnt = 0;
 668                 }
 669         }
 670 }
 671
 672 static int qedr_db_recovery_add(struct qedr_dev *dev,
 673                                 void __iomem *db_addr,
 674                                 void *db_data,
 675                                 enum qed_db_rec_width db_width,
 676                                 enum qed_db_rec_space db_space)
 677 {
 678         if (!db_data) {
 679                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 680                 return 0;
 681         }
 682
 683         return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 684                                                  db_width, db_space);
 685 }
 686
 687 static void qedr_db_recovery_del(struct qedr_dev *dev,
 688                                  void __iomem *db_addr,
 689                                  void *db_data)
 690 {
 691         if (!db_data) {
 692                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 693                 return;
 694         }
 695
 696         /* Ignore return code as there is not much we can do about it. Error
 697          * log will be printed inside.
 698          */
 699         dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 700 }
 701
 702 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 703                               struct qedr_cq *cq, struct ib_udata *udata,
 704                               u32 db_offset)
 705 {
 706         struct qedr_create_cq_uresp uresp;
 707         int rc;
 708
 709         memset(&uresp, 0, sizeof(uresp));
 710
 711         uresp.db_offset = db_offset;
 712         uresp.icid = cq->icid;
 713         if (cq->q.db_mmap_entry)
 714                 uresp.db_rec_addr =
 715                         rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 716
 717         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 718         if (rc)
 719                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 720
 721         return rc;
 722 }
 723
 724 static void consume_cqe(struct qedr_cq *cq)
 725 {
 726         if (cq->latest_cqe == cq->toggle_cqe)
 727                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 728
 729         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 730 }
 731
 732 static inline int qedr_align_cq_entries(int entries)
 733 {
 734         u64 size, aligned_size;
 735
 736         /* We allocate an extra entry that we don't report to the FW. */
 737         size = (entries + 1) * QEDR_CQE_SIZE;
 738         aligned_size = ALIGN(size, PAGE_SIZE);
 739
 740         return aligned_size / QEDR_CQE_SIZE;
 741 }
 742
 743 static int qedr_init_user_db_rec(struct ib_udata *udata,
 744                                  struct qedr_dev *dev, struct qedr_userq *q,
 745                                  bool requires_db_rec)
 746 {
 747         struct qedr_ucontext *uctx =
 748                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 749                                           ibucontext);
 750         struct qedr_user_mmap_entry *entry;
 751         int rc;
 752
 753         /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 754         if (requires_db_rec == 0 || !uctx->db_rec)
 755                 return 0;
 756
 757         /* Allocate a page for doorbell recovery, add to mmap */
 758         q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 759         if (!q->db_rec_data) {
 760                 DP_ERR(dev, "get_zeroed_page failed\n");
 761                 return -ENOMEM;
 762         }
 763
 764         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 765         if (!entry)
 766                 goto err_free_db_data;
 767
 768         entry->address = q->db_rec_data;
 769         entry->length = PAGE_SIZE;
 770         entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 771         rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 772                                          &entry->rdma_entry,
 773                                          PAGE_SIZE);
 774         if (rc)
 775                 goto err_free_entry;
 776
 777         q->db_mmap_entry = &entry->rdma_entry;
 778
 779         return 0;
 780
 781 err_free_entry:
 782         kfree(entry);
 783
 784 err_free_db_data:
 785         free_page((unsigned long)q->db_rec_data);
 786         q->db_rec_data = NULL;
 787         return -ENOMEM;
 788 }
 789
 790 static inline int qedr_init_user_queue(struct ib_udata *udata,
 791                                        struct qedr_dev *dev,
 792                                        struct qedr_userq *q, u64 buf_addr,
 793                                        size_t buf_len, bool requires_db_rec,
 794                                        int access,
 795                                        int alloc_and_init)
 796 {
 797         u32 fw_pages;
 798         int rc;
 799
 800         q->buf_addr = buf_addr;
 801         q->buf_len = buf_len;
 802         q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 803         if (IS_ERR(q->umem)) {
 804                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 805                        PTR_ERR(q->umem));
 806                 return PTR_ERR(q->umem);
 807         }
 808
 809         fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
 810         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 811         if (rc)
 812                 goto err0;
 813
 814         if (alloc_and_init) {
 815                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 816                 if (IS_ERR(q->pbl_tbl)) {
 817                         rc = PTR_ERR(q->pbl_tbl);
 818                         goto err0;
 819                 }
 820                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 821                                    FW_PAGE_SHIFT);
 822         } else {
 823                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 824                 if (!q->pbl_tbl) {
 825                         rc = -ENOMEM;
 826                         goto err0;
 827                 }
 828         }
 829
 830         /* mmap the user address used to store doorbell data for recovery */
 831         return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 832
 833 err0:
 834         ib_umem_release(q->umem);
 835         q->umem = NULL;
 836
 837         return rc;
 838 }
 839
 840 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 841                                        struct qedr_ucontext *ctx,
 842                                        struct qedr_dev *dev, int vector,
 843                                        int chain_entries, int page_cnt,
 844                                        u64 pbl_ptr,
 845                                        struct qed_rdma_create_cq_in_params
 846                                        *params)
 847 {
 848         memset(params, 0, sizeof(*params));
 849         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 850         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 851         params->cnq_id = vector;
 852         params->cq_size = chain_entries - 1;
 853         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 854         params->pbl_num_pages = page_cnt;
 855         params->pbl_ptr = pbl_ptr;
 856         params->pbl_two_level = 0;
 857 }
 858
 859 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 860 {
 861         cq->db.data.agg_flags = flags;
 862         cq->db.data.value = cpu_to_le32(cons);
 863         writeq(cq->db.raw, cq->db_addr);
 864 }
 865
 866 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 867 {
 868         struct qedr_cq *cq = get_qedr_cq(ibcq);
 869         unsigned long sflags;
 870         struct qedr_dev *dev;
 871
 872         dev = get_qedr_dev(ibcq->device);
 873
 874         if (cq->destroyed) {
 875                 DP_ERR(dev,
 876                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 877                        cq, cq->icid);
 878                 return -EINVAL;
 879         }
 880
 881
 882         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 883                 return 0;
 884
 885         spin_lock_irqsave(&cq->cq_lock, sflags);
 886
 887         cq->arm_flags = 0;
 888
 889         if (flags & IB_CQ_SOLICITED)
 890                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 891
 892         if (flags & IB_CQ_NEXT_COMP)
 893                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 894
 895         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 896
 897         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 898
 899         return 0;
 900 }
 901
 902 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 903                    struct uverbs_attr_bundle *attrs)
 904 {
 905         struct ib_udata *udata = &attrs->driver_udata;
 906         struct ib_device *ibdev = ibcq->device;
 907         struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 908                 udata, struct qedr_ucontext, ibucontext);
 909         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 910         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 911         struct qed_chain_init_params chain_params = {
 912                 .mode           = QED_CHAIN_MODE_PBL,
 913                 .intended_use   = QED_CHAIN_USE_TO_CONSUME,
 914                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
 915                 .elem_size      = sizeof(union rdma_cqe),
 916         };
 917         struct qedr_dev *dev = get_qedr_dev(ibdev);
 918         struct qed_rdma_create_cq_in_params params;
 919         struct qedr_create_cq_ureq ureq = {};
 920         int vector = attr->comp_vector;
 921         int entries = attr->cqe;
 922         struct qedr_cq *cq = get_qedr_cq(ibcq);
 923         int chain_entries;
 924         u32 db_offset;
 925         int page_cnt;
 926         u64 pbl_ptr;
 927         u16 icid;
 928         int rc;
 929
 930         DP_DEBUG(dev, QEDR_MSG_INIT,
 931                  "create_cq: called from %s. entries=%d, vector=%d\n",
 932                  udata ? "User Lib" : "Kernel", entries, vector);
 933
 934         if (attr->flags)
 935                 return -EOPNOTSUPP;
 936
 937         if (entries > QEDR_MAX_CQES) {
 938                 DP_ERR(dev,
 939                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 940                        entries, QEDR_MAX_CQES);
 941                 return -EINVAL;
 942         }
 943
 944         chain_entries = qedr_align_cq_entries(entries);
 945         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 946         chain_params.num_elems = chain_entries;
 947
 948         /* calc db offset. user will add DPI base, kernel will add db addr */
 949         db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 950
 951         if (udata) {
 952                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 953                                                          udata->inlen))) {
 954                         DP_ERR(dev,
 955                                "create cq: problem copying data from user space\n");
 956                         goto err0;
 957                 }
 958
 959                 if (!ureq.len) {
 960                         DP_ERR(dev,
 961                                "create cq: cannot create a cq with 0 entries\n");
 962                         goto err0;
 963                 }
 964
 965                 cq->cq_type = QEDR_CQ_TYPE_USER;
 966
 967                 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 968                                           ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 969                                           1);
 970                 if (rc)
 971                         goto err0;
 972
 973                 pbl_ptr = cq->q.pbl_tbl->pa;
 974                 page_cnt = cq->q.pbl_info.num_pbes;
 975
 976                 cq->ibcq.cqe = chain_entries;
 977                 cq->q.db_addr = ctx->dpi_addr + db_offset;
 978         } else {
 979                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 980
 981                 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
 982                                                    &chain_params);
 983                 if (rc)
 984                         goto err0;
 985
 986                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 987                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 988                 cq->ibcq.cqe = cq->pbl.capacity;
 989         }
 990
 991         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 992                             pbl_ptr, &params);
 993
 994         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 995         if (rc)
 996                 goto err1;
 997
 998         cq->icid = icid;
 999         cq->sig = QEDR_CQ_MAGIC_NUMBER;
1000         spin_lock_init(&cq->cq_lock);
1001
1002         if (udata) {
1003                 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1004                 if (rc)
1005                         goto err2;
1006
1007                 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1008                                           &cq->q.db_rec_data->db_data,
1009                                           DB_REC_WIDTH_64B,
1010                                           DB_REC_USER);
1011                 if (rc)
1012                         goto err2;
1013
1014         } else {
1015                 /* Generate doorbell address. */
1016                 cq->db.data.icid = cq->icid;
1017                 cq->db_addr = dev->db_addr + db_offset;
1018                 cq->db.data.params = DB_AGG_CMD_MAX <<
1019                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1020
1021                 /* point to the very last element, passing it we will toggle */
1022                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1023                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1024                 cq->latest_cqe = NULL;
1025                 consume_cqe(cq);
1026                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1027
1028                 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1029                                           DB_REC_WIDTH_64B, DB_REC_KERNEL);
1030                 if (rc)
1031                         goto err2;
1032         }
1033
1034         DP_DEBUG(dev, QEDR_MSG_CQ,
1035                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1036                  cq->icid, cq, params.cq_size);
1037
1038         return 0;
1039
1040 err2:
1041         destroy_iparams.icid = cq->icid;
1042         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1043                                   &destroy_oparams);
1044 err1:
1045         if (udata) {
1046                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1047                 ib_umem_release(cq->q.umem);
1048                 if (cq->q.db_mmap_entry)
1049                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1050         } else {
1051                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1052         }
1053 err0:
1054         return -EINVAL;
1055 }
1056
1057 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1058 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1059
1060 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1061 {
1062         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1063         struct qed_rdma_destroy_cq_out_params oparams;
1064         struct qed_rdma_destroy_cq_in_params iparams;
1065         struct qedr_cq *cq = get_qedr_cq(ibcq);
1066         int iter;
1067
1068         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1069
1070         cq->destroyed = 1;
1071
1072         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1073         if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1074                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1075                 return 0;
1076         }
1077
1078         iparams.icid = cq->icid;
1079         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1080         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1081
1082         if (udata) {
1083                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1084                 ib_umem_release(cq->q.umem);
1085
1086                 if (cq->q.db_rec_data) {
1087                         qedr_db_recovery_del(dev, cq->q.db_addr,
1088                                              &cq->q.db_rec_data->db_data);
1089                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1090                 }
1091         } else {
1092                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1093         }
1094
1095         /* We don't want the IRQ handler to handle a non-existing CQ so we
1096          * wait until all CNQ interrupts, if any, are received. This will always
1097          * happen and will always happen very fast. If not, then a serious error
1098          * has occured. That is why we can use a long delay.
1099          * We spin for a short time so we don’t lose time on context switching
1100          * in case all the completions are handled in that span. Otherwise
1101          * we sleep for a while and check again. Since the CNQ may be
1102          * associated with (only) the current CPU we use msleep to allow the
1103          * current CPU to be freed.
1104          * The CNQ notification is increased in qedr_irq_handler().
1105          */
1106         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1107         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1108                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1109                 iter--;
1110         }
1111
1112         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1113         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1114                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1115                 iter--;
1116         }
1117
1118         /* Note that we don't need to have explicit code to wait for the
1119          * completion of the event handler because it is invoked from the EQ.
1120          * Since the destroy CQ ramrod has also been received on the EQ we can
1121          * be certain that there's no event handler in process.
1122          */
1123         return 0;
1124 }
1125
1126 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1127                                           struct ib_qp_attr *attr,
1128                                           int attr_mask,
1129                                           struct qed_rdma_modify_qp_in_params
1130                                           *qp_params)
1131 {
1132         const struct ib_gid_attr *gid_attr;
1133         enum rdma_network_type nw_type;
1134         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1135         u32 ipv4_addr;
1136         int ret;
1137         int i;
1138
1139         gid_attr = grh->sgid_attr;
1140         ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1141         if (ret)
1142                 return ret;
1143
1144         nw_type = rdma_gid_attr_network_type(gid_attr);
1145         switch (nw_type) {
1146         case RDMA_NETWORK_IPV6:
1147                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1148                        sizeof(qp_params->sgid));
1149                 memcpy(&qp_params->dgid.bytes[0],
1150                        &grh->dgid,
1151                        sizeof(qp_params->dgid));
1152                 qp_params->roce_mode = ROCE_V2_IPV6;
1153                 SET_FIELD(qp_params->modify_flags,
1154                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1155                 break;
1156         case RDMA_NETWORK_ROCE_V1:
1157                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1158                        sizeof(qp_params->sgid));
1159                 memcpy(&qp_params->dgid.bytes[0],
1160                        &grh->dgid,
1161                        sizeof(qp_params->dgid));
1162                 qp_params->roce_mode = ROCE_V1;
1163                 break;
1164         case RDMA_NETWORK_IPV4:
1165                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1166                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1167                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1168                 qp_params->sgid.ipv4_addr = ipv4_addr;
1169                 ipv4_addr =
1170                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1171                 qp_params->dgid.ipv4_addr = ipv4_addr;
1172                 SET_FIELD(qp_params->modify_flags,
1173                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1174                 qp_params->roce_mode = ROCE_V2_IPV4;
1175                 break;
1176         default:
1177                 return -EINVAL;
1178         }
1179
1180         for (i = 0; i < 4; i++) {
1181                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1182                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1183         }
1184
1185         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1186                 qp_params->vlan_id = 0;
1187
1188         return 0;
1189 }
1190
1191 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1192                                struct ib_qp_init_attr *attrs,
1193                                struct ib_udata *udata)
1194 {
1195         struct qedr_device_attr *qattr = &dev->attr;
1196
1197         /* QP0... attrs->qp_type == IB_QPT_GSI */
1198         if (attrs->qp_type != IB_QPT_RC &&
1199             attrs->qp_type != IB_QPT_GSI &&
1200             attrs->qp_type != IB_QPT_XRC_INI &&
1201             attrs->qp_type != IB_QPT_XRC_TGT) {
1202                 DP_DEBUG(dev, QEDR_MSG_QP,
1203                          "create qp: unsupported qp type=0x%x requested\n",
1204                          attrs->qp_type);
1205                 return -EOPNOTSUPP;
1206         }
1207
1208         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1209                 DP_ERR(dev,
1210                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1211                        attrs->cap.max_send_wr, qattr->max_sqe);
1212                 return -EINVAL;
1213         }
1214
1215         if (attrs->cap.max_inline_data > qattr->max_inline) {
1216                 DP_ERR(dev,
1217                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1218                        attrs->cap.max_inline_data, qattr->max_inline);
1219                 return -EINVAL;
1220         }
1221
1222         if (attrs->cap.max_send_sge > qattr->max_sge) {
1223                 DP_ERR(dev,
1224                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1225                        attrs->cap.max_send_sge, qattr->max_sge);
1226                 return -EINVAL;
1227         }
1228
1229         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1230                 DP_ERR(dev,
1231                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1232                        attrs->cap.max_recv_sge, qattr->max_sge);
1233                 return -EINVAL;
1234         }
1235
1236         /* verify consumer QPs are not trying to use GSI QP's CQ.
1237          * TGT QP isn't associated with RQ/SQ
1238          */
1239         if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1240             (attrs->qp_type != IB_QPT_XRC_TGT) &&
1241             (attrs->qp_type != IB_QPT_XRC_INI)) {
1242                 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1243                 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1244
1245                 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1246                     (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1247                         DP_ERR(dev,
1248                                "create qp: consumer QP cannot use GSI CQs.\n");
1249                         return -EINVAL;
1250                 }
1251         }
1252
1253         return 0;
1254 }
1255
1256 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1257                                struct qedr_srq *srq, struct ib_udata *udata)
1258 {
1259         struct qedr_create_srq_uresp uresp = {};
1260         int rc;
1261
1262         uresp.srq_id = srq->srq_id;
1263
1264         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1265         if (rc)
1266                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1267
1268         return rc;
1269 }
1270
1271 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1272                                struct qedr_create_qp_uresp *uresp,
1273                                struct qedr_qp *qp)
1274 {
1275         /* iWARP requires two doorbells per RQ. */
1276         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1277                 uresp->rq_db_offset =
1278                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1279                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1280         } else {
1281                 uresp->rq_db_offset =
1282                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1283         }
1284
1285         uresp->rq_icid = qp->icid;
1286         if (qp->urq.db_mmap_entry)
1287                 uresp->rq_db_rec_addr =
1288                         rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1289 }
1290
1291 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1292                                struct qedr_create_qp_uresp *uresp,
1293                                struct qedr_qp *qp)
1294 {
1295         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1296
1297         /* iWARP uses the same cid for rq and sq */
1298         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1299                 uresp->sq_icid = qp->icid;
1300         else
1301                 uresp->sq_icid = qp->icid + 1;
1302
1303         if (qp->usq.db_mmap_entry)
1304                 uresp->sq_db_rec_addr =
1305                         rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1306 }
1307
1308 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1309                               struct qedr_qp *qp, struct ib_udata *udata,
1310                               struct qedr_create_qp_uresp *uresp)
1311 {
1312         int rc;
1313
1314         memset(uresp, 0, sizeof(*uresp));
1315
1316         if (qedr_qp_has_sq(qp))
1317                 qedr_copy_sq_uresp(dev, uresp, qp);
1318
1319         if (qedr_qp_has_rq(qp))
1320                 qedr_copy_rq_uresp(dev, uresp, qp);
1321
1322         uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1323         uresp->qp_id = qp->qp_id;
1324
1325         rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1326         if (rc)
1327                 DP_ERR(dev,
1328                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1329                        qp->icid);
1330
1331         return rc;
1332 }
1333
1334 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1335 {
1336         qed_chain_reset(&qph->pbl);
1337         qph->prod = 0;
1338         qph->cons = 0;
1339         qph->wqe_cons = 0;
1340         qph->db_data.data.value = cpu_to_le16(0);
1341 }
1342
1343 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1344                                       struct qedr_qp *qp,
1345                                       struct qedr_pd *pd,
1346                                       struct ib_qp_init_attr *attrs)
1347 {
1348         spin_lock_init(&qp->q_lock);
1349         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1350                 kref_init(&qp->refcnt);
1351                 init_completion(&qp->iwarp_cm_comp);
1352                 init_completion(&qp->qp_rel_comp);
1353         }
1354
1355         qp->pd = pd;
1356         qp->qp_type = attrs->qp_type;
1357         qp->max_inline_data = attrs->cap.max_inline_data;
1358         qp->state = QED_ROCE_QP_STATE_RESET;
1359
1360         qp->prev_wqe_size = 0;
1361
1362         qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1363         qp->dev = dev;
1364         if (qedr_qp_has_sq(qp)) {
1365                 qedr_reset_qp_hwq_info(&qp->sq);
1366                 qp->sq.max_sges = attrs->cap.max_send_sge;
1367                 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1368                 DP_DEBUG(dev, QEDR_MSG_QP,
1369                          "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1370                          qp->sq.max_sges, qp->sq_cq->icid);
1371         }
1372
1373         if (attrs->srq)
1374                 qp->srq = get_qedr_srq(attrs->srq);
1375
1376         if (qedr_qp_has_rq(qp)) {
1377                 qedr_reset_qp_hwq_info(&qp->rq);
1378                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1379                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1380                 DP_DEBUG(dev, QEDR_MSG_QP,
1381                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1382                          qp->rq.max_sges, qp->rq_cq->icid);
1383         }
1384
1385         DP_DEBUG(dev, QEDR_MSG_QP,
1386                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1387                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1388                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1389         DP_DEBUG(dev, QEDR_MSG_QP,
1390                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1391                  qp->sq.max_sges, qp->sq_cq->icid);
1392 }
1393
1394 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1395 {
1396         int rc = 0;
1397
1398         if (qedr_qp_has_sq(qp)) {
1399                 qp->sq.db = dev->db_addr +
1400                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1401                 qp->sq.db_data.data.icid = qp->icid + 1;
1402                 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1403                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1404                 if (rc)
1405                         return rc;
1406         }
1407
1408         if (qedr_qp_has_rq(qp)) {
1409                 qp->rq.db = dev->db_addr +
1410                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1411                 qp->rq.db_data.data.icid = qp->icid;
1412                 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1413                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1414                 if (rc && qedr_qp_has_sq(qp))
1415                         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1416         }
1417
1418         return rc;
1419 }
1420
1421 static int qedr_check_srq_params(struct qedr_dev *dev,
1422                                  struct ib_srq_init_attr *attrs,
1423                                  struct ib_udata *udata)
1424 {
1425         struct qedr_device_attr *qattr = &dev->attr;
1426
1427         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1428                 DP_ERR(dev,
1429                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1430                        attrs->attr.max_wr, qattr->max_srq_wr);
1431                 return -EINVAL;
1432         }
1433
1434         if (attrs->attr.max_sge > qattr->max_sge) {
1435                 DP_ERR(dev,
1436                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1437                        attrs->attr.max_sge, qattr->max_sge);
1438         }
1439
1440         if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1441                 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1442                 return -EINVAL;
1443         }
1444
1445         return 0;
1446 }
1447
1448 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1449 {
1450         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1451         ib_umem_release(srq->usrq.umem);
1452         ib_umem_release(srq->prod_umem);
1453 }
1454
1455 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1456 {
1457         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1458         struct qedr_dev *dev = srq->dev;
1459
1460         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1461
1462         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1463                           hw_srq->virt_prod_pair_addr,
1464                           hw_srq->phy_prod_pair_addr);
1465 }
1466
1467 static int qedr_init_srq_user_params(struct ib_udata *udata,
1468                                      struct qedr_srq *srq,
1469                                      struct qedr_create_srq_ureq *ureq,
1470                                      int access)
1471 {
1472         struct scatterlist *sg;
1473         int rc;
1474
1475         rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1476                                   ureq->srq_len, false, access, 1);
1477         if (rc)
1478                 return rc;
1479
1480         srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1481                                      sizeof(struct rdma_srq_producers), access);
1482         if (IS_ERR(srq->prod_umem)) {
1483                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1484                 ib_umem_release(srq->usrq.umem);
1485                 DP_ERR(srq->dev,
1486                        "create srq: failed ib_umem_get for producer, got %ld\n",
1487                        PTR_ERR(srq->prod_umem));
1488                 return PTR_ERR(srq->prod_umem);
1489         }
1490
1491         sg = srq->prod_umem->sgt_append.sgt.sgl;
1492         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1493
1494         return 0;
1495 }
1496
1497 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1498                                         struct qedr_dev *dev,
1499                                         struct ib_srq_init_attr *init_attr)
1500 {
1501         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1502         struct qed_chain_init_params params = {
1503                 .mode           = QED_CHAIN_MODE_PBL,
1504                 .intended_use   = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1505                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
1506                 .elem_size      = QEDR_SRQ_WQE_ELEM_SIZE,
1507         };
1508         dma_addr_t phy_prod_pair_addr;
1509         u32 num_elems;
1510         void *va;
1511         int rc;
1512
1513         va = dma_alloc_coherent(&dev->pdev->dev,
1514                                 sizeof(struct rdma_srq_producers),
1515                                 &phy_prod_pair_addr, GFP_KERNEL);
1516         if (!va) {
1517                 DP_ERR(dev,
1518                        "create srq: failed to allocate dma memory for producer\n");
1519                 return -ENOMEM;
1520         }
1521
1522         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1523         hw_srq->virt_prod_pair_addr = va;
1524
1525         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1526         params.num_elems = num_elems;
1527
1528         rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1529         if (rc)
1530                 goto err0;
1531
1532         hw_srq->num_elems = num_elems;
1533
1534         return 0;
1535
1536 err0:
1537         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1538                           va, phy_prod_pair_addr);
1539         return rc;
1540 }
1541
1542 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1543                     struct ib_udata *udata)
1544 {
1545         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1546         struct qed_rdma_create_srq_in_params in_params = {};
1547         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1548         struct qed_rdma_create_srq_out_params out_params;
1549         struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1550         struct qedr_create_srq_ureq ureq = {};
1551         u64 pbl_base_addr, phy_prod_pair_addr;
1552         struct qedr_srq_hwq_info *hw_srq;
1553         u32 page_cnt, page_size;
1554         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1555         int rc = 0;
1556
1557         DP_DEBUG(dev, QEDR_MSG_QP,
1558                  "create SRQ called from %s (pd %p)\n",
1559                  (udata) ? "User lib" : "kernel", pd);
1560
1561         if (init_attr->srq_type != IB_SRQT_BASIC &&
1562             init_attr->srq_type != IB_SRQT_XRC)
1563                 return -EOPNOTSUPP;
1564
1565         rc = qedr_check_srq_params(dev, init_attr, udata);
1566         if (rc)
1567                 return -EINVAL;
1568
1569         srq->dev = dev;
1570         srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1571         hw_srq = &srq->hw_srq;
1572         spin_lock_init(&srq->lock);
1573
1574         hw_srq->max_wr = init_attr->attr.max_wr;
1575         hw_srq->max_sges = init_attr->attr.max_sge;
1576
1577         if (udata) {
1578                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1579                                                          udata->inlen))) {
1580                         DP_ERR(dev,
1581                                "create srq: problem copying data from user space\n");
1582                         goto err0;
1583                 }
1584
1585                 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1586                 if (rc)
1587                         goto err0;
1588
1589                 page_cnt = srq->usrq.pbl_info.num_pbes;
1590                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1591                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1592                 page_size = PAGE_SIZE;
1593         } else {
1594                 struct qed_chain *pbl;
1595
1596                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1597                 if (rc)
1598                         goto err0;
1599
1600                 pbl = &hw_srq->pbl;
1601                 page_cnt = qed_chain_get_page_cnt(pbl);
1602                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1603                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1604                 page_size = QED_CHAIN_PAGE_SIZE;
1605         }
1606
1607         in_params.pd_id = pd->pd_id;
1608         in_params.pbl_base_addr = pbl_base_addr;
1609         in_params.prod_pair_addr = phy_prod_pair_addr;
1610         in_params.num_pages = page_cnt;
1611         in_params.page_size = page_size;
1612         if (srq->is_xrc) {
1613                 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1614                 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1615
1616                 in_params.is_xrc = 1;
1617                 in_params.xrcd_id = xrcd->xrcd_id;
1618                 in_params.cq_cid = cq->icid;
1619         }
1620
1621         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1622         if (rc)
1623                 goto err1;
1624
1625         srq->srq_id = out_params.srq_id;
1626
1627         if (udata) {
1628                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1629                 if (rc)
1630                         goto err2;
1631         }
1632
1633         rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1634         if (rc)
1635                 goto err2;
1636
1637         DP_DEBUG(dev, QEDR_MSG_SRQ,
1638                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1639         return 0;
1640
1641 err2:
1642         destroy_in_params.srq_id = srq->srq_id;
1643
1644         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1645 err1:
1646         if (udata)
1647                 qedr_free_srq_user_params(srq);
1648         else
1649                 qedr_free_srq_kernel_params(srq);
1650 err0:
1651         return -EFAULT;
1652 }
1653
1654 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1655 {
1656         struct qed_rdma_destroy_srq_in_params in_params = {};
1657         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1658         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1659
1660         xa_erase_irq(&dev->srqs, srq->srq_id);
1661         in_params.srq_id = srq->srq_id;
1662         in_params.is_xrc = srq->is_xrc;
1663         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1664
1665         if (ibsrq->uobject)
1666                 qedr_free_srq_user_params(srq);
1667         else
1668                 qedr_free_srq_kernel_params(srq);
1669
1670         DP_DEBUG(dev, QEDR_MSG_SRQ,
1671                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1672                  srq->srq_id);
1673         return 0;
1674 }
1675
1676 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1677                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1678 {
1679         struct qed_rdma_modify_srq_in_params in_params = {};
1680         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1681         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1682         int rc;
1683
1684         if (attr_mask & IB_SRQ_MAX_WR) {
1685                 DP_ERR(dev,
1686                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1687                        attr_mask, srq);
1688                 return -EINVAL;
1689         }
1690
1691         if (attr_mask & IB_SRQ_LIMIT) {
1692                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1693                         DP_ERR(dev,
1694                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1695                                attr->srq_limit, srq->hw_srq.max_wr);
1696                         return -EINVAL;
1697                 }
1698
1699                 in_params.srq_id = srq->srq_id;
1700                 in_params.wqe_limit = attr->srq_limit;
1701                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1702                 if (rc)
1703                         return rc;
1704         }
1705
1706         srq->srq_limit = attr->srq_limit;
1707
1708         DP_DEBUG(dev, QEDR_MSG_SRQ,
1709                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1710
1711         return 0;
1712 }
1713
1714 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1715 {
1716         switch (ib_qp_type) {
1717         case IB_QPT_RC:
1718                 return QED_RDMA_QP_TYPE_RC;
1719         case IB_QPT_XRC_INI:
1720                 return QED_RDMA_QP_TYPE_XRC_INI;
1721         case IB_QPT_XRC_TGT:
1722                 return QED_RDMA_QP_TYPE_XRC_TGT;
1723         default:
1724                 return QED_RDMA_QP_TYPE_INVAL;
1725         }
1726 }
1727
1728 static inline void
1729 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1730                               struct qedr_pd *pd,
1731                               struct qedr_qp *qp,
1732                               struct ib_qp_init_attr *attrs,
1733                               bool fmr_and_reserved_lkey,
1734                               struct qed_rdma_create_qp_in_params *params)
1735 {
1736         /* QP handle to be written in an async event */
1737         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1738         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1739
1740         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1741         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1742         params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1743         params->stats_queue = 0;
1744
1745         if (pd) {
1746                 params->pd = pd->pd_id;
1747                 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1748         }
1749
1750         if (qedr_qp_has_sq(qp))
1751                 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1752
1753         if (qedr_qp_has_rq(qp))
1754                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1755
1756         if (qedr_qp_has_srq(qp)) {
1757                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1758                 params->srq_id = qp->srq->srq_id;
1759                 params->use_srq = true;
1760         } else {
1761                 params->srq_id = 0;
1762                 params->use_srq = false;
1763         }
1764 }
1765
1766 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1767 {
1768         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1769                  "qp=%p. "
1770                  "sq_addr=0x%llx, "
1771                  "sq_len=%zd, "
1772                  "rq_addr=0x%llx, "
1773                  "rq_len=%zd"
1774                  "\n",
1775                  qp,
1776                  qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1777                  qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1778                  qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1779                  qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1780 }
1781
1782 static inline void
1783 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1784                             struct qedr_qp *qp,
1785                             struct qed_rdma_create_qp_out_params *out_params)
1786 {
1787         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1788         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1789
1790         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1791                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1792         if (!qp->srq) {
1793                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1794                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1795         }
1796
1797         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1798                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1799 }
1800
1801 static void qedr_cleanup_user(struct qedr_dev *dev,
1802                               struct qedr_ucontext *ctx,
1803                               struct qedr_qp *qp)
1804 {
1805         if (qedr_qp_has_sq(qp)) {
1806                 ib_umem_release(qp->usq.umem);
1807                 qp->usq.umem = NULL;
1808         }
1809
1810         if (qedr_qp_has_rq(qp)) {
1811                 ib_umem_release(qp->urq.umem);
1812                 qp->urq.umem = NULL;
1813         }
1814
1815         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1816                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1817                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1818         } else {
1819                 kfree(qp->usq.pbl_tbl);
1820                 kfree(qp->urq.pbl_tbl);
1821         }
1822
1823         if (qp->usq.db_rec_data) {
1824                 qedr_db_recovery_del(dev, qp->usq.db_addr,
1825                                      &qp->usq.db_rec_data->db_data);
1826                 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1827         }
1828
1829         if (qp->urq.db_rec_data) {
1830                 qedr_db_recovery_del(dev, qp->urq.db_addr,
1831                                      &qp->urq.db_rec_data->db_data);
1832                 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1833         }
1834
1835         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1836                 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1837                                      &qp->urq.db_rec_db2_data);
1838 }
1839
1840 static int qedr_create_user_qp(struct qedr_dev *dev,
1841                                struct qedr_qp *qp,
1842                                struct ib_pd *ibpd,
1843                                struct ib_udata *udata,
1844                                struct ib_qp_init_attr *attrs)
1845 {
1846         struct qed_rdma_create_qp_in_params in_params;
1847         struct qed_rdma_create_qp_out_params out_params;
1848         struct qedr_create_qp_uresp uresp = {};
1849         struct qedr_create_qp_ureq ureq = {};
1850         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1851         struct qedr_ucontext *ctx = NULL;
1852         struct qedr_pd *pd = NULL;
1853         int rc = 0;
1854
1855         qp->create_type = QEDR_QP_CREATE_USER;
1856
1857         if (ibpd) {
1858                 pd = get_qedr_pd(ibpd);
1859                 ctx = pd->uctx;
1860         }
1861
1862         if (udata) {
1863                 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1864                                         udata->inlen));
1865                 if (rc) {
1866                         DP_ERR(dev, "Problem copying data from user space\n");
1867                         return rc;
1868                 }
1869         }
1870
1871         if (qedr_qp_has_sq(qp)) {
1872                 /* SQ - read access only (0) */
1873                 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1874                                           ureq.sq_len, true, 0, alloc_and_init);
1875                 if (rc)
1876                         return rc;
1877         }
1878
1879         if (qedr_qp_has_rq(qp)) {
1880                 /* RQ - read access only (0) */
1881                 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1882                                           ureq.rq_len, true, 0, alloc_and_init);
1883                 if (rc) {
1884                         ib_umem_release(qp->usq.umem);
1885                         qp->usq.umem = NULL;
1886                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1887                                 qedr_free_pbl(dev, &qp->usq.pbl_info,
1888                                               qp->usq.pbl_tbl);
1889                         } else {
1890                                 kfree(qp->usq.pbl_tbl);
1891                         }
1892                         return rc;
1893                 }
1894         }
1895
1896         memset(&in_params, 0, sizeof(in_params));
1897         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1898         in_params.qp_handle_lo = ureq.qp_handle_lo;
1899         in_params.qp_handle_hi = ureq.qp_handle_hi;
1900
1901         if (qp->qp_type == IB_QPT_XRC_TGT) {
1902                 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1903
1904                 in_params.xrcd_id = xrcd->xrcd_id;
1905                 in_params.qp_handle_lo = qp->qp_id;
1906                 in_params.use_srq = 1;
1907         }
1908
1909         if (qedr_qp_has_sq(qp)) {
1910                 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1911                 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1912         }
1913
1914         if (qedr_qp_has_rq(qp)) {
1915                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1916                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1917         }
1918
1919         if (ctx)
1920                 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1921
1922         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1923                                               &in_params, &out_params);
1924
1925         if (!qp->qed_qp) {
1926                 rc = -ENOMEM;
1927                 goto err1;
1928         }
1929
1930         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1931                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1932
1933         qp->qp_id = out_params.qp_id;
1934         qp->icid = out_params.icid;
1935
1936         if (udata) {
1937                 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1938                 if (rc)
1939                         goto err;
1940         }
1941
1942         /* db offset was calculated in copy_qp_uresp, now set in the user q */
1943         if (qedr_qp_has_sq(qp)) {
1944                 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1945                 qp->sq.max_wr = attrs->cap.max_send_wr;
1946                 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1947                                           &qp->usq.db_rec_data->db_data,
1948                                           DB_REC_WIDTH_32B,
1949                                           DB_REC_USER);
1950                 if (rc)
1951                         goto err;
1952         }
1953
1954         if (qedr_qp_has_rq(qp)) {
1955                 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1956                 qp->rq.max_wr = attrs->cap.max_recv_wr;
1957                 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1958                                           &qp->urq.db_rec_data->db_data,
1959                                           DB_REC_WIDTH_32B,
1960                                           DB_REC_USER);
1961                 if (rc)
1962                         goto err;
1963         }
1964
1965         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1966                 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1967
1968                 /* calculate the db_rec_db2 data since it is constant so no
1969                  * need to reflect from user
1970                  */
1971                 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1972                 qp->urq.db_rec_db2_data.data.value =
1973                         cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1974
1975                 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1976                                           &qp->urq.db_rec_db2_data,
1977                                           DB_REC_WIDTH_32B,
1978                                           DB_REC_USER);
1979                 if (rc)
1980                         goto err;
1981         }
1982         qedr_qp_user_print(dev, qp);
1983         return rc;
1984 err:
1985         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1986         if (rc)
1987                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1988
1989 err1:
1990         qedr_cleanup_user(dev, ctx, qp);
1991         return rc;
1992 }
1993
1994 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1995 {
1996         int rc;
1997
1998         qp->sq.db = dev->db_addr +
1999             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2000         qp->sq.db_data.data.icid = qp->icid;
2001
2002         rc = qedr_db_recovery_add(dev, qp->sq.db,
2003                                   &qp->sq.db_data,
2004                                   DB_REC_WIDTH_32B,
2005                                   DB_REC_KERNEL);
2006         if (rc)
2007                 return rc;
2008
2009         qp->rq.db = dev->db_addr +
2010                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2011         qp->rq.db_data.data.icid = qp->icid;
2012         qp->rq.iwarp_db2 = dev->db_addr +
2013                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2014         qp->rq.iwarp_db2_data.data.icid = qp->icid;
2015         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2016
2017         rc = qedr_db_recovery_add(dev, qp->rq.db,
2018                                   &qp->rq.db_data,
2019                                   DB_REC_WIDTH_32B,
2020                                   DB_REC_KERNEL);
2021         if (rc)
2022                 return rc;
2023
2024         rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2025                                   &qp->rq.iwarp_db2_data,
2026                                   DB_REC_WIDTH_32B,
2027                                   DB_REC_KERNEL);
2028         return rc;
2029 }
2030
2031 static int
2032 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2033                            struct qedr_qp *qp,
2034                            struct qed_rdma_create_qp_in_params *in_params,
2035                            u32 n_sq_elems, u32 n_rq_elems)
2036 {
2037         struct qed_rdma_create_qp_out_params out_params;
2038         struct qed_chain_init_params params = {
2039                 .mode           = QED_CHAIN_MODE_PBL,
2040                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2041         };
2042         int rc;
2043
2044         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2045         params.num_elems = n_sq_elems;
2046         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2047
2048         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2049         if (rc)
2050                 return rc;
2051
2052         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2053         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2054
2055         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2056         params.num_elems = n_rq_elems;
2057         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2058
2059         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2060         if (rc)
2061                 return rc;
2062
2063         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2064         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2065
2066         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2067                                               in_params, &out_params);
2068
2069         if (!qp->qed_qp)
2070                 return -EINVAL;
2071
2072         qp->qp_id = out_params.qp_id;
2073         qp->icid = out_params.icid;
2074
2075         return qedr_set_roce_db_info(dev, qp);
2076 }
2077
2078 static int
2079 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2080                             struct qedr_qp *qp,
2081                             struct qed_rdma_create_qp_in_params *in_params,
2082                             u32 n_sq_elems, u32 n_rq_elems)
2083 {
2084         struct qed_rdma_create_qp_out_params out_params;
2085         struct qed_chain_init_params params = {
2086                 .mode           = QED_CHAIN_MODE_PBL,
2087                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2088         };
2089         int rc;
2090
2091         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2092                                                      QEDR_SQE_ELEMENT_SIZE,
2093                                                      QED_CHAIN_PAGE_SIZE,
2094                                                      QED_CHAIN_MODE_PBL);
2095         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2096                                                      QEDR_RQE_ELEMENT_SIZE,
2097                                                      QED_CHAIN_PAGE_SIZE,
2098                                                      QED_CHAIN_MODE_PBL);
2099
2100         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2101                                               in_params, &out_params);
2102
2103         if (!qp->qed_qp)
2104                 return -EINVAL;
2105
2106         /* Now we allocate the chain */
2107
2108         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2109         params.num_elems = n_sq_elems;
2110         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2111         params.ext_pbl_virt = out_params.sq_pbl_virt;
2112         params.ext_pbl_phys = out_params.sq_pbl_phys;
2113
2114         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2115         if (rc)
2116                 goto err;
2117
2118         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2119         params.num_elems = n_rq_elems;
2120         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2121         params.ext_pbl_virt = out_params.rq_pbl_virt;
2122         params.ext_pbl_phys = out_params.rq_pbl_phys;
2123
2124         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2125         if (rc)
2126                 goto err;
2127
2128         qp->qp_id = out_params.qp_id;
2129         qp->icid = out_params.icid;
2130
2131         return qedr_set_iwarp_db_info(dev, qp);
2132
2133 err:
2134         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2135
2136         return rc;
2137 }
2138
2139 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2140 {
2141         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2142         kfree(qp->wqe_wr_id);
2143
2144         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2145         kfree(qp->rqe_wr_id);
2146
2147         /* GSI qp is not registered to db mechanism so no need to delete */
2148         if (qp->qp_type == IB_QPT_GSI)
2149                 return;
2150
2151         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2152
2153         if (!qp->srq) {
2154                 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2155
2156                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2157                         qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2158                                              &qp->rq.iwarp_db2_data);
2159         }
2160 }
2161
2162 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2163                                  struct qedr_qp *qp,
2164                                  struct ib_pd *ibpd,
2165                                  struct ib_qp_init_attr *attrs)
2166 {
2167         struct qed_rdma_create_qp_in_params in_params;
2168         struct qedr_pd *pd = get_qedr_pd(ibpd);
2169         int rc = -EINVAL;
2170         u32 n_rq_elems;
2171         u32 n_sq_elems;
2172         u32 n_sq_entries;
2173
2174         memset(&in_params, 0, sizeof(in_params));
2175         qp->create_type = QEDR_QP_CREATE_KERNEL;
2176
2177         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2178          * the ring. The ring should allow at least a single WR, even if the
2179          * user requested none, due to allocation issues.
2180          * We should add an extra WR since the prod and cons indices of
2181          * wqe_wr_id are managed in such a way that the WQ is considered full
2182          * when (prod+1)%max_wr==cons. We currently don't do that because we
2183          * double the number of entries due an iSER issue that pushes far more
2184          * WRs than indicated. If we decline its ib_post_send() then we get
2185          * error prints in the dmesg we'd like to avoid.
2186          */
2187         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2188                               dev->attr.max_sqe);
2189
2190         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2191                                 GFP_KERNEL);
2192         if (!qp->wqe_wr_id) {
2193                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2194                 return -ENOMEM;
2195         }
2196
2197         /* QP handle to be written in CQE */
2198         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2199         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2200
2201         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2202          * the ring. There ring should allow at least a single WR, even if the
2203          * user requested none, due to allocation issues.
2204          */
2205         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2206
2207         /* Allocate driver internal RQ array */
2208         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2209                                 GFP_KERNEL);
2210         if (!qp->rqe_wr_id) {
2211                 DP_ERR(dev,
2212                        "create qp: failed RQ shadow memory allocation\n");
2213                 kfree(qp->wqe_wr_id);
2214                 return -ENOMEM;
2215         }
2216
2217         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2218
2219         n_sq_entries = attrs->cap.max_send_wr;
2220         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2221         n_sq_entries = max_t(u32, n_sq_entries, 1);
2222         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2223
2224         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2225
2226         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2227                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2228                                                  n_sq_elems, n_rq_elems);
2229         else
2230                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2231                                                 n_sq_elems, n_rq_elems);
2232         if (rc)
2233                 qedr_cleanup_kernel(dev, qp);
2234
2235         return rc;
2236 }
2237
2238 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2239                                   struct ib_udata *udata)
2240 {
2241         struct qedr_ucontext *ctx =
2242                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2243                                           ibucontext);
2244         int rc;
2245
2246         if (qp->qp_type != IB_QPT_GSI) {
2247                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2248                 if (rc)
2249                         return rc;
2250         }
2251
2252         if (qp->create_type == QEDR_QP_CREATE_USER)
2253                 qedr_cleanup_user(dev, ctx, qp);
2254         else
2255                 qedr_cleanup_kernel(dev, qp);
2256
2257         return 0;
2258 }
2259
2260 int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2261                    struct ib_udata *udata)
2262 {
2263         struct qedr_xrcd *xrcd = NULL;
2264         struct ib_pd *ibpd = ibqp->pd;
2265         struct qedr_pd *pd = get_qedr_pd(ibpd);
2266         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2267         struct qedr_qp *qp = get_qedr_qp(ibqp);
2268         int rc = 0;
2269
2270         if (attrs->create_flags)
2271                 return -EOPNOTSUPP;
2272
2273         if (attrs->qp_type == IB_QPT_XRC_TGT)
2274                 xrcd = get_qedr_xrcd(attrs->xrcd);
2275         else
2276                 pd = get_qedr_pd(ibpd);
2277
2278         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2279                  udata ? "user library" : "kernel", pd);
2280
2281         rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2282         if (rc)
2283                 return rc;
2284
2285         DP_DEBUG(dev, QEDR_MSG_QP,
2286                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2287                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2288                  get_qedr_cq(attrs->send_cq),
2289                  get_qedr_cq(attrs->send_cq)->icid,
2290                  get_qedr_cq(attrs->recv_cq),
2291                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2292
2293         qedr_set_common_qp_params(dev, qp, pd, attrs);
2294
2295         if (attrs->qp_type == IB_QPT_GSI)
2296                 return qedr_create_gsi_qp(dev, attrs, qp);
2297
2298         if (udata || xrcd)
2299                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2300         else
2301                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2302
2303         if (rc)
2304                 return rc;
2305
2306         qp->ibqp.qp_num = qp->qp_id;
2307
2308         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2309                 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2310                 if (rc)
2311                         goto out_free_qp_resources;
2312         }
2313
2314         return 0;
2315
2316 out_free_qp_resources:
2317         qedr_free_qp_resources(dev, qp, udata);
2318         return -EFAULT;
2319 }
2320
2321 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2322 {
2323         switch (qp_state) {
2324         case QED_ROCE_QP_STATE_RESET:
2325                 return IB_QPS_RESET;
2326         case QED_ROCE_QP_STATE_INIT:
2327                 return IB_QPS_INIT;
2328         case QED_ROCE_QP_STATE_RTR:
2329                 return IB_QPS_RTR;
2330         case QED_ROCE_QP_STATE_RTS:
2331                 return IB_QPS_RTS;
2332         case QED_ROCE_QP_STATE_SQD:
2333                 return IB_QPS_SQD;
2334         case QED_ROCE_QP_STATE_ERR:
2335                 return IB_QPS_ERR;
2336         case QED_ROCE_QP_STATE_SQE:
2337                 return IB_QPS_SQE;
2338         }
2339         return IB_QPS_ERR;
2340 }
2341
2342 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2343                                         enum ib_qp_state qp_state)
2344 {
2345         switch (qp_state) {
2346         case IB_QPS_RESET:
2347                 return QED_ROCE_QP_STATE_RESET;
2348         case IB_QPS_INIT:
2349                 return QED_ROCE_QP_STATE_INIT;
2350         case IB_QPS_RTR:
2351                 return QED_ROCE_QP_STATE_RTR;
2352         case IB_QPS_RTS:
2353                 return QED_ROCE_QP_STATE_RTS;
2354         case IB_QPS_SQD:
2355                 return QED_ROCE_QP_STATE_SQD;
2356         case IB_QPS_ERR:
2357                 return QED_ROCE_QP_STATE_ERR;
2358         default:
2359                 return QED_ROCE_QP_STATE_ERR;
2360         }
2361 }
2362
2363 static int qedr_update_qp_state(struct qedr_dev *dev,
2364                                 struct qedr_qp *qp,
2365                                 enum qed_roce_qp_state cur_state,
2366                                 enum qed_roce_qp_state new_state)
2367 {
2368         int status = 0;
2369
2370         if (new_state == cur_state)
2371                 return 0;
2372
2373         switch (cur_state) {
2374         case QED_ROCE_QP_STATE_RESET:
2375                 switch (new_state) {
2376                 case QED_ROCE_QP_STATE_INIT:
2377                         break;
2378                 default:
2379                         status = -EINVAL;
2380                         break;
2381                 }
2382                 break;
2383         case QED_ROCE_QP_STATE_INIT:
2384                 switch (new_state) {
2385                 case QED_ROCE_QP_STATE_RTR:
2386                         /* Update doorbell (in case post_recv was
2387                          * done before move to RTR)
2388                          */
2389
2390                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2391                                 writel(qp->rq.db_data.raw, qp->rq.db);
2392                         }
2393                         break;
2394                 case QED_ROCE_QP_STATE_ERR:
2395                         break;
2396                 default:
2397                         /* Invalid state change. */
2398                         status = -EINVAL;
2399                         break;
2400                 }
2401                 break;
2402         case QED_ROCE_QP_STATE_RTR:
2403                 /* RTR->XXX */
2404                 switch (new_state) {
2405                 case QED_ROCE_QP_STATE_RTS:
2406                         break;
2407                 case QED_ROCE_QP_STATE_ERR:
2408                         break;
2409                 default:
2410                         /* Invalid state change. */
2411                         status = -EINVAL;
2412                         break;
2413                 }
2414                 break;
2415         case QED_ROCE_QP_STATE_RTS:
2416                 /* RTS->XXX */
2417                 switch (new_state) {
2418                 case QED_ROCE_QP_STATE_SQD:
2419                         break;
2420                 case QED_ROCE_QP_STATE_ERR:
2421                         break;
2422                 default:
2423                         /* Invalid state change. */
2424                         status = -EINVAL;
2425                         break;
2426                 }
2427                 break;
2428         case QED_ROCE_QP_STATE_SQD:
2429                 /* SQD->XXX */
2430                 switch (new_state) {
2431                 case QED_ROCE_QP_STATE_RTS:
2432                 case QED_ROCE_QP_STATE_ERR:
2433                         break;
2434                 default:
2435                         /* Invalid state change. */
2436                         status = -EINVAL;
2437                         break;
2438                 }
2439                 break;
2440         case QED_ROCE_QP_STATE_ERR:
2441                 /* ERR->XXX */
2442                 switch (new_state) {
2443                 case QED_ROCE_QP_STATE_RESET:
2444                         if ((qp->rq.prod != qp->rq.cons) ||
2445                             (qp->sq.prod != qp->sq.cons)) {
2446                                 DP_NOTICE(dev,
2447                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2448                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2449                                           qp->sq.cons);
2450                                 status = -EINVAL;
2451                         }
2452                         break;
2453                 default:
2454                         status = -EINVAL;
2455                         break;
2456                 }
2457                 break;
2458         default:
2459                 status = -EINVAL;
2460                 break;
2461         }
2462
2463         return status;
2464 }
2465
2466 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2467                    int attr_mask, struct ib_udata *udata)
2468 {
2469         struct qedr_qp *qp = get_qedr_qp(ibqp);
2470         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2471         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2472         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2473         enum ib_qp_state old_qp_state, new_qp_state;
2474         enum qed_roce_qp_state cur_state;
2475         int rc = 0;
2476
2477         DP_DEBUG(dev, QEDR_MSG_QP,
2478                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2479                  attr->qp_state);
2480
2481         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2482                 return -EOPNOTSUPP;
2483
2484         old_qp_state = qedr_get_ibqp_state(qp->state);
2485         if (attr_mask & IB_QP_STATE)
2486                 new_qp_state = attr->qp_state;
2487         else
2488                 new_qp_state = old_qp_state;
2489
2490         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2491                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2492                                         ibqp->qp_type, attr_mask)) {
2493                         DP_ERR(dev,
2494                                "modify qp: invalid attribute mask=0x%x specified for\n"
2495                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2496                                attr_mask, qp->qp_id, ibqp->qp_type,
2497                                old_qp_state, new_qp_state);
2498                         rc = -EINVAL;
2499                         goto err;
2500                 }
2501         }
2502
2503         /* Translate the masks... */
2504         if (attr_mask & IB_QP_STATE) {
2505                 SET_FIELD(qp_params.modify_flags,
2506                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2507                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2508         }
2509
2510         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2511                 qp_params.sqd_async = true;
2512
2513         if (attr_mask & IB_QP_PKEY_INDEX) {
2514                 SET_FIELD(qp_params.modify_flags,
2515                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2516                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2517                         rc = -EINVAL;
2518                         goto err;
2519                 }
2520
2521                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2522         }
2523
2524         if (attr_mask & IB_QP_QKEY)
2525                 qp->qkey = attr->qkey;
2526
2527         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2528                 SET_FIELD(qp_params.modify_flags,
2529                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2530                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2531                                                   IB_ACCESS_REMOTE_READ;
2532                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2533                                                    IB_ACCESS_REMOTE_WRITE;
2534                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2535                                                IB_ACCESS_REMOTE_ATOMIC;
2536         }
2537
2538         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2539                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2540                         return -EINVAL;
2541
2542                 if (attr_mask & IB_QP_PATH_MTU) {
2543                         if (attr->path_mtu < IB_MTU_256 ||
2544                             attr->path_mtu > IB_MTU_4096) {
2545                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2546                                 rc = -EINVAL;
2547                                 goto err;
2548                         }
2549                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2550                                       ib_mtu_enum_to_int(iboe_get_mtu
2551                                                          (dev->ndev->mtu)));
2552                 }
2553
2554                 if (!qp->mtu) {
2555                         qp->mtu =
2556                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2557                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2558                 }
2559
2560                 SET_FIELD(qp_params.modify_flags,
2561                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2562
2563                 qp_params.traffic_class_tos = grh->traffic_class;
2564                 qp_params.flow_label = grh->flow_label;
2565                 qp_params.hop_limit_ttl = grh->hop_limit;
2566
2567                 qp->sgid_idx = grh->sgid_index;
2568
2569                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2570                 if (rc) {
2571                         DP_ERR(dev,
2572                                "modify qp: problems with GID index %d (rc=%d)\n",
2573                                grh->sgid_index, rc);
2574                         return rc;
2575                 }
2576
2577                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2578                                    qp_params.remote_mac_addr);
2579                 if (rc)
2580                         return rc;
2581
2582                 qp_params.use_local_mac = true;
2583                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2584
2585                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2586                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2587                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2588                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2589                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2590                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2591                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2592                          qp_params.remote_mac_addr);
2593
2594                 qp_params.mtu = qp->mtu;
2595                 qp_params.lb_indication = false;
2596         }
2597
2598         if (!qp_params.mtu) {
2599                 /* Stay with current MTU */
2600                 if (qp->mtu)
2601                         qp_params.mtu = qp->mtu;
2602                 else
2603                         qp_params.mtu =
2604                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2605         }
2606
2607         if (attr_mask & IB_QP_TIMEOUT) {
2608                 SET_FIELD(qp_params.modify_flags,
2609                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2610
2611                 /* The received timeout value is an exponent used like this:
2612                  *    "12.7.34 LOCAL ACK TIMEOUT
2613                  *    Value representing the transport (ACK) timeout for use by
2614                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2615                  * The FW expects timeout in msec so we need to divide the usec
2616                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2617                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2618                  * The value of zero means infinite so we use a 'max_t' to make
2619                  * sure that sub 1 msec values will be configured as 1 msec.
2620                  */
2621                 if (attr->timeout)
2622                         qp_params.ack_timeout =
2623                                         1 << max_t(int, attr->timeout - 8, 0);
2624                 else
2625                         qp_params.ack_timeout = 0;
2626
2627                 qp->timeout = attr->timeout;
2628         }
2629
2630         if (attr_mask & IB_QP_RETRY_CNT) {
2631                 SET_FIELD(qp_params.modify_flags,
2632                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2633                 qp_params.retry_cnt = attr->retry_cnt;
2634         }
2635
2636         if (attr_mask & IB_QP_RNR_RETRY) {
2637                 SET_FIELD(qp_params.modify_flags,
2638                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2639                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2640         }
2641
2642         if (attr_mask & IB_QP_RQ_PSN) {
2643                 SET_FIELD(qp_params.modify_flags,
2644                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2645                 qp_params.rq_psn = attr->rq_psn;
2646                 qp->rq_psn = attr->rq_psn;
2647         }
2648
2649         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2650                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2651                         rc = -EINVAL;
2652                         DP_ERR(dev,
2653                                "unsupported max_rd_atomic=%d, supported=%d\n",
2654                                attr->max_rd_atomic,
2655                                dev->attr.max_qp_req_rd_atomic_resc);
2656                         goto err;
2657                 }
2658
2659                 SET_FIELD(qp_params.modify_flags,
2660                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2661                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2662         }
2663
2664         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2665                 SET_FIELD(qp_params.modify_flags,
2666                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2667                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2668         }
2669
2670         if (attr_mask & IB_QP_SQ_PSN) {
2671                 SET_FIELD(qp_params.modify_flags,
2672                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2673                 qp_params.sq_psn = attr->sq_psn;
2674                 qp->sq_psn = attr->sq_psn;
2675         }
2676
2677         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2678                 if (attr->max_dest_rd_atomic >
2679                     dev->attr.max_qp_resp_rd_atomic_resc) {
2680                         DP_ERR(dev,
2681                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2682                                attr->max_dest_rd_atomic,
2683                                dev->attr.max_qp_resp_rd_atomic_resc);
2684
2685                         rc = -EINVAL;
2686                         goto err;
2687                 }
2688
2689                 SET_FIELD(qp_params.modify_flags,
2690                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2691                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2692         }
2693
2694         if (attr_mask & IB_QP_DEST_QPN) {
2695                 SET_FIELD(qp_params.modify_flags,
2696                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2697
2698                 qp_params.dest_qp = attr->dest_qp_num;
2699                 qp->dest_qp_num = attr->dest_qp_num;
2700         }
2701
2702         cur_state = qp->state;
2703
2704         /* Update the QP state before the actual ramrod to prevent a race with
2705          * fast path. Modifying the QP state to error will cause the device to
2706          * flush the CQEs and while polling the flushed CQEs will considered as
2707          * a potential issue if the QP isn't in error state.
2708          */
2709         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2710             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2711                 qp->state = QED_ROCE_QP_STATE_ERR;
2712
2713         if (qp->qp_type != IB_QPT_GSI)
2714                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2715                                               qp->qed_qp, &qp_params);
2716
2717         if (attr_mask & IB_QP_STATE) {
2718                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2719                         rc = qedr_update_qp_state(dev, qp, cur_state,
2720                                                   qp_params.new_state);
2721                 qp->state = qp_params.new_state;
2722         }
2723
2724 err:
2725         return rc;
2726 }
2727
2728 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2729 {
2730         int ib_qp_acc_flags = 0;
2731
2732         if (params->incoming_rdma_write_en)
2733                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2734         if (params->incoming_rdma_read_en)
2735                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2736         if (params->incoming_atomic_en)
2737                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2738         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2739         return ib_qp_acc_flags;
2740 }
2741
2742 int qedr_query_qp(struct ib_qp *ibqp,
2743                   struct ib_qp_attr *qp_attr,
2744                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2745 {
2746         struct qed_rdma_query_qp_out_params params;
2747         struct qedr_qp *qp = get_qedr_qp(ibqp);
2748         struct qedr_dev *dev = qp->dev;
2749         int rc = 0;
2750
2751         memset(&params, 0, sizeof(params));
2752         memset(qp_attr, 0, sizeof(*qp_attr));
2753         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2754
2755         if (qp->qp_type != IB_QPT_GSI) {
2756                 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2757                 if (rc)
2758                         goto err;
2759                 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2760         } else {
2761                 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2762         }
2763
2764         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2765         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2766         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2767         qp_attr->rq_psn = params.rq_psn;
2768         qp_attr->sq_psn = params.sq_psn;
2769         qp_attr->dest_qp_num = params.dest_qp;
2770
2771         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2772
2773         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2774         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2775         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2776         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2777         qp_attr->cap.max_inline_data = dev->attr.max_inline;
2778         qp_init_attr->cap = qp_attr->cap;
2779
2780         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2781         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2782                         params.flow_label, qp->sgid_idx,
2783                         params.hop_limit_ttl, params.traffic_class_tos);
2784         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2785         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2786         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2787         qp_attr->timeout = qp->timeout;
2788         qp_attr->rnr_retry = params.rnr_retry;
2789         qp_attr->retry_cnt = params.retry_cnt;
2790         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2791         qp_attr->pkey_index = params.pkey_index;
2792         qp_attr->port_num = 1;
2793         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2794         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2795         qp_attr->alt_pkey_index = 0;
2796         qp_attr->alt_port_num = 0;
2797         qp_attr->alt_timeout = 0;
2798         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2799
2800         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2801         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2802         qp_attr->max_rd_atomic = params.max_rd_atomic;
2803         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2804
2805         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2806                  qp_attr->cap.max_inline_data);
2807
2808 err:
2809         return rc;
2810 }
2811
2812 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2813 {
2814         struct qedr_qp *qp = get_qedr_qp(ibqp);
2815         struct qedr_dev *dev = qp->dev;
2816         struct ib_qp_attr attr;
2817         int attr_mask = 0;
2818
2819         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2820                  qp, qp->qp_type);
2821
2822         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2823                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2824                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2825                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2826
2827                         attr.qp_state = IB_QPS_ERR;
2828                         attr_mask |= IB_QP_STATE;
2829
2830                         /* Change the QP state to ERROR */
2831                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2832                 }
2833         } else {
2834                 /* If connection establishment started the WAIT_FOR_CONNECT
2835                  * bit will be on and we need to Wait for the establishment
2836                  * to complete before destroying the qp.
2837                  */
2838                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2839                                      &qp->iwarp_cm_flags))
2840                         wait_for_completion(&qp->iwarp_cm_comp);
2841
2842                 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2843                  * bit will be on, and we need to wait for the disconnect to
2844                  * complete before continuing. We can use the same completion,
2845                  * iwarp_cm_comp, since this is the only place that waits for
2846                  * this completion and it is sequential. In addition,
2847                  * disconnect can't occur before the connection is fully
2848                  * established, therefore if WAIT_FOR_DISCONNECT is on it
2849                  * means WAIT_FOR_CONNECT is also on and the completion for
2850                  * CONNECT already occurred.
2851                  */
2852                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2853                                      &qp->iwarp_cm_flags))
2854                         wait_for_completion(&qp->iwarp_cm_comp);
2855         }
2856
2857         if (qp->qp_type == IB_QPT_GSI)
2858                 qedr_destroy_gsi_qp(dev);
2859
2860         /* We need to remove the entry from the xarray before we release the
2861          * qp_id to avoid a race of the qp_id being reallocated and failing
2862          * on xa_insert
2863          */
2864         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2865                 xa_erase(&dev->qps, qp->qp_id);
2866
2867         qedr_free_qp_resources(dev, qp, udata);
2868
2869         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2870                 qedr_iw_qp_rem_ref(&qp->ibqp);
2871                 wait_for_completion(&qp->qp_rel_comp);
2872         }
2873
2874         return 0;
2875 }
2876
2877 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2878                    struct ib_udata *udata)
2879 {
2880         struct qedr_ah *ah = get_qedr_ah(ibah);
2881
2882         rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2883
2884         return 0;
2885 }
2886
2887 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2888 {
2889         struct qedr_ah *ah = get_qedr_ah(ibah);
2890
2891         rdma_destroy_ah_attr(&ah->attr);
2892         return 0;
2893 }
2894
2895 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2896 {
2897         struct qedr_pbl *pbl, *tmp;
2898
2899         if (info->pbl_table)
2900                 list_add_tail(&info->pbl_table->list_entry,
2901                               &info->free_pbl_list);
2902
2903         if (!list_empty(&info->inuse_pbl_list))
2904                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2905
2906         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2907                 list_del(&pbl->list_entry);
2908                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2909         }
2910 }
2911
2912 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2913                         size_t page_list_len, bool two_layered)
2914 {
2915         struct qedr_pbl *tmp;
2916         int rc;
2917
2918         INIT_LIST_HEAD(&info->free_pbl_list);
2919         INIT_LIST_HEAD(&info->inuse_pbl_list);
2920
2921         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2922                                   page_list_len, two_layered);
2923         if (rc)
2924                 goto done;
2925
2926         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2927         if (IS_ERR(info->pbl_table)) {
2928                 rc = PTR_ERR(info->pbl_table);
2929                 goto done;
2930         }
2931
2932         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2933                  &info->pbl_table->pa);
2934
2935         /* in usual case we use 2 PBLs, so we add one to free
2936          * list and allocating another one
2937          */
2938         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2939         if (IS_ERR(tmp)) {
2940                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2941                 goto done;
2942         }
2943
2944         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2945
2946         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2947
2948 done:
2949         if (rc)
2950                 free_mr_info(dev, info);
2951
2952         return rc;
2953 }
2954
2955 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2956                                u64 usr_addr, int acc, struct ib_udata *udata)
2957 {
2958         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2959         struct qedr_mr *mr;
2960         struct qedr_pd *pd;
2961         int rc = -ENOMEM;
2962
2963         pd = get_qedr_pd(ibpd);
2964         DP_DEBUG(dev, QEDR_MSG_MR,
2965                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2966                  pd->pd_id, start, len, usr_addr, acc);
2967
2968         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2969                 return ERR_PTR(-EINVAL);
2970
2971         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2972         if (!mr)
2973                 return ERR_PTR(rc);
2974
2975         mr->type = QEDR_MR_USER;
2976
2977         mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2978         if (IS_ERR(mr->umem)) {
2979                 rc = -EFAULT;
2980                 goto err0;
2981         }
2982
2983         rc = init_mr_info(dev, &mr->info,
2984                           ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2985         if (rc)
2986                 goto err1;
2987
2988         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2989                            &mr->info.pbl_info, PAGE_SHIFT);
2990
2991         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2992         if (rc) {
2993                 if (rc == -EINVAL)
2994                         DP_ERR(dev, "Out of MR resources\n");
2995                 else
2996                         DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2997
2998                 goto err1;
2999         }
3000
3001         /* Index only, 18 bit long, lkey = itid << 8 | key */
3002         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3003         mr->hw_mr.key = 0;
3004         mr->hw_mr.pd = pd->pd_id;
3005         mr->hw_mr.local_read = 1;
3006         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3007         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3008         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3009         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3010         mr->hw_mr.mw_bind = false;
3011         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3012         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3013         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3014         mr->hw_mr.page_size_log = PAGE_SHIFT;
3015         mr->hw_mr.length = len;
3016         mr->hw_mr.vaddr = usr_addr;
3017         mr->hw_mr.phy_mr = false;
3018         mr->hw_mr.dma_mr = false;
3019
3020         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3021         if (rc) {
3022                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3023                 goto err2;
3024         }
3025
3026         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3027         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3028             mr->hw_mr.remote_atomic)
3029                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3030
3031         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3032                  mr->ibmr.lkey);
3033         return &mr->ibmr;
3034
3035 err2:
3036         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3037 err1:
3038         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3039 err0:
3040         kfree(mr);
3041         return ERR_PTR(rc);
3042 }
3043
3044 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3045 {
3046         struct qedr_mr *mr = get_qedr_mr(ib_mr);
3047         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3048         int rc = 0;
3049
3050         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3051         if (rc)
3052                 return rc;
3053
3054         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3055
3056         if (mr->type != QEDR_MR_DMA)
3057                 free_mr_info(dev, &mr->info);
3058
3059         /* it could be user registered memory. */
3060         ib_umem_release(mr->umem);
3061
3062         kfree(mr);
3063
3064         return rc;
3065 }
3066
3067 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3068                                        int max_page_list_len)
3069 {
3070         struct qedr_pd *pd = get_qedr_pd(ibpd);
3071         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3072         struct qedr_mr *mr;
3073         int rc = -ENOMEM;
3074
3075         DP_DEBUG(dev, QEDR_MSG_MR,
3076                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3077                  max_page_list_len);
3078
3079         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3080         if (!mr)
3081                 return ERR_PTR(rc);
3082
3083         mr->dev = dev;
3084         mr->type = QEDR_MR_FRMR;
3085
3086         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3087         if (rc)
3088                 goto err0;
3089
3090         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3091         if (rc) {
3092                 if (rc == -EINVAL)
3093                         DP_ERR(dev, "Out of MR resources\n");
3094                 else
3095                         DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3096
3097                 goto err1;
3098         }
3099
3100         /* Index only, 18 bit long, lkey = itid << 8 | key */
3101         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3102         mr->hw_mr.key = 0;
3103         mr->hw_mr.pd = pd->pd_id;
3104         mr->hw_mr.local_read = 1;
3105         mr->hw_mr.local_write = 0;
3106         mr->hw_mr.remote_read = 0;
3107         mr->hw_mr.remote_write = 0;
3108         mr->hw_mr.remote_atomic = 0;
3109         mr->hw_mr.mw_bind = false;
3110         mr->hw_mr.pbl_ptr = 0;
3111         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3112         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3113         mr->hw_mr.length = 0;
3114         mr->hw_mr.vaddr = 0;
3115         mr->hw_mr.phy_mr = true;
3116         mr->hw_mr.dma_mr = false;
3117
3118         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3119         if (rc) {
3120                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3121                 goto err2;
3122         }
3123
3124         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3125         mr->ibmr.rkey = mr->ibmr.lkey;
3126
3127         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3128         return mr;
3129
3130 err2:
3131         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3132 err1:
3133         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3134 err0:
3135         kfree(mr);
3136         return ERR_PTR(rc);
3137 }
3138
3139 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3140                             u32 max_num_sg)
3141 {
3142         struct qedr_mr *mr;
3143
3144         if (mr_type != IB_MR_TYPE_MEM_REG)
3145                 return ERR_PTR(-EINVAL);
3146
3147         mr = __qedr_alloc_mr(ibpd, max_num_sg);
3148
3149         if (IS_ERR(mr))
3150                 return ERR_PTR(-EINVAL);
3151
3152         return &mr->ibmr;
3153 }
3154
3155 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3156 {
3157         struct qedr_mr *mr = get_qedr_mr(ibmr);
3158         struct qedr_pbl *pbl_table;
3159         struct regpair *pbe;
3160         u32 pbes_in_page;
3161
3162         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3163                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3164                 return -ENOMEM;
3165         }
3166
3167         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3168                  mr->npages, addr);
3169
3170         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3171         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3172         pbe = (struct regpair *)pbl_table->va;
3173         pbe +=  mr->npages % pbes_in_page;
3174         pbe->lo = cpu_to_le32((u32)addr);
3175         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3176
3177         mr->npages++;
3178
3179         return 0;
3180 }
3181
3182 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3183 {
3184         int work = info->completed - info->completed_handled - 1;
3185
3186         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3187         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3188                 struct qedr_pbl *pbl;
3189
3190                 /* Free all the page list that are possible to be freed
3191                  * (all the ones that were invalidated), under the assumption
3192                  * that if an FMR was completed successfully that means that
3193                  * if there was an invalidate operation before it also ended
3194                  */
3195                 pbl = list_first_entry(&info->inuse_pbl_list,
3196                                        struct qedr_pbl, list_entry);
3197                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3198                 info->completed_handled++;
3199         }
3200 }
3201
3202 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3203                    int sg_nents, unsigned int *sg_offset)
3204 {
3205         struct qedr_mr *mr = get_qedr_mr(ibmr);
3206
3207         mr->npages = 0;
3208
3209         handle_completed_mrs(mr->dev, &mr->info);
3210         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3211 }
3212
3213 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3214 {
3215         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3216         struct qedr_pd *pd = get_qedr_pd(ibpd);
3217         struct qedr_mr *mr;
3218         int rc;
3219
3220         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3221         if (!mr)
3222                 return ERR_PTR(-ENOMEM);
3223
3224         mr->type = QEDR_MR_DMA;
3225
3226         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3227         if (rc) {
3228                 if (rc == -EINVAL)
3229                         DP_ERR(dev, "Out of MR resources\n");
3230                 else
3231                         DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3232
3233                 goto err1;
3234         }
3235
3236         /* index only, 18 bit long, lkey = itid << 8 | key */
3237         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3238         mr->hw_mr.pd = pd->pd_id;
3239         mr->hw_mr.local_read = 1;
3240         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3241         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3242         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3243         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3244         mr->hw_mr.dma_mr = true;
3245
3246         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3247         if (rc) {
3248                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3249                 goto err2;
3250         }
3251
3252         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3253         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3254             mr->hw_mr.remote_atomic)
3255                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3256
3257         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3258         return &mr->ibmr;
3259
3260 err2:
3261         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3262 err1:
3263         kfree(mr);
3264         return ERR_PTR(rc);
3265 }
3266
3267 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3268 {
3269         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3270 }
3271
3272 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3273 {
3274         int i, len = 0;
3275
3276         for (i = 0; i < num_sge; i++)
3277                 len += sg_list[i].length;
3278
3279         return len;
3280 }
3281
3282 static void swap_wqe_data64(u64 *p)
3283 {
3284         int i;
3285
3286         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3287                 *p = cpu_to_be64(cpu_to_le64(*p));
3288 }
3289
3290 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3291                                        struct qedr_qp *qp, u8 *wqe_size,
3292                                        const struct ib_send_wr *wr,
3293                                        const struct ib_send_wr **bad_wr,
3294                                        u8 *bits, u8 bit)
3295 {
3296         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3297         char *seg_prt, *wqe;
3298         int i, seg_siz;
3299
3300         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3301                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3302                 *bad_wr = wr;
3303                 return 0;
3304         }
3305
3306         if (!data_size)
3307                 return data_size;
3308
3309         *bits |= bit;
3310
3311         seg_prt = NULL;
3312         wqe = NULL;
3313         seg_siz = 0;
3314
3315         /* Copy data inline */
3316         for (i = 0; i < wr->num_sge; i++) {
3317                 u32 len = wr->sg_list[i].length;
3318                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3319
3320                 while (len > 0) {
3321                         u32 cur;
3322
3323                         /* New segment required */
3324                         if (!seg_siz) {
3325                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3326                                 seg_prt = wqe;
3327                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3328                                 (*wqe_size)++;
3329                         }
3330
3331                         /* Calculate currently allowed length */
3332                         cur = min_t(u32, len, seg_siz);
3333                         memcpy(seg_prt, src, cur);
3334
3335                         /* Update segment variables */
3336                         seg_prt += cur;
3337                         seg_siz -= cur;
3338
3339                         /* Update sge variables */
3340                         src += cur;
3341                         len -= cur;
3342
3343                         /* Swap fully-completed segments */
3344                         if (!seg_siz)
3345                                 swap_wqe_data64((u64 *)wqe);
3346                 }
3347         }
3348
3349         /* swap last not completed segment */
3350         if (seg_siz)
3351                 swap_wqe_data64((u64 *)wqe);
3352
3353         return data_size;
3354 }
3355
3356 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3357         do {                                                    \
3358                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3359                 (sge)->length = cpu_to_le32(vlength);           \
3360                 (sge)->flags = cpu_to_le32(vflags);             \
3361         } while (0)
3362
3363 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3364         do {                                                    \
3365                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3366                 (hdr)->num_sges = num_sge;                      \
3367         } while (0)
3368
3369 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3370         do {                                                    \
3371                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3372                 (sge)->length = cpu_to_le32(vlength);           \
3373                 (sge)->l_key = cpu_to_le32(vlkey);              \
3374         } while (0)
3375
3376 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3377                                 const struct ib_send_wr *wr)
3378 {
3379         u32 data_size = 0;
3380         int i;
3381
3382         for (i = 0; i < wr->num_sge; i++) {
3383                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3384
3385                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3386                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3387                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3388                 data_size += wr->sg_list[i].length;
3389         }
3390
3391         if (wqe_size)
3392                 *wqe_size += wr->num_sge;
3393
3394         return data_size;
3395 }
3396
3397 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3398                                      struct qedr_qp *qp,
3399                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3400                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3401                                      const struct ib_send_wr *wr,
3402                                      const struct ib_send_wr **bad_wr)
3403 {
3404         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3405         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3406
3407         if (wr->send_flags & IB_SEND_INLINE &&
3408             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3409              wr->opcode == IB_WR_RDMA_WRITE)) {
3410                 u8 flags = 0;
3411
3412                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3413                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3414                                                    bad_wr, &rwqe->flags, flags);
3415         }
3416
3417         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3418 }
3419
3420 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3421                                      struct qedr_qp *qp,
3422                                      struct rdma_sq_send_wqe_1st *swqe,
3423                                      struct rdma_sq_send_wqe_2st *swqe2,
3424                                      const struct ib_send_wr *wr,
3425                                      const struct ib_send_wr **bad_wr)
3426 {
3427         memset(swqe2, 0, sizeof(*swqe2));
3428         if (wr->send_flags & IB_SEND_INLINE) {
3429                 u8 flags = 0;
3430
3431                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3432                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3433                                                    bad_wr, &swqe->flags, flags);
3434         }
3435
3436         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3437 }
3438
3439 static int qedr_prepare_reg(struct qedr_qp *qp,
3440                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3441                             const struct ib_reg_wr *wr)
3442 {
3443         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3444         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3445
3446         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3447         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3448         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3449         fwqe1->l_key = wr->key;
3450
3451         fwqe2->access_ctrl = 0;
3452
3453         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3454                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3455         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3456                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3457         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3458                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3459         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3460         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3461                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3462         fwqe2->fmr_ctrl = 0;
3463
3464         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3465                    ilog2(mr->ibmr.page_size) - 12);
3466
3467         fwqe2->length_hi = 0;
3468         fwqe2->length_lo = mr->ibmr.length;
3469         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3470         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3471
3472         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3473
3474         return 0;
3475 }
3476
3477 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3478 {
3479         switch (opcode) {
3480         case IB_WR_RDMA_WRITE:
3481         case IB_WR_RDMA_WRITE_WITH_IMM:
3482                 return IB_WC_RDMA_WRITE;
3483         case IB_WR_SEND_WITH_IMM:
3484         case IB_WR_SEND:
3485         case IB_WR_SEND_WITH_INV:
3486                 return IB_WC_SEND;
3487         case IB_WR_RDMA_READ:
3488         case IB_WR_RDMA_READ_WITH_INV:
3489                 return IB_WC_RDMA_READ;
3490         case IB_WR_ATOMIC_CMP_AND_SWP:
3491                 return IB_WC_COMP_SWAP;
3492         case IB_WR_ATOMIC_FETCH_AND_ADD:
3493                 return IB_WC_FETCH_ADD;
3494         case IB_WR_REG_MR:
3495                 return IB_WC_REG_MR;
3496         case IB_WR_LOCAL_INV:
3497                 return IB_WC_LOCAL_INV;
3498         default:
3499                 return IB_WC_SEND;
3500         }
3501 }
3502
3503 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3504                                       const struct ib_send_wr *wr)
3505 {
3506         int wq_is_full, err_wr, pbl_is_full;
3507         struct qedr_dev *dev = qp->dev;
3508
3509         /* prevent SQ overflow and/or processing of a bad WR */
3510         err_wr = wr->num_sge > qp->sq.max_sges;
3511         wq_is_full = qedr_wq_is_full(&qp->sq);
3512         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3513                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3514         if (wq_is_full || err_wr || pbl_is_full) {
3515                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3516                         DP_ERR(dev,
3517                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3518                                qp);
3519                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3520                 }
3521
3522                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3523                         DP_ERR(dev,
3524                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3525                                qp);
3526                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3527                 }
3528
3529                 if (pbl_is_full &&
3530                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3531                         DP_ERR(dev,
3532                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3533                                qp);
3534                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3535                 }
3536                 return false;
3537         }
3538         return true;
3539 }
3540
3541 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3542                             const struct ib_send_wr **bad_wr)
3543 {
3544         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3545         struct qedr_qp *qp = get_qedr_qp(ibqp);
3546         struct rdma_sq_atomic_wqe_1st *awqe1;
3547         struct rdma_sq_atomic_wqe_2nd *awqe2;
3548         struct rdma_sq_atomic_wqe_3rd *awqe3;
3549         struct rdma_sq_send_wqe_2st *swqe2;
3550         struct rdma_sq_local_inv_wqe *iwqe;
3551         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3552         struct rdma_sq_send_wqe_1st *swqe;
3553         struct rdma_sq_rdma_wqe_1st *rwqe;
3554         struct rdma_sq_fmr_wqe_1st *fwqe1;
3555         struct rdma_sq_common_wqe *wqe;
3556         u32 length;
3557         int rc = 0;
3558         bool comp;
3559
3560         if (!qedr_can_post_send(qp, wr)) {
3561                 *bad_wr = wr;
3562                 return -ENOMEM;
3563         }
3564
3565         wqe = qed_chain_produce(&qp->sq.pbl);
3566         qp->wqe_wr_id[qp->sq.prod].signaled =
3567                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3568
3569         wqe->flags = 0;
3570         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3571                    !!(wr->send_flags & IB_SEND_SOLICITED));
3572         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3573         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3574         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3575                    !!(wr->send_flags & IB_SEND_FENCE));
3576         wqe->prev_wqe_size = qp->prev_wqe_size;
3577
3578         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3579
3580         switch (wr->opcode) {
3581         case IB_WR_SEND_WITH_IMM:
3582                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3583                         rc = -EINVAL;
3584                         *bad_wr = wr;
3585                         break;
3586                 }
3587                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3588                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3589                 swqe->wqe_size = 2;
3590                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3591
3592                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3593                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3594                                                    wr, bad_wr);
3595                 swqe->length = cpu_to_le32(length);
3596                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3597                 qp->prev_wqe_size = swqe->wqe_size;
3598                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3599                 break;
3600         case IB_WR_SEND:
3601                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3602                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3603
3604                 swqe->wqe_size = 2;
3605                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3606                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3607                                                    wr, bad_wr);
3608                 swqe->length = cpu_to_le32(length);
3609                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3610                 qp->prev_wqe_size = swqe->wqe_size;
3611                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3612                 break;
3613         case IB_WR_SEND_WITH_INV:
3614                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3615                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3616                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3617                 swqe->wqe_size = 2;
3618                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3619                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3620                                                    wr, bad_wr);
3621                 swqe->length = cpu_to_le32(length);
3622                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3623                 qp->prev_wqe_size = swqe->wqe_size;
3624                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3625                 break;
3626
3627         case IB_WR_RDMA_WRITE_WITH_IMM:
3628                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3629                         rc = -EINVAL;
3630                         *bad_wr = wr;
3631                         break;
3632                 }
3633                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3634                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3635
3636                 rwqe->wqe_size = 2;
3637                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3638                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3639                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3640                                                    wr, bad_wr);
3641                 rwqe->length = cpu_to_le32(length);
3642                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3643                 qp->prev_wqe_size = rwqe->wqe_size;
3644                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3645                 break;
3646         case IB_WR_RDMA_WRITE:
3647                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3648                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3649
3650                 rwqe->wqe_size = 2;
3651                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3652                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3653                                                    wr, bad_wr);
3654                 rwqe->length = cpu_to_le32(length);
3655                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3656                 qp->prev_wqe_size = rwqe->wqe_size;
3657                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3658                 break;
3659         case IB_WR_RDMA_READ_WITH_INV:
3660                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3661                 fallthrough;    /* same is identical to RDMA READ */
3662
3663         case IB_WR_RDMA_READ:
3664                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3665                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3666
3667                 rwqe->wqe_size = 2;
3668                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3669                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3670                                                    wr, bad_wr);
3671                 rwqe->length = cpu_to_le32(length);
3672                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3673                 qp->prev_wqe_size = rwqe->wqe_size;
3674                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3675                 break;
3676
3677         case IB_WR_ATOMIC_CMP_AND_SWP:
3678         case IB_WR_ATOMIC_FETCH_AND_ADD:
3679                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3680                 awqe1->wqe_size = 4;
3681
3682                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3683                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3684                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3685
3686                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3687
3688                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3689                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3690                         DMA_REGPAIR_LE(awqe3->swap_data,
3691                                        atomic_wr(wr)->compare_add);
3692                 } else {
3693                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3694                         DMA_REGPAIR_LE(awqe3->swap_data,
3695                                        atomic_wr(wr)->swap);
3696                         DMA_REGPAIR_LE(awqe3->cmp_data,
3697                                        atomic_wr(wr)->compare_add);
3698                 }
3699
3700                 qedr_prepare_sq_sges(qp, NULL, wr);
3701
3702                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3703                 qp->prev_wqe_size = awqe1->wqe_size;
3704                 break;
3705
3706         case IB_WR_LOCAL_INV:
3707                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3708                 iwqe->wqe_size = 1;
3709
3710                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3711                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3712                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3713                 qp->prev_wqe_size = iwqe->wqe_size;
3714                 break;
3715         case IB_WR_REG_MR:
3716                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3717                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3718                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3719                 fwqe1->wqe_size = 2;
3720
3721                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3722                 if (rc) {
3723                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3724                         *bad_wr = wr;
3725                         break;
3726                 }
3727
3728                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3729                 qp->prev_wqe_size = fwqe1->wqe_size;
3730                 break;
3731         default:
3732                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3733                 rc = -EINVAL;
3734                 *bad_wr = wr;
3735                 break;
3736         }
3737
3738         if (*bad_wr) {
3739                 u16 value;
3740
3741                 /* Restore prod to its position before
3742                  * this WR was processed
3743                  */
3744                 value = le16_to_cpu(qp->sq.db_data.data.value);
3745                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3746
3747                 /* Restore prev_wqe_size */
3748                 qp->prev_wqe_size = wqe->prev_wqe_size;
3749                 rc = -EINVAL;
3750                 DP_ERR(dev, "POST SEND FAILED\n");
3751         }
3752
3753         return rc;
3754 }
3755
3756 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3757                    const struct ib_send_wr **bad_wr)
3758 {
3759         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3760         struct qedr_qp *qp = get_qedr_qp(ibqp);
3761         unsigned long flags;
3762         int rc = 0;
3763
3764         *bad_wr = NULL;
3765
3766         if (qp->qp_type == IB_QPT_GSI)
3767                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3768
3769         spin_lock_irqsave(&qp->q_lock, flags);
3770
3771         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3772                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3773                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3774                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3775                         spin_unlock_irqrestore(&qp->q_lock, flags);
3776                         *bad_wr = wr;
3777                         DP_DEBUG(dev, QEDR_MSG_CQ,
3778                                  "QP in wrong state! QP icid=0x%x state %d\n",
3779                                  qp->icid, qp->state);
3780                         return -EINVAL;
3781                 }
3782         }
3783
3784         while (wr) {
3785                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3786                 if (rc)
3787                         break;
3788
3789                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3790
3791                 qedr_inc_sw_prod(&qp->sq);
3792
3793                 qp->sq.db_data.data.value++;
3794
3795                 wr = wr->next;
3796         }
3797
3798         /* Trigger doorbell
3799          * If there was a failure in the first WR then it will be triggered in
3800          * vane. However this is not harmful (as long as the producer value is
3801          * unchanged). For performance reasons we avoid checking for this
3802          * redundant doorbell.
3803          *
3804          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3805          * soon as we give the doorbell, we could get a completion
3806          * for this wr, therefore we need to make sure that the
3807          * memory is updated before giving the doorbell.
3808          * During qedr_poll_cq, rmb is called before accessing the
3809          * cqe. This covers for the smp_rmb as well.
3810          */
3811         smp_wmb();
3812         writel(qp->sq.db_data.raw, qp->sq.db);
3813
3814         spin_unlock_irqrestore(&qp->q_lock, flags);
3815
3816         return rc;
3817 }
3818
3819 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3820 {
3821         u32 used;
3822
3823         /* Calculate number of elements used based on producer
3824          * count and consumer count and subtract it from max
3825          * work request supported so that we get elements left.
3826          */
3827         used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3828
3829         return hw_srq->max_wr - used;
3830 }
3831
3832 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3833                        const struct ib_recv_wr **bad_wr)
3834 {
3835         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3836         struct qedr_srq_hwq_info *hw_srq;
3837         struct qedr_dev *dev = srq->dev;
3838         struct qed_chain *pbl;
3839         unsigned long flags;
3840         int status = 0;
3841         u32 num_sge;
3842
3843         spin_lock_irqsave(&srq->lock, flags);
3844
3845         hw_srq = &srq->hw_srq;
3846         pbl = &srq->hw_srq.pbl;
3847         while (wr) {
3848                 struct rdma_srq_wqe_header *hdr;
3849                 int i;
3850
3851                 if (!qedr_srq_elem_left(hw_srq) ||
3852                     wr->num_sge > srq->hw_srq.max_sges) {
3853                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3854                                hw_srq->wr_prod_cnt,
3855                                atomic_read(&hw_srq->wr_cons_cnt),
3856                                wr->num_sge, srq->hw_srq.max_sges);
3857                         status = -ENOMEM;
3858                         *bad_wr = wr;
3859                         break;
3860                 }
3861
3862                 hdr = qed_chain_produce(pbl);
3863                 num_sge = wr->num_sge;
3864                 /* Set number of sge and work request id in header */
3865                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3866
3867                 srq->hw_srq.wr_prod_cnt++;
3868                 hw_srq->wqe_prod++;
3869                 hw_srq->sge_prod++;
3870
3871                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3872                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3873                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3874
3875                 for (i = 0; i < wr->num_sge; i++) {
3876                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3877
3878                         /* Set SGE length, lkey and address */
3879                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3880                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3881
3882                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3883                                  "[%d]: len %d key %x addr %x:%x\n",
3884                                  i, srq_sge->length, srq_sge->l_key,
3885                                  srq_sge->addr.hi, srq_sge->addr.lo);
3886                         hw_srq->sge_prod++;
3887                 }
3888
3889                 /* Update WQE and SGE information before
3890                  * updating producer.
3891                  */
3892                 dma_wmb();
3893
3894                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3895                  * in first 4 bytes and need to update WQE producer in
3896                  * next 4 bytes.
3897                  */
3898                 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3899                 /* Make sure sge producer is updated first */
3900                 dma_wmb();
3901                 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3902
3903                 wr = wr->next;
3904         }
3905
3906         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3907                  qed_chain_get_elem_left(pbl));
3908         spin_unlock_irqrestore(&srq->lock, flags);
3909
3910         return status;
3911 }
3912
3913 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3914                    const struct ib_recv_wr **bad_wr)
3915 {
3916         struct qedr_qp *qp = get_qedr_qp(ibqp);
3917         struct qedr_dev *dev = qp->dev;
3918         unsigned long flags;
3919         int status = 0;
3920
3921         if (qp->qp_type == IB_QPT_GSI)
3922                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3923
3924         spin_lock_irqsave(&qp->q_lock, flags);
3925
3926         while (wr) {
3927                 int i;
3928
3929                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3930                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3931                     wr->num_sge > qp->rq.max_sges) {
3932                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3933                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3934                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3935                                qp->rq.max_sges);
3936                         status = -ENOMEM;
3937                         *bad_wr = wr;
3938                         break;
3939                 }
3940                 for (i = 0; i < wr->num_sge; i++) {
3941                         u32 flags = 0;
3942                         struct rdma_rq_sge *rqe =
3943                             qed_chain_produce(&qp->rq.pbl);
3944
3945                         /* First one must include the number
3946                          * of SGE in the list
3947                          */
3948                         if (!i)
3949                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3950                                           wr->num_sge);
3951
3952                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3953                                   wr->sg_list[i].lkey);
3954
3955                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3956                                    wr->sg_list[i].length, flags);
3957                 }
3958
3959                 /* Special case of no sges. FW requires between 1-4 sges...
3960                  * in this case we need to post 1 sge with length zero. this is
3961                  * because rdma write with immediate consumes an RQ.
3962                  */
3963                 if (!wr->num_sge) {
3964                         u32 flags = 0;
3965                         struct rdma_rq_sge *rqe =
3966                             qed_chain_produce(&qp->rq.pbl);
3967
3968                         /* First one must include the number
3969                          * of SGE in the list
3970                          */
3971                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3972                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3973
3974                         RQ_SGE_SET(rqe, 0, 0, flags);
3975                         i = 1;
3976                 }
3977
3978                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3979                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3980
3981                 qedr_inc_sw_prod(&qp->rq);
3982
3983                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3984                  * soon as we give the doorbell, we could get a completion
3985                  * for this wr, therefore we need to make sure that the
3986                  * memory is update before giving the doorbell.
3987                  * During qedr_poll_cq, rmb is called before accessing the
3988                  * cqe. This covers for the smp_rmb as well.
3989                  */
3990                 smp_wmb();
3991
3992                 qp->rq.db_data.data.value++;
3993
3994                 writel(qp->rq.db_data.raw, qp->rq.db);
3995
3996                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3997                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3998                 }
3999
4000                 wr = wr->next;
4001         }
4002
4003         spin_unlock_irqrestore(&qp->q_lock, flags);
4004
4005         return status;
4006 }
4007
4008 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
4009 {
4010         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4011
4012         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4013                 cq->pbl_toggle;
4014 }
4015
4016 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4017 {
4018         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4019         struct qedr_qp *qp;
4020
4021         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4022                                                    resp_cqe->qp_handle.lo,
4023                                                    u64);
4024         return qp;
4025 }
4026
4027 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4028 {
4029         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4030
4031         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4032 }
4033
4034 /* Return latest CQE (needs processing) */
4035 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4036 {
4037         return cq->latest_cqe;
4038 }
4039
4040 /* In fmr we need to increase the number of fmr completed counter for the fmr
4041  * algorithm determining whether we can free a pbl or not.
4042  * we need to perform this whether the work request was signaled or not. for
4043  * this purpose we call this function from the condition that checks if a wr
4044  * should be skipped, to make sure we don't miss it ( possibly this fmr
4045  * operation was not signalted)
4046  */
4047 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4048 {
4049         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4050                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4051 }
4052
4053 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4054                        struct qedr_cq *cq, int num_entries,
4055                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4056                        int force)
4057 {
4058         u16 cnt = 0;
4059
4060         while (num_entries && qp->sq.wqe_cons != hw_cons) {
4061                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4062                         qedr_chk_if_fmr(qp);
4063                         /* skip WC */
4064                         goto next_cqe;
4065                 }
4066
4067                 /* fill WC */
4068                 wc->status = status;
4069                 wc->vendor_err = 0;
4070                 wc->wc_flags = 0;
4071                 wc->src_qp = qp->id;
4072                 wc->qp = &qp->ibqp;
4073
4074                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4075                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4076
4077                 switch (wc->opcode) {
4078                 case IB_WC_RDMA_WRITE:
4079                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4080                         break;
4081                 case IB_WC_COMP_SWAP:
4082                 case IB_WC_FETCH_ADD:
4083                         wc->byte_len = 8;
4084                         break;
4085                 case IB_WC_REG_MR:
4086                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4087                         break;
4088                 case IB_WC_RDMA_READ:
4089                 case IB_WC_SEND:
4090                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4091                         break;
4092                 default:
4093                         break;
4094                 }
4095
4096                 num_entries--;
4097                 wc++;
4098                 cnt++;
4099 next_cqe:
4100                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4101                         qed_chain_consume(&qp->sq.pbl);
4102                 qedr_inc_sw_cons(&qp->sq);
4103         }
4104
4105         return cnt;
4106 }
4107
4108 static int qedr_poll_cq_req(struct qedr_dev *dev,
4109                             struct qedr_qp *qp, struct qedr_cq *cq,
4110                             int num_entries, struct ib_wc *wc,
4111                             struct rdma_cqe_requester *req)
4112 {
4113         int cnt = 0;
4114
4115         switch (req->status) {
4116         case RDMA_CQE_REQ_STS_OK:
4117                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4118                                   IB_WC_SUCCESS, 0);
4119                 break;
4120         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4121                 if (qp->state != QED_ROCE_QP_STATE_ERR)
4122                         DP_DEBUG(dev, QEDR_MSG_CQ,
4123                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4124                                  cq->icid, qp->icid);
4125                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4126                                   IB_WC_WR_FLUSH_ERR, 1);
4127                 break;
4128         default:
4129                 /* process all WQE before the cosumer */
4130                 qp->state = QED_ROCE_QP_STATE_ERR;
4131                 cnt = process_req(dev, qp, cq, num_entries, wc,
4132                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
4133                 wc += cnt;
4134                 /* if we have extra WC fill it with actual error info */
4135                 if (cnt < num_entries) {
4136                         enum ib_wc_status wc_status;
4137
4138                         switch (req->status) {
4139                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4140                                 DP_ERR(dev,
4141                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4142                                        cq->icid, qp->icid);
4143                                 wc_status = IB_WC_BAD_RESP_ERR;
4144                                 break;
4145                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4146                                 DP_ERR(dev,
4147                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4148                                        cq->icid, qp->icid);
4149                                 wc_status = IB_WC_LOC_LEN_ERR;
4150                                 break;
4151                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4152                                 DP_ERR(dev,
4153                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4154                                        cq->icid, qp->icid);
4155                                 wc_status = IB_WC_LOC_QP_OP_ERR;
4156                                 break;
4157                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4158                                 DP_ERR(dev,
4159                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4160                                        cq->icid, qp->icid);
4161                                 wc_status = IB_WC_LOC_PROT_ERR;
4162                                 break;
4163                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4164                                 DP_ERR(dev,
4165                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4166                                        cq->icid, qp->icid);
4167                                 wc_status = IB_WC_MW_BIND_ERR;
4168                                 break;
4169                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4170                                 DP_ERR(dev,
4171                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4172                                        cq->icid, qp->icid);
4173                                 wc_status = IB_WC_REM_INV_REQ_ERR;
4174                                 break;
4175                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4176                                 DP_ERR(dev,
4177                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4178                                        cq->icid, qp->icid);
4179                                 wc_status = IB_WC_REM_ACCESS_ERR;
4180                                 break;
4181                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4182                                 DP_ERR(dev,
4183                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4184                                        cq->icid, qp->icid);
4185                                 wc_status = IB_WC_REM_OP_ERR;
4186                                 break;
4187                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4188                                 DP_ERR(dev,
4189                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4190                                        cq->icid, qp->icid);
4191                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4192                                 break;
4193                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4194                                 DP_ERR(dev,
4195                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4196                                        cq->icid, qp->icid);
4197                                 wc_status = IB_WC_RETRY_EXC_ERR;
4198                                 break;
4199                         default:
4200                                 DP_ERR(dev,
4201                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4202                                        cq->icid, qp->icid);
4203                                 wc_status = IB_WC_GENERAL_ERR;
4204                         }
4205                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4206                                            wc_status, 1);
4207                 }
4208         }
4209
4210         return cnt;
4211 }
4212
4213 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4214 {
4215         switch (status) {
4216         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4217                 return IB_WC_LOC_ACCESS_ERR;
4218         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4219                 return IB_WC_LOC_LEN_ERR;
4220         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4221                 return IB_WC_LOC_QP_OP_ERR;
4222         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4223                 return IB_WC_LOC_PROT_ERR;
4224         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4225                 return IB_WC_MW_BIND_ERR;
4226         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4227                 return IB_WC_REM_INV_RD_REQ_ERR;
4228         case RDMA_CQE_RESP_STS_OK:
4229                 return IB_WC_SUCCESS;
4230         default:
4231                 return IB_WC_GENERAL_ERR;
4232         }
4233 }
4234
4235 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4236                                           struct ib_wc *wc)
4237 {
4238         wc->status = IB_WC_SUCCESS;
4239         wc->byte_len = le32_to_cpu(resp->length);
4240
4241         if (resp->flags & QEDR_RESP_IMM) {
4242                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4243                 wc->wc_flags |= IB_WC_WITH_IMM;
4244
4245                 if (resp->flags & QEDR_RESP_RDMA)
4246                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4247
4248                 if (resp->flags & QEDR_RESP_INV)
4249                         return -EINVAL;
4250
4251         } else if (resp->flags & QEDR_RESP_INV) {
4252                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4253                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4254
4255                 if (resp->flags & QEDR_RESP_RDMA)
4256                         return -EINVAL;
4257
4258         } else if (resp->flags & QEDR_RESP_RDMA) {
4259                 return -EINVAL;
4260         }
4261
4262         return 0;
4263 }
4264
4265 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4266                                struct qedr_cq *cq, struct ib_wc *wc,
4267                                struct rdma_cqe_responder *resp, u64 wr_id)
4268 {
4269         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4270         wc->opcode = IB_WC_RECV;
4271         wc->wc_flags = 0;
4272
4273         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4274                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4275                         DP_ERR(dev,
4276                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4277                                cq, cq->icid, resp->flags);
4278
4279         } else {
4280                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4281                 if (wc->status == IB_WC_GENERAL_ERR)
4282                         DP_ERR(dev,
4283                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4284                                cq, cq->icid, resp->status);
4285         }
4286
4287         /* Fill the rest of the WC */
4288         wc->vendor_err = 0;
4289         wc->src_qp = qp->id;
4290         wc->qp = &qp->ibqp;
4291         wc->wr_id = wr_id;
4292 }
4293
4294 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4295                                 struct qedr_cq *cq, struct ib_wc *wc,
4296                                 struct rdma_cqe_responder *resp)
4297 {
4298         struct qedr_srq *srq = qp->srq;
4299         u64 wr_id;
4300
4301         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4302                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4303
4304         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4305                 wc->status = IB_WC_WR_FLUSH_ERR;
4306                 wc->vendor_err = 0;
4307                 wc->wr_id = wr_id;
4308                 wc->byte_len = 0;
4309                 wc->src_qp = qp->id;
4310                 wc->qp = &qp->ibqp;
4311                 wc->wr_id = wr_id;
4312         } else {
4313                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4314         }
4315         atomic_inc(&srq->hw_srq.wr_cons_cnt);
4316
4317         return 1;
4318 }
4319 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4320                             struct qedr_cq *cq, struct ib_wc *wc,
4321                             struct rdma_cqe_responder *resp)
4322 {
4323         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4324
4325         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4326
4327         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4328                 qed_chain_consume(&qp->rq.pbl);
4329         qedr_inc_sw_cons(&qp->rq);
4330
4331         return 1;
4332 }
4333
4334 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4335                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4336 {
4337         u16 cnt = 0;
4338
4339         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4340                 /* fill WC */
4341                 wc->status = IB_WC_WR_FLUSH_ERR;
4342                 wc->vendor_err = 0;
4343                 wc->wc_flags = 0;
4344                 wc->src_qp = qp->id;
4345                 wc->byte_len = 0;
4346                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4347                 wc->qp = &qp->ibqp;
4348                 num_entries--;
4349                 wc++;
4350                 cnt++;
4351                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4352                         qed_chain_consume(&qp->rq.pbl);
4353                 qedr_inc_sw_cons(&qp->rq);
4354         }
4355
4356         return cnt;
4357 }
4358
4359 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4360                                  struct rdma_cqe_responder *resp, int *update)
4361 {
4362         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4363                 consume_cqe(cq);
4364                 *update |= 1;
4365         }
4366 }
4367
4368 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4369                                  struct qedr_cq *cq, int num_entries,
4370                                  struct ib_wc *wc,
4371                                  struct rdma_cqe_responder *resp)
4372 {
4373         int cnt;
4374
4375         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4376         consume_cqe(cq);
4377
4378         return cnt;
4379 }
4380
4381 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4382                              struct qedr_cq *cq, int num_entries,
4383                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4384                              int *update)
4385 {
4386         int cnt;
4387
4388         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4389                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4390                                          resp->rq_cons_or_srq_id);
4391                 try_consume_resp_cqe(cq, qp, resp, update);
4392         } else {
4393                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4394                 consume_cqe(cq);
4395                 *update |= 1;
4396         }
4397
4398         return cnt;
4399 }
4400
4401 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4402                                 struct rdma_cqe_requester *req, int *update)
4403 {
4404         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4405                 consume_cqe(cq);
4406                 *update |= 1;
4407         }
4408 }
4409
4410 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4411 {
4412         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4413         struct qedr_cq *cq = get_qedr_cq(ibcq);
4414         union rdma_cqe *cqe;
4415         u32 old_cons, new_cons;
4416         unsigned long flags;
4417         int update = 0;
4418         int done = 0;
4419
4420         if (cq->destroyed) {
4421                 DP_ERR(dev,
4422                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4423                        cq, cq->icid);
4424                 return 0;
4425         }
4426
4427         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4428                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4429
4430         spin_lock_irqsave(&cq->cq_lock, flags);
4431         cqe = cq->latest_cqe;
4432         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4433         while (num_entries && is_valid_cqe(cq, cqe)) {
4434                 struct qedr_qp *qp;
4435                 int cnt = 0;
4436
4437                 /* prevent speculative reads of any field of CQE */
4438                 rmb();
4439
4440                 qp = cqe_get_qp(cqe);
4441                 if (!qp) {
4442                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4443                         break;
4444                 }
4445
4446                 wc->qp = &qp->ibqp;
4447
4448                 switch (cqe_get_type(cqe)) {
4449                 case RDMA_CQE_TYPE_REQUESTER:
4450                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4451                                                &cqe->req);
4452                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4453                         break;
4454                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4455                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4456                                                 &cqe->resp, &update);
4457                         break;
4458                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4459                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4460                                                     wc, &cqe->resp);
4461                         update = 1;
4462                         break;
4463                 case RDMA_CQE_TYPE_INVALID:
4464                 default:
4465                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4466                                cqe_get_type(cqe));
4467                 }
4468                 num_entries -= cnt;
4469                 wc += cnt;
4470                 done += cnt;
4471
4472                 cqe = get_cqe(cq);
4473         }
4474         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4475
4476         cq->cq_cons += new_cons - old_cons;
4477
4478         if (update)
4479                 /* doorbell notifies abount latest VALID entry,
4480                  * but chain already point to the next INVALID one
4481                  */
4482                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4483
4484         spin_unlock_irqrestore(&cq->cq_lock, flags);
4485         return done;
4486 }
4487
4488 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4489                      u32 port_num, const struct ib_wc *in_wc,
4490                      const struct ib_grh *in_grh, const struct ib_mad *in,
4491                      struct ib_mad *out_mad, size_t *out_mad_size,
4492                      u16 *out_mad_pkey_index)
4493 {
4494         return IB_MAD_RESULT_SUCCESS;
4495 }