drivers/infiniband/hw/cxgb3/iwch_provider.c

   1 /*
   2  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/device.h>
  35 #include <linux/netdevice.h>
  36 #include <linux/etherdevice.h>
  37 #include <linux/delay.h>
  38 #include <linux/errno.h>
  39 #include <linux/list.h>
  40 #include <linux/sched/mm.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/ethtool.h>
  43 #include <linux/rtnetlink.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/slab.h>
  46
  47 #include <asm/io.h>
  48 #include <asm/irq.h>
  49 #include <asm/byteorder.h>
  50
  51 #include <rdma/iw_cm.h>
  52 #include <rdma/ib_verbs.h>
  53 #include <rdma/ib_smi.h>
  54 #include <rdma/ib_umem.h>
  55 #include <rdma/ib_user_verbs.h>
  56
  57 #include "cxio_hal.h"
  58 #include "iwch.h"
  59 #include "iwch_provider.h"
  60 #include "iwch_cm.h"
  61 #include <rdma/cxgb3-abi.h>
  62 #include "common.h"
  63
  64 static struct ib_ah *iwch_ah_create(struct ib_pd *pd,
  65                                     struct rdma_ah_attr *ah_attr,
  66                                     struct ib_udata *udata)
  67 {
  68         return ERR_PTR(-ENOSYS);
  69 }
  70
  71 static int iwch_ah_destroy(struct ib_ah *ah)
  72 {
  73         return -ENOSYS;
  74 }
  75
  76 static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
  77 {
  78         return -ENOSYS;
  79 }
  80
  81 static int iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
  82 {
  83         return -ENOSYS;
  84 }
  85
  86 static int iwch_process_mad(struct ib_device *ibdev,
  87                             int mad_flags,
  88                             u8 port_num,
  89                             const struct ib_wc *in_wc,
  90                             const struct ib_grh *in_grh,
  91                             const struct ib_mad_hdr *in_mad,
  92                             size_t in_mad_size,
  93                             struct ib_mad_hdr *out_mad,
  94                             size_t *out_mad_size,
  95                             u16 *out_mad_pkey_index)
  96 {
  97         return -ENOSYS;
  98 }
  99
 100 static int iwch_dealloc_ucontext(struct ib_ucontext *context)
 101 {
 102         struct iwch_dev *rhp = to_iwch_dev(context->device);
 103         struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
 104         struct iwch_mm_entry *mm, *tmp;
 105
 106         pr_debug("%s context %p\n", __func__, context);
 107         list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
 108                 kfree(mm);
 109         cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
 110         kfree(ucontext);
 111         return 0;
 112 }
 113
 114 static struct ib_ucontext *iwch_alloc_ucontext(struct ib_device *ibdev,
 115                                         struct ib_udata *udata)
 116 {
 117         struct iwch_ucontext *context;
 118         struct iwch_dev *rhp = to_iwch_dev(ibdev);
 119
 120         pr_debug("%s ibdev %p\n", __func__, ibdev);
 121         context = kzalloc(sizeof(*context), GFP_KERNEL);
 122         if (!context)
 123                 return ERR_PTR(-ENOMEM);
 124         cxio_init_ucontext(&rhp->rdev, &context->uctx);
 125         INIT_LIST_HEAD(&context->mmaps);
 126         spin_lock_init(&context->mmap_lock);
 127         return &context->ibucontext;
 128 }
 129
 130 static int iwch_destroy_cq(struct ib_cq *ib_cq)
 131 {
 132         struct iwch_cq *chp;
 133
 134         pr_debug("%s ib_cq %p\n", __func__, ib_cq);
 135         chp = to_iwch_cq(ib_cq);
 136
 137         remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
 138         atomic_dec(&chp->refcnt);
 139         wait_event(chp->wait, !atomic_read(&chp->refcnt));
 140
 141         cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 142         kfree(chp);
 143         return 0;
 144 }
 145
 146 static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 147                                     const struct ib_cq_init_attr *attr,
 148                                     struct ib_ucontext *ib_context,
 149                                     struct ib_udata *udata)
 150 {
 151         int entries = attr->cqe;
 152         struct iwch_dev *rhp;
 153         struct iwch_cq *chp;
 154         struct iwch_create_cq_resp uresp;
 155         struct iwch_create_cq_req ureq;
 156         struct iwch_ucontext *ucontext = NULL;
 157         static int warned;
 158         size_t resplen;
 159
 160         pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
 161         if (attr->flags)
 162                 return ERR_PTR(-EINVAL);
 163
 164         rhp = to_iwch_dev(ibdev);
 165         chp = kzalloc(sizeof(*chp), GFP_KERNEL);
 166         if (!chp)
 167                 return ERR_PTR(-ENOMEM);
 168
 169         if (ib_context) {
 170                 ucontext = to_iwch_ucontext(ib_context);
 171                 if (!t3a_device(rhp)) {
 172                         if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
 173                                 kfree(chp);
 174                                 return ERR_PTR(-EFAULT);
 175                         }
 176                         chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;
 177                 }
 178         }
 179
 180         if (t3a_device(rhp)) {
 181
 182                 /*
 183                  * T3A: Add some fluff to handle extra CQEs inserted
 184                  * for various errors.
 185                  * Additional CQE possibilities:
 186                  *      TERMINATE,
 187                  *      incoming RDMA WRITE Failures
 188                  *      incoming RDMA READ REQUEST FAILUREs
 189                  * NOTE: We cannot ensure the CQ won't overflow.
 190                  */
 191                 entries += 16;
 192         }
 193         entries = roundup_pow_of_two(entries);
 194         chp->cq.size_log2 = ilog2(entries);
 195
 196         if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
 197                 kfree(chp);
 198                 return ERR_PTR(-ENOMEM);
 199         }
 200         chp->rhp = rhp;
 201         chp->ibcq.cqe = 1 << chp->cq.size_log2;
 202         spin_lock_init(&chp->lock);
 203         spin_lock_init(&chp->comp_handler_lock);
 204         atomic_set(&chp->refcnt, 1);
 205         init_waitqueue_head(&chp->wait);
 206         if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
 207                 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 208                 kfree(chp);
 209                 return ERR_PTR(-ENOMEM);
 210         }
 211
 212         if (ucontext) {
 213                 struct iwch_mm_entry *mm;
 214
 215                 mm = kmalloc(sizeof *mm, GFP_KERNEL);
 216                 if (!mm) {
 217                         iwch_destroy_cq(&chp->ibcq);
 218                         return ERR_PTR(-ENOMEM);
 219                 }
 220                 uresp.cqid = chp->cq.cqid;
 221                 uresp.size_log2 = chp->cq.size_log2;
 222                 spin_lock(&ucontext->mmap_lock);
 223                 uresp.key = ucontext->key;
 224                 ucontext->key += PAGE_SIZE;
 225                 spin_unlock(&ucontext->mmap_lock);
 226                 mm->key = uresp.key;
 227                 mm->addr = virt_to_phys(chp->cq.queue);
 228                 if (udata->outlen < sizeof uresp) {
 229                         if (!warned++)
 230                                 pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n");
 231                         mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
 232                                              sizeof(struct t3_cqe));
 233                         resplen = sizeof(struct iwch_create_cq_resp_v0);
 234                 } else {
 235                         mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
 236                                              sizeof(struct t3_cqe));
 237                         uresp.memsize = mm->len;
 238                         uresp.reserved = 0;
 239                         resplen = sizeof uresp;
 240                 }
 241                 if (ib_copy_to_udata(udata, &uresp, resplen)) {
 242                         kfree(mm);
 243                         iwch_destroy_cq(&chp->ibcq);
 244                         return ERR_PTR(-EFAULT);
 245                 }
 246                 insert_mmap(ucontext, mm);
 247         }
 248         pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
 249                  chp->cq.cqid, chp, (1 << chp->cq.size_log2),
 250                  (unsigned long long)chp->cq.dma_addr);
 251         return &chp->ibcq;
 252 }
 253
 254 static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
 255 {
 256 #ifdef notyet
 257         struct iwch_cq *chp = to_iwch_cq(cq);
 258         struct t3_cq oldcq, newcq;
 259         int ret;
 260
 261         pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe);
 262
 263         /* We don't downsize... */
 264         if (cqe <= cq->cqe)
 265                 return 0;
 266
 267         /* create new t3_cq with new size */
 268         cqe = roundup_pow_of_two(cqe+1);
 269         newcq.size_log2 = ilog2(cqe);
 270
 271         /* Dont allow resize to less than the current wce count */
 272         if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
 273                 return -ENOMEM;
 274         }
 275
 276         /* Quiesce all QPs using this CQ */
 277         ret = iwch_quiesce_qps(chp);
 278         if (ret) {
 279                 return ret;
 280         }
 281
 282         ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
 283         if (ret) {
 284                 return ret;
 285         }
 286
 287         /* copy CQEs */
 288         memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
 289                                         sizeof(struct t3_cqe));
 290
 291         /* old iwch_qp gets new t3_cq but keeps old cqid */
 292         oldcq = chp->cq;
 293         chp->cq = newcq;
 294         chp->cq.cqid = oldcq.cqid;
 295
 296         /* resize new t3_cq to update the HW context */
 297         ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
 298         if (ret) {
 299                 chp->cq = oldcq;
 300                 return ret;
 301         }
 302         chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
 303
 304         /* destroy old t3_cq */
 305         oldcq.cqid = newcq.cqid;
 306         ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
 307         if (ret) {
 308                 pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret);
 309         }
 310
 311         /* add user hooks here */
 312
 313         /* resume qps */
 314         ret = iwch_resume_qps(chp);
 315         return ret;
 316 #else
 317         return -ENOSYS;
 318 #endif
 319 }
 320
 321 static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 322 {
 323         struct iwch_dev *rhp;
 324         struct iwch_cq *chp;
 325         enum t3_cq_opcode cq_op;
 326         int err;
 327         unsigned long flag;
 328         u32 rptr;
 329
 330         chp = to_iwch_cq(ibcq);
 331         rhp = chp->rhp;
 332         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
 333                 cq_op = CQ_ARM_SE;
 334         else
 335                 cq_op = CQ_ARM_AN;
 336         if (chp->user_rptr_addr) {
 337                 if (get_user(rptr, chp->user_rptr_addr))
 338                         return -EFAULT;
 339                 spin_lock_irqsave(&chp->lock, flag);
 340                 chp->cq.rptr = rptr;
 341         } else
 342                 spin_lock_irqsave(&chp->lock, flag);
 343         pr_debug("%s rptr 0x%x\n", __func__, chp->cq.rptr);
 344         err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
 345         spin_unlock_irqrestore(&chp->lock, flag);
 346         if (err < 0)
 347                 pr_err("Error %d rearming CQID 0x%x\n", err, chp->cq.cqid);
 348         if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
 349                 err = 0;
 350         return err;
 351 }
 352
 353 static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 354 {
 355         int len = vma->vm_end - vma->vm_start;
 356         u32 key = vma->vm_pgoff << PAGE_SHIFT;
 357         struct cxio_rdev *rdev_p;
 358         int ret = 0;
 359         struct iwch_mm_entry *mm;
 360         struct iwch_ucontext *ucontext;
 361         u64 addr;
 362
 363         pr_debug("%s pgoff 0x%lx key 0x%x len %d\n", __func__, vma->vm_pgoff,
 364                  key, len);
 365
 366         if (vma->vm_start & (PAGE_SIZE-1)) {
 367                 return -EINVAL;
 368         }
 369
 370         rdev_p = &(to_iwch_dev(context->device)->rdev);
 371         ucontext = to_iwch_ucontext(context);
 372
 373         mm = remove_mmap(ucontext, key, len);
 374         if (!mm)
 375                 return -EINVAL;
 376         addr = mm->addr;
 377         kfree(mm);
 378
 379         if ((addr >= rdev_p->rnic_info.udbell_physbase) &&
 380             (addr < (rdev_p->rnic_info.udbell_physbase +
 381                        rdev_p->rnic_info.udbell_len))) {
 382
 383                 /*
 384                  * Map T3 DB register.
 385                  */
 386                 if (vma->vm_flags & VM_READ) {
 387                         return -EPERM;
 388                 }
 389
 390                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 391                 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
 392                 vma->vm_flags &= ~VM_MAYREAD;
 393                 ret = io_remap_pfn_range(vma, vma->vm_start,
 394                                          addr >> PAGE_SHIFT,
 395                                          len, vma->vm_page_prot);
 396         } else {
 397
 398                 /*
 399                  * Map WQ or CQ contig dma memory...
 400                  */
 401                 ret = remap_pfn_range(vma, vma->vm_start,
 402                                       addr >> PAGE_SHIFT,
 403                                       len, vma->vm_page_prot);
 404         }
 405
 406         return ret;
 407 }
 408
 409 static int iwch_deallocate_pd(struct ib_pd *pd)
 410 {
 411         struct iwch_dev *rhp;
 412         struct iwch_pd *php;
 413
 414         php = to_iwch_pd(pd);
 415         rhp = php->rhp;
 416         pr_debug("%s ibpd %p pdid 0x%x\n", __func__, pd, php->pdid);
 417         cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
 418         kfree(php);
 419         return 0;
 420 }
 421
 422 static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
 423                                struct ib_ucontext *context,
 424                                struct ib_udata *udata)
 425 {
 426         struct iwch_pd *php;
 427         u32 pdid;
 428         struct iwch_dev *rhp;
 429
 430         pr_debug("%s ibdev %p\n", __func__, ibdev);
 431         rhp = (struct iwch_dev *) ibdev;
 432         pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
 433         if (!pdid)
 434                 return ERR_PTR(-EINVAL);
 435         php = kzalloc(sizeof(*php), GFP_KERNEL);
 436         if (!php) {
 437                 cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
 438                 return ERR_PTR(-ENOMEM);
 439         }
 440         php->pdid = pdid;
 441         php->rhp = rhp;
 442         if (context) {
 443                 if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) {
 444                         iwch_deallocate_pd(&php->ibpd);
 445                         return ERR_PTR(-EFAULT);
 446                 }
 447         }
 448         pr_debug("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);
 449         return &php->ibpd;
 450 }
 451
 452 static int iwch_dereg_mr(struct ib_mr *ib_mr)
 453 {
 454         struct iwch_dev *rhp;
 455         struct iwch_mr *mhp;
 456         u32 mmid;
 457
 458         pr_debug("%s ib_mr %p\n", __func__, ib_mr);
 459
 460         mhp = to_iwch_mr(ib_mr);
 461         kfree(mhp->pages);
 462         rhp = mhp->rhp;
 463         mmid = mhp->attr.stag >> 8;
 464         cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
 465                        mhp->attr.pbl_addr);
 466         iwch_free_pbl(mhp);
 467         remove_handle(rhp, &rhp->mmidr, mmid);
 468         if (mhp->kva)
 469                 kfree((void *) (unsigned long) mhp->kva);
 470         if (mhp->umem)
 471                 ib_umem_release(mhp->umem);
 472         pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
 473         kfree(mhp);
 474         return 0;
 475 }
 476
 477 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
 478 {
 479         const u64 total_size = 0xffffffff;
 480         const u64 mask = (total_size + PAGE_SIZE - 1) & PAGE_MASK;
 481         struct iwch_pd *php = to_iwch_pd(pd);
 482         struct iwch_dev *rhp = php->rhp;
 483         struct iwch_mr *mhp;
 484         __be64 *page_list;
 485         int shift = 26, npages, ret, i;
 486
 487         pr_debug("%s ib_pd %p\n", __func__, pd);
 488
 489         /*
 490          * T3 only supports 32 bits of size.
 491          */
 492         if (sizeof(phys_addr_t) > 4) {
 493                 pr_warn_once("Cannot support dma_mrs on this platform\n");
 494                 return ERR_PTR(-ENOTSUPP);
 495         }
 496
 497         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 498         if (!mhp)
 499                 return ERR_PTR(-ENOMEM);
 500
 501         mhp->rhp = rhp;
 502
 503         npages = (total_size + (1ULL << shift) - 1) >> shift;
 504         if (!npages) {
 505                 ret = -EINVAL;
 506                 goto err;
 507         }
 508
 509         page_list = kmalloc_array(npages, sizeof(u64), GFP_KERNEL);
 510         if (!page_list) {
 511                 ret = -ENOMEM;
 512                 goto err;
 513         }
 514
 515         for (i = 0; i < npages; i++)
 516                 page_list[i] = cpu_to_be64((u64)i << shift);
 517
 518         pr_debug("%s mask 0x%llx shift %d len %lld pbl_size %d\n",
 519                  __func__, mask, shift, total_size, npages);
 520
 521         ret = iwch_alloc_pbl(mhp, npages);
 522         if (ret) {
 523                 kfree(page_list);
 524                 goto err_pbl;
 525         }
 526
 527         ret = iwch_write_pbl(mhp, page_list, npages, 0);
 528         kfree(page_list);
 529         if (ret)
 530                 goto err_pbl;
 531
 532         mhp->attr.pdid = php->pdid;
 533         mhp->attr.zbva = 0;
 534
 535         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 536         mhp->attr.va_fbo = 0;
 537         mhp->attr.page_size = shift - 12;
 538
 539         mhp->attr.len = (u32) total_size;
 540         mhp->attr.pbl_size = npages;
 541         ret = iwch_register_mem(rhp, php, mhp, shift);
 542         if (ret)
 543                 goto err_pbl;
 544
 545         return &mhp->ibmr;
 546
 547 err_pbl:
 548         iwch_free_pbl(mhp);
 549
 550 err:
 551         kfree(mhp);
 552         return ERR_PTR(ret);
 553 }
 554
 555 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 556                                       u64 virt, int acc, struct ib_udata *udata)
 557 {
 558         __be64 *pages;
 559         int shift, n, len;
 560         int i, k, entry;
 561         int err = 0;
 562         struct iwch_dev *rhp;
 563         struct iwch_pd *php;
 564         struct iwch_mr *mhp;
 565         struct iwch_reg_user_mr_resp uresp;
 566         struct scatterlist *sg;
 567         pr_debug("%s ib_pd %p\n", __func__, pd);
 568
 569         php = to_iwch_pd(pd);
 570         rhp = php->rhp;
 571         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 572         if (!mhp)
 573                 return ERR_PTR(-ENOMEM);
 574
 575         mhp->rhp = rhp;
 576
 577         mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
 578         if (IS_ERR(mhp->umem)) {
 579                 err = PTR_ERR(mhp->umem);
 580                 kfree(mhp);
 581                 return ERR_PTR(err);
 582         }
 583
 584         shift = mhp->umem->page_shift;
 585
 586         n = mhp->umem->nmap;
 587
 588         err = iwch_alloc_pbl(mhp, n);
 589         if (err)
 590                 goto err;
 591
 592         pages = (__be64 *) __get_free_page(GFP_KERNEL);
 593         if (!pages) {
 594                 err = -ENOMEM;
 595                 goto err_pbl;
 596         }
 597
 598         i = n = 0;
 599
 600         for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
 601                         len = sg_dma_len(sg) >> shift;
 602                         for (k = 0; k < len; ++k) {
 603                                 pages[i++] = cpu_to_be64(sg_dma_address(sg) +
 604                                                          (k << shift));
 605                                 if (i == PAGE_SIZE / sizeof *pages) {
 606                                         err = iwch_write_pbl(mhp, pages, i, n);
 607                                         if (err)
 608                                                 goto pbl_done;
 609                                         n += i;
 610                                         i = 0;
 611                                 }
 612                         }
 613         }
 614
 615         if (i)
 616                 err = iwch_write_pbl(mhp, pages, i, n);
 617
 618 pbl_done:
 619         free_page((unsigned long) pages);
 620         if (err)
 621                 goto err_pbl;
 622
 623         mhp->attr.pdid = php->pdid;
 624         mhp->attr.zbva = 0;
 625         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 626         mhp->attr.va_fbo = virt;
 627         mhp->attr.page_size = shift - 12;
 628         mhp->attr.len = (u32) length;
 629
 630         err = iwch_register_mem(rhp, php, mhp, shift);
 631         if (err)
 632                 goto err_pbl;
 633
 634         if (udata && !t3a_device(rhp)) {
 635                 uresp.pbl_addr = (mhp->attr.pbl_addr -
 636                                  rhp->rdev.rnic_info.pbl_base) >> 3;
 637                 pr_debug("%s user resp pbl_addr 0x%x\n", __func__,
 638                          uresp.pbl_addr);
 639
 640                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 641                         iwch_dereg_mr(&mhp->ibmr);
 642                         err = -EFAULT;
 643                         goto err;
 644                 }
 645         }
 646
 647         return &mhp->ibmr;
 648
 649 err_pbl:
 650         iwch_free_pbl(mhp);
 651
 652 err:
 653         ib_umem_release(mhp->umem);
 654         kfree(mhp);
 655         return ERR_PTR(err);
 656 }
 657
 658 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 659                                    struct ib_udata *udata)
 660 {
 661         struct iwch_dev *rhp;
 662         struct iwch_pd *php;
 663         struct iwch_mw *mhp;
 664         u32 mmid;
 665         u32 stag = 0;
 666         int ret;
 667
 668         if (type != IB_MW_TYPE_1)
 669                 return ERR_PTR(-EINVAL);
 670
 671         php = to_iwch_pd(pd);
 672         rhp = php->rhp;
 673         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 674         if (!mhp)
 675                 return ERR_PTR(-ENOMEM);
 676         ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
 677         if (ret) {
 678                 kfree(mhp);
 679                 return ERR_PTR(ret);
 680         }
 681         mhp->rhp = rhp;
 682         mhp->attr.pdid = php->pdid;
 683         mhp->attr.type = TPT_MW;
 684         mhp->attr.stag = stag;
 685         mmid = (stag) >> 8;
 686         mhp->ibmw.rkey = stag;
 687         if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
 688                 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 689                 kfree(mhp);
 690                 return ERR_PTR(-ENOMEM);
 691         }
 692         pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 693         return &(mhp->ibmw);
 694 }
 695
 696 static int iwch_dealloc_mw(struct ib_mw *mw)
 697 {
 698         struct iwch_dev *rhp;
 699         struct iwch_mw *mhp;
 700         u32 mmid;
 701
 702         mhp = to_iwch_mw(mw);
 703         rhp = mhp->rhp;
 704         mmid = (mw->rkey) >> 8;
 705         cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 706         remove_handle(rhp, &rhp->mmidr, mmid);
 707         pr_debug("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
 708         kfree(mhp);
 709         return 0;
 710 }
 711
 712 static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd,
 713                                    enum ib_mr_type mr_type,
 714                                    u32 max_num_sg)
 715 {
 716         struct iwch_dev *rhp;
 717         struct iwch_pd *php;
 718         struct iwch_mr *mhp;
 719         u32 mmid;
 720         u32 stag = 0;
 721         int ret = -ENOMEM;
 722
 723         if (mr_type != IB_MR_TYPE_MEM_REG ||
 724             max_num_sg > T3_MAX_FASTREG_DEPTH)
 725                 return ERR_PTR(-EINVAL);
 726
 727         php = to_iwch_pd(pd);
 728         rhp = php->rhp;
 729         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 730         if (!mhp)
 731                 goto err;
 732
 733         mhp->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
 734         if (!mhp->pages)
 735                 goto pl_err;
 736
 737         mhp->rhp = rhp;
 738         ret = iwch_alloc_pbl(mhp, max_num_sg);
 739         if (ret)
 740                 goto err1;
 741         mhp->attr.pbl_size = max_num_sg;
 742         ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,
 743                                  mhp->attr.pbl_size, mhp->attr.pbl_addr);
 744         if (ret)
 745                 goto err2;
 746         mhp->attr.pdid = php->pdid;
 747         mhp->attr.type = TPT_NON_SHARED_MR;
 748         mhp->attr.stag = stag;
 749         mhp->attr.state = 1;
 750         mmid = (stag) >> 8;
 751         mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
 752         ret = insert_handle(rhp, &rhp->mmidr, mhp, mmid);
 753         if (ret)
 754                 goto err3;
 755
 756         pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 757         return &(mhp->ibmr);
 758 err3:
 759         cxio_dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
 760                        mhp->attr.pbl_addr);
 761 err2:
 762         iwch_free_pbl(mhp);
 763 err1:
 764         kfree(mhp->pages);
 765 pl_err:
 766         kfree(mhp);
 767 err:
 768         return ERR_PTR(ret);
 769 }
 770
 771 static int iwch_set_page(struct ib_mr *ibmr, u64 addr)
 772 {
 773         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 774
 775         if (unlikely(mhp->npages == mhp->attr.pbl_size))
 776                 return -ENOMEM;
 777
 778         mhp->pages[mhp->npages++] = addr;
 779
 780         return 0;
 781 }
 782
 783 static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 784                           int sg_nents, unsigned int *sg_offset)
 785 {
 786         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 787
 788         mhp->npages = 0;
 789
 790         return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, iwch_set_page);
 791 }
 792
 793 static int iwch_destroy_qp(struct ib_qp *ib_qp)
 794 {
 795         struct iwch_dev *rhp;
 796         struct iwch_qp *qhp;
 797         struct iwch_qp_attributes attrs;
 798         struct iwch_ucontext *ucontext;
 799
 800         qhp = to_iwch_qp(ib_qp);
 801         rhp = qhp->rhp;
 802
 803         attrs.next_state = IWCH_QP_STATE_ERROR;
 804         iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
 805         wait_event(qhp->wait, !qhp->ep);
 806
 807         remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
 808
 809         atomic_dec(&qhp->refcnt);
 810         wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
 811
 812         ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
 813                                   : NULL;
 814         cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 815                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 816
 817         pr_debug("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__,
 818                  ib_qp, qhp->wq.qpid, qhp);
 819         kfree(qhp);
 820         return 0;
 821 }
 822
 823 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
 824                              struct ib_qp_init_attr *attrs,
 825                              struct ib_udata *udata)
 826 {
 827         struct iwch_dev *rhp;
 828         struct iwch_qp *qhp;
 829         struct iwch_pd *php;
 830         struct iwch_cq *schp;
 831         struct iwch_cq *rchp;
 832         struct iwch_create_qp_resp uresp;
 833         int wqsize, sqsize, rqsize;
 834         struct iwch_ucontext *ucontext;
 835
 836         pr_debug("%s ib_pd %p\n", __func__, pd);
 837         if (attrs->qp_type != IB_QPT_RC)
 838                 return ERR_PTR(-EINVAL);
 839         php = to_iwch_pd(pd);
 840         rhp = php->rhp;
 841         schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
 842         rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
 843         if (!schp || !rchp)
 844                 return ERR_PTR(-EINVAL);
 845
 846         /* The RQT size must be # of entries + 1 rounded up to a power of two */
 847         rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
 848         if (rqsize == attrs->cap.max_recv_wr)
 849                 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
 850
 851         /* T3 doesn't support RQT depth < 16 */
 852         if (rqsize < 16)
 853                 rqsize = 16;
 854
 855         if (rqsize > T3_MAX_RQ_SIZE)
 856                 return ERR_PTR(-EINVAL);
 857
 858         if (attrs->cap.max_inline_data > T3_MAX_INLINE)
 859                 return ERR_PTR(-EINVAL);
 860
 861         /*
 862          * NOTE: The SQ and total WQ sizes don't need to be
 863          * a power of two.  However, all the code assumes
 864          * they are. EG: Q_FREECNT() and friends.
 865          */
 866         sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
 867         wqsize = roundup_pow_of_two(rqsize + sqsize);
 868
 869         /*
 870          * Kernel users need more wq space for fastreg WRs which can take
 871          * 2 WR fragments.
 872          */
 873         ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
 874         if (!ucontext && wqsize < (rqsize + (2 * sqsize)))
 875                 wqsize = roundup_pow_of_two(rqsize +
 876                                 roundup_pow_of_two(attrs->cap.max_send_wr * 2));
 877         pr_debug("%s wqsize %d sqsize %d rqsize %d\n", __func__,
 878                  wqsize, sqsize, rqsize);
 879         qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
 880         if (!qhp)
 881                 return ERR_PTR(-ENOMEM);
 882         qhp->wq.size_log2 = ilog2(wqsize);
 883         qhp->wq.rq_size_log2 = ilog2(rqsize);
 884         qhp->wq.sq_size_log2 = ilog2(sqsize);
 885         if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
 886                            ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
 887                 kfree(qhp);
 888                 return ERR_PTR(-ENOMEM);
 889         }
 890
 891         attrs->cap.max_recv_wr = rqsize - 1;
 892         attrs->cap.max_send_wr = sqsize;
 893         attrs->cap.max_inline_data = T3_MAX_INLINE;
 894
 895         qhp->rhp = rhp;
 896         qhp->attr.pd = php->pdid;
 897         qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
 898         qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
 899         qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
 900         qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
 901         qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
 902         qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
 903         qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
 904         qhp->attr.state = IWCH_QP_STATE_IDLE;
 905         qhp->attr.next_state = IWCH_QP_STATE_IDLE;
 906
 907         /*
 908          * XXX - These don't get passed in from the openib user
 909          * at create time.  The CM sets them via a QP modify.
 910          * Need to fix...  I think the CM should
 911          */
 912         qhp->attr.enable_rdma_read = 1;
 913         qhp->attr.enable_rdma_write = 1;
 914         qhp->attr.enable_bind = 1;
 915         qhp->attr.max_ord = 1;
 916         qhp->attr.max_ird = 1;
 917
 918         spin_lock_init(&qhp->lock);
 919         init_waitqueue_head(&qhp->wait);
 920         atomic_set(&qhp->refcnt, 1);
 921
 922         if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
 923                 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 924                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 925                 kfree(qhp);
 926                 return ERR_PTR(-ENOMEM);
 927         }
 928
 929         if (udata) {
 930
 931                 struct iwch_mm_entry *mm1, *mm2;
 932
 933                 mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
 934                 if (!mm1) {
 935                         iwch_destroy_qp(&qhp->ibqp);
 936                         return ERR_PTR(-ENOMEM);
 937                 }
 938
 939                 mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
 940                 if (!mm2) {
 941                         kfree(mm1);
 942                         iwch_destroy_qp(&qhp->ibqp);
 943                         return ERR_PTR(-ENOMEM);
 944                 }
 945
 946                 uresp.qpid = qhp->wq.qpid;
 947                 uresp.size_log2 = qhp->wq.size_log2;
 948                 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
 949                 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
 950                 spin_lock(&ucontext->mmap_lock);
 951                 uresp.key = ucontext->key;
 952                 ucontext->key += PAGE_SIZE;
 953                 uresp.db_key = ucontext->key;
 954                 ucontext->key += PAGE_SIZE;
 955                 spin_unlock(&ucontext->mmap_lock);
 956                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 957                         kfree(mm1);
 958                         kfree(mm2);
 959                         iwch_destroy_qp(&qhp->ibqp);
 960                         return ERR_PTR(-EFAULT);
 961                 }
 962                 mm1->key = uresp.key;
 963                 mm1->addr = virt_to_phys(qhp->wq.queue);
 964                 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
 965                 insert_mmap(ucontext, mm1);
 966                 mm2->key = uresp.db_key;
 967                 mm2->addr = qhp->wq.udb & PAGE_MASK;
 968                 mm2->len = PAGE_SIZE;
 969                 insert_mmap(ucontext, mm2);
 970         }
 971         qhp->ibqp.qp_num = qhp->wq.qpid;
 972         init_timer(&(qhp->timer));
 973         pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
 974                  __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
 975                  qhp->wq.qpid, qhp, (unsigned long long)qhp->wq.dma_addr,
 976                  1 << qhp->wq.size_log2, qhp->wq.rq_addr);
 977         return &qhp->ibqp;
 978 }
 979
 980 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 981                       int attr_mask, struct ib_udata *udata)
 982 {
 983         struct iwch_dev *rhp;
 984         struct iwch_qp *qhp;
 985         enum iwch_qp_attr_mask mask = 0;
 986         struct iwch_qp_attributes attrs;
 987
 988         pr_debug("%s ib_qp %p\n", __func__, ibqp);
 989
 990         /* iwarp does not support the RTR state */
 991         if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
 992                 attr_mask &= ~IB_QP_STATE;
 993
 994         /* Make sure we still have something left to do */
 995         if (!attr_mask)
 996                 return 0;
 997
 998         memset(&attrs, 0, sizeof attrs);
 999         qhp = to_iwch_qp(ibqp);
1000         rhp = qhp->rhp;
1001
1002         attrs.next_state = iwch_convert_state(attr->qp_state);
1003         attrs.enable_rdma_read = (attr->qp_access_flags &
1004                                IB_ACCESS_REMOTE_READ) ?  1 : 0;
1005         attrs.enable_rdma_write = (attr->qp_access_flags &
1006                                 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1007         attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
1008
1009
1010         mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
1011         mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
1012                         (IWCH_QP_ATTR_ENABLE_RDMA_READ |
1013                          IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
1014                          IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
1015
1016         return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
1017 }
1018
1019 void iwch_qp_add_ref(struct ib_qp *qp)
1020 {
1021         pr_debug("%s ib_qp %p\n", __func__, qp);
1022         atomic_inc(&(to_iwch_qp(qp)->refcnt));
1023 }
1024
1025 void iwch_qp_rem_ref(struct ib_qp *qp)
1026 {
1027         pr_debug("%s ib_qp %p\n", __func__, qp);
1028         if (atomic_dec_and_test(&(to_iwch_qp(qp)->refcnt)))
1029                 wake_up(&(to_iwch_qp(qp)->wait));
1030 }
1031
1032 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
1033 {
1034         pr_debug("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
1035         return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
1036 }
1037
1038
1039 static int iwch_query_pkey(struct ib_device *ibdev,
1040                            u8 port, u16 index, u16 * pkey)
1041 {
1042         pr_debug("%s ibdev %p\n", __func__, ibdev);
1043         *pkey = 0;
1044         return 0;
1045 }
1046
1047 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
1048                           int index, union ib_gid *gid)
1049 {
1050         struct iwch_dev *dev;
1051
1052         pr_debug("%s ibdev %p, port %d, index %d, gid %p\n",
1053                  __func__, ibdev, port, index, gid);
1054         dev = to_iwch_dev(ibdev);
1055         BUG_ON(port == 0 || port > 2);
1056         memset(&(gid->raw[0]), 0, sizeof(gid->raw));
1057         memcpy(&(gid->raw[0]), dev->rdev.port_info.lldevs[port-1]->dev_addr, 6);
1058         return 0;
1059 }
1060
1061 static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)
1062 {
1063         struct ethtool_drvinfo info;
1064         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1065         char *cp, *next;
1066         unsigned fw_maj, fw_min, fw_mic;
1067
1068         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1069
1070         next = info.fw_version + 1;
1071         cp = strsep(&next, ".");
1072         sscanf(cp, "%i", &fw_maj);
1073         cp = strsep(&next, ".");
1074         sscanf(cp, "%i", &fw_min);
1075         cp = strsep(&next, ".");
1076         sscanf(cp, "%i", &fw_mic);
1077
1078         return (((u64)fw_maj & 0xffff) << 32) | ((fw_min & 0xffff) << 16) |
1079                (fw_mic & 0xffff);
1080 }
1081
1082 static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
1083                              struct ib_udata *uhw)
1084 {
1085
1086         struct iwch_dev *dev;
1087
1088         pr_debug("%s ibdev %p\n", __func__, ibdev);
1089
1090         if (uhw->inlen || uhw->outlen)
1091                 return -EINVAL;
1092
1093         dev = to_iwch_dev(ibdev);
1094         memset(props, 0, sizeof *props);
1095         memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1096         props->hw_ver = dev->rdev.t3cdev_p->type;
1097         props->fw_ver = fw_vers_string_to_u64(dev);
1098         props->device_cap_flags = dev->device_cap_flags;
1099         props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1100         props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
1101         props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
1102         props->max_mr_size = dev->attr.max_mr_size;
1103         props->max_qp = dev->attr.max_qps;
1104         props->max_qp_wr = dev->attr.max_wrs;
1105         props->max_sge = dev->attr.max_sge_per_wr;
1106         props->max_sge_rd = 1;
1107         props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1108         props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1109         props->max_cq = dev->attr.max_cqs;
1110         props->max_cqe = dev->attr.max_cqes_per_cq;
1111         props->max_mr = dev->attr.max_mem_regs;
1112         props->max_pd = dev->attr.max_pds;
1113         props->local_ca_ack_delay = 0;
1114         props->max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;
1115
1116         return 0;
1117 }
1118
1119 static int iwch_query_port(struct ib_device *ibdev,
1120                            u8 port, struct ib_port_attr *props)
1121 {
1122         struct iwch_dev *dev;
1123         struct net_device *netdev;
1124         struct in_device *inetdev;
1125
1126         pr_debug("%s ibdev %p\n", __func__, ibdev);
1127
1128         dev = to_iwch_dev(ibdev);
1129         netdev = dev->rdev.port_info.lldevs[port-1];
1130
1131         /* props being zeroed by the caller, avoid zeroing it here */
1132         props->max_mtu = IB_MTU_4096;
1133         props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
1134
1135         if (!netif_carrier_ok(netdev))
1136                 props->state = IB_PORT_DOWN;
1137         else {
1138                 inetdev = in_dev_get(netdev);
1139                 if (inetdev) {
1140                         if (inetdev->ifa_list)
1141                                 props->state = IB_PORT_ACTIVE;
1142                         else
1143                                 props->state = IB_PORT_INIT;
1144                         in_dev_put(inetdev);
1145                 } else
1146                         props->state = IB_PORT_INIT;
1147         }
1148
1149         props->port_cap_flags =
1150             IB_PORT_CM_SUP |
1151             IB_PORT_SNMP_TUNNEL_SUP |
1152             IB_PORT_REINIT_SUP |
1153             IB_PORT_DEVICE_MGMT_SUP |
1154             IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1155         props->gid_tbl_len = 1;
1156         props->pkey_tbl_len = 1;
1157         props->active_width = 2;
1158         props->active_speed = IB_SPEED_DDR;
1159         props->max_msg_sz = -1;
1160
1161         return 0;
1162 }
1163
1164 static ssize_t show_rev(struct device *dev, struct device_attribute *attr,
1165                         char *buf)
1166 {
1167         struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1168                                                  ibdev.dev);
1169         pr_debug("%s dev 0x%p\n", __func__, dev);
1170         return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);
1171 }
1172
1173 static ssize_t show_hca(struct device *dev, struct device_attribute *attr,
1174                         char *buf)
1175 {
1176         struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1177                                                  ibdev.dev);
1178         struct ethtool_drvinfo info;
1179         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1180
1181         pr_debug("%s dev 0x%p\n", __func__, dev);
1182         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1183         return sprintf(buf, "%s\n", info.driver);
1184 }
1185
1186 static ssize_t show_board(struct device *dev, struct device_attribute *attr,
1187                           char *buf)
1188 {
1189         struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1190                                                  ibdev.dev);
1191         pr_debug("%s dev 0x%p\n", __func__, dev);
1192         return sprintf(buf, "%x.%x\n", iwch_dev->rdev.rnic_info.pdev->vendor,
1193                        iwch_dev->rdev.rnic_info.pdev->device);
1194 }
1195
1196 enum counters {
1197         IPINRECEIVES,
1198         IPINHDRERRORS,
1199         IPINADDRERRORS,
1200         IPINUNKNOWNPROTOS,
1201         IPINDISCARDS,
1202         IPINDELIVERS,
1203         IPOUTREQUESTS,
1204         IPOUTDISCARDS,
1205         IPOUTNOROUTES,
1206         IPREASMTIMEOUT,
1207         IPREASMREQDS,
1208         IPREASMOKS,
1209         IPREASMFAILS,
1210         TCPACTIVEOPENS,
1211         TCPPASSIVEOPENS,
1212         TCPATTEMPTFAILS,
1213         TCPESTABRESETS,
1214         TCPCURRESTAB,
1215         TCPINSEGS,
1216         TCPOUTSEGS,
1217         TCPRETRANSSEGS,
1218         TCPINERRS,
1219         TCPOUTRSTS,
1220         TCPRTOMIN,
1221         TCPRTOMAX,
1222         NR_COUNTERS
1223 };
1224
1225 static const char * const names[] = {
1226         [IPINRECEIVES] = "ipInReceives",
1227         [IPINHDRERRORS] = "ipInHdrErrors",
1228         [IPINADDRERRORS] = "ipInAddrErrors",
1229         [IPINUNKNOWNPROTOS] = "ipInUnknownProtos",
1230         [IPINDISCARDS] = "ipInDiscards",
1231         [IPINDELIVERS] = "ipInDelivers",
1232         [IPOUTREQUESTS] = "ipOutRequests",
1233         [IPOUTDISCARDS] = "ipOutDiscards",
1234         [IPOUTNOROUTES] = "ipOutNoRoutes",
1235         [IPREASMTIMEOUT] = "ipReasmTimeout",
1236         [IPREASMREQDS] = "ipReasmReqds",
1237         [IPREASMOKS] = "ipReasmOKs",
1238         [IPREASMFAILS] = "ipReasmFails",
1239         [TCPACTIVEOPENS] = "tcpActiveOpens",
1240         [TCPPASSIVEOPENS] = "tcpPassiveOpens",
1241         [TCPATTEMPTFAILS] = "tcpAttemptFails",
1242         [TCPESTABRESETS] = "tcpEstabResets",
1243         [TCPCURRESTAB] = "tcpCurrEstab",
1244         [TCPINSEGS] = "tcpInSegs",
1245         [TCPOUTSEGS] = "tcpOutSegs",
1246         [TCPRETRANSSEGS] = "tcpRetransSegs",
1247         [TCPINERRS] = "tcpInErrs",
1248         [TCPOUTRSTS] = "tcpOutRsts",
1249         [TCPRTOMIN] = "tcpRtoMin",
1250         [TCPRTOMAX] = "tcpRtoMax",
1251 };
1252
1253 static struct rdma_hw_stats *iwch_alloc_stats(struct ib_device *ibdev,
1254                                               u8 port_num)
1255 {
1256         BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);
1257
1258         /* Our driver only supports device level stats */
1259         if (port_num != 0)
1260                 return NULL;
1261
1262         return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,
1263                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
1264 }
1265
1266 static int iwch_get_mib(struct ib_device *ibdev, struct rdma_hw_stats *stats,
1267                         u8 port, int index)
1268 {
1269         struct iwch_dev *dev;
1270         struct tp_mib_stats m;
1271         int ret;
1272
1273         if (port != 0 || !stats)
1274                 return -ENOSYS;
1275
1276         pr_debug("%s ibdev %p\n", __func__, ibdev);
1277         dev = to_iwch_dev(ibdev);
1278         ret = dev->rdev.t3cdev_p->ctl(dev->rdev.t3cdev_p, RDMA_GET_MIB, &m);
1279         if (ret)
1280                 return -ENOSYS;
1281
1282         stats->value[IPINRECEIVES] = ((u64)m.ipInReceive_hi << 32) +    m.ipInReceive_lo;
1283         stats->value[IPINHDRERRORS] = ((u64)m.ipInHdrErrors_hi << 32) + m.ipInHdrErrors_lo;
1284         stats->value[IPINADDRERRORS] = ((u64)m.ipInAddrErrors_hi << 32) + m.ipInAddrErrors_lo;
1285         stats->value[IPINUNKNOWNPROTOS] = ((u64)m.ipInUnknownProtos_hi << 32) + m.ipInUnknownProtos_lo;
1286         stats->value[IPINDISCARDS] = ((u64)m.ipInDiscards_hi << 32) + m.ipInDiscards_lo;
1287         stats->value[IPINDELIVERS] = ((u64)m.ipInDelivers_hi << 32) + m.ipInDelivers_lo;
1288         stats->value[IPOUTREQUESTS] = ((u64)m.ipOutRequests_hi << 32) + m.ipOutRequests_lo;
1289         stats->value[IPOUTDISCARDS] = ((u64)m.ipOutDiscards_hi << 32) + m.ipOutDiscards_lo;
1290         stats->value[IPOUTNOROUTES] = ((u64)m.ipOutNoRoutes_hi << 32) + m.ipOutNoRoutes_lo;
1291         stats->value[IPREASMTIMEOUT] =  m.ipReasmTimeout;
1292         stats->value[IPREASMREQDS] = m.ipReasmReqds;
1293         stats->value[IPREASMOKS] = m.ipReasmOKs;
1294         stats->value[IPREASMFAILS] = m.ipReasmFails;
1295         stats->value[TCPACTIVEOPENS] =  m.tcpActiveOpens;
1296         stats->value[TCPPASSIVEOPENS] = m.tcpPassiveOpens;
1297         stats->value[TCPATTEMPTFAILS] = m.tcpAttemptFails;
1298         stats->value[TCPESTABRESETS] = m.tcpEstabResets;
1299         stats->value[TCPCURRESTAB] = m.tcpOutRsts;
1300         stats->value[TCPINSEGS] = m.tcpCurrEstab;
1301         stats->value[TCPOUTSEGS] = ((u64)m.tcpInSegs_hi << 32) + m.tcpInSegs_lo;
1302         stats->value[TCPRETRANSSEGS] = ((u64)m.tcpOutSegs_hi << 32) + m.tcpOutSegs_lo;
1303         stats->value[TCPINERRS] = ((u64)m.tcpRetransSeg_hi << 32) + m.tcpRetransSeg_lo,
1304         stats->value[TCPOUTRSTS] = ((u64)m.tcpInErrs_hi << 32) + m.tcpInErrs_lo;
1305         stats->value[TCPRTOMIN] = m.tcpRtoMin;
1306         stats->value[TCPRTOMAX] = m.tcpRtoMax;
1307
1308         return stats->num_counters;
1309 }
1310
1311 static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
1312 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
1313 static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
1314
1315 static struct device_attribute *iwch_class_attributes[] = {
1316         &dev_attr_hw_rev,
1317         &dev_attr_hca_type,
1318         &dev_attr_board_id,
1319 };
1320
1321 static int iwch_port_immutable(struct ib_device *ibdev, u8 port_num,
1322                                struct ib_port_immutable *immutable)
1323 {
1324         struct ib_port_attr attr;
1325         int err;
1326
1327         immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1328
1329         err = ib_query_port(ibdev, port_num, &attr);
1330         if (err)
1331                 return err;
1332
1333         immutable->pkey_tbl_len = attr.pkey_tbl_len;
1334         immutable->gid_tbl_len = attr.gid_tbl_len;
1335
1336         return 0;
1337 }
1338
1339 static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str,
1340                                size_t str_len)
1341 {
1342         struct iwch_dev *iwch_dev = to_iwch_dev(ibdev);
1343         struct ethtool_drvinfo info;
1344         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1345
1346         pr_debug("%s dev 0x%p\n", __func__, iwch_dev);
1347         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1348         snprintf(str, str_len, "%s", info.fw_version);
1349 }
1350
1351 int iwch_register_device(struct iwch_dev *dev)
1352 {
1353         int ret;
1354         int i;
1355
1356         pr_debug("%s iwch_dev %p\n", __func__, dev);
1357         strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
1358         memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1359         memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1360         dev->ibdev.owner = THIS_MODULE;
1361         dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
1362                                 IB_DEVICE_MEM_WINDOW |
1363                                 IB_DEVICE_MEM_MGT_EXTENSIONS;
1364
1365         /* cxgb3 supports STag 0. */
1366         dev->ibdev.local_dma_lkey = 0;
1367
1368         dev->ibdev.uverbs_cmd_mask =
1369             (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1370             (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1371             (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1372             (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1373             (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1374             (1ull << IB_USER_VERBS_CMD_REG_MR) |
1375             (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1376             (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1377             (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1378             (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1379             (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1380             (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1381             (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1382             (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1383             (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1384             (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1385             (1ull << IB_USER_VERBS_CMD_POST_RECV);
1386         dev->ibdev.node_type = RDMA_NODE_RNIC;
1387         BUILD_BUG_ON(sizeof(IWCH_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
1388         memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1389         dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
1390         dev->ibdev.num_comp_vectors = 1;
1391         dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
1392         dev->ibdev.query_device = iwch_query_device;
1393         dev->ibdev.query_port = iwch_query_port;
1394         dev->ibdev.query_pkey = iwch_query_pkey;
1395         dev->ibdev.query_gid = iwch_query_gid;
1396         dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
1397         dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
1398         dev->ibdev.mmap = iwch_mmap;
1399         dev->ibdev.alloc_pd = iwch_allocate_pd;
1400         dev->ibdev.dealloc_pd = iwch_deallocate_pd;
1401         dev->ibdev.create_ah = iwch_ah_create;
1402         dev->ibdev.destroy_ah = iwch_ah_destroy;
1403         dev->ibdev.create_qp = iwch_create_qp;
1404         dev->ibdev.modify_qp = iwch_ib_modify_qp;
1405         dev->ibdev.destroy_qp = iwch_destroy_qp;
1406         dev->ibdev.create_cq = iwch_create_cq;
1407         dev->ibdev.destroy_cq = iwch_destroy_cq;
1408         dev->ibdev.resize_cq = iwch_resize_cq;
1409         dev->ibdev.poll_cq = iwch_poll_cq;
1410         dev->ibdev.get_dma_mr = iwch_get_dma_mr;
1411         dev->ibdev.reg_user_mr = iwch_reg_user_mr;
1412         dev->ibdev.dereg_mr = iwch_dereg_mr;
1413         dev->ibdev.alloc_mw = iwch_alloc_mw;
1414         dev->ibdev.dealloc_mw = iwch_dealloc_mw;
1415         dev->ibdev.alloc_mr = iwch_alloc_mr;
1416         dev->ibdev.map_mr_sg = iwch_map_mr_sg;
1417         dev->ibdev.attach_mcast = iwch_multicast_attach;
1418         dev->ibdev.detach_mcast = iwch_multicast_detach;
1419         dev->ibdev.process_mad = iwch_process_mad;
1420         dev->ibdev.req_notify_cq = iwch_arm_cq;
1421         dev->ibdev.post_send = iwch_post_send;
1422         dev->ibdev.post_recv = iwch_post_receive;
1423         dev->ibdev.alloc_hw_stats = iwch_alloc_stats;
1424         dev->ibdev.get_hw_stats = iwch_get_mib;
1425         dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1426         dev->ibdev.get_port_immutable = iwch_port_immutable;
1427         dev->ibdev.get_dev_fw_str = get_dev_fw_ver_str;
1428
1429         dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
1430         if (!dev->ibdev.iwcm)
1431                 return -ENOMEM;
1432
1433         dev->ibdev.iwcm->connect = iwch_connect;
1434         dev->ibdev.iwcm->accept = iwch_accept_cr;
1435         dev->ibdev.iwcm->reject = iwch_reject_cr;
1436         dev->ibdev.iwcm->create_listen = iwch_create_listen;
1437         dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1438         dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1439         dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1440         dev->ibdev.iwcm->get_qp = iwch_get_qp;
1441         memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev->name,
1442                sizeof(dev->ibdev.iwcm->ifname));
1443
1444         ret = ib_register_device(&dev->ibdev, NULL);
1445         if (ret)
1446                 goto bail1;
1447
1448         for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i) {
1449                 ret = device_create_file(&dev->ibdev.dev,
1450                                          iwch_class_attributes[i]);
1451                 if (ret) {
1452                         goto bail2;
1453                 }
1454         }
1455         return 0;
1456 bail2:
1457         ib_unregister_device(&dev->ibdev);
1458 bail1:
1459         kfree(dev->ibdev.iwcm);
1460         return ret;
1461 }
1462
1463 void iwch_unregister_device(struct iwch_dev *dev)
1464 {
1465         int i;
1466
1467         pr_debug("%s iwch_dev %p\n", __func__, dev);
1468         for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i)
1469                 device_remove_file(&dev->ibdev.dev,
1470                                    iwch_class_attributes[i]);
1471         ib_unregister_device(&dev->ibdev);
1472         kfree(dev->ibdev.iwcm);
1473         return;
1474 }