drivers/infiniband/core/uverbs_cmd.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
   4  * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
   5  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/file.h>
  37 #include <linux/fs.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40
  41 #include <linux/uaccess.h>
  42
  43 #include <rdma/uverbs_types.h>
  44 #include <rdma/uverbs_std_types.h>
  45 #include "rdma_core.h"
  46
  47 #include "uverbs.h"
  48 #include "core_priv.h"
  49
  50 /*
  51  * Copy a response to userspace. If the provided 'resp' is larger than the
  52  * user buffer it is silently truncated. If the user provided a larger buffer
  53  * then the trailing portion is zero filled.
  54  *
  55  * These semantics are intended to support future extension of the output
  56  * structures.
  57  */
  58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,
  59                            size_t resp_len)
  60 {
  61         int ret;
  62
  63         if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
  64                 return uverbs_copy_to_struct_or_zero(
  65                         attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);
  66
  67         if (copy_to_user(attrs->ucore.outbuf, resp,
  68                          min(attrs->ucore.outlen, resp_len)))
  69                 return -EFAULT;
  70
  71         if (resp_len < attrs->ucore.outlen) {
  72                 /*
  73                  * Zero fill any extra memory that user
  74                  * space might have provided.
  75                  */
  76                 ret = clear_user(attrs->ucore.outbuf + resp_len,
  77                                  attrs->ucore.outlen - resp_len);
  78                 if (ret)
  79                         return -EFAULT;
  80         }
  81
  82         return 0;
  83 }
  84
  85 /*
  86  * Copy a request from userspace. If the provided 'req' is larger than the
  87  * user buffer then the user buffer is zero extended into the 'req'. If 'req'
  88  * is smaller than the user buffer then the uncopied bytes in the user buffer
  89  * must be zero.
  90  */
  91 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,
  92                           size_t req_len)
  93 {
  94         if (copy_from_user(req, attrs->ucore.inbuf,
  95                            min(attrs->ucore.inlen, req_len)))
  96                 return -EFAULT;
  97
  98         if (attrs->ucore.inlen < req_len) {
  99                 memset(req + attrs->ucore.inlen, 0,
 100                        req_len - attrs->ucore.inlen);
 101         } else if (attrs->ucore.inlen > req_len) {
 102                 if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
 103                                           attrs->ucore.inlen - req_len))
 104                         return -EOPNOTSUPP;
 105         }
 106         return 0;
 107 }
 108
 109 /*
 110  * Generate the value for the 'response_length' protocol used by write_ex.
 111  * This is the number of bytes the kernel actually wrote. Userspace can use
 112  * this to detect what structure members in the response the kernel
 113  * understood.
 114  */
 115 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
 116                                   size_t resp_len)
 117 {
 118         return min_t(size_t, attrs->ucore.outlen, resp_len);
 119 }
 120
 121 /*
 122  * The iterator version of the request interface is for handlers that need to
 123  * step over a flex array at the end of a command header.
 124  */
 125 struct uverbs_req_iter {
 126         const void __user *cur;
 127         const void __user *end;
 128 };
 129
 130 static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
 131                                 struct uverbs_req_iter *iter,
 132                                 void *req,
 133                                 size_t req_len)
 134 {
 135         if (attrs->ucore.inlen < req_len)
 136                 return -ENOSPC;
 137
 138         if (copy_from_user(req, attrs->ucore.inbuf, req_len))
 139                 return -EFAULT;
 140
 141         iter->cur = attrs->ucore.inbuf + req_len;
 142         iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
 143         return 0;
 144 }
 145
 146 static int uverbs_request_next(struct uverbs_req_iter *iter, void *val,
 147                                size_t len)
 148 {
 149         if (iter->cur + len > iter->end)
 150                 return -ENOSPC;
 151
 152         if (copy_from_user(val, iter->cur, len))
 153                 return -EFAULT;
 154
 155         iter->cur += len;
 156         return 0;
 157 }
 158
 159 static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,
 160                                                   size_t len)
 161 {
 162         const void __user *res = iter->cur;
 163
 164         if (iter->cur + len > iter->end)
 165                 return (void __force __user *)ERR_PTR(-ENOSPC);
 166         iter->cur += len;
 167         return res;
 168 }
 169
 170 static int uverbs_request_finish(struct uverbs_req_iter *iter)
 171 {
 172         if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))
 173                 return -EOPNOTSUPP;
 174         return 0;
 175 }
 176
 177 static struct ib_uverbs_completion_event_file *
 178 _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
 179 {
 180         struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
 181                                                fd, attrs);
 182
 183         if (IS_ERR(uobj))
 184                 return (void *)uobj;
 185
 186         uverbs_uobject_get(uobj);
 187         uobj_put_read(uobj);
 188
 189         return container_of(uobj, struct ib_uverbs_completion_event_file,
 190                             uobj);
 191 }
 192 #define ib_uverbs_lookup_comp_file(_fd, _ufile)                                \
 193         _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
 194
 195 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
 196 {
 197         struct ib_uverbs_file *file = attrs->ufile;
 198         struct ib_uverbs_get_context      cmd;
 199         struct ib_uverbs_get_context_resp resp;
 200         struct ib_ucontext               *ucontext;
 201         struct file                      *filp;
 202         struct ib_rdmacg_object          cg_obj;
 203         struct ib_device *ib_dev;
 204         int ret;
 205
 206         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 207         if (ret)
 208                 return ret;
 209
 210         mutex_lock(&file->ucontext_lock);
 211         ib_dev = srcu_dereference(file->device->ib_dev,
 212                                   &file->device->disassociate_srcu);
 213         if (!ib_dev) {
 214                 ret = -EIO;
 215                 goto err;
 216         }
 217
 218         if (file->ucontext) {
 219                 ret = -EINVAL;
 220                 goto err;
 221         }
 222
 223         ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
 224         if (ret)
 225                 goto err;
 226
 227         ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);
 228         if (!ucontext) {
 229                 ret = -ENOMEM;
 230                 goto err_alloc;
 231         }
 232
 233         attrs->context = ucontext;
 234
 235         ucontext->res.type = RDMA_RESTRACK_CTX;
 236         ucontext->device = ib_dev;
 237         ucontext->cg_obj = cg_obj;
 238         /* ufile is required when some objects are released */
 239         ucontext->ufile = file;
 240
 241         ucontext->closing = false;
 242         ucontext->cleanup_retryable = false;
 243
 244         mutex_init(&ucontext->per_mm_list_lock);
 245         INIT_LIST_HEAD(&ucontext->per_mm_list);
 246
 247         ret = get_unused_fd_flags(O_CLOEXEC);
 248         if (ret < 0)
 249                 goto err_free;
 250         resp.async_fd = ret;
 251
 252         filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
 253         if (IS_ERR(filp)) {
 254                 ret = PTR_ERR(filp);
 255                 goto err_fd;
 256         }
 257
 258         resp.num_comp_vectors = file->device->num_comp_vectors;
 259
 260         ret = uverbs_response(attrs, &resp, sizeof(resp));
 261         if (ret)
 262                 goto err_file;
 263
 264         ret = ib_dev->ops.alloc_ucontext(ucontext, &attrs->driver_udata);
 265         if (ret)
 266                 goto err_file;
 267         if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
 268                 ucontext->invalidate_range = NULL;
 269
 270         rdma_restrack_uadd(&ucontext->res);
 271
 272         fd_install(resp.async_fd, filp);
 273
 274         /*
 275          * Make sure that ib_uverbs_get_ucontext() sees the pointer update
 276          * only after all writes to setup the ucontext have completed
 277          */
 278         smp_store_release(&file->ucontext, ucontext);
 279
 280         mutex_unlock(&file->ucontext_lock);
 281
 282         return 0;
 283
 284 err_file:
 285         ib_uverbs_free_async_event_file(file);
 286         fput(filp);
 287
 288 err_fd:
 289         put_unused_fd(resp.async_fd);
 290
 291 err_free:
 292         kfree(ucontext);
 293
 294 err_alloc:
 295         ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
 296
 297 err:
 298         mutex_unlock(&file->ucontext_lock);
 299         return ret;
 300 }
 301
 302 static void copy_query_dev_fields(struct ib_ucontext *ucontext,
 303                                   struct ib_uverbs_query_device_resp *resp,
 304                                   struct ib_device_attr *attr)
 305 {
 306         struct ib_device *ib_dev = ucontext->device;
 307
 308         resp->fw_ver            = attr->fw_ver;
 309         resp->node_guid         = ib_dev->node_guid;
 310         resp->sys_image_guid    = attr->sys_image_guid;
 311         resp->max_mr_size       = attr->max_mr_size;
 312         resp->page_size_cap     = attr->page_size_cap;
 313         resp->vendor_id         = attr->vendor_id;
 314         resp->vendor_part_id    = attr->vendor_part_id;
 315         resp->hw_ver            = attr->hw_ver;
 316         resp->max_qp            = attr->max_qp;
 317         resp->max_qp_wr         = attr->max_qp_wr;
 318         resp->device_cap_flags  = lower_32_bits(attr->device_cap_flags);
 319         resp->max_sge           = min(attr->max_send_sge, attr->max_recv_sge);
 320         resp->max_sge_rd        = attr->max_sge_rd;
 321         resp->max_cq            = attr->max_cq;
 322         resp->max_cqe           = attr->max_cqe;
 323         resp->max_mr            = attr->max_mr;
 324         resp->max_pd            = attr->max_pd;
 325         resp->max_qp_rd_atom    = attr->max_qp_rd_atom;
 326         resp->max_ee_rd_atom    = attr->max_ee_rd_atom;
 327         resp->max_res_rd_atom   = attr->max_res_rd_atom;
 328         resp->max_qp_init_rd_atom       = attr->max_qp_init_rd_atom;
 329         resp->max_ee_init_rd_atom       = attr->max_ee_init_rd_atom;
 330         resp->atomic_cap                = attr->atomic_cap;
 331         resp->max_ee                    = attr->max_ee;
 332         resp->max_rdd                   = attr->max_rdd;
 333         resp->max_mw                    = attr->max_mw;
 334         resp->max_raw_ipv6_qp           = attr->max_raw_ipv6_qp;
 335         resp->max_raw_ethy_qp           = attr->max_raw_ethy_qp;
 336         resp->max_mcast_grp             = attr->max_mcast_grp;
 337         resp->max_mcast_qp_attach       = attr->max_mcast_qp_attach;
 338         resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
 339         resp->max_ah                    = attr->max_ah;
 340         resp->max_fmr                   = attr->max_fmr;
 341         resp->max_map_per_fmr           = attr->max_map_per_fmr;
 342         resp->max_srq                   = attr->max_srq;
 343         resp->max_srq_wr                = attr->max_srq_wr;
 344         resp->max_srq_sge               = attr->max_srq_sge;
 345         resp->max_pkeys                 = attr->max_pkeys;
 346         resp->local_ca_ack_delay        = attr->local_ca_ack_delay;
 347         resp->phys_port_cnt             = ib_dev->phys_port_cnt;
 348 }
 349
 350 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
 351 {
 352         struct ib_uverbs_query_device      cmd;
 353         struct ib_uverbs_query_device_resp resp;
 354         struct ib_ucontext *ucontext;
 355         int ret;
 356
 357         ucontext = ib_uverbs_get_ucontext(attrs);
 358         if (IS_ERR(ucontext))
 359                 return PTR_ERR(ucontext);
 360
 361         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 362         if (ret)
 363                 return ret;
 364
 365         memset(&resp, 0, sizeof resp);
 366         copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);
 367
 368         return uverbs_response(attrs, &resp, sizeof(resp));
 369 }
 370
 371 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
 372 {
 373         struct ib_uverbs_query_port      cmd;
 374         struct ib_uverbs_query_port_resp resp;
 375         struct ib_port_attr              attr;
 376         int                              ret;
 377         struct ib_ucontext *ucontext;
 378         struct ib_device *ib_dev;
 379
 380         ucontext = ib_uverbs_get_ucontext(attrs);
 381         if (IS_ERR(ucontext))
 382                 return PTR_ERR(ucontext);
 383         ib_dev = ucontext->device;
 384
 385         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 386         if (ret)
 387                 return ret;
 388
 389         ret = ib_query_port(ib_dev, cmd.port_num, &attr);
 390         if (ret)
 391                 return ret;
 392
 393         memset(&resp, 0, sizeof resp);
 394         copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num);
 395
 396         return uverbs_response(attrs, &resp, sizeof(resp));
 397 }
 398
 399 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
 400 {
 401         struct ib_uverbs_alloc_pd      cmd;
 402         struct ib_uverbs_alloc_pd_resp resp;
 403         struct ib_uobject             *uobj;
 404         struct ib_pd                  *pd;
 405         int                            ret;
 406         struct ib_device *ib_dev;
 407
 408         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 409         if (ret)
 410                 return ret;
 411
 412         uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
 413         if (IS_ERR(uobj))
 414                 return PTR_ERR(uobj);
 415
 416         pd = rdma_zalloc_drv_obj(ib_dev, ib_pd);
 417         if (!pd) {
 418                 ret = -ENOMEM;
 419                 goto err;
 420         }
 421
 422         pd->device  = ib_dev;
 423         pd->uobject = uobj;
 424         pd->__internal_mr = NULL;
 425         atomic_set(&pd->usecnt, 0);
 426         pd->res.type = RDMA_RESTRACK_PD;
 427
 428         ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
 429         if (ret)
 430                 goto err_alloc;
 431
 432         uobj->object = pd;
 433         memset(&resp, 0, sizeof resp);
 434         resp.pd_handle = uobj->id;
 435         rdma_restrack_uadd(&pd->res);
 436
 437         ret = uverbs_response(attrs, &resp, sizeof(resp));
 438         if (ret)
 439                 goto err_copy;
 440
 441         return uobj_alloc_commit(uobj, attrs);
 442
 443 err_copy:
 444         ib_dealloc_pd_user(pd, &attrs->driver_udata);
 445         pd = NULL;
 446 err_alloc:
 447         kfree(pd);
 448 err:
 449         uobj_alloc_abort(uobj, attrs);
 450         return ret;
 451 }
 452
 453 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
 454 {
 455         struct ib_uverbs_dealloc_pd cmd;
 456         int ret;
 457
 458         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 459         if (ret)
 460                 return ret;
 461
 462         return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
 463 }
 464
 465 struct xrcd_table_entry {
 466         struct rb_node  node;
 467         struct ib_xrcd *xrcd;
 468         struct inode   *inode;
 469 };
 470
 471 static int xrcd_table_insert(struct ib_uverbs_device *dev,
 472                             struct inode *inode,
 473                             struct ib_xrcd *xrcd)
 474 {
 475         struct xrcd_table_entry *entry, *scan;
 476         struct rb_node **p = &dev->xrcd_tree.rb_node;
 477         struct rb_node *parent = NULL;
 478
 479         entry = kmalloc(sizeof *entry, GFP_KERNEL);
 480         if (!entry)
 481                 return -ENOMEM;
 482
 483         entry->xrcd  = xrcd;
 484         entry->inode = inode;
 485
 486         while (*p) {
 487                 parent = *p;
 488                 scan = rb_entry(parent, struct xrcd_table_entry, node);
 489
 490                 if (inode < scan->inode) {
 491                         p = &(*p)->rb_left;
 492                 } else if (inode > scan->inode) {
 493                         p = &(*p)->rb_right;
 494                 } else {
 495                         kfree(entry);
 496                         return -EEXIST;
 497                 }
 498         }
 499
 500         rb_link_node(&entry->node, parent, p);
 501         rb_insert_color(&entry->node, &dev->xrcd_tree);
 502         igrab(inode);
 503         return 0;
 504 }
 505
 506 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
 507                                                   struct inode *inode)
 508 {
 509         struct xrcd_table_entry *entry;
 510         struct rb_node *p = dev->xrcd_tree.rb_node;
 511
 512         while (p) {
 513                 entry = rb_entry(p, struct xrcd_table_entry, node);
 514
 515                 if (inode < entry->inode)
 516                         p = p->rb_left;
 517                 else if (inode > entry->inode)
 518                         p = p->rb_right;
 519                 else
 520                         return entry;
 521         }
 522
 523         return NULL;
 524 }
 525
 526 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
 527 {
 528         struct xrcd_table_entry *entry;
 529
 530         entry = xrcd_table_search(dev, inode);
 531         if (!entry)
 532                 return NULL;
 533
 534         return entry->xrcd;
 535 }
 536
 537 static void xrcd_table_delete(struct ib_uverbs_device *dev,
 538                               struct inode *inode)
 539 {
 540         struct xrcd_table_entry *entry;
 541
 542         entry = xrcd_table_search(dev, inode);
 543         if (entry) {
 544                 iput(inode);
 545                 rb_erase(&entry->node, &dev->xrcd_tree);
 546                 kfree(entry);
 547         }
 548 }
 549
 550 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
 551 {
 552         struct ib_uverbs_device *ibudev = attrs->ufile->device;
 553         struct ib_uverbs_open_xrcd      cmd;
 554         struct ib_uverbs_open_xrcd_resp resp;
 555         struct ib_uxrcd_object         *obj;
 556         struct ib_xrcd                 *xrcd = NULL;
 557         struct fd                       f = {NULL, 0};
 558         struct inode                   *inode = NULL;
 559         int                             ret = 0;
 560         int                             new_xrcd = 0;
 561         struct ib_device *ib_dev;
 562
 563         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 564         if (ret)
 565                 return ret;
 566
 567         mutex_lock(&ibudev->xrcd_tree_mutex);
 568
 569         if (cmd.fd != -1) {
 570                 /* search for file descriptor */
 571                 f = fdget(cmd.fd);
 572                 if (!f.file) {
 573                         ret = -EBADF;
 574                         goto err_tree_mutex_unlock;
 575                 }
 576
 577                 inode = file_inode(f.file);
 578                 xrcd = find_xrcd(ibudev, inode);
 579                 if (!xrcd && !(cmd.oflags & O_CREAT)) {
 580                         /* no file descriptor. Need CREATE flag */
 581                         ret = -EAGAIN;
 582                         goto err_tree_mutex_unlock;
 583                 }
 584
 585                 if (xrcd && cmd.oflags & O_EXCL) {
 586                         ret = -EINVAL;
 587                         goto err_tree_mutex_unlock;
 588                 }
 589         }
 590
 591         obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
 592                                                    &ib_dev);
 593         if (IS_ERR(obj)) {
 594                 ret = PTR_ERR(obj);
 595                 goto err_tree_mutex_unlock;
 596         }
 597
 598         if (!xrcd) {
 599                 xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata);
 600                 if (IS_ERR(xrcd)) {
 601                         ret = PTR_ERR(xrcd);
 602                         goto err;
 603                 }
 604
 605                 xrcd->inode   = inode;
 606                 xrcd->device  = ib_dev;
 607                 atomic_set(&xrcd->usecnt, 0);
 608                 mutex_init(&xrcd->tgt_qp_mutex);
 609                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
 610                 new_xrcd = 1;
 611         }
 612
 613         atomic_set(&obj->refcnt, 0);
 614         obj->uobject.object = xrcd;
 615         memset(&resp, 0, sizeof resp);
 616         resp.xrcd_handle = obj->uobject.id;
 617
 618         if (inode) {
 619                 if (new_xrcd) {
 620                         /* create new inode/xrcd table entry */
 621                         ret = xrcd_table_insert(ibudev, inode, xrcd);
 622                         if (ret)
 623                                 goto err_dealloc_xrcd;
 624                 }
 625                 atomic_inc(&xrcd->usecnt);
 626         }
 627
 628         ret = uverbs_response(attrs, &resp, sizeof(resp));
 629         if (ret)
 630                 goto err_copy;
 631
 632         if (f.file)
 633                 fdput(f);
 634
 635         mutex_unlock(&ibudev->xrcd_tree_mutex);
 636
 637         return uobj_alloc_commit(&obj->uobject, attrs);
 638
 639 err_copy:
 640         if (inode) {
 641                 if (new_xrcd)
 642                         xrcd_table_delete(ibudev, inode);
 643                 atomic_dec(&xrcd->usecnt);
 644         }
 645
 646 err_dealloc_xrcd:
 647         ib_dealloc_xrcd(xrcd, &attrs->driver_udata);
 648
 649 err:
 650         uobj_alloc_abort(&obj->uobject, attrs);
 651
 652 err_tree_mutex_unlock:
 653         if (f.file)
 654                 fdput(f);
 655
 656         mutex_unlock(&ibudev->xrcd_tree_mutex);
 657
 658         return ret;
 659 }
 660
 661 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
 662 {
 663         struct ib_uverbs_close_xrcd cmd;
 664         int ret;
 665
 666         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 667         if (ret)
 668                 return ret;
 669
 670         return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
 671 }
 672
 673 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
 674                            enum rdma_remove_reason why,
 675                            struct uverbs_attr_bundle *attrs)
 676 {
 677         struct inode *inode;
 678         int ret;
 679         struct ib_uverbs_device *dev = attrs->ufile->device;
 680
 681         inode = xrcd->inode;
 682         if (inode && !atomic_dec_and_test(&xrcd->usecnt))
 683                 return 0;
 684
 685         ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata);
 686
 687         if (ib_is_destroy_retryable(ret, why, uobject)) {
 688                 atomic_inc(&xrcd->usecnt);
 689                 return ret;
 690         }
 691
 692         if (inode)
 693                 xrcd_table_delete(dev, inode);
 694
 695         return ret;
 696 }
 697
 698 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
 699 {
 700         struct ib_uverbs_reg_mr      cmd;
 701         struct ib_uverbs_reg_mr_resp resp;
 702         struct ib_uobject           *uobj;
 703         struct ib_pd                *pd;
 704         struct ib_mr                *mr;
 705         int                          ret;
 706         struct ib_device *ib_dev;
 707
 708         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 709         if (ret)
 710                 return ret;
 711
 712         if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
 713                 return -EINVAL;
 714
 715         ret = ib_check_mr_access(cmd.access_flags);
 716         if (ret)
 717                 return ret;
 718
 719         uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
 720         if (IS_ERR(uobj))
 721                 return PTR_ERR(uobj);
 722
 723         pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
 724         if (!pd) {
 725                 ret = -EINVAL;
 726                 goto err_free;
 727         }
 728
 729         if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
 730                 if (!(pd->device->attrs.device_cap_flags &
 731                       IB_DEVICE_ON_DEMAND_PAGING)) {
 732                         pr_debug("ODP support not available\n");
 733                         ret = -EINVAL;
 734                         goto err_put;
 735                 }
 736         }
 737
 738         mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
 739                                          cmd.access_flags,
 740                                          &attrs->driver_udata);
 741         if (IS_ERR(mr)) {
 742                 ret = PTR_ERR(mr);
 743                 goto err_put;
 744         }
 745
 746         mr->device  = pd->device;
 747         mr->pd      = pd;
 748         mr->dm      = NULL;
 749         mr->uobject = uobj;
 750         atomic_inc(&pd->usecnt);
 751         mr->res.type = RDMA_RESTRACK_MR;
 752         rdma_restrack_uadd(&mr->res);
 753
 754         uobj->object = mr;
 755
 756         memset(&resp, 0, sizeof resp);
 757         resp.lkey      = mr->lkey;
 758         resp.rkey      = mr->rkey;
 759         resp.mr_handle = uobj->id;
 760
 761         ret = uverbs_response(attrs, &resp, sizeof(resp));
 762         if (ret)
 763                 goto err_copy;
 764
 765         uobj_put_obj_read(pd);
 766
 767         return uobj_alloc_commit(uobj, attrs);
 768
 769 err_copy:
 770         ib_dereg_mr_user(mr, &attrs->driver_udata);
 771
 772 err_put:
 773         uobj_put_obj_read(pd);
 774
 775 err_free:
 776         uobj_alloc_abort(uobj, attrs);
 777         return ret;
 778 }
 779
 780 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
 781 {
 782         struct ib_uverbs_rereg_mr      cmd;
 783         struct ib_uverbs_rereg_mr_resp resp;
 784         struct ib_pd                *pd = NULL;
 785         struct ib_mr                *mr;
 786         struct ib_pd                *old_pd;
 787         int                          ret;
 788         struct ib_uobject           *uobj;
 789
 790         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 791         if (ret)
 792                 return ret;
 793
 794         if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
 795                 return -EINVAL;
 796
 797         if ((cmd.flags & IB_MR_REREG_TRANS) &&
 798             (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
 799              (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
 800                         return -EINVAL;
 801
 802         uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
 803         if (IS_ERR(uobj))
 804                 return PTR_ERR(uobj);
 805
 806         mr = uobj->object;
 807
 808         if (mr->dm) {
 809                 ret = -EINVAL;
 810                 goto put_uobjs;
 811         }
 812
 813         if (cmd.flags & IB_MR_REREG_ACCESS) {
 814                 ret = ib_check_mr_access(cmd.access_flags);
 815                 if (ret)
 816                         goto put_uobjs;
 817         }
 818
 819         if (cmd.flags & IB_MR_REREG_PD) {
 820                 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
 821                                        attrs);
 822                 if (!pd) {
 823                         ret = -EINVAL;
 824                         goto put_uobjs;
 825                 }
 826         }
 827
 828         old_pd = mr->pd;
 829         ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start,
 830                                             cmd.length, cmd.hca_va,
 831                                             cmd.access_flags, pd,
 832                                             &attrs->driver_udata);
 833         if (ret)
 834                 goto put_uobj_pd;
 835
 836         if (cmd.flags & IB_MR_REREG_PD) {
 837                 atomic_inc(&pd->usecnt);
 838                 mr->pd = pd;
 839                 atomic_dec(&old_pd->usecnt);
 840         }
 841
 842         memset(&resp, 0, sizeof(resp));
 843         resp.lkey      = mr->lkey;
 844         resp.rkey      = mr->rkey;
 845
 846         ret = uverbs_response(attrs, &resp, sizeof(resp));
 847
 848 put_uobj_pd:
 849         if (cmd.flags & IB_MR_REREG_PD)
 850                 uobj_put_obj_read(pd);
 851
 852 put_uobjs:
 853         uobj_put_write(uobj);
 854
 855         return ret;
 856 }
 857
 858 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
 859 {
 860         struct ib_uverbs_dereg_mr cmd;
 861         int ret;
 862
 863         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 864         if (ret)
 865                 return ret;
 866
 867         return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
 868 }
 869
 870 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
 871 {
 872         struct ib_uverbs_alloc_mw      cmd;
 873         struct ib_uverbs_alloc_mw_resp resp;
 874         struct ib_uobject             *uobj;
 875         struct ib_pd                  *pd;
 876         struct ib_mw                  *mw;
 877         int                            ret;
 878         struct ib_device *ib_dev;
 879
 880         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 881         if (ret)
 882                 return ret;
 883
 884         uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
 885         if (IS_ERR(uobj))
 886                 return PTR_ERR(uobj);
 887
 888         pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
 889         if (!pd) {
 890                 ret = -EINVAL;
 891                 goto err_free;
 892         }
 893
 894         if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) {
 895                 ret = -EINVAL;
 896                 goto err_put;
 897         }
 898
 899         mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata);
 900         if (IS_ERR(mw)) {
 901                 ret = PTR_ERR(mw);
 902                 goto err_put;
 903         }
 904
 905         mw->device  = pd->device;
 906         mw->pd      = pd;
 907         mw->uobject = uobj;
 908         atomic_inc(&pd->usecnt);
 909
 910         uobj->object = mw;
 911
 912         memset(&resp, 0, sizeof(resp));
 913         resp.rkey      = mw->rkey;
 914         resp.mw_handle = uobj->id;
 915
 916         ret = uverbs_response(attrs, &resp, sizeof(resp));
 917         if (ret)
 918                 goto err_copy;
 919
 920         uobj_put_obj_read(pd);
 921         return uobj_alloc_commit(uobj, attrs);
 922
 923 err_copy:
 924         uverbs_dealloc_mw(mw);
 925 err_put:
 926         uobj_put_obj_read(pd);
 927 err_free:
 928         uobj_alloc_abort(uobj, attrs);
 929         return ret;
 930 }
 931
 932 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
 933 {
 934         struct ib_uverbs_dealloc_mw cmd;
 935         int ret;
 936
 937         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 938         if (ret)
 939                 return ret;
 940
 941         return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
 942 }
 943
 944 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
 945 {
 946         struct ib_uverbs_create_comp_channel       cmd;
 947         struct ib_uverbs_create_comp_channel_resp  resp;
 948         struct ib_uobject                         *uobj;
 949         struct ib_uverbs_completion_event_file    *ev_file;
 950         struct ib_device *ib_dev;
 951         int ret;
 952
 953         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
 954         if (ret)
 955                 return ret;
 956
 957         uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
 958         if (IS_ERR(uobj))
 959                 return PTR_ERR(uobj);
 960
 961         resp.fd = uobj->id;
 962
 963         ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
 964                                uobj);
 965         ib_uverbs_init_event_queue(&ev_file->ev_queue);
 966
 967         ret = uverbs_response(attrs, &resp, sizeof(resp));
 968         if (ret) {
 969                 uobj_alloc_abort(uobj, attrs);
 970                 return ret;
 971         }
 972
 973         return uobj_alloc_commit(uobj, attrs);
 974 }
 975
 976 static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
 977                                        struct ib_uverbs_ex_create_cq *cmd)
 978 {
 979         struct ib_ucq_object           *obj;
 980         struct ib_uverbs_completion_event_file    *ev_file = NULL;
 981         struct ib_cq                   *cq;
 982         int                             ret;
 983         struct ib_uverbs_ex_create_cq_resp resp;
 984         struct ib_cq_init_attr attr = {};
 985         struct ib_device *ib_dev;
 986
 987         if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
 988                 return ERR_PTR(-EINVAL);
 989
 990         obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
 991                                                  &ib_dev);
 992         if (IS_ERR(obj))
 993                 return obj;
 994
 995         if (cmd->comp_channel >= 0) {
 996                 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
 997                 if (IS_ERR(ev_file)) {
 998                         ret = PTR_ERR(ev_file);
 999                         goto err;
1000                 }
1001         }
1002
1003         obj->uobject.user_handle = cmd->user_handle;
1004         obj->comp_events_reported  = 0;
1005         obj->async_events_reported = 0;
1006         INIT_LIST_HEAD(&obj->comp_list);
1007         INIT_LIST_HEAD(&obj->async_list);
1008
1009         attr.cqe = cmd->cqe;
1010         attr.comp_vector = cmd->comp_vector;
1011         attr.flags = cmd->flags;
1012
1013         cq = ib_dev->ops.create_cq(ib_dev, &attr, &attrs->driver_udata);
1014         if (IS_ERR(cq)) {
1015                 ret = PTR_ERR(cq);
1016                 goto err_file;
1017         }
1018
1019         cq->device        = ib_dev;
1020         cq->uobject       = &obj->uobject;
1021         cq->comp_handler  = ib_uverbs_comp_handler;
1022         cq->event_handler = ib_uverbs_cq_event_handler;
1023         cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
1024         atomic_set(&cq->usecnt, 0);
1025
1026         obj->uobject.object = cq;
1027         memset(&resp, 0, sizeof resp);
1028         resp.base.cq_handle = obj->uobject.id;
1029         resp.base.cqe       = cq->cqe;
1030         resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1031
1032         cq->res.type = RDMA_RESTRACK_CQ;
1033         rdma_restrack_uadd(&cq->res);
1034
1035         ret = uverbs_response(attrs, &resp, sizeof(resp));
1036         if (ret)
1037                 goto err_cb;
1038
1039         ret = uobj_alloc_commit(&obj->uobject, attrs);
1040         if (ret)
1041                 return ERR_PTR(ret);
1042         return obj;
1043
1044 err_cb:
1045         ib_destroy_cq(cq);
1046
1047 err_file:
1048         if (ev_file)
1049                 ib_uverbs_release_ucq(attrs->ufile, ev_file, obj);
1050
1051 err:
1052         uobj_alloc_abort(&obj->uobject, attrs);
1053
1054         return ERR_PTR(ret);
1055 }
1056
1057 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1058 {
1059         struct ib_uverbs_create_cq      cmd;
1060         struct ib_uverbs_ex_create_cq   cmd_ex;
1061         struct ib_ucq_object           *obj;
1062         int ret;
1063
1064         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1065         if (ret)
1066                 return ret;
1067
1068         memset(&cmd_ex, 0, sizeof(cmd_ex));
1069         cmd_ex.user_handle = cmd.user_handle;
1070         cmd_ex.cqe = cmd.cqe;
1071         cmd_ex.comp_vector = cmd.comp_vector;
1072         cmd_ex.comp_channel = cmd.comp_channel;
1073
1074         obj = create_cq(attrs, &cmd_ex);
1075         return PTR_ERR_OR_ZERO(obj);
1076 }
1077
1078 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1079 {
1080         struct ib_uverbs_ex_create_cq  cmd;
1081         struct ib_ucq_object           *obj;
1082         int ret;
1083
1084         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1085         if (ret)
1086                 return ret;
1087
1088         if (cmd.comp_mask)
1089                 return -EINVAL;
1090
1091         if (cmd.reserved)
1092                 return -EINVAL;
1093
1094         obj = create_cq(attrs, &cmd);
1095         return PTR_ERR_OR_ZERO(obj);
1096 }
1097
1098 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
1099 {
1100         struct ib_uverbs_resize_cq      cmd;
1101         struct ib_uverbs_resize_cq_resp resp = {};
1102         struct ib_cq                    *cq;
1103         int                             ret = -EINVAL;
1104
1105         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1106         if (ret)
1107                 return ret;
1108
1109         cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1110         if (!cq)
1111                 return -EINVAL;
1112
1113         ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
1114         if (ret)
1115                 goto out;
1116
1117         resp.cqe = cq->cqe;
1118
1119         ret = uverbs_response(attrs, &resp, sizeof(resp));
1120 out:
1121         uobj_put_obj_read(cq);
1122
1123         return ret;
1124 }
1125
1126 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1127                            struct ib_wc *wc)
1128 {
1129         struct ib_uverbs_wc tmp;
1130
1131         tmp.wr_id               = wc->wr_id;
1132         tmp.status              = wc->status;
1133         tmp.opcode              = wc->opcode;
1134         tmp.vendor_err          = wc->vendor_err;
1135         tmp.byte_len            = wc->byte_len;
1136         tmp.ex.imm_data         = wc->ex.imm_data;
1137         tmp.qp_num              = wc->qp->qp_num;
1138         tmp.src_qp              = wc->src_qp;
1139         tmp.wc_flags            = wc->wc_flags;
1140         tmp.pkey_index          = wc->pkey_index;
1141         if (rdma_cap_opa_ah(ib_dev, wc->port_num))
1142                 tmp.slid        = OPA_TO_IB_UCAST_LID(wc->slid);
1143         else
1144                 tmp.slid        = ib_lid_cpu16(wc->slid);
1145         tmp.sl                  = wc->sl;
1146         tmp.dlid_path_bits      = wc->dlid_path_bits;
1147         tmp.port_num            = wc->port_num;
1148         tmp.reserved            = 0;
1149
1150         if (copy_to_user(dest, &tmp, sizeof tmp))
1151                 return -EFAULT;
1152
1153         return 0;
1154 }
1155
1156 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
1157 {
1158         struct ib_uverbs_poll_cq       cmd;
1159         struct ib_uverbs_poll_cq_resp  resp;
1160         u8 __user                     *header_ptr;
1161         u8 __user                     *data_ptr;
1162         struct ib_cq                  *cq;
1163         struct ib_wc                   wc;
1164         int                            ret;
1165
1166         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1167         if (ret)
1168                 return ret;
1169
1170         cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1171         if (!cq)
1172                 return -EINVAL;
1173
1174         /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1175         header_ptr = attrs->ucore.outbuf;
1176         data_ptr = header_ptr + sizeof resp;
1177
1178         memset(&resp, 0, sizeof resp);
1179         while (resp.count < cmd.ne) {
1180                 ret = ib_poll_cq(cq, 1, &wc);
1181                 if (ret < 0)
1182                         goto out_put;
1183                 if (!ret)
1184                         break;
1185
1186                 ret = copy_wc_to_user(cq->device, data_ptr, &wc);
1187                 if (ret)
1188                         goto out_put;
1189
1190                 data_ptr += sizeof(struct ib_uverbs_wc);
1191                 ++resp.count;
1192         }
1193
1194         if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1195                 ret = -EFAULT;
1196                 goto out_put;
1197         }
1198         ret = 0;
1199
1200         if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
1201                 ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);
1202
1203 out_put:
1204         uobj_put_obj_read(cq);
1205         return ret;
1206 }
1207
1208 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
1209 {
1210         struct ib_uverbs_req_notify_cq cmd;
1211         struct ib_cq                  *cq;
1212         int ret;
1213
1214         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1215         if (ret)
1216                 return ret;
1217
1218         cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1219         if (!cq)
1220                 return -EINVAL;
1221
1222         ib_req_notify_cq(cq, cmd.solicited_only ?
1223                          IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1224
1225         uobj_put_obj_read(cq);
1226
1227         return 0;
1228 }
1229
1230 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
1231 {
1232         struct ib_uverbs_destroy_cq      cmd;
1233         struct ib_uverbs_destroy_cq_resp resp;
1234         struct ib_uobject               *uobj;
1235         struct ib_ucq_object            *obj;
1236         int ret;
1237
1238         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1239         if (ret)
1240                 return ret;
1241
1242         uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1243         if (IS_ERR(uobj))
1244                 return PTR_ERR(uobj);
1245
1246         obj = container_of(uobj, struct ib_ucq_object, uobject);
1247         memset(&resp, 0, sizeof(resp));
1248         resp.comp_events_reported  = obj->comp_events_reported;
1249         resp.async_events_reported = obj->async_events_reported;
1250
1251         uobj_put_destroy(uobj);
1252
1253         return uverbs_response(attrs, &resp, sizeof(resp));
1254 }
1255
1256 static int create_qp(struct uverbs_attr_bundle *attrs,
1257                      struct ib_uverbs_ex_create_qp *cmd)
1258 {
1259         struct ib_uqp_object            *obj;
1260         struct ib_device                *device;
1261         struct ib_pd                    *pd = NULL;
1262         struct ib_xrcd                  *xrcd = NULL;
1263         struct ib_uobject               *xrcd_uobj = ERR_PTR(-ENOENT);
1264         struct ib_cq                    *scq = NULL, *rcq = NULL;
1265         struct ib_srq                   *srq = NULL;
1266         struct ib_qp                    *qp;
1267         struct ib_qp_init_attr          attr = {};
1268         struct ib_uverbs_ex_create_qp_resp resp;
1269         int                             ret;
1270         struct ib_rwq_ind_table *ind_tbl = NULL;
1271         bool has_sq = true;
1272         struct ib_device *ib_dev;
1273
1274         if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1275                 return -EPERM;
1276
1277         obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1278                                                  &ib_dev);
1279         if (IS_ERR(obj))
1280                 return PTR_ERR(obj);
1281         obj->uxrcd = NULL;
1282         obj->uevent.uobject.user_handle = cmd->user_handle;
1283         mutex_init(&obj->mcast_lock);
1284
1285         if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
1286                 ind_tbl = uobj_get_obj_read(rwq_ind_table,
1287                                             UVERBS_OBJECT_RWQ_IND_TBL,
1288                                             cmd->rwq_ind_tbl_handle, attrs);
1289                 if (!ind_tbl) {
1290                         ret = -EINVAL;
1291                         goto err_put;
1292                 }
1293
1294                 attr.rwq_ind_tbl = ind_tbl;
1295         }
1296
1297         if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1298                 ret = -EINVAL;
1299                 goto err_put;
1300         }
1301
1302         if (ind_tbl && !cmd->max_send_wr)
1303                 has_sq = false;
1304
1305         if (cmd->qp_type == IB_QPT_XRC_TGT) {
1306                 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
1307                                           attrs);
1308
1309                 if (IS_ERR(xrcd_uobj)) {
1310                         ret = -EINVAL;
1311                         goto err_put;
1312                 }
1313
1314                 xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1315                 if (!xrcd) {
1316                         ret = -EINVAL;
1317                         goto err_put;
1318                 }
1319                 device = xrcd->device;
1320         } else {
1321                 if (cmd->qp_type == IB_QPT_XRC_INI) {
1322                         cmd->max_recv_wr = 0;
1323                         cmd->max_recv_sge = 0;
1324                 } else {
1325                         if (cmd->is_srq) {
1326                                 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
1327                                                         cmd->srq_handle, attrs);
1328                                 if (!srq || srq->srq_type == IB_SRQT_XRC) {
1329                                         ret = -EINVAL;
1330                                         goto err_put;
1331                                 }
1332                         }
1333
1334                         if (!ind_tbl) {
1335                                 if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1336                                         rcq = uobj_get_obj_read(
1337                                                 cq, UVERBS_OBJECT_CQ,
1338                                                 cmd->recv_cq_handle, attrs);
1339                                         if (!rcq) {
1340                                                 ret = -EINVAL;
1341                                                 goto err_put;
1342                                         }
1343                                 }
1344                         }
1345                 }
1346
1347                 if (has_sq)
1348                         scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
1349                                                 cmd->send_cq_handle, attrs);
1350                 if (!ind_tbl)
1351                         rcq = rcq ?: scq;
1352                 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
1353                                        attrs);
1354                 if (!pd || (!scq && has_sq)) {
1355                         ret = -EINVAL;
1356                         goto err_put;
1357                 }
1358
1359                 device = pd->device;
1360         }
1361
1362         attr.event_handler = ib_uverbs_qp_event_handler;
1363         attr.qp_context    = attrs->ufile;
1364         attr.send_cq       = scq;
1365         attr.recv_cq       = rcq;
1366         attr.srq           = srq;
1367         attr.xrcd          = xrcd;
1368         attr.sq_sig_type   = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1369                                               IB_SIGNAL_REQ_WR;
1370         attr.qp_type       = cmd->qp_type;
1371         attr.create_flags  = 0;
1372
1373         attr.cap.max_send_wr     = cmd->max_send_wr;
1374         attr.cap.max_recv_wr     = cmd->max_recv_wr;
1375         attr.cap.max_send_sge    = cmd->max_send_sge;
1376         attr.cap.max_recv_sge    = cmd->max_recv_sge;
1377         attr.cap.max_inline_data = cmd->max_inline_data;
1378
1379         obj->uevent.events_reported     = 0;
1380         INIT_LIST_HEAD(&obj->uevent.event_list);
1381         INIT_LIST_HEAD(&obj->mcast_list);
1382
1383         attr.create_flags = cmd->create_flags;
1384         if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1385                                 IB_QP_CREATE_CROSS_CHANNEL |
1386                                 IB_QP_CREATE_MANAGED_SEND |
1387                                 IB_QP_CREATE_MANAGED_RECV |
1388                                 IB_QP_CREATE_SCATTER_FCS |
1389                                 IB_QP_CREATE_CVLAN_STRIPPING |
1390                                 IB_QP_CREATE_SOURCE_QPN |
1391                                 IB_QP_CREATE_PCI_WRITE_END_PADDING)) {
1392                 ret = -EINVAL;
1393                 goto err_put;
1394         }
1395
1396         if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1397                 if (!capable(CAP_NET_RAW)) {
1398                         ret = -EPERM;
1399                         goto err_put;
1400                 }
1401
1402                 attr.source_qpn = cmd->source_qpn;
1403         }
1404
1405         if (cmd->qp_type == IB_QPT_XRC_TGT)
1406                 qp = ib_create_qp(pd, &attr);
1407         else
1408                 qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
1409                                    &obj->uevent.uobject);
1410
1411         if (IS_ERR(qp)) {
1412                 ret = PTR_ERR(qp);
1413                 goto err_put;
1414         }
1415
1416         if (cmd->qp_type != IB_QPT_XRC_TGT) {
1417                 ret = ib_create_qp_security(qp, device);
1418                 if (ret)
1419                         goto err_cb;
1420
1421                 qp->pd            = pd;
1422                 qp->send_cq       = attr.send_cq;
1423                 qp->recv_cq       = attr.recv_cq;
1424                 qp->srq           = attr.srq;
1425                 qp->rwq_ind_tbl   = ind_tbl;
1426                 qp->event_handler = attr.event_handler;
1427                 qp->qp_context    = attr.qp_context;
1428                 qp->qp_type       = attr.qp_type;
1429                 atomic_set(&qp->usecnt, 0);
1430                 atomic_inc(&pd->usecnt);
1431                 qp->port = 0;
1432                 if (attr.send_cq)
1433                         atomic_inc(&attr.send_cq->usecnt);
1434                 if (attr.recv_cq)
1435                         atomic_inc(&attr.recv_cq->usecnt);
1436                 if (attr.srq)
1437                         atomic_inc(&attr.srq->usecnt);
1438                 if (ind_tbl)
1439                         atomic_inc(&ind_tbl->usecnt);
1440         } else {
1441                 /* It is done in _ib_create_qp for other QP types */
1442                 qp->uobject = &obj->uevent.uobject;
1443         }
1444
1445         obj->uevent.uobject.object = qp;
1446
1447         memset(&resp, 0, sizeof resp);
1448         resp.base.qpn             = qp->qp_num;
1449         resp.base.qp_handle       = obj->uevent.uobject.id;
1450         resp.base.max_recv_sge    = attr.cap.max_recv_sge;
1451         resp.base.max_send_sge    = attr.cap.max_send_sge;
1452         resp.base.max_recv_wr     = attr.cap.max_recv_wr;
1453         resp.base.max_send_wr     = attr.cap.max_send_wr;
1454         resp.base.max_inline_data = attr.cap.max_inline_data;
1455         resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1456
1457         ret = uverbs_response(attrs, &resp, sizeof(resp));
1458         if (ret)
1459                 goto err_cb;
1460
1461         if (xrcd) {
1462                 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1463                                           uobject);
1464                 atomic_inc(&obj->uxrcd->refcnt);
1465                 uobj_put_read(xrcd_uobj);
1466         }
1467
1468         if (pd)
1469                 uobj_put_obj_read(pd);
1470         if (scq)
1471                 uobj_put_obj_read(scq);
1472         if (rcq && rcq != scq)
1473                 uobj_put_obj_read(rcq);
1474         if (srq)
1475                 uobj_put_obj_read(srq);
1476         if (ind_tbl)
1477                 uobj_put_obj_read(ind_tbl);
1478
1479         return uobj_alloc_commit(&obj->uevent.uobject, attrs);
1480 err_cb:
1481         ib_destroy_qp(qp);
1482
1483 err_put:
1484         if (!IS_ERR(xrcd_uobj))
1485                 uobj_put_read(xrcd_uobj);
1486         if (pd)
1487                 uobj_put_obj_read(pd);
1488         if (scq)
1489                 uobj_put_obj_read(scq);
1490         if (rcq && rcq != scq)
1491                 uobj_put_obj_read(rcq);
1492         if (srq)
1493                 uobj_put_obj_read(srq);
1494         if (ind_tbl)
1495                 uobj_put_obj_read(ind_tbl);
1496
1497         uobj_alloc_abort(&obj->uevent.uobject, attrs);
1498         return ret;
1499 }
1500
1501 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
1502 {
1503         struct ib_uverbs_create_qp      cmd;
1504         struct ib_uverbs_ex_create_qp   cmd_ex;
1505         int ret;
1506
1507         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1508         if (ret)
1509                 return ret;
1510
1511         memset(&cmd_ex, 0, sizeof(cmd_ex));
1512         cmd_ex.user_handle = cmd.user_handle;
1513         cmd_ex.pd_handle = cmd.pd_handle;
1514         cmd_ex.send_cq_handle = cmd.send_cq_handle;
1515         cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1516         cmd_ex.srq_handle = cmd.srq_handle;
1517         cmd_ex.max_send_wr = cmd.max_send_wr;
1518         cmd_ex.max_recv_wr = cmd.max_recv_wr;
1519         cmd_ex.max_send_sge = cmd.max_send_sge;
1520         cmd_ex.max_recv_sge = cmd.max_recv_sge;
1521         cmd_ex.max_inline_data = cmd.max_inline_data;
1522         cmd_ex.sq_sig_all = cmd.sq_sig_all;
1523         cmd_ex.qp_type = cmd.qp_type;
1524         cmd_ex.is_srq = cmd.is_srq;
1525
1526         return create_qp(attrs, &cmd_ex);
1527 }
1528
1529 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
1530 {
1531         struct ib_uverbs_ex_create_qp cmd;
1532         int ret;
1533
1534         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1535         if (ret)
1536                 return ret;
1537
1538         if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1539                 return -EINVAL;
1540
1541         if (cmd.reserved)
1542                 return -EINVAL;
1543
1544         return create_qp(attrs, &cmd);
1545 }
1546
1547 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1548 {
1549         struct ib_uverbs_open_qp        cmd;
1550         struct ib_uverbs_create_qp_resp resp;
1551         struct ib_uqp_object           *obj;
1552         struct ib_xrcd                 *xrcd;
1553         struct ib_uobject              *uninitialized_var(xrcd_uobj);
1554         struct ib_qp                   *qp;
1555         struct ib_qp_open_attr          attr;
1556         int ret;
1557         struct ib_device *ib_dev;
1558
1559         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1560         if (ret)
1561                 return ret;
1562
1563         obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1564                                                  &ib_dev);
1565         if (IS_ERR(obj))
1566                 return PTR_ERR(obj);
1567
1568         xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
1569         if (IS_ERR(xrcd_uobj)) {
1570                 ret = -EINVAL;
1571                 goto err_put;
1572         }
1573
1574         xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1575         if (!xrcd) {
1576                 ret = -EINVAL;
1577                 goto err_xrcd;
1578         }
1579
1580         attr.event_handler = ib_uverbs_qp_event_handler;
1581         attr.qp_context    = attrs->ufile;
1582         attr.qp_num        = cmd.qpn;
1583         attr.qp_type       = cmd.qp_type;
1584
1585         obj->uevent.events_reported = 0;
1586         INIT_LIST_HEAD(&obj->uevent.event_list);
1587         INIT_LIST_HEAD(&obj->mcast_list);
1588
1589         qp = ib_open_qp(xrcd, &attr);
1590         if (IS_ERR(qp)) {
1591                 ret = PTR_ERR(qp);
1592                 goto err_xrcd;
1593         }
1594
1595         obj->uevent.uobject.object = qp;
1596         obj->uevent.uobject.user_handle = cmd.user_handle;
1597
1598         memset(&resp, 0, sizeof resp);
1599         resp.qpn       = qp->qp_num;
1600         resp.qp_handle = obj->uevent.uobject.id;
1601
1602         ret = uverbs_response(attrs, &resp, sizeof(resp));
1603         if (ret)
1604                 goto err_destroy;
1605
1606         obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1607         atomic_inc(&obj->uxrcd->refcnt);
1608         qp->uobject = &obj->uevent.uobject;
1609         uobj_put_read(xrcd_uobj);
1610
1611         return uobj_alloc_commit(&obj->uevent.uobject, attrs);
1612
1613 err_destroy:
1614         ib_destroy_qp(qp);
1615 err_xrcd:
1616         uobj_put_read(xrcd_uobj);
1617 err_put:
1618         uobj_alloc_abort(&obj->uevent.uobject, attrs);
1619         return ret;
1620 }
1621
1622 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1623                                    struct rdma_ah_attr *rdma_attr)
1624 {
1625         const struct ib_global_route   *grh;
1626
1627         uverb_attr->dlid              = rdma_ah_get_dlid(rdma_attr);
1628         uverb_attr->sl                = rdma_ah_get_sl(rdma_attr);
1629         uverb_attr->src_path_bits     = rdma_ah_get_path_bits(rdma_attr);
1630         uverb_attr->static_rate       = rdma_ah_get_static_rate(rdma_attr);
1631         uverb_attr->is_global         = !!(rdma_ah_get_ah_flags(rdma_attr) &
1632                                          IB_AH_GRH);
1633         if (uverb_attr->is_global) {
1634                 grh = rdma_ah_read_grh(rdma_attr);
1635                 memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1636                 uverb_attr->flow_label        = grh->flow_label;
1637                 uverb_attr->sgid_index        = grh->sgid_index;
1638                 uverb_attr->hop_limit         = grh->hop_limit;
1639                 uverb_attr->traffic_class     = grh->traffic_class;
1640         }
1641         uverb_attr->port_num          = rdma_ah_get_port_num(rdma_attr);
1642 }
1643
1644 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
1645 {
1646         struct ib_uverbs_query_qp      cmd;
1647         struct ib_uverbs_query_qp_resp resp;
1648         struct ib_qp                   *qp;
1649         struct ib_qp_attr              *attr;
1650         struct ib_qp_init_attr         *init_attr;
1651         int                            ret;
1652
1653         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1654         if (ret)
1655                 return ret;
1656
1657         attr      = kmalloc(sizeof *attr, GFP_KERNEL);
1658         init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1659         if (!attr || !init_attr) {
1660                 ret = -ENOMEM;
1661                 goto out;
1662         }
1663
1664         qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1665         if (!qp) {
1666                 ret = -EINVAL;
1667                 goto out;
1668         }
1669
1670         ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1671
1672         uobj_put_obj_read(qp);
1673
1674         if (ret)
1675                 goto out;
1676
1677         memset(&resp, 0, sizeof resp);
1678
1679         resp.qp_state               = attr->qp_state;
1680         resp.cur_qp_state           = attr->cur_qp_state;
1681         resp.path_mtu               = attr->path_mtu;
1682         resp.path_mig_state         = attr->path_mig_state;
1683         resp.qkey                   = attr->qkey;
1684         resp.rq_psn                 = attr->rq_psn;
1685         resp.sq_psn                 = attr->sq_psn;
1686         resp.dest_qp_num            = attr->dest_qp_num;
1687         resp.qp_access_flags        = attr->qp_access_flags;
1688         resp.pkey_index             = attr->pkey_index;
1689         resp.alt_pkey_index         = attr->alt_pkey_index;
1690         resp.sq_draining            = attr->sq_draining;
1691         resp.max_rd_atomic          = attr->max_rd_atomic;
1692         resp.max_dest_rd_atomic     = attr->max_dest_rd_atomic;
1693         resp.min_rnr_timer          = attr->min_rnr_timer;
1694         resp.port_num               = attr->port_num;
1695         resp.timeout                = attr->timeout;
1696         resp.retry_cnt              = attr->retry_cnt;
1697         resp.rnr_retry              = attr->rnr_retry;
1698         resp.alt_port_num           = attr->alt_port_num;
1699         resp.alt_timeout            = attr->alt_timeout;
1700
1701         copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
1702         copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
1703
1704         resp.max_send_wr            = init_attr->cap.max_send_wr;
1705         resp.max_recv_wr            = init_attr->cap.max_recv_wr;
1706         resp.max_send_sge           = init_attr->cap.max_send_sge;
1707         resp.max_recv_sge           = init_attr->cap.max_recv_sge;
1708         resp.max_inline_data        = init_attr->cap.max_inline_data;
1709         resp.sq_sig_all             = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1710
1711         ret = uverbs_response(attrs, &resp, sizeof(resp));
1712
1713 out:
1714         kfree(attr);
1715         kfree(init_attr);
1716
1717         return ret;
1718 }
1719
1720 /* Remove ignored fields set in the attribute mask */
1721 static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1722 {
1723         switch (qp_type) {
1724         case IB_QPT_XRC_INI:
1725                 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1726         case IB_QPT_XRC_TGT:
1727                 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1728                                 IB_QP_RNR_RETRY);
1729         default:
1730                 return mask;
1731         }
1732 }
1733
1734 static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1735                                      struct rdma_ah_attr *rdma_attr,
1736                                      struct ib_uverbs_qp_dest *uverb_attr)
1737 {
1738         rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
1739         if (uverb_attr->is_global) {
1740                 rdma_ah_set_grh(rdma_attr, NULL,
1741                                 uverb_attr->flow_label,
1742                                 uverb_attr->sgid_index,
1743                                 uverb_attr->hop_limit,
1744                                 uverb_attr->traffic_class);
1745                 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
1746         } else {
1747                 rdma_ah_set_ah_flags(rdma_attr, 0);
1748         }
1749         rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
1750         rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
1751         rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
1752         rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
1753         rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
1754         rdma_ah_set_make_grd(rdma_attr, false);
1755 }
1756
1757 static int modify_qp(struct uverbs_attr_bundle *attrs,
1758                      struct ib_uverbs_ex_modify_qp *cmd)
1759 {
1760         struct ib_qp_attr *attr;
1761         struct ib_qp *qp;
1762         int ret;
1763
1764         attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1765         if (!attr)
1766                 return -ENOMEM;
1767
1768         qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
1769                                attrs);
1770         if (!qp) {
1771                 ret = -EINVAL;
1772                 goto out;
1773         }
1774
1775         if ((cmd->base.attr_mask & IB_QP_PORT) &&
1776             !rdma_is_port_valid(qp->device, cmd->base.port_num)) {
1777                 ret = -EINVAL;
1778                 goto release_qp;
1779         }
1780
1781         if ((cmd->base.attr_mask & IB_QP_AV)) {
1782                 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
1783                         ret = -EINVAL;
1784                         goto release_qp;
1785                 }
1786
1787                 if (cmd->base.attr_mask & IB_QP_STATE &&
1788                     cmd->base.qp_state == IB_QPS_RTR) {
1789                 /* We are in INIT->RTR TRANSITION (if we are not,
1790                  * this transition will be rejected in subsequent checks).
1791                  * In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1792                  * but the IB_QP_STATE flag is required.
1793                  *
1794                  * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1795                  * when IB_QP_AV is set, has required inclusion of a valid
1796                  * port number in the primary AV. (AVs are created and handled
1797                  * differently for infiniband and ethernet (RoCE) ports).
1798                  *
1799                  * Check the port number included in the primary AV against
1800                  * the port number in the qp struct, which was set (and saved)
1801                  * in the RST->INIT transition.
1802                  */
1803                         if (cmd->base.dest.port_num != qp->real_qp->port) {
1804                                 ret = -EINVAL;
1805                                 goto release_qp;
1806                         }
1807                 } else {
1808                 /* We are in SQD->SQD. (If we are not, this transition will
1809                  * be rejected later in the verbs layer checks).
1810                  * Check for both IB_QP_PORT and IB_QP_AV, these can be set
1811                  * together in the SQD->SQD transition.
1812                  *
1813                  * If only IP_QP_AV was set, add in IB_QP_PORT as well (the
1814                  * verbs layer driver does not track primary port changes
1815                  * resulting from path migration. Thus, in SQD, if the primary
1816                  * AV is modified, the primary port should also be modified).
1817                  *
1818                  * Note that in this transition, the IB_QP_STATE flag
1819                  * is not allowed.
1820                  */
1821                         if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1822                              == (IB_QP_AV | IB_QP_PORT)) &&
1823                             cmd->base.port_num != cmd->base.dest.port_num) {
1824                                 ret = -EINVAL;
1825                                 goto release_qp;
1826                         }
1827                         if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1828                             == IB_QP_AV) {
1829                                 cmd->base.attr_mask |= IB_QP_PORT;
1830                                 cmd->base.port_num = cmd->base.dest.port_num;
1831                         }
1832                 }
1833         }
1834
1835         if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
1836             (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
1837             !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) ||
1838             cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {
1839                 ret = -EINVAL;
1840                 goto release_qp;
1841         }
1842
1843         if ((cmd->base.attr_mask & IB_QP_CUR_STATE &&
1844             cmd->base.cur_qp_state > IB_QPS_ERR) ||
1845             (cmd->base.attr_mask & IB_QP_STATE &&
1846             cmd->base.qp_state > IB_QPS_ERR)) {
1847                 ret = -EINVAL;
1848                 goto release_qp;
1849         }
1850
1851         if (cmd->base.attr_mask & IB_QP_STATE)
1852                 attr->qp_state = cmd->base.qp_state;
1853         if (cmd->base.attr_mask & IB_QP_CUR_STATE)
1854                 attr->cur_qp_state = cmd->base.cur_qp_state;
1855         if (cmd->base.attr_mask & IB_QP_PATH_MTU)
1856                 attr->path_mtu = cmd->base.path_mtu;
1857         if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)
1858                 attr->path_mig_state = cmd->base.path_mig_state;
1859         if (cmd->base.attr_mask & IB_QP_QKEY)
1860                 attr->qkey = cmd->base.qkey;
1861         if (cmd->base.attr_mask & IB_QP_RQ_PSN)
1862                 attr->rq_psn = cmd->base.rq_psn;
1863         if (cmd->base.attr_mask & IB_QP_SQ_PSN)
1864                 attr->sq_psn = cmd->base.sq_psn;
1865         if (cmd->base.attr_mask & IB_QP_DEST_QPN)
1866                 attr->dest_qp_num = cmd->base.dest_qp_num;
1867         if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS)
1868                 attr->qp_access_flags = cmd->base.qp_access_flags;
1869         if (cmd->base.attr_mask & IB_QP_PKEY_INDEX)
1870                 attr->pkey_index = cmd->base.pkey_index;
1871         if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1872                 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
1873         if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
1874                 attr->max_rd_atomic = cmd->base.max_rd_atomic;
1875         if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1876                 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic;
1877         if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER)
1878                 attr->min_rnr_timer = cmd->base.min_rnr_timer;
1879         if (cmd->base.attr_mask & IB_QP_PORT)
1880                 attr->port_num = cmd->base.port_num;
1881         if (cmd->base.attr_mask & IB_QP_TIMEOUT)
1882                 attr->timeout = cmd->base.timeout;
1883         if (cmd->base.attr_mask & IB_QP_RETRY_CNT)
1884                 attr->retry_cnt = cmd->base.retry_cnt;
1885         if (cmd->base.attr_mask & IB_QP_RNR_RETRY)
1886                 attr->rnr_retry = cmd->base.rnr_retry;
1887         if (cmd->base.attr_mask & IB_QP_ALT_PATH) {
1888                 attr->alt_port_num = cmd->base.alt_port_num;
1889                 attr->alt_timeout = cmd->base.alt_timeout;
1890                 attr->alt_pkey_index = cmd->base.alt_pkey_index;
1891         }
1892         if (cmd->base.attr_mask & IB_QP_RATE_LIMIT)
1893                 attr->rate_limit = cmd->rate_limit;
1894
1895         if (cmd->base.attr_mask & IB_QP_AV)
1896                 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
1897                                          &cmd->base.dest);
1898
1899         if (cmd->base.attr_mask & IB_QP_ALT_PATH)
1900                 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
1901                                          &cmd->base.alt_dest);
1902
1903         ret = ib_modify_qp_with_udata(qp, attr,
1904                                       modify_qp_mask(qp->qp_type,
1905                                                      cmd->base.attr_mask),
1906                                       &attrs->driver_udata);
1907
1908 release_qp:
1909         uobj_put_obj_read(qp);
1910 out:
1911         kfree(attr);
1912
1913         return ret;
1914 }
1915
1916 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
1917 {
1918         struct ib_uverbs_ex_modify_qp cmd;
1919         int ret;
1920
1921         ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));
1922         if (ret)
1923                 return ret;
1924
1925         if (cmd.base.attr_mask &
1926             ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
1927                 return -EOPNOTSUPP;
1928
1929         return modify_qp(attrs, &cmd);
1930 }
1931
1932 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
1933 {
1934         struct ib_uverbs_ex_modify_qp cmd;
1935         struct ib_uverbs_ex_modify_qp_resp resp = {
1936                 .response_length = uverbs_response_length(attrs, sizeof(resp))
1937         };
1938         int ret;
1939
1940         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1941         if (ret)
1942                 return ret;
1943
1944         /*
1945          * Last bit is reserved for extending the attr_mask by
1946          * using another field.
1947          */
1948         BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
1949
1950         if (cmd.base.attr_mask &
1951             ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
1952                 return -EOPNOTSUPP;
1953
1954         ret = modify_qp(attrs, &cmd);
1955         if (ret)
1956                 return ret;
1957
1958         return uverbs_response(attrs, &resp, sizeof(resp));
1959 }
1960
1961 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
1962 {
1963         struct ib_uverbs_destroy_qp      cmd;
1964         struct ib_uverbs_destroy_qp_resp resp;
1965         struct ib_uobject               *uobj;
1966         struct ib_uqp_object            *obj;
1967         int ret;
1968
1969         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1970         if (ret)
1971                 return ret;
1972
1973         uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1974         if (IS_ERR(uobj))
1975                 return PTR_ERR(uobj);
1976
1977         obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
1978         memset(&resp, 0, sizeof(resp));
1979         resp.events_reported = obj->uevent.events_reported;
1980
1981         uobj_put_destroy(uobj);
1982
1983         return uverbs_response(attrs, &resp, sizeof(resp));
1984 }
1985
1986 static void *alloc_wr(size_t wr_size, __u32 num_sge)
1987 {
1988         if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) /
1989                        sizeof (struct ib_sge))
1990                 return NULL;
1991
1992         return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +
1993                          num_sge * sizeof (struct ib_sge), GFP_KERNEL);
1994 }
1995
1996 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
1997 {
1998         struct ib_uverbs_post_send      cmd;
1999         struct ib_uverbs_post_send_resp resp;
2000         struct ib_uverbs_send_wr       *user_wr;
2001         struct ib_send_wr              *wr = NULL, *last, *next;
2002         const struct ib_send_wr        *bad_wr;
2003         struct ib_qp                   *qp;
2004         int                             i, sg_ind;
2005         int                             is_ud;
2006         int ret, ret2;
2007         size_t                          next_size;
2008         const struct ib_sge __user *sgls;
2009         const void __user *wqes;
2010         struct uverbs_req_iter iter;
2011
2012         ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2013         if (ret)
2014                 return ret;
2015         wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);
2016         if (IS_ERR(wqes))
2017                 return PTR_ERR(wqes);
2018         sgls = uverbs_request_next_ptr(
2019                 &iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));
2020         if (IS_ERR(sgls))
2021                 return PTR_ERR(sgls);
2022         ret = uverbs_request_finish(&iter);
2023         if (ret)
2024                 return ret;
2025
2026         user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2027         if (!user_wr)
2028                 return -ENOMEM;
2029
2030         qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2031         if (!qp) {
2032                 ret = -EINVAL;
2033                 goto out;
2034         }
2035
2036         is_ud = qp->qp_type == IB_QPT_UD;
2037         sg_ind = 0;
2038         last = NULL;
2039         for (i = 0; i < cmd.wr_count; ++i) {
2040                 if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,
2041                                    cmd.wqe_size)) {
2042                         ret = -EFAULT;
2043                         goto out_put;
2044                 }
2045
2046                 if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2047                         ret = -EINVAL;
2048                         goto out_put;
2049                 }
2050
2051                 if (is_ud) {
2052                         struct ib_ud_wr *ud;
2053
2054                         if (user_wr->opcode != IB_WR_SEND &&
2055                             user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2056                                 ret = -EINVAL;
2057                                 goto out_put;
2058                         }
2059
2060                         next_size = sizeof(*ud);
2061                         ud = alloc_wr(next_size, user_wr->num_sge);
2062                         if (!ud) {
2063                                 ret = -ENOMEM;
2064                                 goto out_put;
2065                         }
2066
2067                         ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
2068                                                    user_wr->wr.ud.ah, attrs);
2069                         if (!ud->ah) {
2070                                 kfree(ud);
2071                                 ret = -EINVAL;
2072                                 goto out_put;
2073                         }
2074                         ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2075                         ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2076
2077                         next = &ud->wr;
2078                 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2079                            user_wr->opcode == IB_WR_RDMA_WRITE ||
2080                            user_wr->opcode == IB_WR_RDMA_READ) {
2081                         struct ib_rdma_wr *rdma;
2082
2083                         next_size = sizeof(*rdma);
2084                         rdma = alloc_wr(next_size, user_wr->num_sge);
2085                         if (!rdma) {
2086                                 ret = -ENOMEM;
2087                                 goto out_put;
2088                         }
2089
2090                         rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2091                         rdma->rkey = user_wr->wr.rdma.rkey;
2092
2093                         next = &rdma->wr;
2094                 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2095                            user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2096                         struct ib_atomic_wr *atomic;
2097
2098                         next_size = sizeof(*atomic);
2099                         atomic = alloc_wr(next_size, user_wr->num_sge);
2100                         if (!atomic) {
2101                                 ret = -ENOMEM;
2102                                 goto out_put;
2103                         }
2104
2105                         atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2106                         atomic->compare_add = user_wr->wr.atomic.compare_add;
2107                         atomic->swap = user_wr->wr.atomic.swap;
2108                         atomic->rkey = user_wr->wr.atomic.rkey;
2109
2110                         next = &atomic->wr;
2111                 } else if (user_wr->opcode == IB_WR_SEND ||
2112                            user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2113                            user_wr->opcode == IB_WR_SEND_WITH_INV) {
2114                         next_size = sizeof(*next);
2115                         next = alloc_wr(next_size, user_wr->num_sge);
2116                         if (!next) {
2117                                 ret = -ENOMEM;
2118                                 goto out_put;
2119                         }
2120                 } else {
2121                         ret = -EINVAL;
2122                         goto out_put;
2123                 }
2124
2125                 if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2126                     user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2127                         next->ex.imm_data =
2128                                         (__be32 __force) user_wr->ex.imm_data;
2129                 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2130                         next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2131                 }
2132
2133                 if (!last)
2134                         wr = next;
2135                 else
2136                         last->next = next;
2137                 last = next;
2138
2139                 next->next       = NULL;
2140                 next->wr_id      = user_wr->wr_id;
2141                 next->num_sge    = user_wr->num_sge;
2142                 next->opcode     = user_wr->opcode;
2143                 next->send_flags = user_wr->send_flags;
2144
2145                 if (next->num_sge) {
2146                         next->sg_list = (void *) next +
2147                                 ALIGN(next_size, sizeof(struct ib_sge));
2148                         if (copy_from_user(next->sg_list, sgls + sg_ind,
2149                                            next->num_sge *
2150                                                    sizeof(struct ib_sge))) {
2151                                 ret = -EFAULT;
2152                                 goto out_put;
2153                         }
2154                         sg_ind += next->num_sge;
2155                 } else
2156                         next->sg_list = NULL;
2157         }
2158
2159         resp.bad_wr = 0;
2160         ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
2161         if (ret)
2162                 for (next = wr; next; next = next->next) {
2163                         ++resp.bad_wr;
2164                         if (next == bad_wr)
2165                                 break;
2166                 }
2167
2168         ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2169         if (ret2)
2170                 ret = ret2;
2171
2172 out_put:
2173         uobj_put_obj_read(qp);
2174
2175         while (wr) {
2176                 if (is_ud && ud_wr(wr)->ah)
2177                         uobj_put_obj_read(ud_wr(wr)->ah);
2178                 next = wr->next;
2179                 kfree(wr);
2180                 wr = next;
2181         }
2182
2183 out:
2184         kfree(user_wr);
2185
2186         return ret;
2187 }
2188
2189 static struct ib_recv_wr *
2190 ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
2191                           u32 wqe_size, u32 sge_count)
2192 {
2193         struct ib_uverbs_recv_wr *user_wr;
2194         struct ib_recv_wr        *wr = NULL, *last, *next;
2195         int                       sg_ind;
2196         int                       i;
2197         int                       ret;
2198         const struct ib_sge __user *sgls;
2199         const void __user *wqes;
2200
2201         if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2202                 return ERR_PTR(-EINVAL);
2203
2204         wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);
2205         if (IS_ERR(wqes))
2206                 return ERR_CAST(wqes);
2207         sgls = uverbs_request_next_ptr(
2208                 iter, sge_count * sizeof(struct ib_uverbs_sge));
2209         if (IS_ERR(sgls))
2210                 return ERR_CAST(sgls);
2211         ret = uverbs_request_finish(iter);
2212         if (ret)
2213                 return ERR_PTR(ret);
2214
2215         user_wr = kmalloc(wqe_size, GFP_KERNEL);
2216         if (!user_wr)
2217                 return ERR_PTR(-ENOMEM);
2218
2219         sg_ind = 0;
2220         last = NULL;
2221         for (i = 0; i < wr_count; ++i) {
2222                 if (copy_from_user(user_wr, wqes + i * wqe_size,
2223                                    wqe_size)) {
2224                         ret = -EFAULT;
2225                         goto err;
2226                 }
2227
2228                 if (user_wr->num_sge + sg_ind > sge_count) {
2229                         ret = -EINVAL;
2230                         goto err;
2231                 }
2232
2233                 if (user_wr->num_sge >=
2234                     (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) /
2235                     sizeof (struct ib_sge)) {
2236                         ret = -EINVAL;
2237                         goto err;
2238                 }
2239
2240                 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2241                                user_wr->num_sge * sizeof (struct ib_sge),
2242                                GFP_KERNEL);
2243                 if (!next) {
2244                         ret = -ENOMEM;
2245                         goto err;
2246                 }
2247
2248                 if (!last)
2249                         wr = next;
2250                 else
2251                         last->next = next;
2252                 last = next;
2253
2254                 next->next       = NULL;
2255                 next->wr_id      = user_wr->wr_id;
2256                 next->num_sge    = user_wr->num_sge;
2257
2258                 if (next->num_sge) {
2259                         next->sg_list = (void *) next +
2260                                 ALIGN(sizeof *next, sizeof (struct ib_sge));
2261                         if (copy_from_user(next->sg_list, sgls + sg_ind,
2262                                            next->num_sge *
2263                                                    sizeof(struct ib_sge))) {
2264                                 ret = -EFAULT;
2265                                 goto err;
2266                         }
2267                         sg_ind += next->num_sge;
2268                 } else
2269                         next->sg_list = NULL;
2270         }
2271
2272         kfree(user_wr);
2273         return wr;
2274
2275 err:
2276         kfree(user_wr);
2277
2278         while (wr) {
2279                 next = wr->next;
2280                 kfree(wr);
2281                 wr = next;
2282         }
2283
2284         return ERR_PTR(ret);
2285 }
2286
2287 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
2288 {
2289         struct ib_uverbs_post_recv      cmd;
2290         struct ib_uverbs_post_recv_resp resp;
2291         struct ib_recv_wr              *wr, *next;
2292         const struct ib_recv_wr        *bad_wr;
2293         struct ib_qp                   *qp;
2294         int ret, ret2;
2295         struct uverbs_req_iter iter;
2296
2297         ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2298         if (ret)
2299                 return ret;
2300
2301         wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2302                                        cmd.sge_count);
2303         if (IS_ERR(wr))
2304                 return PTR_ERR(wr);
2305
2306         qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2307         if (!qp) {
2308                 ret = -EINVAL;
2309                 goto out;
2310         }
2311
2312         resp.bad_wr = 0;
2313         ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
2314
2315         uobj_put_obj_read(qp);
2316         if (ret) {
2317                 for (next = wr; next; next = next->next) {
2318                         ++resp.bad_wr;
2319                         if (next == bad_wr)
2320                                 break;
2321                 }
2322         }
2323
2324         ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2325         if (ret2)
2326                 ret = ret2;
2327 out:
2328         while (wr) {
2329                 next = wr->next;
2330                 kfree(wr);
2331                 wr = next;
2332         }
2333
2334         return ret;
2335 }
2336
2337 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
2338 {
2339         struct ib_uverbs_post_srq_recv      cmd;
2340         struct ib_uverbs_post_srq_recv_resp resp;
2341         struct ib_recv_wr                  *wr, *next;
2342         const struct ib_recv_wr            *bad_wr;
2343         struct ib_srq                      *srq;
2344         int ret, ret2;
2345         struct uverbs_req_iter iter;
2346
2347         ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2348         if (ret)
2349                 return ret;
2350
2351         wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2352                                        cmd.sge_count);
2353         if (IS_ERR(wr))
2354                 return PTR_ERR(wr);
2355
2356         srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
2357         if (!srq) {
2358                 ret = -EINVAL;
2359                 goto out;
2360         }
2361
2362         resp.bad_wr = 0;
2363         ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
2364
2365         uobj_put_obj_read(srq);
2366
2367         if (ret)
2368                 for (next = wr; next; next = next->next) {
2369                         ++resp.bad_wr;
2370                         if (next == bad_wr)
2371                                 break;
2372                 }
2373
2374         ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2375         if (ret2)
2376                 ret = ret2;
2377
2378 out:
2379         while (wr) {
2380                 next = wr->next;
2381                 kfree(wr);
2382                 wr = next;
2383         }
2384
2385         return ret;
2386 }
2387
2388 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
2389 {
2390         struct ib_uverbs_create_ah       cmd;
2391         struct ib_uverbs_create_ah_resp  resp;
2392         struct ib_uobject               *uobj;
2393         struct ib_pd                    *pd;
2394         struct ib_ah                    *ah;
2395         struct rdma_ah_attr             attr = {};
2396         int ret;
2397         struct ib_device *ib_dev;
2398
2399         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2400         if (ret)
2401                 return ret;
2402
2403         uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
2404         if (IS_ERR(uobj))
2405                 return PTR_ERR(uobj);
2406
2407         if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) {
2408                 ret = -EINVAL;
2409                 goto err;
2410         }
2411
2412         pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2413         if (!pd) {
2414                 ret = -EINVAL;
2415                 goto err;
2416         }
2417
2418         attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
2419         rdma_ah_set_make_grd(&attr, false);
2420         rdma_ah_set_dlid(&attr, cmd.attr.dlid);
2421         rdma_ah_set_sl(&attr, cmd.attr.sl);
2422         rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
2423         rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);
2424         rdma_ah_set_port_num(&attr, cmd.attr.port_num);
2425
2426         if (cmd.attr.is_global) {
2427                 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,
2428                                 cmd.attr.grh.sgid_index,
2429                                 cmd.attr.grh.hop_limit,
2430                                 cmd.attr.grh.traffic_class);
2431                 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);
2432         } else {
2433                 rdma_ah_set_ah_flags(&attr, 0);
2434         }
2435
2436         ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);
2437         if (IS_ERR(ah)) {
2438                 ret = PTR_ERR(ah);
2439                 goto err_put;
2440         }
2441
2442         ah->uobject  = uobj;
2443         uobj->user_handle = cmd.user_handle;
2444         uobj->object = ah;
2445
2446         resp.ah_handle = uobj->id;
2447
2448         ret = uverbs_response(attrs, &resp, sizeof(resp));
2449         if (ret)
2450                 goto err_copy;
2451
2452         uobj_put_obj_read(pd);
2453         return uobj_alloc_commit(uobj, attrs);
2454
2455 err_copy:
2456         rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
2457
2458 err_put:
2459         uobj_put_obj_read(pd);
2460
2461 err:
2462         uobj_alloc_abort(uobj, attrs);
2463         return ret;
2464 }
2465
2466 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
2467 {
2468         struct ib_uverbs_destroy_ah cmd;
2469         int ret;
2470
2471         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2472         if (ret)
2473                 return ret;
2474
2475         return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
2476 }
2477
2478 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
2479 {
2480         struct ib_uverbs_attach_mcast cmd;
2481         struct ib_qp                 *qp;
2482         struct ib_uqp_object         *obj;
2483         struct ib_uverbs_mcast_entry *mcast;
2484         int                           ret;
2485
2486         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2487         if (ret)
2488                 return ret;
2489
2490         qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2491         if (!qp)
2492                 return -EINVAL;
2493
2494         obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2495
2496         mutex_lock(&obj->mcast_lock);
2497         list_for_each_entry(mcast, &obj->mcast_list, list)
2498                 if (cmd.mlid == mcast->lid &&
2499                     !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2500                         ret = 0;
2501                         goto out_put;
2502                 }
2503
2504         mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2505         if (!mcast) {
2506                 ret = -ENOMEM;
2507                 goto out_put;
2508         }
2509
2510         mcast->lid = cmd.mlid;
2511         memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2512
2513         ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2514         if (!ret)
2515                 list_add_tail(&mcast->list, &obj->mcast_list);
2516         else
2517                 kfree(mcast);
2518
2519 out_put:
2520         mutex_unlock(&obj->mcast_lock);
2521         uobj_put_obj_read(qp);
2522
2523         return ret;
2524 }
2525
2526 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
2527 {
2528         struct ib_uverbs_detach_mcast cmd;
2529         struct ib_uqp_object         *obj;
2530         struct ib_qp                 *qp;
2531         struct ib_uverbs_mcast_entry *mcast;
2532         int                           ret = -EINVAL;
2533         bool                          found = false;
2534
2535         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2536         if (ret)
2537                 return ret;
2538
2539         qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2540         if (!qp)
2541                 return -EINVAL;
2542
2543         obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2544         mutex_lock(&obj->mcast_lock);
2545
2546         list_for_each_entry(mcast, &obj->mcast_list, list)
2547                 if (cmd.mlid == mcast->lid &&
2548                     !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2549                         list_del(&mcast->list);
2550                         kfree(mcast);
2551                         found = true;
2552                         break;
2553                 }
2554
2555         if (!found) {
2556                 ret = -EINVAL;
2557                 goto out_put;
2558         }
2559
2560         ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);
2561
2562 out_put:
2563         mutex_unlock(&obj->mcast_lock);
2564         uobj_put_obj_read(qp);
2565         return ret;
2566 }
2567
2568 struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
2569 {
2570         struct ib_uflow_resources *resources;
2571
2572         resources = kzalloc(sizeof(*resources), GFP_KERNEL);
2573
2574         if (!resources)
2575                 return NULL;
2576
2577         if (!num_specs)
2578                 goto out;
2579
2580         resources->counters =
2581                 kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL);
2582         resources->collection =
2583                 kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL);
2584
2585         if (!resources->counters || !resources->collection)
2586                 goto err;
2587
2588 out:
2589         resources->max = num_specs;
2590         return resources;
2591
2592 err:
2593         kfree(resources->counters);
2594         kfree(resources);
2595
2596         return NULL;
2597 }
2598 EXPORT_SYMBOL(flow_resources_alloc);
2599
2600 void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2601 {
2602         unsigned int i;
2603
2604         if (!uflow_res)
2605                 return;
2606
2607         for (i = 0; i < uflow_res->collection_num; i++)
2608                 atomic_dec(&uflow_res->collection[i]->usecnt);
2609
2610         for (i = 0; i < uflow_res->counters_num; i++)
2611                 atomic_dec(&uflow_res->counters[i]->usecnt);
2612
2613         kfree(uflow_res->collection);
2614         kfree(uflow_res->counters);
2615         kfree(uflow_res);
2616 }
2617 EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
2618
2619 void flow_resources_add(struct ib_uflow_resources *uflow_res,
2620                         enum ib_flow_spec_type type,
2621                         void *ibobj)
2622 {
2623         WARN_ON(uflow_res->num >= uflow_res->max);
2624
2625         switch (type) {
2626         case IB_FLOW_SPEC_ACTION_HANDLE:
2627                 atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);
2628                 uflow_res->collection[uflow_res->collection_num++] =
2629                         (struct ib_flow_action *)ibobj;
2630                 break;
2631         case IB_FLOW_SPEC_ACTION_COUNT:
2632                 atomic_inc(&((struct ib_counters *)ibobj)->usecnt);
2633                 uflow_res->counters[uflow_res->counters_num++] =
2634                         (struct ib_counters *)ibobj;
2635                 break;
2636         default:
2637                 WARN_ON(1);
2638         }
2639
2640         uflow_res->num++;
2641 }
2642 EXPORT_SYMBOL(flow_resources_add);
2643
2644 static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,
2645                                        struct ib_uverbs_flow_spec *kern_spec,
2646                                        union ib_flow_spec *ib_spec,
2647                                        struct ib_uflow_resources *uflow_res)
2648 {
2649         ib_spec->type = kern_spec->type;
2650         switch (ib_spec->type) {
2651         case IB_FLOW_SPEC_ACTION_TAG:
2652                 if (kern_spec->flow_tag.size !=
2653                     sizeof(struct ib_uverbs_flow_spec_action_tag))
2654                         return -EINVAL;
2655
2656                 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2657                 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2658                 break;
2659         case IB_FLOW_SPEC_ACTION_DROP:
2660                 if (kern_spec->drop.size !=
2661                     sizeof(struct ib_uverbs_flow_spec_action_drop))
2662                         return -EINVAL;
2663
2664                 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2665                 break;
2666         case IB_FLOW_SPEC_ACTION_HANDLE:
2667                 if (kern_spec->action.size !=
2668                     sizeof(struct ib_uverbs_flow_spec_action_handle))
2669                         return -EOPNOTSUPP;
2670                 ib_spec->action.act = uobj_get_obj_read(flow_action,
2671                                                         UVERBS_OBJECT_FLOW_ACTION,
2672                                                         kern_spec->action.handle,
2673                                                         attrs);
2674                 if (!ib_spec->action.act)
2675                         return -EINVAL;
2676                 ib_spec->action.size =
2677                         sizeof(struct ib_flow_spec_action_handle);
2678                 flow_resources_add(uflow_res,
2679                                    IB_FLOW_SPEC_ACTION_HANDLE,
2680                                    ib_spec->action.act);
2681                 uobj_put_obj_read(ib_spec->action.act);
2682                 break;
2683         case IB_FLOW_SPEC_ACTION_COUNT:
2684                 if (kern_spec->flow_count.size !=
2685                         sizeof(struct ib_uverbs_flow_spec_action_count))
2686                         return -EINVAL;
2687                 ib_spec->flow_count.counters =
2688                         uobj_get_obj_read(counters,
2689                                           UVERBS_OBJECT_COUNTERS,
2690                                           kern_spec->flow_count.handle,
2691                                           attrs);
2692                 if (!ib_spec->flow_count.counters)
2693                         return -EINVAL;
2694                 ib_spec->flow_count.size =
2695                                 sizeof(struct ib_flow_spec_action_count);
2696                 flow_resources_add(uflow_res,
2697                                    IB_FLOW_SPEC_ACTION_COUNT,
2698                                    ib_spec->flow_count.counters);
2699                 uobj_put_obj_read(ib_spec->flow_count.counters);
2700                 break;
2701         default:
2702                 return -EINVAL;
2703         }
2704         return 0;
2705 }
2706
2707 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec)
2708 {
2709         /* Returns user space filter size, includes padding */
2710         return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;
2711 }
2712
2713 static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,
2714                                 u16 ib_real_filter_sz)
2715 {
2716         /*
2717          * User space filter structures must be 64 bit aligned, otherwise this
2718          * may pass, but we won't handle additional new attributes.
2719          */
2720
2721         if (kern_filter_size > ib_real_filter_sz) {
2722                 if (memchr_inv(kern_spec_filter +
2723                                ib_real_filter_sz, 0,
2724                                kern_filter_size - ib_real_filter_sz))
2725                         return -EINVAL;
2726                 return ib_real_filter_sz;
2727         }
2728         return kern_filter_size;
2729 }
2730
2731 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
2732                                           const void *kern_spec_mask,
2733                                           const void *kern_spec_val,
2734                                           size_t kern_filter_sz,
2735                                           union ib_flow_spec *ib_spec)
2736 {
2737         ssize_t actual_filter_sz;
2738         ssize_t ib_filter_sz;
2739
2740         /* User flow spec size must be aligned to 4 bytes */
2741         if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2742                 return -EINVAL;
2743
2744         ib_spec->type = type;
2745
2746         if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2747                 return -EINVAL;
2748
2749         switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2750         case IB_FLOW_SPEC_ETH:
2751                 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
2752                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2753                                                     kern_filter_sz,
2754                                                     ib_filter_sz);
2755                 if (actual_filter_sz <= 0)
2756                         return -EINVAL;
2757                 ib_spec->size = sizeof(struct ib_flow_spec_eth);
2758                 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2759                 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
2760                 break;
2761         case IB_FLOW_SPEC_IPV4:
2762                 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);
2763                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2764                                                     kern_filter_sz,
2765                                                     ib_filter_sz);
2766                 if (actual_filter_sz <= 0)
2767                         return -EINVAL;
2768                 ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2769                 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2770                 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
2771                 break;
2772         case IB_FLOW_SPEC_IPV6:
2773                 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);
2774                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2775                                                     kern_filter_sz,
2776                                                     ib_filter_sz);
2777                 if (actual_filter_sz <= 0)
2778                         return -EINVAL;
2779                 ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2780                 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2781                 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
2782
2783                 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2784                     (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2785                         return -EINVAL;
2786                 break;
2787         case IB_FLOW_SPEC_TCP:
2788         case IB_FLOW_SPEC_UDP:
2789                 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);
2790                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2791                                                     kern_filter_sz,
2792                                                     ib_filter_sz);
2793                 if (actual_filter_sz <= 0)
2794                         return -EINVAL;
2795                 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2796                 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2797                 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
2798                 break;
2799         case IB_FLOW_SPEC_VXLAN_TUNNEL:
2800                 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);
2801                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2802                                                     kern_filter_sz,
2803                                                     ib_filter_sz);
2804                 if (actual_filter_sz <= 0)
2805                         return -EINVAL;
2806                 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2807                 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2808                 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2809
2810                 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2811                     (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2812                         return -EINVAL;
2813                 break;
2814         case IB_FLOW_SPEC_ESP:
2815                 ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz);
2816                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2817                                                     kern_filter_sz,
2818                                                     ib_filter_sz);
2819                 if (actual_filter_sz <= 0)
2820                         return -EINVAL;
2821                 ib_spec->esp.size = sizeof(struct ib_flow_spec_esp);
2822                 memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);
2823                 memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);
2824                 break;
2825         case IB_FLOW_SPEC_GRE:
2826                 ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz);
2827                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2828                                                     kern_filter_sz,
2829                                                     ib_filter_sz);
2830                 if (actual_filter_sz <= 0)
2831                         return -EINVAL;
2832                 ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);
2833                 memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);
2834                 memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);
2835                 break;
2836         case IB_FLOW_SPEC_MPLS:
2837                 ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz);
2838                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2839                                                     kern_filter_sz,
2840                                                     ib_filter_sz);
2841                 if (actual_filter_sz <= 0)
2842                         return -EINVAL;
2843                 ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);
2844                 memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);
2845                 memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);
2846                 break;
2847         default:
2848                 return -EINVAL;
2849         }
2850         return 0;
2851 }
2852
2853 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2854                                        union ib_flow_spec *ib_spec)
2855 {
2856         ssize_t kern_filter_sz;
2857         void *kern_spec_mask;
2858         void *kern_spec_val;
2859
2860         kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr);
2861
2862         kern_spec_val = (void *)kern_spec +
2863                 sizeof(struct ib_uverbs_flow_spec_hdr);
2864         kern_spec_mask = kern_spec_val + kern_filter_sz;
2865
2866         return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type,
2867                                                      kern_spec_mask,
2868                                                      kern_spec_val,
2869                                                      kern_filter_sz, ib_spec);
2870 }
2871
2872 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
2873                                 struct ib_uverbs_flow_spec *kern_spec,
2874                                 union ib_flow_spec *ib_spec,
2875                                 struct ib_uflow_resources *uflow_res)
2876 {
2877         if (kern_spec->reserved)
2878                 return -EINVAL;
2879
2880         if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
2881                 return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
2882                                                    uflow_res);
2883         else
2884                 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2885 }
2886
2887 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
2888 {
2889         struct ib_uverbs_ex_create_wq cmd;
2890         struct ib_uverbs_ex_create_wq_resp resp = {};
2891         struct ib_uwq_object           *obj;
2892         int err = 0;
2893         struct ib_cq *cq;
2894         struct ib_pd *pd;
2895         struct ib_wq *wq;
2896         struct ib_wq_init_attr wq_init_attr = {};
2897         struct ib_device *ib_dev;
2898
2899         err = uverbs_request(attrs, &cmd, sizeof(cmd));
2900         if (err)
2901                 return err;
2902
2903         if (cmd.comp_mask)
2904                 return -EOPNOTSUPP;
2905
2906         obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
2907                                                  &ib_dev);
2908         if (IS_ERR(obj))
2909                 return PTR_ERR(obj);
2910
2911         pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2912         if (!pd) {
2913                 err = -EINVAL;
2914                 goto err_uobj;
2915         }
2916
2917         cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
2918         if (!cq) {
2919                 err = -EINVAL;
2920                 goto err_put_pd;
2921         }
2922
2923         wq_init_attr.cq = cq;
2924         wq_init_attr.max_sge = cmd.max_sge;
2925         wq_init_attr.max_wr = cmd.max_wr;
2926         wq_init_attr.wq_context = attrs->ufile;
2927         wq_init_attr.wq_type = cmd.wq_type;
2928         wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
2929         wq_init_attr.create_flags = cmd.create_flags;
2930         obj->uevent.events_reported = 0;
2931         INIT_LIST_HEAD(&obj->uevent.event_list);
2932
2933         wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
2934         if (IS_ERR(wq)) {
2935                 err = PTR_ERR(wq);
2936                 goto err_put_cq;
2937         }
2938
2939         wq->uobject = &obj->uevent.uobject;
2940         obj->uevent.uobject.object = wq;
2941         wq->wq_type = wq_init_attr.wq_type;
2942         wq->cq = cq;
2943         wq->pd = pd;
2944         wq->device = pd->device;
2945         wq->wq_context = wq_init_attr.wq_context;
2946         atomic_set(&wq->usecnt, 0);
2947         atomic_inc(&pd->usecnt);
2948         atomic_inc(&cq->usecnt);
2949         wq->uobject = &obj->uevent.uobject;
2950         obj->uevent.uobject.object = wq;
2951
2952         memset(&resp, 0, sizeof(resp));
2953         resp.wq_handle = obj->uevent.uobject.id;
2954         resp.max_sge = wq_init_attr.max_sge;
2955         resp.max_wr = wq_init_attr.max_wr;
2956         resp.wqn = wq->wq_num;
2957         resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2958         err = uverbs_response(attrs, &resp, sizeof(resp));
2959         if (err)
2960                 goto err_copy;
2961
2962         uobj_put_obj_read(pd);
2963         uobj_put_obj_read(cq);
2964         return uobj_alloc_commit(&obj->uevent.uobject, attrs);
2965
2966 err_copy:
2967         ib_destroy_wq(wq, &attrs->driver_udata);
2968 err_put_cq:
2969         uobj_put_obj_read(cq);
2970 err_put_pd:
2971         uobj_put_obj_read(pd);
2972 err_uobj:
2973         uobj_alloc_abort(&obj->uevent.uobject, attrs);
2974
2975         return err;
2976 }
2977
2978 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
2979 {
2980         struct ib_uverbs_ex_destroy_wq  cmd;
2981         struct ib_uverbs_ex_destroy_wq_resp     resp = {};
2982         struct ib_uobject               *uobj;
2983         struct ib_uwq_object            *obj;
2984         int                             ret;
2985
2986         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2987         if (ret)
2988                 return ret;
2989
2990         if (cmd.comp_mask)
2991                 return -EOPNOTSUPP;
2992
2993         resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2994         uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
2995         if (IS_ERR(uobj))
2996                 return PTR_ERR(uobj);
2997
2998         obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
2999         resp.events_reported = obj->uevent.events_reported;
3000
3001         uobj_put_destroy(uobj);
3002
3003         return uverbs_response(attrs, &resp, sizeof(resp));
3004 }
3005
3006 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
3007 {
3008         struct ib_uverbs_ex_modify_wq cmd;
3009         struct ib_wq *wq;
3010         struct ib_wq_attr wq_attr = {};
3011         int ret;
3012
3013         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3014         if (ret)
3015                 return ret;
3016
3017         if (!cmd.attr_mask)
3018                 return -EINVAL;
3019
3020         if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
3021                 return -EINVAL;
3022
3023         wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3024         if (!wq)
3025                 return -EINVAL;
3026
3027         wq_attr.curr_wq_state = cmd.curr_wq_state;
3028         wq_attr.wq_state = cmd.wq_state;
3029         if (cmd.attr_mask & IB_WQ_FLAGS) {
3030                 wq_attr.flags = cmd.flags;
3031                 wq_attr.flags_mask = cmd.flags_mask;
3032         }
3033         ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
3034                                         &attrs->driver_udata);
3035         uobj_put_obj_read(wq);
3036         return ret;
3037 }
3038
3039 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3040 {
3041         struct ib_uverbs_ex_create_rwq_ind_table cmd;
3042         struct ib_uverbs_ex_create_rwq_ind_table_resp  resp = {};
3043         struct ib_uobject                 *uobj;
3044         int err;
3045         struct ib_rwq_ind_table_init_attr init_attr = {};
3046         struct ib_rwq_ind_table *rwq_ind_tbl;
3047         struct ib_wq    **wqs = NULL;
3048         u32 *wqs_handles = NULL;
3049         struct ib_wq    *wq = NULL;
3050         int i, j, num_read_wqs;
3051         u32 num_wq_handles;
3052         struct uverbs_req_iter iter;
3053         struct ib_device *ib_dev;
3054
3055         err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3056         if (err)
3057                 return err;
3058
3059         if (cmd.comp_mask)
3060                 return -EOPNOTSUPP;
3061
3062         if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3063                 return -EINVAL;
3064
3065         num_wq_handles = 1 << cmd.log_ind_tbl_size;
3066         wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3067                               GFP_KERNEL);
3068         if (!wqs_handles)
3069                 return -ENOMEM;
3070
3071         err = uverbs_request_next(&iter, wqs_handles,
3072                                   num_wq_handles * sizeof(__u32));
3073         if (err)
3074                 goto err_free;
3075
3076         err = uverbs_request_finish(&iter);
3077         if (err)
3078                 goto err_free;
3079
3080         wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);
3081         if (!wqs) {
3082                 err = -ENOMEM;
3083                 goto  err_free;
3084         }
3085
3086         for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3087                         num_read_wqs++) {
3088                 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
3089                                        wqs_handles[num_read_wqs], attrs);
3090                 if (!wq) {
3091                         err = -EINVAL;
3092                         goto put_wqs;
3093                 }
3094
3095                 wqs[num_read_wqs] = wq;
3096         }
3097
3098         uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
3099         if (IS_ERR(uobj)) {
3100                 err = PTR_ERR(uobj);
3101                 goto put_wqs;
3102         }
3103
3104         init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3105         init_attr.ind_tbl = wqs;
3106
3107         rwq_ind_tbl = ib_dev->ops.create_rwq_ind_table(ib_dev, &init_attr,
3108                                                        &attrs->driver_udata);
3109
3110         if (IS_ERR(rwq_ind_tbl)) {
3111                 err = PTR_ERR(rwq_ind_tbl);
3112                 goto err_uobj;
3113         }
3114
3115         rwq_ind_tbl->ind_tbl = wqs;
3116         rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3117         rwq_ind_tbl->uobject = uobj;
3118         uobj->object = rwq_ind_tbl;
3119         rwq_ind_tbl->device = ib_dev;
3120         atomic_set(&rwq_ind_tbl->usecnt, 0);
3121
3122         for (i = 0; i < num_wq_handles; i++)
3123                 atomic_inc(&wqs[i]->usecnt);
3124
3125         resp.ind_tbl_handle = uobj->id;
3126         resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3127         resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3128
3129         err = uverbs_response(attrs, &resp, sizeof(resp));
3130         if (err)
3131                 goto err_copy;
3132
3133         kfree(wqs_handles);
3134
3135         for (j = 0; j < num_read_wqs; j++)
3136                 uobj_put_obj_read(wqs[j]);
3137
3138         return uobj_alloc_commit(uobj, attrs);
3139
3140 err_copy:
3141         ib_destroy_rwq_ind_table(rwq_ind_tbl);
3142 err_uobj:
3143         uobj_alloc_abort(uobj, attrs);
3144 put_wqs:
3145         for (j = 0; j < num_read_wqs; j++)
3146                 uobj_put_obj_read(wqs[j]);
3147 err_free:
3148         kfree(wqs_handles);
3149         kfree(wqs);
3150         return err;
3151 }
3152
3153 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3154 {
3155         struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
3156         int ret;
3157
3158         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3159         if (ret)
3160                 return ret;
3161
3162         if (cmd.comp_mask)
3163                 return -EOPNOTSUPP;
3164
3165         return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
3166                                     cmd.ind_tbl_handle, attrs);
3167 }
3168
3169 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3170 {
3171         struct ib_uverbs_create_flow      cmd;
3172         struct ib_uverbs_create_flow_resp resp;
3173         struct ib_uobject                 *uobj;
3174         struct ib_flow                    *flow_id;
3175         struct ib_uverbs_flow_attr        *kern_flow_attr;
3176         struct ib_flow_attr               *flow_attr;
3177         struct ib_qp                      *qp;
3178         struct ib_uflow_resources         *uflow_res;
3179         struct ib_uverbs_flow_spec_hdr    *kern_spec;
3180         struct uverbs_req_iter iter;
3181         int err;
3182         void *ib_spec;
3183         int i;
3184         struct ib_device *ib_dev;
3185
3186         err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3187         if (err)
3188                 return err;
3189
3190         if (cmd.comp_mask)
3191                 return -EINVAL;
3192
3193         if (!capable(CAP_NET_RAW))
3194                 return -EPERM;
3195
3196         if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3197                 return -EINVAL;
3198
3199         if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3200             ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3201              (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3202                 return -EINVAL;
3203
3204         if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3205                 return -EINVAL;
3206
3207         if (cmd.flow_attr.size >
3208             (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3209                 return -EINVAL;
3210
3211         if (cmd.flow_attr.reserved[0] ||
3212             cmd.flow_attr.reserved[1])
3213                 return -EINVAL;
3214
3215         if (cmd.flow_attr.num_of_specs) {
3216                 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3217                                          GFP_KERNEL);
3218                 if (!kern_flow_attr)
3219                         return -ENOMEM;
3220
3221                 *kern_flow_attr = cmd.flow_attr;
3222                 err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,
3223                                           cmd.flow_attr.size);
3224                 if (err)
3225                         goto err_free_attr;
3226         } else {
3227                 kern_flow_attr = &cmd.flow_attr;
3228         }
3229
3230         err = uverbs_request_finish(&iter);
3231         if (err)
3232                 goto err_free_attr;
3233
3234         uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
3235         if (IS_ERR(uobj)) {
3236                 err = PTR_ERR(uobj);
3237                 goto err_free_attr;
3238         }
3239
3240         qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
3241         if (!qp) {
3242                 err = -EINVAL;
3243                 goto err_uobj;
3244         }
3245
3246         if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3247                 err = -EINVAL;
3248                 goto err_put;
3249         }
3250
3251         flow_attr = kzalloc(struct_size(flow_attr, flows,
3252                                 cmd.flow_attr.num_of_specs), GFP_KERNEL);
3253         if (!flow_attr) {
3254                 err = -ENOMEM;
3255                 goto err_put;
3256         }
3257         uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs);
3258         if (!uflow_res) {
3259                 err = -ENOMEM;
3260                 goto err_free_flow_attr;
3261         }
3262
3263         flow_attr->type = kern_flow_attr->type;
3264         flow_attr->priority = kern_flow_attr->priority;
3265         flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3266         flow_attr->port = kern_flow_attr->port;
3267         flow_attr->flags = kern_flow_attr->flags;
3268         flow_attr->size = sizeof(*flow_attr);
3269
3270         kern_spec = kern_flow_attr->flow_specs;
3271         ib_spec = flow_attr + 1;
3272         for (i = 0; i < flow_attr->num_of_specs &&
3273                         cmd.flow_attr.size >= sizeof(*kern_spec) &&
3274                         cmd.flow_attr.size >= kern_spec->size;
3275              i++) {
3276                 err = kern_spec_to_ib_spec(
3277                                 attrs, (struct ib_uverbs_flow_spec *)kern_spec,
3278                                 ib_spec, uflow_res);
3279                 if (err)
3280                         goto err_free;
3281
3282                 flow_attr->size +=
3283                         ((union ib_flow_spec *) ib_spec)->size;
3284                 cmd.flow_attr.size -= kern_spec->size;
3285                 kern_spec = ((void *)kern_spec) + kern_spec->size;
3286                 ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3287         }
3288         if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3289                 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3290                         i, cmd.flow_attr.size);
3291                 err = -EINVAL;
3292                 goto err_free;
3293         }
3294
3295         flow_id = qp->device->ops.create_flow(
3296                 qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata);
3297
3298         if (IS_ERR(flow_id)) {
3299                 err = PTR_ERR(flow_id);
3300                 goto err_free;
3301         }
3302
3303         ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
3304
3305         memset(&resp, 0, sizeof(resp));
3306         resp.flow_handle = uobj->id;
3307
3308         err = uverbs_response(attrs, &resp, sizeof(resp));
3309         if (err)
3310                 goto err_copy;
3311
3312         uobj_put_obj_read(qp);
3313         kfree(flow_attr);
3314         if (cmd.flow_attr.num_of_specs)
3315                 kfree(kern_flow_attr);
3316         return uobj_alloc_commit(uobj, attrs);
3317 err_copy:
3318         if (!qp->device->ops.destroy_flow(flow_id))
3319                 atomic_dec(&qp->usecnt);
3320 err_free:
3321         ib_uverbs_flow_resources_free(uflow_res);
3322 err_free_flow_attr:
3323         kfree(flow_attr);
3324 err_put:
3325         uobj_put_obj_read(qp);
3326 err_uobj:
3327         uobj_alloc_abort(uobj, attrs);
3328 err_free_attr:
3329         if (cmd.flow_attr.num_of_specs)
3330                 kfree(kern_flow_attr);
3331         return err;
3332 }
3333
3334 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
3335 {
3336         struct ib_uverbs_destroy_flow   cmd;
3337         int                             ret;
3338
3339         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3340         if (ret)
3341                 return ret;
3342
3343         if (cmd.comp_mask)
3344                 return -EINVAL;
3345
3346         return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
3347 }
3348
3349 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
3350                                 struct ib_uverbs_create_xsrq *cmd,
3351                                 struct ib_udata *udata)
3352 {
3353         struct ib_uverbs_create_srq_resp resp;
3354         struct ib_usrq_object           *obj;
3355         struct ib_pd                    *pd;
3356         struct ib_srq                   *srq;
3357         struct ib_uobject               *uninitialized_var(xrcd_uobj);
3358         struct ib_srq_init_attr          attr;
3359         int ret;
3360         struct ib_device *ib_dev;
3361
3362         obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
3363                                                   &ib_dev);
3364         if (IS_ERR(obj))
3365                 return PTR_ERR(obj);
3366
3367         if (cmd->srq_type == IB_SRQT_TM)
3368                 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3369
3370         if (cmd->srq_type == IB_SRQT_XRC) {
3371                 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
3372                                           attrs);
3373                 if (IS_ERR(xrcd_uobj)) {
3374                         ret = -EINVAL;
3375                         goto err;
3376                 }
3377
3378                 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3379                 if (!attr.ext.xrc.xrcd) {
3380                         ret = -EINVAL;
3381                         goto err_put_xrcd;
3382                 }
3383
3384                 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3385                 atomic_inc(&obj->uxrcd->refcnt);
3386         }
3387
3388         if (ib_srq_has_cq(cmd->srq_type)) {
3389                 attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
3390                                                 cmd->cq_handle, attrs);
3391                 if (!attr.ext.cq) {
3392                         ret = -EINVAL;
3393                         goto err_put_xrcd;
3394                 }
3395         }
3396
3397         pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
3398         if (!pd) {
3399                 ret = -EINVAL;
3400                 goto err_put_cq;
3401         }
3402
3403         attr.event_handler  = ib_uverbs_srq_event_handler;
3404         attr.srq_context    = attrs->ufile;
3405         attr.srq_type       = cmd->srq_type;
3406         attr.attr.max_wr    = cmd->max_wr;
3407         attr.attr.max_sge   = cmd->max_sge;
3408         attr.attr.srq_limit = cmd->srq_limit;
3409
3410         obj->uevent.events_reported = 0;
3411         INIT_LIST_HEAD(&obj->uevent.event_list);
3412
3413         srq = rdma_zalloc_drv_obj(ib_dev, ib_srq);
3414         if (!srq) {
3415                 ret = -ENOMEM;
3416                 goto err_put;
3417         }
3418
3419         srq->device        = pd->device;
3420         srq->pd            = pd;
3421         srq->srq_type      = cmd->srq_type;
3422         srq->uobject       = &obj->uevent.uobject;
3423         srq->event_handler = attr.event_handler;
3424         srq->srq_context   = attr.srq_context;
3425
3426         ret = pd->device->ops.create_srq(srq, &attr, udata);
3427         if (ret)
3428                 goto err_free;
3429
3430         if (ib_srq_has_cq(cmd->srq_type)) {
3431                 srq->ext.cq       = attr.ext.cq;
3432                 atomic_inc(&attr.ext.cq->usecnt);
3433         }
3434
3435         if (cmd->srq_type == IB_SRQT_XRC) {
3436                 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3437                 atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3438         }
3439
3440         atomic_inc(&pd->usecnt);
3441         atomic_set(&srq->usecnt, 0);
3442
3443         obj->uevent.uobject.object = srq;
3444         obj->uevent.uobject.user_handle = cmd->user_handle;
3445
3446         memset(&resp, 0, sizeof resp);
3447         resp.srq_handle = obj->uevent.uobject.id;
3448         resp.max_wr     = attr.attr.max_wr;
3449         resp.max_sge    = attr.attr.max_sge;
3450         if (cmd->srq_type == IB_SRQT_XRC)
3451                 resp.srqn = srq->ext.xrc.srq_num;
3452
3453         ret = uverbs_response(attrs, &resp, sizeof(resp));
3454         if (ret)
3455                 goto err_copy;
3456
3457         if (cmd->srq_type == IB_SRQT_XRC)
3458                 uobj_put_read(xrcd_uobj);
3459
3460         if (ib_srq_has_cq(cmd->srq_type))
3461                 uobj_put_obj_read(attr.ext.cq);
3462
3463         uobj_put_obj_read(pd);
3464         return uobj_alloc_commit(&obj->uevent.uobject, attrs);
3465
3466 err_copy:
3467         ib_destroy_srq_user(srq, &attrs->driver_udata);
3468
3469 err_free:
3470         kfree(srq);
3471 err_put:
3472         uobj_put_obj_read(pd);
3473
3474 err_put_cq:
3475         if (ib_srq_has_cq(cmd->srq_type))
3476                 uobj_put_obj_read(attr.ext.cq);
3477
3478 err_put_xrcd:
3479         if (cmd->srq_type == IB_SRQT_XRC) {
3480                 atomic_dec(&obj->uxrcd->refcnt);
3481                 uobj_put_read(xrcd_uobj);
3482         }
3483
3484 err:
3485         uobj_alloc_abort(&obj->uevent.uobject, attrs);
3486         return ret;
3487 }
3488
3489 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
3490 {
3491         struct ib_uverbs_create_srq      cmd;
3492         struct ib_uverbs_create_xsrq     xcmd;
3493         int ret;
3494
3495         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3496         if (ret)
3497                 return ret;
3498
3499         memset(&xcmd, 0, sizeof(xcmd));
3500         xcmd.response    = cmd.response;
3501         xcmd.user_handle = cmd.user_handle;
3502         xcmd.srq_type    = IB_SRQT_BASIC;
3503         xcmd.pd_handle   = cmd.pd_handle;
3504         xcmd.max_wr      = cmd.max_wr;
3505         xcmd.max_sge     = cmd.max_sge;
3506         xcmd.srq_limit   = cmd.srq_limit;
3507
3508         return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);
3509 }
3510
3511 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
3512 {
3513         struct ib_uverbs_create_xsrq     cmd;
3514         int ret;
3515
3516         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3517         if (ret)
3518                 return ret;
3519
3520         return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);
3521 }
3522
3523 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
3524 {
3525         struct ib_uverbs_modify_srq cmd;
3526         struct ib_srq              *srq;
3527         struct ib_srq_attr          attr;
3528         int                         ret;
3529
3530         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3531         if (ret)
3532                 return ret;
3533
3534         srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3535         if (!srq)
3536                 return -EINVAL;
3537
3538         attr.max_wr    = cmd.max_wr;
3539         attr.srq_limit = cmd.srq_limit;
3540
3541         ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
3542                                           &attrs->driver_udata);
3543
3544         uobj_put_obj_read(srq);
3545
3546         return ret;
3547 }
3548
3549 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
3550 {
3551         struct ib_uverbs_query_srq      cmd;
3552         struct ib_uverbs_query_srq_resp resp;
3553         struct ib_srq_attr              attr;
3554         struct ib_srq                   *srq;
3555         int                             ret;
3556
3557         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3558         if (ret)
3559                 return ret;
3560
3561         srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3562         if (!srq)
3563                 return -EINVAL;
3564
3565         ret = ib_query_srq(srq, &attr);
3566
3567         uobj_put_obj_read(srq);
3568
3569         if (ret)
3570                 return ret;
3571
3572         memset(&resp, 0, sizeof resp);
3573
3574         resp.max_wr    = attr.max_wr;
3575         resp.max_sge   = attr.max_sge;
3576         resp.srq_limit = attr.srq_limit;
3577
3578         return uverbs_response(attrs, &resp, sizeof(resp));
3579 }
3580
3581 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
3582 {
3583         struct ib_uverbs_destroy_srq      cmd;
3584         struct ib_uverbs_destroy_srq_resp resp;
3585         struct ib_uobject                *uobj;
3586         struct ib_uevent_object          *obj;
3587         int ret;
3588
3589         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3590         if (ret)
3591                 return ret;
3592
3593         uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3594         if (IS_ERR(uobj))
3595                 return PTR_ERR(uobj);
3596
3597         obj = container_of(uobj, struct ib_uevent_object, uobject);
3598         memset(&resp, 0, sizeof(resp));
3599         resp.events_reported = obj->events_reported;
3600
3601         uobj_put_destroy(uobj);
3602
3603         return uverbs_response(attrs, &resp, sizeof(resp));
3604 }
3605
3606 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
3607 {
3608         struct ib_uverbs_ex_query_device_resp resp = {};
3609         struct ib_uverbs_ex_query_device  cmd;
3610         struct ib_device_attr attr = {0};
3611         struct ib_ucontext *ucontext;
3612         struct ib_device *ib_dev;
3613         int err;
3614
3615         ucontext = ib_uverbs_get_ucontext(attrs);
3616         if (IS_ERR(ucontext))
3617                 return PTR_ERR(ucontext);
3618         ib_dev = ucontext->device;
3619
3620         err = uverbs_request(attrs, &cmd, sizeof(cmd));
3621         if (err)
3622                 return err;
3623
3624         if (cmd.comp_mask)
3625                 return -EINVAL;
3626
3627         if (cmd.reserved)
3628                 return -EINVAL;
3629
3630         err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
3631         if (err)
3632                 return err;
3633
3634         copy_query_dev_fields(ucontext, &resp.base, &attr);
3635
3636         resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3637         resp.odp_caps.per_transport_caps.rc_odp_caps =
3638                 attr.odp_caps.per_transport_caps.rc_odp_caps;
3639         resp.odp_caps.per_transport_caps.uc_odp_caps =
3640                 attr.odp_caps.per_transport_caps.uc_odp_caps;
3641         resp.odp_caps.per_transport_caps.ud_odp_caps =
3642                 attr.odp_caps.per_transport_caps.ud_odp_caps;
3643         resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps;
3644
3645         resp.timestamp_mask = attr.timestamp_mask;
3646         resp.hca_core_clock = attr.hca_core_clock;
3647         resp.device_cap_flags_ex = attr.device_cap_flags;
3648         resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3649         resp.rss_caps.max_rwq_indirection_tables =
3650                 attr.rss_caps.max_rwq_indirection_tables;
3651         resp.rss_caps.max_rwq_indirection_table_size =
3652                 attr.rss_caps.max_rwq_indirection_table_size;
3653         resp.max_wq_type_rq = attr.max_wq_type_rq;
3654         resp.raw_packet_caps = attr.raw_packet_caps;
3655         resp.tm_caps.max_rndv_hdr_size  = attr.tm_caps.max_rndv_hdr_size;
3656         resp.tm_caps.max_num_tags       = attr.tm_caps.max_num_tags;
3657         resp.tm_caps.max_ops            = attr.tm_caps.max_ops;
3658         resp.tm_caps.max_sge            = attr.tm_caps.max_sge;
3659         resp.tm_caps.flags              = attr.tm_caps.flags;
3660         resp.cq_moderation_caps.max_cq_moderation_count  =
3661                 attr.cq_caps.max_cq_moderation_count;
3662         resp.cq_moderation_caps.max_cq_moderation_period =
3663                 attr.cq_caps.max_cq_moderation_period;
3664         resp.max_dm_size = attr.max_dm_size;
3665         resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3666
3667         return uverbs_response(attrs, &resp, sizeof(resp));
3668 }
3669
3670 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
3671 {
3672         struct ib_uverbs_ex_modify_cq cmd;
3673         struct ib_cq *cq;
3674         int ret;
3675
3676         ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3677         if (ret)
3678                 return ret;
3679
3680         if (!cmd.attr_mask || cmd.reserved)
3681                 return -EINVAL;
3682
3683         if (cmd.attr_mask > IB_CQ_MODERATE)
3684                 return -EOPNOTSUPP;
3685
3686         cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
3687         if (!cq)
3688                 return -EINVAL;
3689
3690         ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
3691
3692         uobj_put_obj_read(cq);
3693
3694         return ret;
3695 }
3696
3697 /*
3698  * Describe the input structs for write(). Some write methods have an input
3699  * only struct, most have an input and output. If the struct has an output then
3700  * the 'response' u64 must be the first field in the request structure.
3701  *
3702  * If udata is present then both the request and response structs have a
3703  * trailing driver_data flex array. In this case the size of the base struct
3704  * cannot be changed.
3705  */
3706 #define offsetof_after(_struct, _member)                                       \
3707         (offsetof(_struct, _member) + sizeof(((_struct *)NULL)->_member))
3708
3709 #define UAPI_DEF_WRITE_IO(req, resp)                                           \
3710         .write.has_resp = 1 +                                                  \
3711                           BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) +    \
3712                           BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) !=    \
3713                                             sizeof(u64)),                      \
3714         .write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
3715
3716 #define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
3717
3718 #define UAPI_DEF_WRITE_UDATA_IO(req, resp)                                     \
3719         UAPI_DEF_WRITE_IO(req, resp),                                          \
3720                 .write.has_udata =                                             \
3721                         1 +                                                    \
3722                         BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=        \
3723                                           sizeof(req)) +                       \
3724                         BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) !=       \
3725                                           sizeof(resp))
3726
3727 #define UAPI_DEF_WRITE_UDATA_I(req)                                            \
3728         UAPI_DEF_WRITE_I(req),                                                 \
3729                 .write.has_udata =                                             \
3730                         1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=    \
3731                                               sizeof(req))
3732
3733 /*
3734  * The _EX versions are for use with WRITE_EX and allow the last struct member
3735  * to be specified. Buffers that do not include that member will be rejected.
3736  */
3737 #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member)     \
3738         .write.has_resp = 1,                                                   \
3739         .write.req_size = offsetof_after(req, req_last_member),                \
3740         .write.resp_size = offsetof_after(resp, resp_last_member)
3741
3742 #define UAPI_DEF_WRITE_I_EX(req, req_last_member)                              \
3743         .write.req_size = offsetof_after(req, req_last_member)
3744
3745 const struct uapi_definition uverbs_def_write_intf[] = {
3746         DECLARE_UVERBS_OBJECT(
3747                 UVERBS_OBJECT_AH,
3748                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
3749                                      ib_uverbs_create_ah,
3750                                      UAPI_DEF_WRITE_UDATA_IO(
3751                                              struct ib_uverbs_create_ah,
3752                                              struct ib_uverbs_create_ah_resp),
3753                                      UAPI_DEF_METHOD_NEEDS_FN(create_ah)),
3754                 DECLARE_UVERBS_WRITE(
3755                         IB_USER_VERBS_CMD_DESTROY_AH,
3756                         ib_uverbs_destroy_ah,
3757                         UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),
3758                         UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),
3759
3760         DECLARE_UVERBS_OBJECT(
3761                 UVERBS_OBJECT_COMP_CHANNEL,
3762                 DECLARE_UVERBS_WRITE(
3763                         IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
3764                         ib_uverbs_create_comp_channel,
3765                         UAPI_DEF_WRITE_IO(
3766                                 struct ib_uverbs_create_comp_channel,
3767                                 struct ib_uverbs_create_comp_channel_resp))),
3768
3769         DECLARE_UVERBS_OBJECT(
3770                 UVERBS_OBJECT_CQ,
3771                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
3772                                      ib_uverbs_create_cq,
3773                                      UAPI_DEF_WRITE_UDATA_IO(
3774                                              struct ib_uverbs_create_cq,
3775                                              struct ib_uverbs_create_cq_resp),
3776                                      UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3777                 DECLARE_UVERBS_WRITE(
3778                         IB_USER_VERBS_CMD_DESTROY_CQ,
3779                         ib_uverbs_destroy_cq,
3780                         UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
3781                                           struct ib_uverbs_destroy_cq_resp),
3782                         UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
3783                 DECLARE_UVERBS_WRITE(
3784                         IB_USER_VERBS_CMD_POLL_CQ,
3785                         ib_uverbs_poll_cq,
3786                         UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
3787                                           struct ib_uverbs_poll_cq_resp),
3788                         UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
3789                 DECLARE_UVERBS_WRITE(
3790                         IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
3791                         ib_uverbs_req_notify_cq,
3792                         UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
3793                         UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
3794                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
3795                                      ib_uverbs_resize_cq,
3796                                      UAPI_DEF_WRITE_UDATA_IO(
3797                                              struct ib_uverbs_resize_cq,
3798                                              struct ib_uverbs_resize_cq_resp),
3799                                      UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
3800                 DECLARE_UVERBS_WRITE_EX(
3801                         IB_USER_VERBS_EX_CMD_CREATE_CQ,
3802                         ib_uverbs_ex_create_cq,
3803                         UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
3804                                              reserved,
3805                                              struct ib_uverbs_ex_create_cq_resp,
3806                                              response_length),
3807                         UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3808                 DECLARE_UVERBS_WRITE_EX(
3809                         IB_USER_VERBS_EX_CMD_MODIFY_CQ,
3810                         ib_uverbs_ex_modify_cq,
3811                         UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
3812                         UAPI_DEF_METHOD_NEEDS_FN(create_cq))),
3813
3814         DECLARE_UVERBS_OBJECT(
3815                 UVERBS_OBJECT_DEVICE,
3816                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
3817                                      ib_uverbs_get_context,
3818                                      UAPI_DEF_WRITE_UDATA_IO(
3819                                              struct ib_uverbs_get_context,
3820                                              struct ib_uverbs_get_context_resp)),
3821                 DECLARE_UVERBS_WRITE(
3822                         IB_USER_VERBS_CMD_QUERY_DEVICE,
3823                         ib_uverbs_query_device,
3824                         UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
3825                                           struct ib_uverbs_query_device_resp)),
3826                 DECLARE_UVERBS_WRITE(
3827                         IB_USER_VERBS_CMD_QUERY_PORT,
3828                         ib_uverbs_query_port,
3829                         UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
3830                                           struct ib_uverbs_query_port_resp),
3831                         UAPI_DEF_METHOD_NEEDS_FN(query_port)),
3832                 DECLARE_UVERBS_WRITE_EX(
3833                         IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
3834                         ib_uverbs_ex_query_device,
3835                         UAPI_DEF_WRITE_IO_EX(
3836                                 struct ib_uverbs_ex_query_device,
3837                                 reserved,
3838                                 struct ib_uverbs_ex_query_device_resp,
3839                                 response_length),
3840                         UAPI_DEF_METHOD_NEEDS_FN(query_device)),
3841                 UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
3842                 UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
3843
3844         DECLARE_UVERBS_OBJECT(
3845                 UVERBS_OBJECT_FLOW,
3846                 DECLARE_UVERBS_WRITE_EX(
3847                         IB_USER_VERBS_EX_CMD_CREATE_FLOW,
3848                         ib_uverbs_ex_create_flow,
3849                         UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
3850                                              flow_attr,
3851                                              struct ib_uverbs_create_flow_resp,
3852                                              flow_handle),
3853                         UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
3854                 DECLARE_UVERBS_WRITE_EX(
3855                         IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
3856                         ib_uverbs_ex_destroy_flow,
3857                         UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
3858                         UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
3859
3860         DECLARE_UVERBS_OBJECT(
3861                 UVERBS_OBJECT_MR,
3862                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
3863                                      ib_uverbs_dereg_mr,
3864                                      UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
3865                                      UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
3866                 DECLARE_UVERBS_WRITE(
3867                         IB_USER_VERBS_CMD_REG_MR,
3868                         ib_uverbs_reg_mr,
3869                         UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
3870                                                 struct ib_uverbs_reg_mr_resp),
3871                         UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
3872                 DECLARE_UVERBS_WRITE(
3873                         IB_USER_VERBS_CMD_REREG_MR,
3874                         ib_uverbs_rereg_mr,
3875                         UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
3876                                                 struct ib_uverbs_rereg_mr_resp),
3877                         UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
3878
3879         DECLARE_UVERBS_OBJECT(
3880                 UVERBS_OBJECT_MW,
3881                 DECLARE_UVERBS_WRITE(
3882                         IB_USER_VERBS_CMD_ALLOC_MW,
3883                         ib_uverbs_alloc_mw,
3884                         UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
3885                                                 struct ib_uverbs_alloc_mw_resp),
3886                         UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
3887                 DECLARE_UVERBS_WRITE(
3888                         IB_USER_VERBS_CMD_DEALLOC_MW,
3889                         ib_uverbs_dealloc_mw,
3890                         UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
3891                         UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
3892
3893         DECLARE_UVERBS_OBJECT(
3894                 UVERBS_OBJECT_PD,
3895                 DECLARE_UVERBS_WRITE(
3896                         IB_USER_VERBS_CMD_ALLOC_PD,
3897                         ib_uverbs_alloc_pd,
3898                         UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
3899                                                 struct ib_uverbs_alloc_pd_resp),
3900                         UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
3901                 DECLARE_UVERBS_WRITE(
3902                         IB_USER_VERBS_CMD_DEALLOC_PD,
3903                         ib_uverbs_dealloc_pd,
3904                         UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
3905                         UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
3906
3907         DECLARE_UVERBS_OBJECT(
3908                 UVERBS_OBJECT_QP,
3909                 DECLARE_UVERBS_WRITE(
3910                         IB_USER_VERBS_CMD_ATTACH_MCAST,
3911                         ib_uverbs_attach_mcast,
3912                         UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
3913                         UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
3914                         UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3915                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
3916                                      ib_uverbs_create_qp,
3917                                      UAPI_DEF_WRITE_UDATA_IO(
3918                                              struct ib_uverbs_create_qp,
3919                                              struct ib_uverbs_create_qp_resp),
3920                                      UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3921                 DECLARE_UVERBS_WRITE(
3922                         IB_USER_VERBS_CMD_DESTROY_QP,
3923                         ib_uverbs_destroy_qp,
3924                         UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
3925                                           struct ib_uverbs_destroy_qp_resp),
3926                         UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
3927                 DECLARE_UVERBS_WRITE(
3928                         IB_USER_VERBS_CMD_DETACH_MCAST,
3929                         ib_uverbs_detach_mcast,
3930                         UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
3931                         UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3932                 DECLARE_UVERBS_WRITE(
3933                         IB_USER_VERBS_CMD_MODIFY_QP,
3934                         ib_uverbs_modify_qp,
3935                         UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
3936                         UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
3937                 DECLARE_UVERBS_WRITE(
3938                         IB_USER_VERBS_CMD_POST_RECV,
3939                         ib_uverbs_post_recv,
3940                         UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
3941                                           struct ib_uverbs_post_recv_resp),
3942                         UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
3943                 DECLARE_UVERBS_WRITE(
3944                         IB_USER_VERBS_CMD_POST_SEND,
3945                         ib_uverbs_post_send,
3946                         UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
3947                                           struct ib_uverbs_post_send_resp),
3948                         UAPI_DEF_METHOD_NEEDS_FN(post_send)),
3949                 DECLARE_UVERBS_WRITE(
3950                         IB_USER_VERBS_CMD_QUERY_QP,
3951                         ib_uverbs_query_qp,
3952                         UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
3953                                           struct ib_uverbs_query_qp_resp),
3954                         UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
3955                 DECLARE_UVERBS_WRITE_EX(
3956                         IB_USER_VERBS_EX_CMD_CREATE_QP,
3957                         ib_uverbs_ex_create_qp,
3958                         UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
3959                                              comp_mask,
3960                                              struct ib_uverbs_ex_create_qp_resp,
3961                                              response_length),
3962                         UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3963                 DECLARE_UVERBS_WRITE_EX(
3964                         IB_USER_VERBS_EX_CMD_MODIFY_QP,
3965                         ib_uverbs_ex_modify_qp,
3966                         UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
3967                                              base,
3968                                              struct ib_uverbs_ex_modify_qp_resp,
3969                                              response_length),
3970                         UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
3971
3972         DECLARE_UVERBS_OBJECT(
3973                 UVERBS_OBJECT_RWQ_IND_TBL,
3974                 DECLARE_UVERBS_WRITE_EX(
3975                         IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
3976                         ib_uverbs_ex_create_rwq_ind_table,
3977                         UAPI_DEF_WRITE_IO_EX(
3978                                 struct ib_uverbs_ex_create_rwq_ind_table,
3979                                 log_ind_tbl_size,
3980                                 struct ib_uverbs_ex_create_rwq_ind_table_resp,
3981                                 ind_tbl_num),
3982                         UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
3983                 DECLARE_UVERBS_WRITE_EX(
3984                         IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
3985                         ib_uverbs_ex_destroy_rwq_ind_table,
3986                         UAPI_DEF_WRITE_I(
3987                                 struct ib_uverbs_ex_destroy_rwq_ind_table),
3988                         UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
3989
3990         DECLARE_UVERBS_OBJECT(
3991                 UVERBS_OBJECT_WQ,
3992                 DECLARE_UVERBS_WRITE_EX(
3993                         IB_USER_VERBS_EX_CMD_CREATE_WQ,
3994                         ib_uverbs_ex_create_wq,
3995                         UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
3996                                              max_sge,
3997                                              struct ib_uverbs_ex_create_wq_resp,
3998                                              wqn),
3999                         UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
4000                 DECLARE_UVERBS_WRITE_EX(
4001                         IB_USER_VERBS_EX_CMD_DESTROY_WQ,
4002                         ib_uverbs_ex_destroy_wq,
4003                         UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
4004                                              wq_handle,
4005                                              struct ib_uverbs_ex_destroy_wq_resp,
4006                                              reserved),
4007                         UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
4008                 DECLARE_UVERBS_WRITE_EX(
4009                         IB_USER_VERBS_EX_CMD_MODIFY_WQ,
4010                         ib_uverbs_ex_modify_wq,
4011                         UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
4012                                             curr_wq_state),
4013                         UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
4014
4015         DECLARE_UVERBS_OBJECT(
4016                 UVERBS_OBJECT_SRQ,
4017                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
4018                                      ib_uverbs_create_srq,
4019                                      UAPI_DEF_WRITE_UDATA_IO(
4020                                              struct ib_uverbs_create_srq,
4021                                              struct ib_uverbs_create_srq_resp),
4022                                      UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4023                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
4024                                      ib_uverbs_create_xsrq,
4025                                      UAPI_DEF_WRITE_UDATA_IO(
4026                                              struct ib_uverbs_create_xsrq,
4027                                              struct ib_uverbs_create_srq_resp),
4028                                      UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4029                 DECLARE_UVERBS_WRITE(
4030                         IB_USER_VERBS_CMD_DESTROY_SRQ,
4031                         ib_uverbs_destroy_srq,
4032                         UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
4033                                           struct ib_uverbs_destroy_srq_resp),
4034                         UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
4035                 DECLARE_UVERBS_WRITE(
4036                         IB_USER_VERBS_CMD_MODIFY_SRQ,
4037                         ib_uverbs_modify_srq,
4038                         UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
4039                         UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
4040                 DECLARE_UVERBS_WRITE(
4041                         IB_USER_VERBS_CMD_POST_SRQ_RECV,
4042                         ib_uverbs_post_srq_recv,
4043                         UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
4044                                           struct ib_uverbs_post_srq_recv_resp),
4045                         UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
4046                 DECLARE_UVERBS_WRITE(
4047                         IB_USER_VERBS_CMD_QUERY_SRQ,
4048                         ib_uverbs_query_srq,
4049                         UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
4050                                           struct ib_uverbs_query_srq_resp),
4051                         UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
4052
4053         DECLARE_UVERBS_OBJECT(
4054                 UVERBS_OBJECT_XRCD,
4055                 DECLARE_UVERBS_WRITE(
4056                         IB_USER_VERBS_CMD_CLOSE_XRCD,
4057                         ib_uverbs_close_xrcd,
4058                         UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),
4059                         UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),
4060                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
4061                                      ib_uverbs_open_qp,
4062                                      UAPI_DEF_WRITE_UDATA_IO(
4063                                              struct ib_uverbs_open_qp,
4064                                              struct ib_uverbs_create_qp_resp)),
4065                 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
4066                                      ib_uverbs_open_xrcd,
4067                                      UAPI_DEF_WRITE_UDATA_IO(
4068                                              struct ib_uverbs_open_xrcd,
4069                                              struct ib_uverbs_open_xrcd_resp),
4070                                      UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),
4071
4072         {},
4073 };