drivers/infiniband/hw/mlx5/devx.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4  */
   5
   6 #include <rdma/ib_user_verbs.h>
   7 #include <rdma/ib_verbs.h>
   8 #include <rdma/uverbs_types.h>
   9 #include <rdma/uverbs_ioctl.h>
  10 #include <rdma/mlx5_user_ioctl_cmds.h>
  11 #include <rdma/mlx5_user_ioctl_verbs.h>
  12 #include <rdma/ib_umem.h>
  13 #include <rdma/uverbs_std_types.h>
  14 #include <linux/mlx5/driver.h>
  15 #include <linux/mlx5/fs.h>
  16 #include "mlx5_ib.h"
  17 #include "qp.h"
  18 #include <linux/xarray.h>
  19
  20 #define UVERBS_MODULE_NAME mlx5_ib
  21 #include <rdma/uverbs_named_ioctl.h>
  22
  23 static void dispatch_event_fd(struct list_head *fd_list, const void *data);
  24
  25 enum devx_obj_flags {
  26         DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
  27         DEVX_OBJ_FLAGS_DCT = 1 << 1,
  28         DEVX_OBJ_FLAGS_CQ = 1 << 2,
  29 };
  30
  31 struct devx_async_data {
  32         struct mlx5_ib_dev *mdev;
  33         struct list_head list;
  34         struct devx_async_cmd_event_file *ev_file;
  35         struct mlx5_async_work cb_work;
  36         u16 cmd_out_len;
  37         /* must be last field in this structure */
  38         struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
  39 };
  40
  41 struct devx_async_event_data {
  42         struct list_head list; /* headed in ev_file->event_list */
  43         struct mlx5_ib_uapi_devx_async_event_hdr hdr;
  44 };
  45
  46 /* first level XA value data structure */
  47 struct devx_event {
  48         struct xarray object_ids; /* second XA level, Key = object id */
  49         struct list_head unaffiliated_list;
  50 };
  51
  52 /* second level XA value data structure */
  53 struct devx_obj_event {
  54         struct rcu_head rcu;
  55         struct list_head obj_sub_list;
  56 };
  57
  58 struct devx_event_subscription {
  59         struct list_head file_list; /* headed in ev_file->
  60                                      * subscribed_events_list
  61                                      */
  62         struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
  63                                    * devx_obj_event->obj_sub_list
  64                                    */
  65         struct list_head obj_list; /* headed in devx_object */
  66         struct list_head event_list; /* headed in ev_file->event_list or in
  67                                       * temp list via subscription
  68                                       */
  69
  70         u8 is_cleaned:1;
  71         u32 xa_key_level1;
  72         u32 xa_key_level2;
  73         struct rcu_head rcu;
  74         u64 cookie;
  75         struct devx_async_event_file *ev_file;
  76         struct eventfd_ctx *eventfd;
  77 };
  78
  79 struct devx_async_event_file {
  80         struct ib_uobject uobj;
  81         /* Head of events that are subscribed to this FD */
  82         struct list_head subscribed_events_list;
  83         spinlock_t lock;
  84         wait_queue_head_t poll_wait;
  85         struct list_head event_list;
  86         struct mlx5_ib_dev *dev;
  87         u8 omit_data:1;
  88         u8 is_overflow_err:1;
  89         u8 is_destroyed:1;
  90 };
  91
  92 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
  93 struct devx_obj {
  94         struct mlx5_ib_dev      *ib_dev;
  95         u64                     obj_id;
  96         u32                     dinlen; /* destroy inbox length */
  97         u32                     dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
  98         u32                     flags;
  99         union {
 100                 struct mlx5_ib_devx_mr  devx_mr;
 101                 struct mlx5_core_dct    core_dct;
 102                 struct mlx5_core_cq     core_cq;
 103                 u32                     flow_counter_bulk_size;
 104         };
 105         struct list_head event_sub; /* holds devx_event_subscription entries */
 106 };
 107
 108 struct devx_umem {
 109         struct mlx5_core_dev            *mdev;
 110         struct ib_umem                  *umem;
 111         u32                             page_offset;
 112         int                             page_shift;
 113         int                             ncont;
 114         u32                             dinlen;
 115         u32                             dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
 116 };
 117
 118 struct devx_umem_reg_cmd {
 119         void                            *in;
 120         u32                             inlen;
 121         u32                             out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
 122 };
 123
 124 static struct mlx5_ib_ucontext *
 125 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
 126 {
 127         return to_mucontext(ib_uverbs_get_ucontext(attrs));
 128 }
 129
 130 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
 131 {
 132         u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
 133         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
 134         void *uctx;
 135         int err;
 136         u16 uid;
 137         u32 cap = 0;
 138
 139         /* 0 means not supported */
 140         if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
 141                 return -EINVAL;
 142
 143         uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
 144         if (is_user && capable(CAP_NET_RAW) &&
 145             (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
 146                 cap |= MLX5_UCTX_CAP_RAW_TX;
 147         if (is_user && capable(CAP_SYS_RAWIO) &&
 148             (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
 149              MLX5_UCTX_CAP_INTERNAL_DEV_RES))
 150                 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
 151
 152         MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
 153         MLX5_SET(uctx, uctx, cap, cap);
 154
 155         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 156         if (err)
 157                 return err;
 158
 159         uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 160         return uid;
 161 }
 162
 163 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
 164 {
 165         u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};
 166         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
 167
 168         MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
 169         MLX5_SET(destroy_uctx_in, in, uid, uid);
 170
 171         mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 172 }
 173
 174 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
 175 {
 176         struct devx_obj *devx_obj = obj;
 177         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 178
 179         switch (opcode) {
 180         case MLX5_CMD_OP_DESTROY_TIR:
 181                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 182                 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
 183                                     obj_id);
 184                 return true;
 185
 186         case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
 187                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 188                 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
 189                                     table_id);
 190                 return true;
 191         default:
 192                 return false;
 193         }
 194 }
 195
 196 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id)
 197 {
 198         struct devx_obj *devx_obj = obj;
 199         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 200
 201         if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
 202
 203                 if (offset && offset >= devx_obj->flow_counter_bulk_size)
 204                         return false;
 205
 206                 *counter_id = MLX5_GET(dealloc_flow_counter_in,
 207                                        devx_obj->dinbox,
 208                                        flow_counter_id);
 209                 *counter_id += offset;
 210                 return true;
 211         }
 212
 213         return false;
 214 }
 215
 216 static bool is_legacy_unaffiliated_event_num(u16 event_num)
 217 {
 218         switch (event_num) {
 219         case MLX5_EVENT_TYPE_PORT_CHANGE:
 220                 return true;
 221         default:
 222                 return false;
 223         }
 224 }
 225
 226 static bool is_legacy_obj_event_num(u16 event_num)
 227 {
 228         switch (event_num) {
 229         case MLX5_EVENT_TYPE_PATH_MIG:
 230         case MLX5_EVENT_TYPE_COMM_EST:
 231         case MLX5_EVENT_TYPE_SQ_DRAINED:
 232         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 233         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 234         case MLX5_EVENT_TYPE_CQ_ERROR:
 235         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 236         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 237         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 238         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 239         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 240         case MLX5_EVENT_TYPE_DCT_DRAINED:
 241         case MLX5_EVENT_TYPE_COMP:
 242         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 243         case MLX5_EVENT_TYPE_XRQ_ERROR:
 244                 return true;
 245         default:
 246                 return false;
 247         }
 248 }
 249
 250 static u16 get_legacy_obj_type(u16 opcode)
 251 {
 252         switch (opcode) {
 253         case MLX5_CMD_OP_CREATE_RQ:
 254                 return MLX5_EVENT_QUEUE_TYPE_RQ;
 255         case MLX5_CMD_OP_CREATE_QP:
 256                 return MLX5_EVENT_QUEUE_TYPE_QP;
 257         case MLX5_CMD_OP_CREATE_SQ:
 258                 return MLX5_EVENT_QUEUE_TYPE_SQ;
 259         case MLX5_CMD_OP_CREATE_DCT:
 260                 return MLX5_EVENT_QUEUE_TYPE_DCT;
 261         default:
 262                 return 0;
 263         }
 264 }
 265
 266 static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
 267 {
 268         u16 opcode;
 269
 270         opcode = (obj->obj_id >> 32) & 0xffff;
 271
 272         if (is_legacy_obj_event_num(event_num))
 273                 return get_legacy_obj_type(opcode);
 274
 275         switch (opcode) {
 276         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 277                 return (obj->obj_id >> 48);
 278         case MLX5_CMD_OP_CREATE_RQ:
 279                 return MLX5_OBJ_TYPE_RQ;
 280         case MLX5_CMD_OP_CREATE_QP:
 281                 return MLX5_OBJ_TYPE_QP;
 282         case MLX5_CMD_OP_CREATE_SQ:
 283                 return MLX5_OBJ_TYPE_SQ;
 284         case MLX5_CMD_OP_CREATE_DCT:
 285                 return MLX5_OBJ_TYPE_DCT;
 286         case MLX5_CMD_OP_CREATE_TIR:
 287                 return MLX5_OBJ_TYPE_TIR;
 288         case MLX5_CMD_OP_CREATE_TIS:
 289                 return MLX5_OBJ_TYPE_TIS;
 290         case MLX5_CMD_OP_CREATE_PSV:
 291                 return MLX5_OBJ_TYPE_PSV;
 292         case MLX5_OBJ_TYPE_MKEY:
 293                 return MLX5_OBJ_TYPE_MKEY;
 294         case MLX5_CMD_OP_CREATE_RMP:
 295                 return MLX5_OBJ_TYPE_RMP;
 296         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 297                 return MLX5_OBJ_TYPE_XRC_SRQ;
 298         case MLX5_CMD_OP_CREATE_XRQ:
 299                 return MLX5_OBJ_TYPE_XRQ;
 300         case MLX5_CMD_OP_CREATE_RQT:
 301                 return MLX5_OBJ_TYPE_RQT;
 302         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 303                 return MLX5_OBJ_TYPE_FLOW_COUNTER;
 304         case MLX5_CMD_OP_CREATE_CQ:
 305                 return MLX5_OBJ_TYPE_CQ;
 306         default:
 307                 return 0;
 308         }
 309 }
 310
 311 static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
 312 {
 313         switch (event_type) {
 314         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 315         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 316         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 317         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 318         case MLX5_EVENT_TYPE_PATH_MIG:
 319         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 320         case MLX5_EVENT_TYPE_COMM_EST:
 321         case MLX5_EVENT_TYPE_SQ_DRAINED:
 322         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 323         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 324                 return eqe->data.qp_srq.type;
 325         case MLX5_EVENT_TYPE_CQ_ERROR:
 326         case MLX5_EVENT_TYPE_XRQ_ERROR:
 327                 return 0;
 328         case MLX5_EVENT_TYPE_DCT_DRAINED:
 329         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 330                 return MLX5_EVENT_QUEUE_TYPE_DCT;
 331         default:
 332                 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
 333         }
 334 }
 335
 336 static u32 get_dec_obj_id(u64 obj_id)
 337 {
 338         return (obj_id & 0xffffffff);
 339 }
 340
 341 /*
 342  * As the obj_id in the firmware is not globally unique the object type
 343  * must be considered upon checking for a valid object id.
 344  * For that the opcode of the creator command is encoded as part of the obj_id.
 345  */
 346 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
 347 {
 348         return ((u64)opcode << 32) | obj_id;
 349 }
 350
 351 static u64 devx_get_obj_id(const void *in)
 352 {
 353         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 354         u64 obj_id;
 355
 356         switch (opcode) {
 357         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 358         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 359                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
 360                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 361                                                  obj_type) << 16,
 362                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 363                                                  obj_id));
 364                 break;
 365         case MLX5_CMD_OP_QUERY_MKEY:
 366                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
 367                                         MLX5_GET(query_mkey_in, in,
 368                                                  mkey_index));
 369                 break;
 370         case MLX5_CMD_OP_QUERY_CQ:
 371                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 372                                         MLX5_GET(query_cq_in, in, cqn));
 373                 break;
 374         case MLX5_CMD_OP_MODIFY_CQ:
 375                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 376                                         MLX5_GET(modify_cq_in, in, cqn));
 377                 break;
 378         case MLX5_CMD_OP_QUERY_SQ:
 379                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 380                                         MLX5_GET(query_sq_in, in, sqn));
 381                 break;
 382         case MLX5_CMD_OP_MODIFY_SQ:
 383                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 384                                         MLX5_GET(modify_sq_in, in, sqn));
 385                 break;
 386         case MLX5_CMD_OP_QUERY_RQ:
 387                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 388                                         MLX5_GET(query_rq_in, in, rqn));
 389                 break;
 390         case MLX5_CMD_OP_MODIFY_RQ:
 391                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 392                                         MLX5_GET(modify_rq_in, in, rqn));
 393                 break;
 394         case MLX5_CMD_OP_QUERY_RMP:
 395                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 396                                         MLX5_GET(query_rmp_in, in, rmpn));
 397                 break;
 398         case MLX5_CMD_OP_MODIFY_RMP:
 399                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 400                                         MLX5_GET(modify_rmp_in, in, rmpn));
 401                 break;
 402         case MLX5_CMD_OP_QUERY_RQT:
 403                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 404                                         MLX5_GET(query_rqt_in, in, rqtn));
 405                 break;
 406         case MLX5_CMD_OP_MODIFY_RQT:
 407                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 408                                         MLX5_GET(modify_rqt_in, in, rqtn));
 409                 break;
 410         case MLX5_CMD_OP_QUERY_TIR:
 411                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 412                                         MLX5_GET(query_tir_in, in, tirn));
 413                 break;
 414         case MLX5_CMD_OP_MODIFY_TIR:
 415                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 416                                         MLX5_GET(modify_tir_in, in, tirn));
 417                 break;
 418         case MLX5_CMD_OP_QUERY_TIS:
 419                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 420                                         MLX5_GET(query_tis_in, in, tisn));
 421                 break;
 422         case MLX5_CMD_OP_MODIFY_TIS:
 423                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 424                                         MLX5_GET(modify_tis_in, in, tisn));
 425                 break;
 426         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 427                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 428                                         MLX5_GET(query_flow_table_in, in,
 429                                                  table_id));
 430                 break;
 431         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 432                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 433                                         MLX5_GET(modify_flow_table_in, in,
 434                                                  table_id));
 435                 break;
 436         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 437                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
 438                                         MLX5_GET(query_flow_group_in, in,
 439                                                  group_id));
 440                 break;
 441         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 442                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 443                                         MLX5_GET(query_fte_in, in,
 444                                                  flow_index));
 445                 break;
 446         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 447                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 448                                         MLX5_GET(set_fte_in, in, flow_index));
 449                 break;
 450         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 451                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
 452                                         MLX5_GET(query_q_counter_in, in,
 453                                                  counter_set_id));
 454                 break;
 455         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 456                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
 457                                         MLX5_GET(query_flow_counter_in, in,
 458                                                  flow_counter_id));
 459                 break;
 460         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 461                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
 462                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 463                                                  obj_id));
 464                 break;
 465         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 466                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 467                                         MLX5_GET(query_scheduling_element_in,
 468                                                  in, scheduling_element_id));
 469                 break;
 470         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 471                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 472                                         MLX5_GET(modify_scheduling_element_in,
 473                                                  in, scheduling_element_id));
 474                 break;
 475         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 476                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
 477                                         MLX5_GET(add_vxlan_udp_dport_in, in,
 478                                                  vxlan_udp_port));
 479                 break;
 480         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 481                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 482                                         MLX5_GET(query_l2_table_entry_in, in,
 483                                                  table_index));
 484                 break;
 485         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 486                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 487                                         MLX5_GET(set_l2_table_entry_in, in,
 488                                                  table_index));
 489                 break;
 490         case MLX5_CMD_OP_QUERY_QP:
 491                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 492                                         MLX5_GET(query_qp_in, in, qpn));
 493                 break;
 494         case MLX5_CMD_OP_RST2INIT_QP:
 495                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 496                                         MLX5_GET(rst2init_qp_in, in, qpn));
 497                 break;
 498         case MLX5_CMD_OP_INIT2INIT_QP:
 499                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 500                                         MLX5_GET(init2init_qp_in, in, qpn));
 501                 break;
 502         case MLX5_CMD_OP_INIT2RTR_QP:
 503                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 504                                         MLX5_GET(init2rtr_qp_in, in, qpn));
 505                 break;
 506         case MLX5_CMD_OP_RTR2RTS_QP:
 507                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 508                                         MLX5_GET(rtr2rts_qp_in, in, qpn));
 509                 break;
 510         case MLX5_CMD_OP_RTS2RTS_QP:
 511                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 512                                         MLX5_GET(rts2rts_qp_in, in, qpn));
 513                 break;
 514         case MLX5_CMD_OP_SQERR2RTS_QP:
 515                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 516                                         MLX5_GET(sqerr2rts_qp_in, in, qpn));
 517                 break;
 518         case MLX5_CMD_OP_2ERR_QP:
 519                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 520                                         MLX5_GET(qp_2err_in, in, qpn));
 521                 break;
 522         case MLX5_CMD_OP_2RST_QP:
 523                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 524                                         MLX5_GET(qp_2rst_in, in, qpn));
 525                 break;
 526         case MLX5_CMD_OP_QUERY_DCT:
 527                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 528                                         MLX5_GET(query_dct_in, in, dctn));
 529                 break;
 530         case MLX5_CMD_OP_QUERY_XRQ:
 531         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 532         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 533                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 534                                         MLX5_GET(query_xrq_in, in, xrqn));
 535                 break;
 536         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 537                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 538                                         MLX5_GET(query_xrc_srq_in, in,
 539                                                  xrc_srqn));
 540                 break;
 541         case MLX5_CMD_OP_ARM_XRC_SRQ:
 542                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 543                                         MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
 544                 break;
 545         case MLX5_CMD_OP_QUERY_SRQ:
 546                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
 547                                         MLX5_GET(query_srq_in, in, srqn));
 548                 break;
 549         case MLX5_CMD_OP_ARM_RQ:
 550                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 551                                         MLX5_GET(arm_rq_in, in, srq_number));
 552                 break;
 553         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 554                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 555                                         MLX5_GET(drain_dct_in, in, dctn));
 556                 break;
 557         case MLX5_CMD_OP_ARM_XRQ:
 558         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 559         case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 560         case MLX5_CMD_OP_MODIFY_XRQ:
 561                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 562                                         MLX5_GET(arm_xrq_in, in, xrqn));
 563                 break;
 564         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 565                 obj_id = get_enc_obj_id
 566                                 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
 567                                  MLX5_GET(query_packet_reformat_context_in,
 568                                           in, packet_reformat_id));
 569                 break;
 570         default:
 571                 obj_id = 0;
 572         }
 573
 574         return obj_id;
 575 }
 576
 577 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
 578                                  struct ib_uobject *uobj, const void *in)
 579 {
 580         struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 581         u64 obj_id = devx_get_obj_id(in);
 582
 583         if (!obj_id)
 584                 return false;
 585
 586         switch (uobj_get_object_id(uobj)) {
 587         case UVERBS_OBJECT_CQ:
 588                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 589                                       to_mcq(uobj->object)->mcq.cqn) ==
 590                                       obj_id;
 591
 592         case UVERBS_OBJECT_SRQ:
 593         {
 594                 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
 595                 u16 opcode;
 596
 597                 switch (srq->common.res) {
 598                 case MLX5_RES_XSRQ:
 599                         opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
 600                         break;
 601                 case MLX5_RES_XRQ:
 602                         opcode = MLX5_CMD_OP_CREATE_XRQ;
 603                         break;
 604                 default:
 605                         if (!dev->mdev->issi)
 606                                 opcode = MLX5_CMD_OP_CREATE_SRQ;
 607                         else
 608                                 opcode = MLX5_CMD_OP_CREATE_RMP;
 609                 }
 610
 611                 return get_enc_obj_id(opcode,
 612                                       to_msrq(uobj->object)->msrq.srqn) ==
 613                                       obj_id;
 614         }
 615
 616         case UVERBS_OBJECT_QP:
 617         {
 618                 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
 619                 enum ib_qp_type qp_type = qp->ibqp.qp_type;
 620
 621                 if (qp_type == IB_QPT_RAW_PACKET ||
 622                     (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
 623                         struct mlx5_ib_raw_packet_qp *raw_packet_qp =
 624                                                          &qp->raw_packet_qp;
 625                         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
 626                         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
 627
 628                         return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 629                                                rq->base.mqp.qpn) == obj_id ||
 630                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 631                                                sq->base.mqp.qpn) == obj_id ||
 632                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 633                                                rq->tirn) == obj_id ||
 634                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 635                                                sq->tisn) == obj_id);
 636                 }
 637
 638                 if (qp_type == MLX5_IB_QPT_DCT)
 639                         return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 640                                               qp->dct.mdct.mqp.qpn) == obj_id;
 641
 642                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 643                                       qp->ibqp.qp_num) == obj_id;
 644         }
 645
 646         case UVERBS_OBJECT_WQ:
 647                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 648                                       to_mrwq(uobj->object)->core_qp.qpn) ==
 649                                       obj_id;
 650
 651         case UVERBS_OBJECT_RWQ_IND_TBL:
 652                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 653                                       to_mrwq_ind_table(uobj->object)->rqtn) ==
 654                                       obj_id;
 655
 656         case MLX5_IB_OBJECT_DEVX_OBJ:
 657                 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
 658
 659         default:
 660                 return false;
 661         }
 662 }
 663
 664 static void devx_set_umem_valid(const void *in)
 665 {
 666         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 667
 668         switch (opcode) {
 669         case MLX5_CMD_OP_CREATE_MKEY:
 670                 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
 671                 break;
 672         case MLX5_CMD_OP_CREATE_CQ:
 673         {
 674                 void *cqc;
 675
 676                 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
 677                 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 678                 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
 679                 break;
 680         }
 681         case MLX5_CMD_OP_CREATE_QP:
 682         {
 683                 void *qpc;
 684
 685                 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 686                 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
 687                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
 688                 break;
 689         }
 690
 691         case MLX5_CMD_OP_CREATE_RQ:
 692         {
 693                 void *rqc, *wq;
 694
 695                 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 696                 wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 697                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 698                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 699                 break;
 700         }
 701
 702         case MLX5_CMD_OP_CREATE_SQ:
 703         {
 704                 void *sqc, *wq;
 705
 706                 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 707                 wq = MLX5_ADDR_OF(sqc, sqc, wq);
 708                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 709                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 710                 break;
 711         }
 712
 713         case MLX5_CMD_OP_MODIFY_CQ:
 714                 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
 715                 break;
 716
 717         case MLX5_CMD_OP_CREATE_RMP:
 718         {
 719                 void *rmpc, *wq;
 720
 721                 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
 722                 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
 723                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 724                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 725                 break;
 726         }
 727
 728         case MLX5_CMD_OP_CREATE_XRQ:
 729         {
 730                 void *xrqc, *wq;
 731
 732                 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
 733                 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
 734                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 735                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 736                 break;
 737         }
 738
 739         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 740         {
 741                 void *xrc_srqc;
 742
 743                 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
 744                 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
 745                                         xrc_srq_context_entry);
 746                 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
 747                 break;
 748         }
 749
 750         default:
 751                 return;
 752         }
 753 }
 754
 755 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
 756 {
 757         *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 758
 759         switch (*opcode) {
 760         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 761         case MLX5_CMD_OP_CREATE_MKEY:
 762         case MLX5_CMD_OP_CREATE_CQ:
 763         case MLX5_CMD_OP_ALLOC_PD:
 764         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 765         case MLX5_CMD_OP_CREATE_RMP:
 766         case MLX5_CMD_OP_CREATE_SQ:
 767         case MLX5_CMD_OP_CREATE_RQ:
 768         case MLX5_CMD_OP_CREATE_RQT:
 769         case MLX5_CMD_OP_CREATE_TIR:
 770         case MLX5_CMD_OP_CREATE_TIS:
 771         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 772         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 773         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 774         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 775         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 776         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 777         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 778         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 779         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 780         case MLX5_CMD_OP_CREATE_QP:
 781         case MLX5_CMD_OP_CREATE_SRQ:
 782         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 783         case MLX5_CMD_OP_CREATE_DCT:
 784         case MLX5_CMD_OP_CREATE_XRQ:
 785         case MLX5_CMD_OP_ATTACH_TO_MCG:
 786         case MLX5_CMD_OP_ALLOC_XRCD:
 787                 return true;
 788         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 789         {
 790                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 791                 if (op_mod == 0)
 792                         return true;
 793                 return false;
 794         }
 795         case MLX5_CMD_OP_CREATE_PSV:
 796         {
 797                 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
 798
 799                 if (num_psv == 1)
 800                         return true;
 801                 return false;
 802         }
 803         default:
 804                 return false;
 805         }
 806 }
 807
 808 static bool devx_is_obj_modify_cmd(const void *in)
 809 {
 810         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 811
 812         switch (opcode) {
 813         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 814         case MLX5_CMD_OP_MODIFY_CQ:
 815         case MLX5_CMD_OP_MODIFY_RMP:
 816         case MLX5_CMD_OP_MODIFY_SQ:
 817         case MLX5_CMD_OP_MODIFY_RQ:
 818         case MLX5_CMD_OP_MODIFY_RQT:
 819         case MLX5_CMD_OP_MODIFY_TIR:
 820         case MLX5_CMD_OP_MODIFY_TIS:
 821         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 822         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 823         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 824         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 825         case MLX5_CMD_OP_RST2INIT_QP:
 826         case MLX5_CMD_OP_INIT2RTR_QP:
 827         case MLX5_CMD_OP_INIT2INIT_QP:
 828         case MLX5_CMD_OP_RTR2RTS_QP:
 829         case MLX5_CMD_OP_RTS2RTS_QP:
 830         case MLX5_CMD_OP_SQERR2RTS_QP:
 831         case MLX5_CMD_OP_2ERR_QP:
 832         case MLX5_CMD_OP_2RST_QP:
 833         case MLX5_CMD_OP_ARM_XRC_SRQ:
 834         case MLX5_CMD_OP_ARM_RQ:
 835         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 836         case MLX5_CMD_OP_ARM_XRQ:
 837         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 838         case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 839         case MLX5_CMD_OP_MODIFY_XRQ:
 840                 return true;
 841         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 842         {
 843                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 844
 845                 if (op_mod == 1)
 846                         return true;
 847                 return false;
 848         }
 849         default:
 850                 return false;
 851         }
 852 }
 853
 854 static bool devx_is_obj_query_cmd(const void *in)
 855 {
 856         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 857
 858         switch (opcode) {
 859         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 860         case MLX5_CMD_OP_QUERY_MKEY:
 861         case MLX5_CMD_OP_QUERY_CQ:
 862         case MLX5_CMD_OP_QUERY_RMP:
 863         case MLX5_CMD_OP_QUERY_SQ:
 864         case MLX5_CMD_OP_QUERY_RQ:
 865         case MLX5_CMD_OP_QUERY_RQT:
 866         case MLX5_CMD_OP_QUERY_TIR:
 867         case MLX5_CMD_OP_QUERY_TIS:
 868         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 869         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 870         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 871         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 872         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 873         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 874         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 875         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 876         case MLX5_CMD_OP_QUERY_QP:
 877         case MLX5_CMD_OP_QUERY_SRQ:
 878         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 879         case MLX5_CMD_OP_QUERY_DCT:
 880         case MLX5_CMD_OP_QUERY_XRQ:
 881         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 882         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 883         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 884                 return true;
 885         default:
 886                 return false;
 887         }
 888 }
 889
 890 static bool devx_is_whitelist_cmd(void *in)
 891 {
 892         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 893
 894         switch (opcode) {
 895         case MLX5_CMD_OP_QUERY_HCA_CAP:
 896         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 897         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 898                 return true;
 899         default:
 900                 return false;
 901         }
 902 }
 903
 904 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
 905 {
 906         if (devx_is_whitelist_cmd(cmd_in)) {
 907                 struct mlx5_ib_dev *dev;
 908
 909                 if (c->devx_uid)
 910                         return c->devx_uid;
 911
 912                 dev = to_mdev(c->ibucontext.device);
 913                 if (dev->devx_whitelist_uid)
 914                         return dev->devx_whitelist_uid;
 915
 916                 return -EOPNOTSUPP;
 917         }
 918
 919         if (!c->devx_uid)
 920                 return -EINVAL;
 921
 922         return c->devx_uid;
 923 }
 924
 925 static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
 926 {
 927         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 928
 929         /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
 930         if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
 931              MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
 932             (opcode >= MLX5_CMD_OP_GENERAL_START &&
 933              opcode < MLX5_CMD_OP_GENERAL_END))
 934                 return true;
 935
 936         switch (opcode) {
 937         case MLX5_CMD_OP_QUERY_HCA_CAP:
 938         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 939         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 940         case MLX5_CMD_OP_QUERY_VPORT_STATE:
 941         case MLX5_CMD_OP_QUERY_ADAPTER:
 942         case MLX5_CMD_OP_QUERY_ISSI:
 943         case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
 944         case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
 945         case MLX5_CMD_OP_QUERY_VNIC_ENV:
 946         case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
 947         case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
 948         case MLX5_CMD_OP_NOP:
 949         case MLX5_CMD_OP_QUERY_CONG_STATUS:
 950         case MLX5_CMD_OP_QUERY_CONG_PARAMS:
 951         case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
 952         case MLX5_CMD_OP_QUERY_LAG:
 953                 return true;
 954         default:
 955                 return false;
 956         }
 957 }
 958
 959 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
 960         struct uverbs_attr_bundle *attrs)
 961 {
 962         struct mlx5_ib_ucontext *c;
 963         struct mlx5_ib_dev *dev;
 964         int user_vector;
 965         int dev_eqn;
 966         unsigned int irqn;
 967         int err;
 968
 969         if (uverbs_copy_from(&user_vector, attrs,
 970                              MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
 971                 return -EFAULT;
 972
 973         c = devx_ufile2uctx(attrs);
 974         if (IS_ERR(c))
 975                 return PTR_ERR(c);
 976         dev = to_mdev(c->ibucontext.device);
 977
 978         err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
 979         if (err < 0)
 980                 return err;
 981
 982         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
 983                            &dev_eqn, sizeof(dev_eqn)))
 984                 return -EFAULT;
 985
 986         return 0;
 987 }
 988
 989 /*
 990  *Security note:
 991  * The hardware protection mechanism works like this: Each device object that
 992  * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
 993  * the device specification manual) upon its creation. Then upon doorbell,
 994  * hardware fetches the object context for which the doorbell was rang, and
 995  * validates that the UAR through which the DB was rang matches the UAR ID
 996  * of the object.
 997  * If no match the doorbell is silently ignored by the hardware. Of course,
 998  * the user cannot ring a doorbell on a UAR that was not mapped to it.
 999  * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
1000  * mailboxes (except tagging them with UID), we expose to the user its UAR
1001  * ID, so it can embed it in these objects in the expected specification
1002  * format. So the only thing the user can do is hurt itself by creating a
1003  * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1004  * may ring a doorbell on its objects.
1005  * The consequence of that will be that another user can schedule a QP/SQ
1006  * of the buggy user for execution (just insert it to the hardware schedule
1007  * queue or arm its CQ for event generation), no further harm is expected.
1008  */
1009 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1010         struct uverbs_attr_bundle *attrs)
1011 {
1012         struct mlx5_ib_ucontext *c;
1013         struct mlx5_ib_dev *dev;
1014         u32 user_idx;
1015         s32 dev_idx;
1016
1017         c = devx_ufile2uctx(attrs);
1018         if (IS_ERR(c))
1019                 return PTR_ERR(c);
1020         dev = to_mdev(c->ibucontext.device);
1021
1022         if (uverbs_copy_from(&user_idx, attrs,
1023                              MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1024                 return -EFAULT;
1025
1026         dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1027         if (dev_idx < 0)
1028                 return dev_idx;
1029
1030         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1031                            &dev_idx, sizeof(dev_idx)))
1032                 return -EFAULT;
1033
1034         return 0;
1035 }
1036
1037 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1038         struct uverbs_attr_bundle *attrs)
1039 {
1040         struct mlx5_ib_ucontext *c;
1041         struct mlx5_ib_dev *dev;
1042         void *cmd_in = uverbs_attr_get_alloced_ptr(
1043                 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1044         int cmd_out_len = uverbs_attr_get_len(attrs,
1045                                         MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1046         void *cmd_out;
1047         int err;
1048         int uid;
1049
1050         c = devx_ufile2uctx(attrs);
1051         if (IS_ERR(c))
1052                 return PTR_ERR(c);
1053         dev = to_mdev(c->ibucontext.device);
1054
1055         uid = devx_get_uid(c, cmd_in);
1056         if (uid < 0)
1057                 return uid;
1058
1059         /* Only white list of some general HCA commands are allowed for this method. */
1060         if (!devx_is_general_cmd(cmd_in, dev))
1061                 return -EINVAL;
1062
1063         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1064         if (IS_ERR(cmd_out))
1065                 return PTR_ERR(cmd_out);
1066
1067         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1068         err = mlx5_cmd_exec(dev->mdev, cmd_in,
1069                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1070                             cmd_out, cmd_out_len);
1071         if (err)
1072                 return err;
1073
1074         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1075                               cmd_out_len);
1076 }
1077
1078 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1079                                        u32 *dinlen,
1080                                        u32 *obj_id)
1081 {
1082         u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
1083         u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1084
1085         *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
1086         *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1087
1088         MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1089         MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1090
1091         switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
1092         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1093                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1094                 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
1095                 break;
1096
1097         case MLX5_CMD_OP_CREATE_UMEM:
1098                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1099                          MLX5_CMD_OP_DESTROY_UMEM);
1100                 break;
1101         case MLX5_CMD_OP_CREATE_MKEY:
1102                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
1103                 break;
1104         case MLX5_CMD_OP_CREATE_CQ:
1105                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1106                 break;
1107         case MLX5_CMD_OP_ALLOC_PD:
1108                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1109                 break;
1110         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1111                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1112                          MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1113                 break;
1114         case MLX5_CMD_OP_CREATE_RMP:
1115                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1116                 break;
1117         case MLX5_CMD_OP_CREATE_SQ:
1118                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1119                 break;
1120         case MLX5_CMD_OP_CREATE_RQ:
1121                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1122                 break;
1123         case MLX5_CMD_OP_CREATE_RQT:
1124                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1125                 break;
1126         case MLX5_CMD_OP_CREATE_TIR:
1127                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1128                 break;
1129         case MLX5_CMD_OP_CREATE_TIS:
1130                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1131                 break;
1132         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1133                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1134                          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1135                 break;
1136         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1137                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1138                 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
1139                 MLX5_SET(destroy_flow_table_in, din, other_vport,
1140                          MLX5_GET(create_flow_table_in,  in, other_vport));
1141                 MLX5_SET(destroy_flow_table_in, din, vport_number,
1142                          MLX5_GET(create_flow_table_in,  in, vport_number));
1143                 MLX5_SET(destroy_flow_table_in, din, table_type,
1144                          MLX5_GET(create_flow_table_in,  in, table_type));
1145                 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1146                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1147                          MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1148                 break;
1149         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1150                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1151                 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
1152                 MLX5_SET(destroy_flow_group_in, din, other_vport,
1153                          MLX5_GET(create_flow_group_in, in, other_vport));
1154                 MLX5_SET(destroy_flow_group_in, din, vport_number,
1155                          MLX5_GET(create_flow_group_in, in, vport_number));
1156                 MLX5_SET(destroy_flow_group_in, din, table_type,
1157                          MLX5_GET(create_flow_group_in, in, table_type));
1158                 MLX5_SET(destroy_flow_group_in, din, table_id,
1159                          MLX5_GET(create_flow_group_in, in, table_id));
1160                 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1161                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1162                          MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1163                 break;
1164         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1165                 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1166                 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
1167                 MLX5_SET(delete_fte_in, din, other_vport,
1168                          MLX5_GET(set_fte_in,  in, other_vport));
1169                 MLX5_SET(delete_fte_in, din, vport_number,
1170                          MLX5_GET(set_fte_in, in, vport_number));
1171                 MLX5_SET(delete_fte_in, din, table_type,
1172                          MLX5_GET(set_fte_in, in, table_type));
1173                 MLX5_SET(delete_fte_in, din, table_id,
1174                          MLX5_GET(set_fte_in, in, table_id));
1175                 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1176                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1177                          MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1178                 break;
1179         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1180                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1181                          MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1182                 break;
1183         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1184                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1185                          MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1186                 break;
1187         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1188                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1189                          MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1190                 break;
1191         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1192                 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1193                 *obj_id = MLX5_GET(create_scheduling_element_out, out,
1194                                    scheduling_element_id);
1195                 MLX5_SET(destroy_scheduling_element_in, din,
1196                          scheduling_hierarchy,
1197                          MLX5_GET(create_scheduling_element_in, in,
1198                                   scheduling_hierarchy));
1199                 MLX5_SET(destroy_scheduling_element_in, din,
1200                          scheduling_element_id, *obj_id);
1201                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1202                          MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1203                 break;
1204         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1205                 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1206                 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
1207                 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1208                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1209                          MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1210                 break;
1211         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1212                 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1213                 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
1214                 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1215                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1216                          MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1217                 break;
1218         case MLX5_CMD_OP_CREATE_QP:
1219                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1220                 break;
1221         case MLX5_CMD_OP_CREATE_SRQ:
1222                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1223                 break;
1224         case MLX5_CMD_OP_CREATE_XRC_SRQ:
1225                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1226                          MLX5_CMD_OP_DESTROY_XRC_SRQ);
1227                 break;
1228         case MLX5_CMD_OP_CREATE_DCT:
1229                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1230                 break;
1231         case MLX5_CMD_OP_CREATE_XRQ:
1232                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1233                 break;
1234         case MLX5_CMD_OP_ATTACH_TO_MCG:
1235                 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1236                 MLX5_SET(detach_from_mcg_in, din, qpn,
1237                          MLX5_GET(attach_to_mcg_in, in, qpn));
1238                 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1239                        MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1240                        MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1241                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
1242                 break;
1243         case MLX5_CMD_OP_ALLOC_XRCD:
1244                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
1245                 break;
1246         case MLX5_CMD_OP_CREATE_PSV:
1247                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1248                          MLX5_CMD_OP_DESTROY_PSV);
1249                 MLX5_SET(destroy_psv_in, din, psvn,
1250                          MLX5_GET(create_psv_out, out, psv0_index));
1251                 break;
1252         default:
1253                 /* The entry must match to one of the devx_is_obj_create_cmd */
1254                 WARN_ON(true);
1255                 break;
1256         }
1257 }
1258
1259 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1260                                      struct mlx5_ib_dev *dev,
1261                                      void *in, void *out)
1262 {
1263         struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
1264         struct mlx5_core_mkey *mkey;
1265         void *mkc;
1266         u8 key;
1267
1268         mkey = &devx_mr->mmkey;
1269         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1270         key = MLX5_GET(mkc, mkc, mkey_7_0);
1271         mkey->key = mlx5_idx_to_mkey(
1272                         MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1273         mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1274         mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1275         mkey->size = MLX5_GET64(mkc, mkc, len);
1276         mkey->pd = MLX5_GET(mkc, mkc, pd);
1277         devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1278
1279         return xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(mkey->key), mkey,
1280                                GFP_KERNEL));
1281 }
1282
1283 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1284                                    struct devx_obj *obj,
1285                                    void *in, int in_len)
1286 {
1287         int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1288                         MLX5_FLD_SZ_BYTES(create_mkey_in,
1289                         memory_key_mkey_entry);
1290         void *mkc;
1291         u8 access_mode;
1292
1293         if (in_len < min_len)
1294                 return -EINVAL;
1295
1296         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1297
1298         access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1299         access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1300
1301         if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1302                 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1303                 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1304                         obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1305                 return 0;
1306         }
1307
1308         MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1309         return 0;
1310 }
1311
1312 static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1313                                       struct devx_event_subscription *sub)
1314 {
1315         struct devx_event *event;
1316         struct devx_obj_event *xa_val_level2;
1317
1318         if (sub->is_cleaned)
1319                 return;
1320
1321         sub->is_cleaned = 1;
1322         list_del_rcu(&sub->xa_list);
1323
1324         if (list_empty(&sub->obj_list))
1325                 return;
1326
1327         list_del_rcu(&sub->obj_list);
1328         /* check whether key level 1 for this obj_sub_list is empty */
1329         event = xa_load(&dev->devx_event_table.event_xa,
1330                         sub->xa_key_level1);
1331         WARN_ON(!event);
1332
1333         xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1334         if (list_empty(&xa_val_level2->obj_sub_list)) {
1335                 xa_erase(&event->object_ids,
1336                          sub->xa_key_level2);
1337                 kfree_rcu(xa_val_level2, rcu);
1338         }
1339 }
1340
1341 static int devx_obj_cleanup(struct ib_uobject *uobject,
1342                             enum rdma_remove_reason why,
1343                             struct uverbs_attr_bundle *attrs)
1344 {
1345         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1346         struct mlx5_devx_event_table *devx_event_table;
1347         struct devx_obj *obj = uobject->object;
1348         struct devx_event_subscription *sub_entry, *tmp;
1349         struct mlx5_ib_dev *dev;
1350         int ret;
1351
1352         dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1353         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1354                 /*
1355                  * The pagefault_single_data_segment() does commands against
1356                  * the mmkey, we must wait for that to stop before freeing the
1357                  * mkey, as another allocation could get the same mkey #.
1358                  */
1359                 xa_erase(&obj->ib_dev->odp_mkeys,
1360                          mlx5_base_mkey(obj->devx_mr.mmkey.key));
1361                 synchronize_srcu(&dev->odp_srcu);
1362         }
1363
1364         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1365                 ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1366         else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1367                 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1368         else
1369                 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1370                                     obj->dinlen, out, sizeof(out));
1371         if (ib_is_destroy_retryable(ret, why, uobject))
1372                 return ret;
1373
1374         devx_event_table = &dev->devx_event_table;
1375
1376         mutex_lock(&devx_event_table->event_xa_lock);
1377         list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1378                 devx_cleanup_subscription(dev, sub_entry);
1379         mutex_unlock(&devx_event_table->event_xa_lock);
1380
1381         kfree(obj);
1382         return ret;
1383 }
1384
1385 static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1386 {
1387         struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1388         struct mlx5_devx_event_table *table;
1389         struct devx_event *event;
1390         struct devx_obj_event *obj_event;
1391         u32 obj_id = mcq->cqn;
1392
1393         table = &obj->ib_dev->devx_event_table;
1394         rcu_read_lock();
1395         event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1396         if (!event)
1397                 goto out;
1398
1399         obj_event = xa_load(&event->object_ids, obj_id);
1400         if (!obj_event)
1401                 goto out;
1402
1403         dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1404 out:
1405         rcu_read_unlock();
1406 }
1407
1408 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1409         struct uverbs_attr_bundle *attrs)
1410 {
1411         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1412         int cmd_out_len =  uverbs_attr_get_len(attrs,
1413                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1414         int cmd_in_len = uverbs_attr_get_len(attrs,
1415                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1416         void *cmd_out;
1417         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1418                 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1419         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1420                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1421         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1422         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1423         struct devx_obj *obj;
1424         u16 obj_type = 0;
1425         int err;
1426         int uid;
1427         u32 obj_id;
1428         u16 opcode;
1429
1430         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1431                 return -EINVAL;
1432
1433         uid = devx_get_uid(c, cmd_in);
1434         if (uid < 0)
1435                 return uid;
1436
1437         if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1438                 return -EINVAL;
1439
1440         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1441         if (IS_ERR(cmd_out))
1442                 return PTR_ERR(cmd_out);
1443
1444         obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1445         if (!obj)
1446                 return -ENOMEM;
1447
1448         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1449         if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1450                 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1451                 if (err)
1452                         goto obj_free;
1453         } else {
1454                 devx_set_umem_valid(cmd_in);
1455         }
1456
1457         if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1458                 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1459                 err = mlx5_core_create_dct(dev, &obj->core_dct, cmd_in,
1460                                            cmd_in_len, cmd_out, cmd_out_len);
1461         } else if (opcode == MLX5_CMD_OP_CREATE_CQ) {
1462                 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1463                 obj->core_cq.comp = devx_cq_comp;
1464                 err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,
1465                                           cmd_in, cmd_in_len, cmd_out,
1466                                           cmd_out_len);
1467         } else {
1468                 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1469                                     cmd_in_len,
1470                                     cmd_out, cmd_out_len);
1471         }
1472
1473         if (err)
1474                 goto obj_free;
1475
1476         if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1477                 u8 bulk = MLX5_GET(alloc_flow_counter_in,
1478                                    cmd_in,
1479                                    flow_counter_bulk);
1480                 obj->flow_counter_bulk_size = 128UL * bulk;
1481         }
1482
1483         uobj->object = obj;
1484         INIT_LIST_HEAD(&obj->event_sub);
1485         obj->ib_dev = dev;
1486         devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1487                                    &obj_id);
1488         WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1489
1490         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1491         if (err)
1492                 goto obj_destroy;
1493
1494         if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1495                 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1496         obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1497
1498         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1499                 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1500                 if (err)
1501                         goto obj_destroy;
1502         }
1503         return 0;
1504
1505 obj_destroy:
1506         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1507                 mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1508         else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1509                 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1510         else
1511                 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1512                               sizeof(out));
1513 obj_free:
1514         kfree(obj);
1515         return err;
1516 }
1517
1518 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1519         struct uverbs_attr_bundle *attrs)
1520 {
1521         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1522         int cmd_out_len = uverbs_attr_get_len(attrs,
1523                                         MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1524         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1525                                                           MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1526         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1527                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1528         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1529         void *cmd_out;
1530         int err;
1531         int uid;
1532
1533         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1534                 return -EINVAL;
1535
1536         uid = devx_get_uid(c, cmd_in);
1537         if (uid < 0)
1538                 return uid;
1539
1540         if (!devx_is_obj_modify_cmd(cmd_in))
1541                 return -EINVAL;
1542
1543         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1544                 return -EINVAL;
1545
1546         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1547         if (IS_ERR(cmd_out))
1548                 return PTR_ERR(cmd_out);
1549
1550         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1551         devx_set_umem_valid(cmd_in);
1552
1553         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1554                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1555                             cmd_out, cmd_out_len);
1556         if (err)
1557                 return err;
1558
1559         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1560                               cmd_out, cmd_out_len);
1561 }
1562
1563 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1564         struct uverbs_attr_bundle *attrs)
1565 {
1566         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1567         int cmd_out_len = uverbs_attr_get_len(attrs,
1568                                               MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1569         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1570                                                           MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1571         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1572                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1573         void *cmd_out;
1574         int err;
1575         int uid;
1576         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1577
1578         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1579                 return -EINVAL;
1580
1581         uid = devx_get_uid(c, cmd_in);
1582         if (uid < 0)
1583                 return uid;
1584
1585         if (!devx_is_obj_query_cmd(cmd_in))
1586                 return -EINVAL;
1587
1588         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1589                 return -EINVAL;
1590
1591         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1592         if (IS_ERR(cmd_out))
1593                 return PTR_ERR(cmd_out);
1594
1595         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1596         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1597                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1598                             cmd_out, cmd_out_len);
1599         if (err)
1600                 return err;
1601
1602         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1603                               cmd_out, cmd_out_len);
1604 }
1605
1606 struct devx_async_event_queue {
1607         spinlock_t              lock;
1608         wait_queue_head_t       poll_wait;
1609         struct list_head        event_list;
1610         atomic_t                bytes_in_use;
1611         u8                      is_destroyed:1;
1612 };
1613
1614 struct devx_async_cmd_event_file {
1615         struct ib_uobject               uobj;
1616         struct devx_async_event_queue   ev_queue;
1617         struct mlx5_async_ctx           async_ctx;
1618 };
1619
1620 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1621 {
1622         spin_lock_init(&ev_queue->lock);
1623         INIT_LIST_HEAD(&ev_queue->event_list);
1624         init_waitqueue_head(&ev_queue->poll_wait);
1625         atomic_set(&ev_queue->bytes_in_use, 0);
1626         ev_queue->is_destroyed = 0;
1627 }
1628
1629 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1630         struct uverbs_attr_bundle *attrs)
1631 {
1632         struct devx_async_cmd_event_file *ev_file;
1633
1634         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1635                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1636         struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1637
1638         ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1639                                uobj);
1640         devx_init_event_queue(&ev_file->ev_queue);
1641         mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1642         return 0;
1643 }
1644
1645 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1646         struct uverbs_attr_bundle *attrs)
1647 {
1648         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1649                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1650         struct devx_async_event_file *ev_file;
1651         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1652                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1653         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1654         u32 flags;
1655         int err;
1656
1657         err = uverbs_get_flags32(&flags, attrs,
1658                 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1659                 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1660
1661         if (err)
1662                 return err;
1663
1664         ev_file = container_of(uobj, struct devx_async_event_file,
1665                                uobj);
1666         spin_lock_init(&ev_file->lock);
1667         INIT_LIST_HEAD(&ev_file->event_list);
1668         init_waitqueue_head(&ev_file->poll_wait);
1669         if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1670                 ev_file->omit_data = 1;
1671         INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1672         ev_file->dev = dev;
1673         get_device(&dev->ib_dev.dev);
1674         return 0;
1675 }
1676
1677 static void devx_query_callback(int status, struct mlx5_async_work *context)
1678 {
1679         struct devx_async_data *async_data =
1680                 container_of(context, struct devx_async_data, cb_work);
1681         struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1682         struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1683         unsigned long flags;
1684
1685         /*
1686          * Note that if the struct devx_async_cmd_event_file uobj begins to be
1687          * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1688          * routine returns, ensuring that it always remains valid here.
1689          */
1690         spin_lock_irqsave(&ev_queue->lock, flags);
1691         list_add_tail(&async_data->list, &ev_queue->event_list);
1692         spin_unlock_irqrestore(&ev_queue->lock, flags);
1693
1694         wake_up_interruptible(&ev_queue->poll_wait);
1695 }
1696
1697 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1698
1699 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1700         struct uverbs_attr_bundle *attrs)
1701 {
1702         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1703                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1704         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1705                                 attrs,
1706                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1707         u16 cmd_out_len;
1708         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1709                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1710         struct ib_uobject *fd_uobj;
1711         int err;
1712         int uid;
1713         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1714         struct devx_async_cmd_event_file *ev_file;
1715         struct devx_async_data *async_data;
1716
1717         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1718                 return -EINVAL;
1719
1720         uid = devx_get_uid(c, cmd_in);
1721         if (uid < 0)
1722                 return uid;
1723
1724         if (!devx_is_obj_query_cmd(cmd_in))
1725                 return -EINVAL;
1726
1727         err = uverbs_get_const(&cmd_out_len, attrs,
1728                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1729         if (err)
1730                 return err;
1731
1732         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1733                 return -EINVAL;
1734
1735         fd_uobj = uverbs_attr_get_uobject(attrs,
1736                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1737         if (IS_ERR(fd_uobj))
1738                 return PTR_ERR(fd_uobj);
1739
1740         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1741                                uobj);
1742
1743         if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1744                         MAX_ASYNC_BYTES_IN_USE) {
1745                 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1746                 return -EAGAIN;
1747         }
1748
1749         async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1750                                           cmd_out_len), GFP_KERNEL);
1751         if (!async_data) {
1752                 err = -ENOMEM;
1753                 goto sub_bytes;
1754         }
1755
1756         err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1757                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1758         if (err)
1759                 goto free_async;
1760
1761         async_data->cmd_out_len = cmd_out_len;
1762         async_data->mdev = mdev;
1763         async_data->ev_file = ev_file;
1764
1765         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1766         err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1767                     uverbs_attr_get_len(attrs,
1768                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1769                     async_data->hdr.out_data,
1770                     async_data->cmd_out_len,
1771                     devx_query_callback, &async_data->cb_work);
1772
1773         if (err)
1774                 goto free_async;
1775
1776         return 0;
1777
1778 free_async:
1779         kvfree(async_data);
1780 sub_bytes:
1781         atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1782         return err;
1783 }
1784
1785 static void
1786 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1787                            u32 key_level1,
1788                            bool is_level2,
1789                            u32 key_level2)
1790 {
1791         struct devx_event *event;
1792         struct devx_obj_event *xa_val_level2;
1793
1794         /* Level 1 is valid for future use, no need to free */
1795         if (!is_level2)
1796                 return;
1797
1798         event = xa_load(&devx_event_table->event_xa, key_level1);
1799         WARN_ON(!event);
1800
1801         xa_val_level2 = xa_load(&event->object_ids,
1802                                 key_level2);
1803         if (list_empty(&xa_val_level2->obj_sub_list)) {
1804                 xa_erase(&event->object_ids,
1805                          key_level2);
1806                 kfree_rcu(xa_val_level2, rcu);
1807         }
1808 }
1809
1810 static int
1811 subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1812                          u32 key_level1,
1813                          bool is_level2,
1814                          u32 key_level2)
1815 {
1816         struct devx_obj_event *obj_event;
1817         struct devx_event *event;
1818         int err;
1819
1820         event = xa_load(&devx_event_table->event_xa, key_level1);
1821         if (!event) {
1822                 event = kzalloc(sizeof(*event), GFP_KERNEL);
1823                 if (!event)
1824                         return -ENOMEM;
1825
1826                 INIT_LIST_HEAD(&event->unaffiliated_list);
1827                 xa_init(&event->object_ids);
1828
1829                 err = xa_insert(&devx_event_table->event_xa,
1830                                 key_level1,
1831                                 event,
1832                                 GFP_KERNEL);
1833                 if (err) {
1834                         kfree(event);
1835                         return err;
1836                 }
1837         }
1838
1839         if (!is_level2)
1840                 return 0;
1841
1842         obj_event = xa_load(&event->object_ids, key_level2);
1843         if (!obj_event) {
1844                 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1845                 if (!obj_event)
1846                         /* Level1 is valid for future use, no need to free */
1847                         return -ENOMEM;
1848
1849                 err = xa_insert(&event->object_ids,
1850                                 key_level2,
1851                                 obj_event,
1852                                 GFP_KERNEL);
1853                 if (err)
1854                         return err;
1855                 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1856         }
1857
1858         return 0;
1859 }
1860
1861 static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1862                                    struct devx_obj *obj)
1863 {
1864         int i;
1865
1866         for (i = 0; i < num_events; i++) {
1867                 if (obj) {
1868                         if (!is_legacy_obj_event_num(event_type_num_list[i]))
1869                                 return false;
1870                 } else if (!is_legacy_unaffiliated_event_num(
1871                                 event_type_num_list[i])) {
1872                         return false;
1873                 }
1874         }
1875
1876         return true;
1877 }
1878
1879 #define MAX_SUPP_EVENT_NUM 255
1880 static bool is_valid_events(struct mlx5_core_dev *dev,
1881                             int num_events, u16 *event_type_num_list,
1882                             struct devx_obj *obj)
1883 {
1884         __be64 *aff_events;
1885         __be64 *unaff_events;
1886         int mask_entry;
1887         int mask_bit;
1888         int i;
1889
1890         if (MLX5_CAP_GEN(dev, event_cap)) {
1891                 aff_events = MLX5_CAP_DEV_EVENT(dev,
1892                                                 user_affiliated_events);
1893                 unaff_events = MLX5_CAP_DEV_EVENT(dev,
1894                                                   user_unaffiliated_events);
1895         } else {
1896                 return is_valid_events_legacy(num_events, event_type_num_list,
1897                                               obj);
1898         }
1899
1900         for (i = 0; i < num_events; i++) {
1901                 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
1902                         return false;
1903
1904                 mask_entry = event_type_num_list[i] / 64;
1905                 mask_bit = event_type_num_list[i] % 64;
1906
1907                 if (obj) {
1908                         /* CQ completion */
1909                         if (event_type_num_list[i] == 0)
1910                                 continue;
1911
1912                         if (!(be64_to_cpu(aff_events[mask_entry]) &
1913                                         (1ull << mask_bit)))
1914                                 return false;
1915
1916                         continue;
1917                 }
1918
1919                 if (!(be64_to_cpu(unaff_events[mask_entry]) &
1920                                 (1ull << mask_bit)))
1921                         return false;
1922         }
1923
1924         return true;
1925 }
1926
1927 #define MAX_NUM_EVENTS 16
1928 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
1929         struct uverbs_attr_bundle *attrs)
1930 {
1931         struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
1932                                 attrs,
1933                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
1934         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1935                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1936         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1937         struct ib_uobject *fd_uobj;
1938         struct devx_obj *obj = NULL;
1939         struct devx_async_event_file *ev_file;
1940         struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
1941         u16 *event_type_num_list;
1942         struct devx_event_subscription *event_sub, *tmp_sub;
1943         struct list_head sub_list;
1944         int redirect_fd;
1945         bool use_eventfd = false;
1946         int num_events;
1947         int num_alloc_xa_entries = 0;
1948         u16 obj_type = 0;
1949         u64 cookie = 0;
1950         u32 obj_id = 0;
1951         int err;
1952         int i;
1953
1954         if (!c->devx_uid)
1955                 return -EINVAL;
1956
1957         if (!IS_ERR(devx_uobj)) {
1958                 obj = (struct devx_obj *)devx_uobj->object;
1959                 if (obj)
1960                         obj_id = get_dec_obj_id(obj->obj_id);
1961         }
1962
1963         fd_uobj = uverbs_attr_get_uobject(attrs,
1964                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
1965         if (IS_ERR(fd_uobj))
1966                 return PTR_ERR(fd_uobj);
1967
1968         ev_file = container_of(fd_uobj, struct devx_async_event_file,
1969                                uobj);
1970
1971         if (uverbs_attr_is_valid(attrs,
1972                                  MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
1973                 err = uverbs_copy_from(&redirect_fd, attrs,
1974                                MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
1975                 if (err)
1976                         return err;
1977
1978                 use_eventfd = true;
1979         }
1980
1981         if (uverbs_attr_is_valid(attrs,
1982                                  MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
1983                 if (use_eventfd)
1984                         return -EINVAL;
1985
1986                 err = uverbs_copy_from(&cookie, attrs,
1987                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
1988                 if (err)
1989                         return err;
1990         }
1991
1992         num_events = uverbs_attr_ptr_get_array_size(
1993                 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
1994                 sizeof(u16));
1995
1996         if (num_events < 0)
1997                 return num_events;
1998
1999         if (num_events > MAX_NUM_EVENTS)
2000                 return -EINVAL;
2001
2002         event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
2003                         MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2004
2005         if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2006                 return -EINVAL;
2007
2008         INIT_LIST_HEAD(&sub_list);
2009
2010         /* Protect from concurrent subscriptions to same XA entries to allow
2011          * both to succeed
2012          */
2013         mutex_lock(&devx_event_table->event_xa_lock);
2014         for (i = 0; i < num_events; i++) {
2015                 u32 key_level1;
2016
2017                 if (obj)
2018                         obj_type = get_dec_obj_type(obj,
2019                                                     event_type_num_list[i]);
2020                 key_level1 = event_type_num_list[i] | obj_type << 16;
2021
2022                 err = subscribe_event_xa_alloc(devx_event_table,
2023                                                key_level1,
2024                                                obj,
2025                                                obj_id);
2026                 if (err)
2027                         goto err;
2028
2029                 num_alloc_xa_entries++;
2030                 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2031                 if (!event_sub)
2032                         goto err;
2033
2034                 list_add_tail(&event_sub->event_list, &sub_list);
2035                 uverbs_uobject_get(&ev_file->uobj);
2036                 if (use_eventfd) {
2037                         event_sub->eventfd =
2038                                 eventfd_ctx_fdget(redirect_fd);
2039
2040                         if (IS_ERR(event_sub->eventfd)) {
2041                                 err = PTR_ERR(event_sub->eventfd);
2042                                 event_sub->eventfd = NULL;
2043                                 goto err;
2044                         }
2045                 }
2046
2047                 event_sub->cookie = cookie;
2048                 event_sub->ev_file = ev_file;
2049                 /* May be needed upon cleanup the devx object/subscription */
2050                 event_sub->xa_key_level1 = key_level1;
2051                 event_sub->xa_key_level2 = obj_id;
2052                 INIT_LIST_HEAD(&event_sub->obj_list);
2053         }
2054
2055         /* Once all the allocations and the XA data insertions were done we
2056          * can go ahead and add all the subscriptions to the relevant lists
2057          * without concern of a failure.
2058          */
2059         list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2060                 struct devx_event *event;
2061                 struct devx_obj_event *obj_event;
2062
2063                 list_del_init(&event_sub->event_list);
2064
2065                 spin_lock_irq(&ev_file->lock);
2066                 list_add_tail_rcu(&event_sub->file_list,
2067                                   &ev_file->subscribed_events_list);
2068                 spin_unlock_irq(&ev_file->lock);
2069
2070                 event = xa_load(&devx_event_table->event_xa,
2071                                 event_sub->xa_key_level1);
2072                 WARN_ON(!event);
2073
2074                 if (!obj) {
2075                         list_add_tail_rcu(&event_sub->xa_list,
2076                                           &event->unaffiliated_list);
2077                         continue;
2078                 }
2079
2080                 obj_event = xa_load(&event->object_ids, obj_id);
2081                 WARN_ON(!obj_event);
2082                 list_add_tail_rcu(&event_sub->xa_list,
2083                                   &obj_event->obj_sub_list);
2084                 list_add_tail_rcu(&event_sub->obj_list,
2085                                   &obj->event_sub);
2086         }
2087
2088         mutex_unlock(&devx_event_table->event_xa_lock);
2089         return 0;
2090
2091 err:
2092         list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2093                 list_del(&event_sub->event_list);
2094
2095                 subscribe_event_xa_dealloc(devx_event_table,
2096                                            event_sub->xa_key_level1,
2097                                            obj,
2098                                            obj_id);
2099
2100                 if (event_sub->eventfd)
2101                         eventfd_ctx_put(event_sub->eventfd);
2102                 uverbs_uobject_put(&event_sub->ev_file->uobj);
2103                 kfree(event_sub);
2104         }
2105
2106         mutex_unlock(&devx_event_table->event_xa_lock);
2107         return err;
2108 }
2109
2110 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2111                          struct uverbs_attr_bundle *attrs,
2112                          struct devx_umem *obj)
2113 {
2114         u64 addr;
2115         size_t size;
2116         u32 access;
2117         int npages;
2118         int err;
2119         u32 page_mask;
2120
2121         if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2122             uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2123                 return -EFAULT;
2124
2125         err = uverbs_get_flags32(&access, attrs,
2126                                  MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2127                                  IB_ACCESS_LOCAL_WRITE |
2128                                  IB_ACCESS_REMOTE_WRITE |
2129                                  IB_ACCESS_REMOTE_READ);
2130         if (err)
2131                 return err;
2132
2133         err = ib_check_mr_access(access);
2134         if (err)
2135                 return err;
2136
2137         obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access);
2138         if (IS_ERR(obj->umem))
2139                 return PTR_ERR(obj->umem);
2140
2141         mlx5_ib_cont_pages(obj->umem, obj->umem->address,
2142                            MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
2143                            &obj->page_shift, &obj->ncont, NULL);
2144
2145         if (!npages) {
2146                 ib_umem_release(obj->umem);
2147                 return -EINVAL;
2148         }
2149
2150         page_mask = (1 << obj->page_shift) - 1;
2151         obj->page_offset = obj->umem->address & page_mask;
2152
2153         return 0;
2154 }
2155
2156 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
2157                                    struct devx_umem *obj,
2158                                    struct devx_umem_reg_cmd *cmd)
2159 {
2160         cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2161                     (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
2162         cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2163         return PTR_ERR_OR_ZERO(cmd->in);
2164 }
2165
2166 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
2167                                     struct devx_umem *obj,
2168                                     struct devx_umem_reg_cmd *cmd)
2169 {
2170         void *umem;
2171         __be64 *mtt;
2172
2173         umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2174         mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2175
2176         MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2177         MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
2178         MLX5_SET(umem, umem, log_page_size, obj->page_shift -
2179                                             MLX5_ADAPTER_PAGE_SHIFT);
2180         MLX5_SET(umem, umem, page_offset, obj->page_offset);
2181         mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
2182                              (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2183                              MLX5_IB_MTT_READ);
2184 }
2185
2186 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2187         struct uverbs_attr_bundle *attrs)
2188 {
2189         struct devx_umem_reg_cmd cmd;
2190         struct devx_umem *obj;
2191         struct ib_uobject *uobj = uverbs_attr_get_uobject(
2192                 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2193         u32 obj_id;
2194         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2195                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2196         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2197         int err;
2198
2199         if (!c->devx_uid)
2200                 return -EINVAL;
2201
2202         obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2203         if (!obj)
2204                 return -ENOMEM;
2205
2206         err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
2207         if (err)
2208                 goto err_obj_free;
2209
2210         err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
2211         if (err)
2212                 goto err_umem_release;
2213
2214         devx_umem_reg_cmd_build(dev, obj, &cmd);
2215
2216         MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2217         err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2218                             sizeof(cmd.out));
2219         if (err)
2220                 goto err_umem_release;
2221
2222         obj->mdev = dev->mdev;
2223         uobj->object = obj;
2224         devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2225         uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2226
2227         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id,
2228                              sizeof(obj_id));
2229         return err;
2230
2231 err_umem_release:
2232         ib_umem_release(obj->umem);
2233 err_obj_free:
2234         kfree(obj);
2235         return err;
2236 }
2237
2238 static int devx_umem_cleanup(struct ib_uobject *uobject,
2239                              enum rdma_remove_reason why,
2240                              struct uverbs_attr_bundle *attrs)
2241 {
2242         struct devx_umem *obj = uobject->object;
2243         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2244         int err;
2245
2246         err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2247         if (ib_is_destroy_retryable(err, why, uobject))
2248                 return err;
2249
2250         ib_umem_release(obj->umem);
2251         kfree(obj);
2252         return 0;
2253 }
2254
2255 static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2256                                   unsigned long event_type)
2257 {
2258         __be64 *unaff_events;
2259         int mask_entry;
2260         int mask_bit;
2261
2262         if (!MLX5_CAP_GEN(dev, event_cap))
2263                 return is_legacy_unaffiliated_event_num(event_type);
2264
2265         unaff_events = MLX5_CAP_DEV_EVENT(dev,
2266                                           user_unaffiliated_events);
2267         WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2268
2269         mask_entry = event_type / 64;
2270         mask_bit = event_type % 64;
2271
2272         if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2273                 return false;
2274
2275         return true;
2276 }
2277
2278 static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2279 {
2280         struct mlx5_eqe *eqe = data;
2281         u32 obj_id = 0;
2282
2283         switch (event_type) {
2284         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2285         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2286         case MLX5_EVENT_TYPE_PATH_MIG:
2287         case MLX5_EVENT_TYPE_COMM_EST:
2288         case MLX5_EVENT_TYPE_SQ_DRAINED:
2289         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2290         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2291         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2292         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2293         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2294                 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2295                 break;
2296         case MLX5_EVENT_TYPE_XRQ_ERROR:
2297                 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2298                 break;
2299         case MLX5_EVENT_TYPE_DCT_DRAINED:
2300         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2301                 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2302                 break;
2303         case MLX5_EVENT_TYPE_CQ_ERROR:
2304                 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2305                 break;
2306         default:
2307                 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2308                 break;
2309         }
2310
2311         return obj_id;
2312 }
2313
2314 static int deliver_event(struct devx_event_subscription *event_sub,
2315                          const void *data)
2316 {
2317         struct devx_async_event_file *ev_file;
2318         struct devx_async_event_data *event_data;
2319         unsigned long flags;
2320
2321         ev_file = event_sub->ev_file;
2322
2323         if (ev_file->omit_data) {
2324                 spin_lock_irqsave(&ev_file->lock, flags);
2325                 if (!list_empty(&event_sub->event_list) ||
2326                     ev_file->is_destroyed) {
2327                         spin_unlock_irqrestore(&ev_file->lock, flags);
2328                         return 0;
2329                 }
2330
2331                 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2332                 spin_unlock_irqrestore(&ev_file->lock, flags);
2333                 wake_up_interruptible(&ev_file->poll_wait);
2334                 return 0;
2335         }
2336
2337         event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2338                              GFP_ATOMIC);
2339         if (!event_data) {
2340                 spin_lock_irqsave(&ev_file->lock, flags);
2341                 ev_file->is_overflow_err = 1;
2342                 spin_unlock_irqrestore(&ev_file->lock, flags);
2343                 return -ENOMEM;
2344         }
2345
2346         event_data->hdr.cookie = event_sub->cookie;
2347         memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2348
2349         spin_lock_irqsave(&ev_file->lock, flags);
2350         if (!ev_file->is_destroyed)
2351                 list_add_tail(&event_data->list, &ev_file->event_list);
2352         else
2353                 kfree(event_data);
2354         spin_unlock_irqrestore(&ev_file->lock, flags);
2355         wake_up_interruptible(&ev_file->poll_wait);
2356
2357         return 0;
2358 }
2359
2360 static void dispatch_event_fd(struct list_head *fd_list,
2361                               const void *data)
2362 {
2363         struct devx_event_subscription *item;
2364
2365         list_for_each_entry_rcu(item, fd_list, xa_list) {
2366                 if (item->eventfd)
2367                         eventfd_signal(item->eventfd, 1);
2368                 else
2369                         deliver_event(item, data);
2370         }
2371 }
2372
2373 static int devx_event_notifier(struct notifier_block *nb,
2374                                unsigned long event_type, void *data)
2375 {
2376         struct mlx5_devx_event_table *table;
2377         struct mlx5_ib_dev *dev;
2378         struct devx_event *event;
2379         struct devx_obj_event *obj_event;
2380         u16 obj_type = 0;
2381         bool is_unaffiliated;
2382         u32 obj_id;
2383
2384         /* Explicit filtering to kernel events which may occur frequently */
2385         if (event_type == MLX5_EVENT_TYPE_CMD ||
2386             event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2387                 return NOTIFY_OK;
2388
2389         table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2390         dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2391         is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2392
2393         if (!is_unaffiliated)
2394                 obj_type = get_event_obj_type(event_type, data);
2395
2396         rcu_read_lock();
2397         event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2398         if (!event) {
2399                 rcu_read_unlock();
2400                 return NOTIFY_DONE;
2401         }
2402
2403         if (is_unaffiliated) {
2404                 dispatch_event_fd(&event->unaffiliated_list, data);
2405                 rcu_read_unlock();
2406                 return NOTIFY_OK;
2407         }
2408
2409         obj_id = devx_get_obj_id_from_event(event_type, data);
2410         obj_event = xa_load(&event->object_ids, obj_id);
2411         if (!obj_event) {
2412                 rcu_read_unlock();
2413                 return NOTIFY_DONE;
2414         }
2415
2416         dispatch_event_fd(&obj_event->obj_sub_list, data);
2417
2418         rcu_read_unlock();
2419         return NOTIFY_OK;
2420 }
2421
2422 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
2423 {
2424         struct mlx5_devx_event_table *table = &dev->devx_event_table;
2425
2426         xa_init(&table->event_xa);
2427         mutex_init(&table->event_xa_lock);
2428         MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2429         mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2430 }
2431
2432 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
2433 {
2434         struct mlx5_devx_event_table *table = &dev->devx_event_table;
2435         struct devx_event_subscription *sub, *tmp;
2436         struct devx_event *event;
2437         void *entry;
2438         unsigned long id;
2439
2440         mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2441         mutex_lock(&dev->devx_event_table.event_xa_lock);
2442         xa_for_each(&table->event_xa, id, entry) {
2443                 event = entry;
2444                 list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
2445                                          xa_list)
2446                         devx_cleanup_subscription(dev, sub);
2447                 kfree(entry);
2448         }
2449         mutex_unlock(&dev->devx_event_table.event_xa_lock);
2450         xa_destroy(&table->event_xa);
2451 }
2452
2453 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2454                                          size_t count, loff_t *pos)
2455 {
2456         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2457         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2458         struct devx_async_data *event;
2459         int ret = 0;
2460         size_t eventsz;
2461
2462         spin_lock_irq(&ev_queue->lock);
2463
2464         while (list_empty(&ev_queue->event_list)) {
2465                 spin_unlock_irq(&ev_queue->lock);
2466
2467                 if (filp->f_flags & O_NONBLOCK)
2468                         return -EAGAIN;
2469
2470                 if (wait_event_interruptible(
2471                             ev_queue->poll_wait,
2472                             (!list_empty(&ev_queue->event_list) ||
2473                              ev_queue->is_destroyed))) {
2474                         return -ERESTARTSYS;
2475                 }
2476
2477                 spin_lock_irq(&ev_queue->lock);
2478                 if (ev_queue->is_destroyed) {
2479                         spin_unlock_irq(&ev_queue->lock);
2480                         return -EIO;
2481                 }
2482         }
2483
2484         event = list_entry(ev_queue->event_list.next,
2485                            struct devx_async_data, list);
2486         eventsz = event->cmd_out_len +
2487                         sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2488
2489         if (eventsz > count) {
2490                 spin_unlock_irq(&ev_queue->lock);
2491                 return -ENOSPC;
2492         }
2493
2494         list_del(ev_queue->event_list.next);
2495         spin_unlock_irq(&ev_queue->lock);
2496
2497         if (copy_to_user(buf, &event->hdr, eventsz))
2498                 ret = -EFAULT;
2499         else
2500                 ret = eventsz;
2501
2502         atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2503         kvfree(event);
2504         return ret;
2505 }
2506
2507 static __poll_t devx_async_cmd_event_poll(struct file *filp,
2508                                               struct poll_table_struct *wait)
2509 {
2510         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2511         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2512         __poll_t pollflags = 0;
2513
2514         poll_wait(filp, &ev_queue->poll_wait, wait);
2515
2516         spin_lock_irq(&ev_queue->lock);
2517         if (ev_queue->is_destroyed)
2518                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2519         else if (!list_empty(&ev_queue->event_list))
2520                 pollflags = EPOLLIN | EPOLLRDNORM;
2521         spin_unlock_irq(&ev_queue->lock);
2522
2523         return pollflags;
2524 }
2525
2526 static const struct file_operations devx_async_cmd_event_fops = {
2527         .owner   = THIS_MODULE,
2528         .read    = devx_async_cmd_event_read,
2529         .poll    = devx_async_cmd_event_poll,
2530         .release = uverbs_uobject_fd_release,
2531         .llseek  = no_llseek,
2532 };
2533
2534 static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2535                                      size_t count, loff_t *pos)
2536 {
2537         struct devx_async_event_file *ev_file = filp->private_data;
2538         struct devx_event_subscription *event_sub;
2539         struct devx_async_event_data *uninitialized_var(event);
2540         int ret = 0;
2541         size_t eventsz;
2542         bool omit_data;
2543         void *event_data;
2544
2545         omit_data = ev_file->omit_data;
2546
2547         spin_lock_irq(&ev_file->lock);
2548
2549         if (ev_file->is_overflow_err) {
2550                 ev_file->is_overflow_err = 0;
2551                 spin_unlock_irq(&ev_file->lock);
2552                 return -EOVERFLOW;
2553         }
2554
2555
2556         while (list_empty(&ev_file->event_list)) {
2557                 spin_unlock_irq(&ev_file->lock);
2558
2559                 if (filp->f_flags & O_NONBLOCK)
2560                         return -EAGAIN;
2561
2562                 if (wait_event_interruptible(ev_file->poll_wait,
2563                             (!list_empty(&ev_file->event_list) ||
2564                              ev_file->is_destroyed))) {
2565                         return -ERESTARTSYS;
2566                 }
2567
2568                 spin_lock_irq(&ev_file->lock);
2569                 if (ev_file->is_destroyed) {
2570                         spin_unlock_irq(&ev_file->lock);
2571                         return -EIO;
2572                 }
2573         }
2574
2575         if (omit_data) {
2576                 event_sub = list_first_entry(&ev_file->event_list,
2577                                         struct devx_event_subscription,
2578                                         event_list);
2579                 eventsz = sizeof(event_sub->cookie);
2580                 event_data = &event_sub->cookie;
2581         } else {
2582                 event = list_first_entry(&ev_file->event_list,
2583                                       struct devx_async_event_data, list);
2584                 eventsz = sizeof(struct mlx5_eqe) +
2585                         sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2586                 event_data = &event->hdr;
2587         }
2588
2589         if (eventsz > count) {
2590                 spin_unlock_irq(&ev_file->lock);
2591                 return -EINVAL;
2592         }
2593
2594         if (omit_data)
2595                 list_del_init(&event_sub->event_list);
2596         else
2597                 list_del(&event->list);
2598
2599         spin_unlock_irq(&ev_file->lock);
2600
2601         if (copy_to_user(buf, event_data, eventsz))
2602                 /* This points to an application issue, not a kernel concern */
2603                 ret = -EFAULT;
2604         else
2605                 ret = eventsz;
2606
2607         if (!omit_data)
2608                 kfree(event);
2609         return ret;
2610 }
2611
2612 static __poll_t devx_async_event_poll(struct file *filp,
2613                                       struct poll_table_struct *wait)
2614 {
2615         struct devx_async_event_file *ev_file = filp->private_data;
2616         __poll_t pollflags = 0;
2617
2618         poll_wait(filp, &ev_file->poll_wait, wait);
2619
2620         spin_lock_irq(&ev_file->lock);
2621         if (ev_file->is_destroyed)
2622                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2623         else if (!list_empty(&ev_file->event_list))
2624                 pollflags = EPOLLIN | EPOLLRDNORM;
2625         spin_unlock_irq(&ev_file->lock);
2626
2627         return pollflags;
2628 }
2629
2630 static void devx_free_subscription(struct rcu_head *rcu)
2631 {
2632         struct devx_event_subscription *event_sub =
2633                 container_of(rcu, struct devx_event_subscription, rcu);
2634
2635         if (event_sub->eventfd)
2636                 eventfd_ctx_put(event_sub->eventfd);
2637         uverbs_uobject_put(&event_sub->ev_file->uobj);
2638         kfree(event_sub);
2639 }
2640
2641 static const struct file_operations devx_async_event_fops = {
2642         .owner   = THIS_MODULE,
2643         .read    = devx_async_event_read,
2644         .poll    = devx_async_event_poll,
2645         .release = uverbs_uobject_fd_release,
2646         .llseek  = no_llseek,
2647 };
2648
2649 static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2650                                              enum rdma_remove_reason why)
2651 {
2652         struct devx_async_cmd_event_file *comp_ev_file =
2653                 container_of(uobj, struct devx_async_cmd_event_file,
2654                              uobj);
2655         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2656         struct devx_async_data *entry, *tmp;
2657
2658         spin_lock_irq(&ev_queue->lock);
2659         ev_queue->is_destroyed = 1;
2660         spin_unlock_irq(&ev_queue->lock);
2661         wake_up_interruptible(&ev_queue->poll_wait);
2662
2663         mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2664
2665         spin_lock_irq(&comp_ev_file->ev_queue.lock);
2666         list_for_each_entry_safe(entry, tmp,
2667                                  &comp_ev_file->ev_queue.event_list, list) {
2668                 list_del(&entry->list);
2669                 kvfree(entry);
2670         }
2671         spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2672         return 0;
2673 };
2674
2675 static int devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2676                                          enum rdma_remove_reason why)
2677 {
2678         struct devx_async_event_file *ev_file =
2679                 container_of(uobj, struct devx_async_event_file,
2680                              uobj);
2681         struct devx_event_subscription *event_sub, *event_sub_tmp;
2682         struct mlx5_ib_dev *dev = ev_file->dev;
2683
2684         spin_lock_irq(&ev_file->lock);
2685         ev_file->is_destroyed = 1;
2686
2687         /* free the pending events allocation */
2688         if (ev_file->omit_data) {
2689                 struct devx_event_subscription *event_sub, *tmp;
2690
2691                 list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2692                                          event_list)
2693                         list_del_init(&event_sub->event_list);
2694
2695         } else {
2696                 struct devx_async_event_data *entry, *tmp;
2697
2698                 list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2699                                          list) {
2700                         list_del(&entry->list);
2701                         kfree(entry);
2702                 }
2703         }
2704
2705         spin_unlock_irq(&ev_file->lock);
2706         wake_up_interruptible(&ev_file->poll_wait);
2707
2708         mutex_lock(&dev->devx_event_table.event_xa_lock);
2709         /* delete the subscriptions which are related to this FD */
2710         list_for_each_entry_safe(event_sub, event_sub_tmp,
2711                                  &ev_file->subscribed_events_list, file_list) {
2712                 devx_cleanup_subscription(dev, event_sub);
2713                 list_del_rcu(&event_sub->file_list);
2714                 /* subscription may not be used by the read API any more */
2715                 call_rcu(&event_sub->rcu, devx_free_subscription);
2716         }
2717         mutex_unlock(&dev->devx_event_table.event_xa_lock);
2718
2719         put_device(&dev->ib_dev.dev);
2720         return 0;
2721 };
2722
2723 DECLARE_UVERBS_NAMED_METHOD(
2724         MLX5_IB_METHOD_DEVX_UMEM_REG,
2725         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2726                         MLX5_IB_OBJECT_DEVX_UMEM,
2727                         UVERBS_ACCESS_NEW,
2728                         UA_MANDATORY),
2729         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
2730                            UVERBS_ATTR_TYPE(u64),
2731                            UA_MANDATORY),
2732         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
2733                            UVERBS_ATTR_TYPE(u64),
2734                            UA_MANDATORY),
2735         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2736                              enum ib_access_flags),
2737         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
2738                             UVERBS_ATTR_TYPE(u32),
2739                             UA_MANDATORY));
2740
2741 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2742         MLX5_IB_METHOD_DEVX_UMEM_DEREG,
2743         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
2744                         MLX5_IB_OBJECT_DEVX_UMEM,
2745                         UVERBS_ACCESS_DESTROY,
2746                         UA_MANDATORY));
2747
2748 DECLARE_UVERBS_NAMED_METHOD(
2749         MLX5_IB_METHOD_DEVX_QUERY_EQN,
2750         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
2751                            UVERBS_ATTR_TYPE(u32),
2752                            UA_MANDATORY),
2753         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
2754                             UVERBS_ATTR_TYPE(u32),
2755                             UA_MANDATORY));
2756
2757 DECLARE_UVERBS_NAMED_METHOD(
2758         MLX5_IB_METHOD_DEVX_QUERY_UAR,
2759         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
2760                            UVERBS_ATTR_TYPE(u32),
2761                            UA_MANDATORY),
2762         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
2763                             UVERBS_ATTR_TYPE(u32),
2764                             UA_MANDATORY));
2765
2766 DECLARE_UVERBS_NAMED_METHOD(
2767         MLX5_IB_METHOD_DEVX_OTHER,
2768         UVERBS_ATTR_PTR_IN(
2769                 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
2770                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2771                 UA_MANDATORY,
2772                 UA_ALLOC_AND_COPY),
2773         UVERBS_ATTR_PTR_OUT(
2774                 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
2775                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2776                 UA_MANDATORY));
2777
2778 DECLARE_UVERBS_NAMED_METHOD(
2779         MLX5_IB_METHOD_DEVX_OBJ_CREATE,
2780         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
2781                         MLX5_IB_OBJECT_DEVX_OBJ,
2782                         UVERBS_ACCESS_NEW,
2783                         UA_MANDATORY),
2784         UVERBS_ATTR_PTR_IN(
2785                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
2786                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2787                 UA_MANDATORY,
2788                 UA_ALLOC_AND_COPY),
2789         UVERBS_ATTR_PTR_OUT(
2790                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
2791                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2792                 UA_MANDATORY));
2793
2794 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2795         MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
2796         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
2797                         MLX5_IB_OBJECT_DEVX_OBJ,
2798                         UVERBS_ACCESS_DESTROY,
2799                         UA_MANDATORY));
2800
2801 DECLARE_UVERBS_NAMED_METHOD(
2802         MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
2803         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
2804                         UVERBS_IDR_ANY_OBJECT,
2805                         UVERBS_ACCESS_WRITE,
2806                         UA_MANDATORY),
2807         UVERBS_ATTR_PTR_IN(
2808                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
2809                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2810                 UA_MANDATORY,
2811                 UA_ALLOC_AND_COPY),
2812         UVERBS_ATTR_PTR_OUT(
2813                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
2814                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2815                 UA_MANDATORY));
2816
2817 DECLARE_UVERBS_NAMED_METHOD(
2818         MLX5_IB_METHOD_DEVX_OBJ_QUERY,
2819         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2820                         UVERBS_IDR_ANY_OBJECT,
2821                         UVERBS_ACCESS_READ,
2822                         UA_MANDATORY),
2823         UVERBS_ATTR_PTR_IN(
2824                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2825                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2826                 UA_MANDATORY,
2827                 UA_ALLOC_AND_COPY),
2828         UVERBS_ATTR_PTR_OUT(
2829                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
2830                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2831                 UA_MANDATORY));
2832
2833 DECLARE_UVERBS_NAMED_METHOD(
2834         MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
2835         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2836                         UVERBS_IDR_ANY_OBJECT,
2837                         UVERBS_ACCESS_READ,
2838                         UA_MANDATORY),
2839         UVERBS_ATTR_PTR_IN(
2840                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2841                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2842                 UA_MANDATORY,
2843                 UA_ALLOC_AND_COPY),
2844         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
2845                 u16, UA_MANDATORY),
2846         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
2847                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2848                 UVERBS_ACCESS_READ,
2849                 UA_MANDATORY),
2850         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
2851                 UVERBS_ATTR_TYPE(u64),
2852                 UA_MANDATORY));
2853
2854 DECLARE_UVERBS_NAMED_METHOD(
2855         MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
2856         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
2857                 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2858                 UVERBS_ACCESS_READ,
2859                 UA_MANDATORY),
2860         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
2861                 MLX5_IB_OBJECT_DEVX_OBJ,
2862                 UVERBS_ACCESS_READ,
2863                 UA_OPTIONAL),
2864         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2865                 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
2866                 UA_MANDATORY,
2867                 UA_ALLOC_AND_COPY),
2868         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
2869                 UVERBS_ATTR_TYPE(u64),
2870                 UA_OPTIONAL),
2871         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
2872                 UVERBS_ATTR_TYPE(u32),
2873                 UA_OPTIONAL));
2874
2875 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
2876                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
2877                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
2878                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
2879                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
2880
2881 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
2882                             UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
2883                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
2884                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
2885                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
2886                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
2887                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
2888
2889 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
2890                             UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
2891                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
2892                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
2893
2894
2895 DECLARE_UVERBS_NAMED_METHOD(
2896         MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
2897         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
2898                         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2899                         UVERBS_ACCESS_NEW,
2900                         UA_MANDATORY));
2901
2902 DECLARE_UVERBS_NAMED_OBJECT(
2903         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2904         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
2905                              devx_async_cmd_event_destroy_uobj,
2906                              &devx_async_cmd_event_fops, "[devx_async_cmd]",
2907                              O_RDONLY),
2908         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
2909
2910 DECLARE_UVERBS_NAMED_METHOD(
2911         MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
2912         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
2913                         MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2914                         UVERBS_ACCESS_NEW,
2915                         UA_MANDATORY),
2916         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
2917                         enum mlx5_ib_uapi_devx_create_event_channel_flags,
2918                         UA_MANDATORY));
2919
2920 DECLARE_UVERBS_NAMED_OBJECT(
2921         MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2922         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
2923                              devx_async_event_destroy_uobj,
2924                              &devx_async_event_fops, "[devx_async_event]",
2925                              O_RDONLY),
2926         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
2927
2928 static bool devx_is_supported(struct ib_device *device)
2929 {
2930         struct mlx5_ib_dev *dev = to_mdev(device);
2931
2932         return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
2933 }
2934
2935 const struct uapi_definition mlx5_ib_devx_defs[] = {
2936         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2937                 MLX5_IB_OBJECT_DEVX,
2938                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2939         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2940                 MLX5_IB_OBJECT_DEVX_OBJ,
2941                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2942         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2943                 MLX5_IB_OBJECT_DEVX_UMEM,
2944                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2945         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2946                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2947                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2948         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2949                 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2950                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2951         {},
2952 };