drivers/infiniband/core/cma.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   4  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   5  * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved.
   6  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   7  */
   8
   9 #include <linux/completion.h>
  10 #include <linux/in.h>
  11 #include <linux/in6.h>
  12 #include <linux/mutex.h>
  13 #include <linux/random.h>
  14 #include <linux/rbtree.h>
  15 #include <linux/igmp.h>
  16 #include <linux/xarray.h>
  17 #include <linux/inetdevice.h>
  18 #include <linux/slab.h>
  19 #include <linux/module.h>
  20 #include <net/route.h>
  21
  22 #include <net/net_namespace.h>
  23 #include <net/netns/generic.h>
  24 #include <net/netevent.h>
  25 #include <net/tcp.h>
  26 #include <net/ipv6.h>
  27 #include <net/ip_fib.h>
  28 #include <net/ip6_route.h>
  29
  30 #include <rdma/rdma_cm.h>
  31 #include <rdma/rdma_cm_ib.h>
  32 #include <rdma/rdma_netlink.h>
  33 #include <rdma/ib.h>
  34 #include <rdma/ib_cache.h>
  35 #include <rdma/ib_cm.h>
  36 #include <rdma/ib_sa.h>
  37 #include <rdma/iw_cm.h>
  38
  39 #include "core_priv.h"
  40 #include "cma_priv.h"
  41 #include "cma_trace.h"
  42
  43 MODULE_AUTHOR("Sean Hefty");
  44 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  45 MODULE_LICENSE("Dual BSD/GPL");
  46
  47 #define CMA_CM_RESPONSE_TIMEOUT 20
  48 #define CMA_MAX_CM_RETRIES 15
  49 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  50 #define CMA_IBOE_PACKET_LIFETIME 16
  51 #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
  52
  53 static const char * const cma_events[] = {
  54         [RDMA_CM_EVENT_ADDR_RESOLVED]    = "address resolved",
  55         [RDMA_CM_EVENT_ADDR_ERROR]       = "address error",
  56         [RDMA_CM_EVENT_ROUTE_RESOLVED]   = "route resolved ",
  57         [RDMA_CM_EVENT_ROUTE_ERROR]      = "route error",
  58         [RDMA_CM_EVENT_CONNECT_REQUEST]  = "connect request",
  59         [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response",
  60         [RDMA_CM_EVENT_CONNECT_ERROR]    = "connect error",
  61         [RDMA_CM_EVENT_UNREACHABLE]      = "unreachable",
  62         [RDMA_CM_EVENT_REJECTED]         = "rejected",
  63         [RDMA_CM_EVENT_ESTABLISHED]      = "established",
  64         [RDMA_CM_EVENT_DISCONNECTED]     = "disconnected",
  65         [RDMA_CM_EVENT_DEVICE_REMOVAL]   = "device removal",
  66         [RDMA_CM_EVENT_MULTICAST_JOIN]   = "multicast join",
  67         [RDMA_CM_EVENT_MULTICAST_ERROR]  = "multicast error",
  68         [RDMA_CM_EVENT_ADDR_CHANGE]      = "address change",
  69         [RDMA_CM_EVENT_TIMEWAIT_EXIT]    = "timewait exit",
  70 };
  71
  72 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
  73                               enum ib_gid_type gid_type);
  74
  75 const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
  76 {
  77         size_t index = event;
  78
  79         return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ?
  80                         cma_events[index] : "unrecognized event";
  81 }
  82 EXPORT_SYMBOL(rdma_event_msg);
  83
  84 const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
  85                                                 int reason)
  86 {
  87         if (rdma_ib_or_roce(id->device, id->port_num))
  88                 return ibcm_reject_msg(reason);
  89
  90         if (rdma_protocol_iwarp(id->device, id->port_num))
  91                 return iwcm_reject_msg(reason);
  92
  93         WARN_ON_ONCE(1);
  94         return "unrecognized transport";
  95 }
  96 EXPORT_SYMBOL(rdma_reject_msg);
  97
  98 /**
  99  * rdma_is_consumer_reject - return true if the consumer rejected the connect
 100  *                           request.
 101  * @id: Communication identifier that received the REJECT event.
 102  * @reason: Value returned in the REJECT event status field.
 103  */
 104 static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
 105 {
 106         if (rdma_ib_or_roce(id->device, id->port_num))
 107                 return reason == IB_CM_REJ_CONSUMER_DEFINED;
 108
 109         if (rdma_protocol_iwarp(id->device, id->port_num))
 110                 return reason == -ECONNREFUSED;
 111
 112         WARN_ON_ONCE(1);
 113         return false;
 114 }
 115
 116 const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
 117                                       struct rdma_cm_event *ev, u8 *data_len)
 118 {
 119         const void *p;
 120
 121         if (rdma_is_consumer_reject(id, ev->status)) {
 122                 *data_len = ev->param.conn.private_data_len;
 123                 p = ev->param.conn.private_data;
 124         } else {
 125                 *data_len = 0;
 126                 p = NULL;
 127         }
 128         return p;
 129 }
 130 EXPORT_SYMBOL(rdma_consumer_reject_data);
 131
 132 /**
 133  * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id.
 134  * @id: Communication Identifier
 135  */
 136 struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)
 137 {
 138         struct rdma_id_private *id_priv;
 139
 140         id_priv = container_of(id, struct rdma_id_private, id);
 141         if (id->device->node_type == RDMA_NODE_RNIC)
 142                 return id_priv->cm_id.iw;
 143         return NULL;
 144 }
 145 EXPORT_SYMBOL(rdma_iw_cm_id);
 146
 147 /**
 148  * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack.
 149  * @res: rdma resource tracking entry pointer
 150  */
 151 struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)
 152 {
 153         struct rdma_id_private *id_priv =
 154                 container_of(res, struct rdma_id_private, res);
 155
 156         return &id_priv->id;
 157 }
 158 EXPORT_SYMBOL(rdma_res_to_id);
 159
 160 static int cma_add_one(struct ib_device *device);
 161 static void cma_remove_one(struct ib_device *device, void *client_data);
 162
 163 static struct ib_client cma_client = {
 164         .name   = "cma",
 165         .add    = cma_add_one,
 166         .remove = cma_remove_one
 167 };
 168
 169 static struct ib_sa_client sa_client;
 170 static LIST_HEAD(dev_list);
 171 static LIST_HEAD(listen_any_list);
 172 static DEFINE_MUTEX(lock);
 173 static struct rb_root id_table = RB_ROOT;
 174 /* Serialize operations of id_table tree */
 175 static DEFINE_SPINLOCK(id_table_lock);
 176 static struct workqueue_struct *cma_wq;
 177 static unsigned int cma_pernet_id;
 178
 179 struct cma_pernet {
 180         struct xarray tcp_ps;
 181         struct xarray udp_ps;
 182         struct xarray ipoib_ps;
 183         struct xarray ib_ps;
 184 };
 185
 186 static struct cma_pernet *cma_pernet(struct net *net)
 187 {
 188         return net_generic(net, cma_pernet_id);
 189 }
 190
 191 static
 192 struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)
 193 {
 194         struct cma_pernet *pernet = cma_pernet(net);
 195
 196         switch (ps) {
 197         case RDMA_PS_TCP:
 198                 return &pernet->tcp_ps;
 199         case RDMA_PS_UDP:
 200                 return &pernet->udp_ps;
 201         case RDMA_PS_IPOIB:
 202                 return &pernet->ipoib_ps;
 203         case RDMA_PS_IB:
 204                 return &pernet->ib_ps;
 205         default:
 206                 return NULL;
 207         }
 208 }
 209
 210 struct id_table_entry {
 211         struct list_head id_list;
 212         struct rb_node rb_node;
 213 };
 214
 215 struct cma_device {
 216         struct list_head        list;
 217         struct ib_device        *device;
 218         struct completion       comp;
 219         refcount_t refcount;
 220         struct list_head        id_list;
 221         enum ib_gid_type        *default_gid_type;
 222         u8                      *default_roce_tos;
 223 };
 224
 225 struct rdma_bind_list {
 226         enum rdma_ucm_port_space ps;
 227         struct hlist_head       owners;
 228         unsigned short          port;
 229 };
 230
 231 static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,
 232                         struct rdma_bind_list *bind_list, int snum)
 233 {
 234         struct xarray *xa = cma_pernet_xa(net, ps);
 235
 236         return xa_insert(xa, snum, bind_list, GFP_KERNEL);
 237 }
 238
 239 static struct rdma_bind_list *cma_ps_find(struct net *net,
 240                                           enum rdma_ucm_port_space ps, int snum)
 241 {
 242         struct xarray *xa = cma_pernet_xa(net, ps);
 243
 244         return xa_load(xa, snum);
 245 }
 246
 247 static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps,
 248                           int snum)
 249 {
 250         struct xarray *xa = cma_pernet_xa(net, ps);
 251
 252         xa_erase(xa, snum);
 253 }
 254
 255 enum {
 256         CMA_OPTION_AFONLY,
 257 };
 258
 259 void cma_dev_get(struct cma_device *cma_dev)
 260 {
 261         refcount_inc(&cma_dev->refcount);
 262 }
 263
 264 void cma_dev_put(struct cma_device *cma_dev)
 265 {
 266         if (refcount_dec_and_test(&cma_dev->refcount))
 267                 complete(&cma_dev->comp);
 268 }
 269
 270 struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter  filter,
 271                                              void               *cookie)
 272 {
 273         struct cma_device *cma_dev;
 274         struct cma_device *found_cma_dev = NULL;
 275
 276         mutex_lock(&lock);
 277
 278         list_for_each_entry(cma_dev, &dev_list, list)
 279                 if (filter(cma_dev->device, cookie)) {
 280                         found_cma_dev = cma_dev;
 281                         break;
 282                 }
 283
 284         if (found_cma_dev)
 285                 cma_dev_get(found_cma_dev);
 286         mutex_unlock(&lock);
 287         return found_cma_dev;
 288 }
 289
 290 int cma_get_default_gid_type(struct cma_device *cma_dev,
 291                              u32 port)
 292 {
 293         if (!rdma_is_port_valid(cma_dev->device, port))
 294                 return -EINVAL;
 295
 296         return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
 297 }
 298
 299 int cma_set_default_gid_type(struct cma_device *cma_dev,
 300                              u32 port,
 301                              enum ib_gid_type default_gid_type)
 302 {
 303         unsigned long supported_gids;
 304
 305         if (!rdma_is_port_valid(cma_dev->device, port))
 306                 return -EINVAL;
 307
 308         if (default_gid_type == IB_GID_TYPE_IB &&
 309             rdma_protocol_roce_eth_encap(cma_dev->device, port))
 310                 default_gid_type = IB_GID_TYPE_ROCE;
 311
 312         supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
 313
 314         if (!(supported_gids & 1 << default_gid_type))
 315                 return -EINVAL;
 316
 317         cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
 318                 default_gid_type;
 319
 320         return 0;
 321 }
 322
 323 int cma_get_default_roce_tos(struct cma_device *cma_dev, u32 port)
 324 {
 325         if (!rdma_is_port_valid(cma_dev->device, port))
 326                 return -EINVAL;
 327
 328         return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)];
 329 }
 330
 331 int cma_set_default_roce_tos(struct cma_device *cma_dev, u32 port,
 332                              u8 default_roce_tos)
 333 {
 334         if (!rdma_is_port_valid(cma_dev->device, port))
 335                 return -EINVAL;
 336
 337         cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] =
 338                  default_roce_tos;
 339
 340         return 0;
 341 }
 342 struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
 343 {
 344         return cma_dev->device;
 345 }
 346
 347 /*
 348  * Device removal can occur at anytime, so we need extra handling to
 349  * serialize notifying the user of device removal with other callbacks.
 350  * We do this by disabling removal notification while a callback is in process,
 351  * and reporting it after the callback completes.
 352  */
 353
 354 struct cma_multicast {
 355         struct rdma_id_private *id_priv;
 356         union {
 357                 struct ib_sa_multicast *sa_mc;
 358                 struct {
 359                         struct work_struct work;
 360                         struct rdma_cm_event event;
 361                 } iboe_join;
 362         };
 363         struct list_head        list;
 364         void                    *context;
 365         struct sockaddr_storage addr;
 366         u8                      join_state;
 367 };
 368
 369 struct cma_work {
 370         struct work_struct      work;
 371         struct rdma_id_private  *id;
 372         enum rdma_cm_state      old_state;
 373         enum rdma_cm_state      new_state;
 374         struct rdma_cm_event    event;
 375 };
 376
 377 union cma_ip_addr {
 378         struct in6_addr ip6;
 379         struct {
 380                 __be32 pad[3];
 381                 __be32 addr;
 382         } ip4;
 383 };
 384
 385 struct cma_hdr {
 386         u8 cma_version;
 387         u8 ip_version;  /* IP version: 7:4 */
 388         __be16 port;
 389         union cma_ip_addr src_addr;
 390         union cma_ip_addr dst_addr;
 391 };
 392
 393 #define CMA_VERSION 0x00
 394
 395 struct cma_req_info {
 396         struct sockaddr_storage listen_addr_storage;
 397         struct sockaddr_storage src_addr_storage;
 398         struct ib_device *device;
 399         union ib_gid local_gid;
 400         __be64 service_id;
 401         int port;
 402         bool has_gid;
 403         u16 pkey;
 404 };
 405
 406 static int cma_comp_exch(struct rdma_id_private *id_priv,
 407                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 408 {
 409         unsigned long flags;
 410         int ret;
 411
 412         /*
 413          * The FSM uses a funny double locking where state is protected by both
 414          * the handler_mutex and the spinlock. State is not allowed to change
 415          * to/from a handler_mutex protected value without also holding
 416          * handler_mutex.
 417          */
 418         if (comp == RDMA_CM_CONNECT || exch == RDMA_CM_CONNECT)
 419                 lockdep_assert_held(&id_priv->handler_mutex);
 420
 421         spin_lock_irqsave(&id_priv->lock, flags);
 422         if ((ret = (id_priv->state == comp)))
 423                 id_priv->state = exch;
 424         spin_unlock_irqrestore(&id_priv->lock, flags);
 425         return ret;
 426 }
 427
 428 static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
 429 {
 430         return hdr->ip_version >> 4;
 431 }
 432
 433 static void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 434 {
 435         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 436 }
 437
 438 static struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
 439 {
 440         return (struct sockaddr *)&id_priv->id.route.addr.src_addr;
 441 }
 442
 443 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
 444 {
 445         return (struct sockaddr *)&id_priv->id.route.addr.dst_addr;
 446 }
 447
 448 static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
 449 {
 450         struct in_device *in_dev = NULL;
 451
 452         if (ndev) {
 453                 rtnl_lock();
 454                 in_dev = __in_dev_get_rtnl(ndev);
 455                 if (in_dev) {
 456                         if (join)
 457                                 ip_mc_inc_group(in_dev,
 458                                                 *(__be32 *)(mgid->raw + 12));
 459                         else
 460                                 ip_mc_dec_group(in_dev,
 461                                                 *(__be32 *)(mgid->raw + 12));
 462                 }
 463                 rtnl_unlock();
 464         }
 465         return (in_dev) ? 0 : -ENODEV;
 466 }
 467
 468 static int compare_netdev_and_ip(int ifindex_a, struct sockaddr *sa,
 469                                  struct id_table_entry *entry_b)
 470 {
 471         struct rdma_id_private *id_priv = list_first_entry(
 472                 &entry_b->id_list, struct rdma_id_private, id_list_entry);
 473         int ifindex_b = id_priv->id.route.addr.dev_addr.bound_dev_if;
 474         struct sockaddr *sb = cma_dst_addr(id_priv);
 475
 476         if (ifindex_a != ifindex_b)
 477                 return (ifindex_a > ifindex_b) ? 1 : -1;
 478
 479         if (sa->sa_family != sb->sa_family)
 480                 return sa->sa_family - sb->sa_family;
 481
 482         if (sa->sa_family == AF_INET &&
 483             __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in)) {
 484                 return memcmp(&((struct sockaddr_in *)sa)->sin_addr,
 485                               &((struct sockaddr_in *)sb)->sin_addr,
 486                               sizeof(((struct sockaddr_in *)sa)->sin_addr));
 487         }
 488
 489         if (sa->sa_family == AF_INET6 &&
 490             __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in6)) {
 491                 return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr,
 492                                      &((struct sockaddr_in6 *)sb)->sin6_addr);
 493         }
 494
 495         return -1;
 496 }
 497
 498 static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv)
 499 {
 500         struct rb_node **new, *parent = NULL;
 501         struct id_table_entry *this, *node;
 502         unsigned long flags;
 503         int result;
 504
 505         node = kzalloc(sizeof(*node), GFP_KERNEL);
 506         if (!node)
 507                 return -ENOMEM;
 508
 509         spin_lock_irqsave(&id_table_lock, flags);
 510         new = &id_table.rb_node;
 511         while (*new) {
 512                 this = container_of(*new, struct id_table_entry, rb_node);
 513                 result = compare_netdev_and_ip(
 514                         node_id_priv->id.route.addr.dev_addr.bound_dev_if,
 515                         cma_dst_addr(node_id_priv), this);
 516
 517                 parent = *new;
 518                 if (result < 0)
 519                         new = &((*new)->rb_left);
 520                 else if (result > 0)
 521                         new = &((*new)->rb_right);
 522                 else {
 523                         list_add_tail(&node_id_priv->id_list_entry,
 524                                       &this->id_list);
 525                         kfree(node);
 526                         goto unlock;
 527                 }
 528         }
 529
 530         INIT_LIST_HEAD(&node->id_list);
 531         list_add_tail(&node_id_priv->id_list_entry, &node->id_list);
 532
 533         rb_link_node(&node->rb_node, parent, new);
 534         rb_insert_color(&node->rb_node, &id_table);
 535
 536 unlock:
 537         spin_unlock_irqrestore(&id_table_lock, flags);
 538         return 0;
 539 }
 540
 541 static struct id_table_entry *
 542 node_from_ndev_ip(struct rb_root *root, int ifindex, struct sockaddr *sa)
 543 {
 544         struct rb_node *node = root->rb_node;
 545         struct id_table_entry *data;
 546         int result;
 547
 548         while (node) {
 549                 data = container_of(node, struct id_table_entry, rb_node);
 550                 result = compare_netdev_and_ip(ifindex, sa, data);
 551                 if (result < 0)
 552                         node = node->rb_left;
 553                 else if (result > 0)
 554                         node = node->rb_right;
 555                 else
 556                         return data;
 557         }
 558
 559         return NULL;
 560 }
 561
 562 static void cma_remove_id_from_tree(struct rdma_id_private *id_priv)
 563 {
 564         struct id_table_entry *data;
 565         unsigned long flags;
 566
 567         spin_lock_irqsave(&id_table_lock, flags);
 568         if (list_empty(&id_priv->id_list_entry))
 569                 goto out;
 570
 571         data = node_from_ndev_ip(&id_table,
 572                                  id_priv->id.route.addr.dev_addr.bound_dev_if,
 573                                  cma_dst_addr(id_priv));
 574         if (!data)
 575                 goto out;
 576
 577         list_del_init(&id_priv->id_list_entry);
 578         if (list_empty(&data->id_list)) {
 579                 rb_erase(&data->rb_node, &id_table);
 580                 kfree(data);
 581         }
 582 out:
 583         spin_unlock_irqrestore(&id_table_lock, flags);
 584 }
 585
 586 static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
 587                                struct cma_device *cma_dev)
 588 {
 589         cma_dev_get(cma_dev);
 590         id_priv->cma_dev = cma_dev;
 591         id_priv->id.device = cma_dev->device;
 592         id_priv->id.route.addr.dev_addr.transport =
 593                 rdma_node_get_transport(cma_dev->device->node_type);
 594         list_add_tail(&id_priv->device_item, &cma_dev->id_list);
 595
 596         trace_cm_id_attach(id_priv, cma_dev->device);
 597 }
 598
 599 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 600                               struct cma_device *cma_dev)
 601 {
 602         _cma_attach_to_dev(id_priv, cma_dev);
 603         id_priv->gid_type =
 604                 cma_dev->default_gid_type[id_priv->id.port_num -
 605                                           rdma_start_port(cma_dev->device)];
 606 }
 607
 608 static void cma_release_dev(struct rdma_id_private *id_priv)
 609 {
 610         mutex_lock(&lock);
 611         list_del_init(&id_priv->device_item);
 612         cma_dev_put(id_priv->cma_dev);
 613         id_priv->cma_dev = NULL;
 614         id_priv->id.device = NULL;
 615         if (id_priv->id.route.addr.dev_addr.sgid_attr) {
 616                 rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
 617                 id_priv->id.route.addr.dev_addr.sgid_attr = NULL;
 618         }
 619         mutex_unlock(&lock);
 620 }
 621
 622 static inline unsigned short cma_family(struct rdma_id_private *id_priv)
 623 {
 624         return id_priv->id.route.addr.src_addr.ss_family;
 625 }
 626
 627 static int cma_set_default_qkey(struct rdma_id_private *id_priv)
 628 {
 629         struct ib_sa_mcmember_rec rec;
 630         int ret = 0;
 631
 632         switch (id_priv->id.ps) {
 633         case RDMA_PS_UDP:
 634         case RDMA_PS_IB:
 635                 id_priv->qkey = RDMA_UDP_QKEY;
 636                 break;
 637         case RDMA_PS_IPOIB:
 638                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 639                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 640                                              id_priv->id.port_num, &rec.mgid,
 641                                              &rec);
 642                 if (!ret)
 643                         id_priv->qkey = be32_to_cpu(rec.qkey);
 644                 break;
 645         default:
 646                 break;
 647         }
 648         return ret;
 649 }
 650
 651 static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
 652 {
 653         if (!qkey ||
 654             (id_priv->qkey && (id_priv->qkey != qkey)))
 655                 return -EINVAL;
 656
 657         id_priv->qkey = qkey;
 658         return 0;
 659 }
 660
 661 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
 662 {
 663         dev_addr->dev_type = ARPHRD_INFINIBAND;
 664         rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
 665         ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
 666 }
 667
 668 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 669 {
 670         int ret;
 671
 672         if (addr->sa_family != AF_IB) {
 673                 ret = rdma_translate_ip(addr, dev_addr);
 674         } else {
 675                 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
 676                 ret = 0;
 677         }
 678
 679         return ret;
 680 }
 681
 682 static const struct ib_gid_attr *
 683 cma_validate_port(struct ib_device *device, u32 port,
 684                   enum ib_gid_type gid_type,
 685                   union ib_gid *gid,
 686                   struct rdma_id_private *id_priv)
 687 {
 688         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 689         int bound_if_index = dev_addr->bound_dev_if;
 690         const struct ib_gid_attr *sgid_attr;
 691         int dev_type = dev_addr->dev_type;
 692         struct net_device *ndev = NULL;
 693
 694         if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net))
 695                 return ERR_PTR(-ENODEV);
 696
 697         if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
 698                 return ERR_PTR(-ENODEV);
 699
 700         if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
 701                 return ERR_PTR(-ENODEV);
 702
 703         if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
 704                 ndev = dev_get_by_index(dev_addr->net, bound_if_index);
 705                 if (!ndev)
 706                         return ERR_PTR(-ENODEV);
 707         } else {
 708                 gid_type = IB_GID_TYPE_IB;
 709         }
 710
 711         sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
 712         dev_put(ndev);
 713         return sgid_attr;
 714 }
 715
 716 static void cma_bind_sgid_attr(struct rdma_id_private *id_priv,
 717                                const struct ib_gid_attr *sgid_attr)
 718 {
 719         WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr);
 720         id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr;
 721 }
 722
 723 /**
 724  * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute
 725  * based on source ip address.
 726  * @id_priv:    cm_id which should be bound to cma device
 727  *
 728  * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute
 729  * based on source IP address. It returns 0 on success or error code otherwise.
 730  * It is applicable to active and passive side cm_id.
 731  */
 732 static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)
 733 {
 734         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 735         const struct ib_gid_attr *sgid_attr;
 736         union ib_gid gid, iboe_gid, *gidp;
 737         struct cma_device *cma_dev;
 738         enum ib_gid_type gid_type;
 739         int ret = -ENODEV;
 740         u32 port;
 741
 742         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 743             id_priv->id.ps == RDMA_PS_IPOIB)
 744                 return -EINVAL;
 745
 746         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 747                     &iboe_gid);
 748
 749         memcpy(&gid, dev_addr->src_dev_addr +
 750                rdma_addr_gid_offset(dev_addr), sizeof(gid));
 751
 752         mutex_lock(&lock);
 753         list_for_each_entry(cma_dev, &dev_list, list) {
 754                 rdma_for_each_port (cma_dev->device, port) {
 755                         gidp = rdma_protocol_roce(cma_dev->device, port) ?
 756                                &iboe_gid : &gid;
 757                         gid_type = cma_dev->default_gid_type[port - 1];
 758                         sgid_attr = cma_validate_port(cma_dev->device, port,
 759                                                       gid_type, gidp, id_priv);
 760                         if (!IS_ERR(sgid_attr)) {
 761                                 id_priv->id.port_num = port;
 762                                 cma_bind_sgid_attr(id_priv, sgid_attr);
 763                                 cma_attach_to_dev(id_priv, cma_dev);
 764                                 ret = 0;
 765                                 goto out;
 766                         }
 767                 }
 768         }
 769 out:
 770         mutex_unlock(&lock);
 771         return ret;
 772 }
 773
 774 /**
 775  * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute
 776  * @id_priv:            cm id to bind to cma device
 777  * @listen_id_priv:     listener cm id to match against
 778  * @req:                Pointer to req structure containaining incoming
 779  *                      request information
 780  * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when
 781  * rdma device matches for listen_id and incoming request. It also verifies
 782  * that a GID table entry is present for the source address.
 783  * Returns 0 on success, or returns error code otherwise.
 784  */
 785 static int cma_ib_acquire_dev(struct rdma_id_private *id_priv,
 786                               const struct rdma_id_private *listen_id_priv,
 787                               struct cma_req_info *req)
 788 {
 789         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 790         const struct ib_gid_attr *sgid_attr;
 791         enum ib_gid_type gid_type;
 792         union ib_gid gid;
 793
 794         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 795             id_priv->id.ps == RDMA_PS_IPOIB)
 796                 return -EINVAL;
 797
 798         if (rdma_protocol_roce(req->device, req->port))
 799                 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 800                             &gid);
 801         else
 802                 memcpy(&gid, dev_addr->src_dev_addr +
 803                        rdma_addr_gid_offset(dev_addr), sizeof(gid));
 804
 805         gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1];
 806         sgid_attr = cma_validate_port(req->device, req->port,
 807                                       gid_type, &gid, id_priv);
 808         if (IS_ERR(sgid_attr))
 809                 return PTR_ERR(sgid_attr);
 810
 811         id_priv->id.port_num = req->port;
 812         cma_bind_sgid_attr(id_priv, sgid_attr);
 813         /* Need to acquire lock to protect against reader
 814          * of cma_dev->id_list such as cma_netdev_callback() and
 815          * cma_process_remove().
 816          */
 817         mutex_lock(&lock);
 818         cma_attach_to_dev(id_priv, listen_id_priv->cma_dev);
 819         mutex_unlock(&lock);
 820         rdma_restrack_add(&id_priv->res);
 821         return 0;
 822 }
 823
 824 static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
 825                               const struct rdma_id_private *listen_id_priv)
 826 {
 827         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 828         const struct ib_gid_attr *sgid_attr;
 829         struct cma_device *cma_dev;
 830         enum ib_gid_type gid_type;
 831         int ret = -ENODEV;
 832         union ib_gid gid;
 833         u32 port;
 834
 835         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 836             id_priv->id.ps == RDMA_PS_IPOIB)
 837                 return -EINVAL;
 838
 839         memcpy(&gid, dev_addr->src_dev_addr +
 840                rdma_addr_gid_offset(dev_addr), sizeof(gid));
 841
 842         mutex_lock(&lock);
 843
 844         cma_dev = listen_id_priv->cma_dev;
 845         port = listen_id_priv->id.port_num;
 846         gid_type = listen_id_priv->gid_type;
 847         sgid_attr = cma_validate_port(cma_dev->device, port,
 848                                       gid_type, &gid, id_priv);
 849         if (!IS_ERR(sgid_attr)) {
 850                 id_priv->id.port_num = port;
 851                 cma_bind_sgid_attr(id_priv, sgid_attr);
 852                 ret = 0;
 853                 goto out;
 854         }
 855
 856         list_for_each_entry(cma_dev, &dev_list, list) {
 857                 rdma_for_each_port (cma_dev->device, port) {
 858                         if (listen_id_priv->cma_dev == cma_dev &&
 859                             listen_id_priv->id.port_num == port)
 860                                 continue;
 861
 862                         gid_type = cma_dev->default_gid_type[port - 1];
 863                         sgid_attr = cma_validate_port(cma_dev->device, port,
 864                                                       gid_type, &gid, id_priv);
 865                         if (!IS_ERR(sgid_attr)) {
 866                                 id_priv->id.port_num = port;
 867                                 cma_bind_sgid_attr(id_priv, sgid_attr);
 868                                 ret = 0;
 869                                 goto out;
 870                         }
 871                 }
 872         }
 873
 874 out:
 875         if (!ret) {
 876                 cma_attach_to_dev(id_priv, cma_dev);
 877                 rdma_restrack_add(&id_priv->res);
 878         }
 879
 880         mutex_unlock(&lock);
 881         return ret;
 882 }
 883
 884 /*
 885  * Select the source IB device and address to reach the destination IB address.
 886  */
 887 static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
 888 {
 889         struct cma_device *cma_dev, *cur_dev;
 890         struct sockaddr_ib *addr;
 891         union ib_gid gid, sgid, *dgid;
 892         unsigned int p;
 893         u16 pkey, index;
 894         enum ib_port_state port_state;
 895         int ret;
 896         int i;
 897
 898         cma_dev = NULL;
 899         addr = (struct sockaddr_ib *) cma_dst_addr(id_priv);
 900         dgid = (union ib_gid *) &addr->sib_addr;
 901         pkey = ntohs(addr->sib_pkey);
 902
 903         mutex_lock(&lock);
 904         list_for_each_entry(cur_dev, &dev_list, list) {
 905                 rdma_for_each_port (cur_dev->device, p) {
 906                         if (!rdma_cap_af_ib(cur_dev->device, p))
 907                                 continue;
 908
 909                         if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index))
 910                                 continue;
 911
 912                         if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
 913                                 continue;
 914
 915                         for (i = 0; i < cur_dev->device->port_data[p].immutable.gid_tbl_len;
 916                              ++i) {
 917                                 ret = rdma_query_gid(cur_dev->device, p, i,
 918                                                      &gid);
 919                                 if (ret)
 920                                         continue;
 921
 922                                 if (!memcmp(&gid, dgid, sizeof(gid))) {
 923                                         cma_dev = cur_dev;
 924                                         sgid = gid;
 925                                         id_priv->id.port_num = p;
 926                                         goto found;
 927                                 }
 928
 929                                 if (!cma_dev && (gid.global.subnet_prefix ==
 930                                     dgid->global.subnet_prefix) &&
 931                                     port_state == IB_PORT_ACTIVE) {
 932                                         cma_dev = cur_dev;
 933                                         sgid = gid;
 934                                         id_priv->id.port_num = p;
 935                                         goto found;
 936                                 }
 937                         }
 938                 }
 939         }
 940         mutex_unlock(&lock);
 941         return -ENODEV;
 942
 943 found:
 944         cma_attach_to_dev(id_priv, cma_dev);
 945         rdma_restrack_add(&id_priv->res);
 946         mutex_unlock(&lock);
 947         addr = (struct sockaddr_ib *)cma_src_addr(id_priv);
 948         memcpy(&addr->sib_addr, &sgid, sizeof(sgid));
 949         cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
 950         return 0;
 951 }
 952
 953 static void cma_id_get(struct rdma_id_private *id_priv)
 954 {
 955         refcount_inc(&id_priv->refcount);
 956 }
 957
 958 static void cma_id_put(struct rdma_id_private *id_priv)
 959 {
 960         if (refcount_dec_and_test(&id_priv->refcount))
 961                 complete(&id_priv->comp);
 962 }
 963
 964 static struct rdma_id_private *
 965 __rdma_create_id(struct net *net, rdma_cm_event_handler event_handler,
 966                  void *context, enum rdma_ucm_port_space ps,
 967                  enum ib_qp_type qp_type, const struct rdma_id_private *parent)
 968 {
 969         struct rdma_id_private *id_priv;
 970
 971         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 972         if (!id_priv)
 973                 return ERR_PTR(-ENOMEM);
 974
 975         id_priv->state = RDMA_CM_IDLE;
 976         id_priv->id.context = context;
 977         id_priv->id.event_handler = event_handler;
 978         id_priv->id.ps = ps;
 979         id_priv->id.qp_type = qp_type;
 980         id_priv->tos_set = false;
 981         id_priv->timeout_set = false;
 982         id_priv->min_rnr_timer_set = false;
 983         id_priv->gid_type = IB_GID_TYPE_IB;
 984         spin_lock_init(&id_priv->lock);
 985         mutex_init(&id_priv->qp_mutex);
 986         init_completion(&id_priv->comp);
 987         refcount_set(&id_priv->refcount, 1);
 988         mutex_init(&id_priv->handler_mutex);
 989         INIT_LIST_HEAD(&id_priv->device_item);
 990         INIT_LIST_HEAD(&id_priv->id_list_entry);
 991         INIT_LIST_HEAD(&id_priv->listen_list);
 992         INIT_LIST_HEAD(&id_priv->mc_list);
 993         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 994         id_priv->id.route.addr.dev_addr.net = get_net(net);
 995         id_priv->seq_num &= 0x00ffffff;
 996
 997         rdma_restrack_new(&id_priv->res, RDMA_RESTRACK_CM_ID);
 998         if (parent)
 999                 rdma_restrack_parent_name(&id_priv->res, &parent->res);
1000
1001         return id_priv;
1002 }
1003
1004 struct rdma_cm_id *
1005 __rdma_create_kernel_id(struct net *net, rdma_cm_event_handler event_handler,
1006                         void *context, enum rdma_ucm_port_space ps,
1007                         enum ib_qp_type qp_type, const char *caller)
1008 {
1009         struct rdma_id_private *ret;
1010
1011         ret = __rdma_create_id(net, event_handler, context, ps, qp_type, NULL);
1012         if (IS_ERR(ret))
1013                 return ERR_CAST(ret);
1014
1015         rdma_restrack_set_name(&ret->res, caller);
1016         return &ret->id;
1017 }
1018 EXPORT_SYMBOL(__rdma_create_kernel_id);
1019
1020 struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler,
1021                                        void *context,
1022                                        enum rdma_ucm_port_space ps,
1023                                        enum ib_qp_type qp_type)
1024 {
1025         struct rdma_id_private *ret;
1026
1027         ret = __rdma_create_id(current->nsproxy->net_ns, event_handler, context,
1028                                ps, qp_type, NULL);
1029         if (IS_ERR(ret))
1030                 return ERR_CAST(ret);
1031
1032         rdma_restrack_set_name(&ret->res, NULL);
1033         return &ret->id;
1034 }
1035 EXPORT_SYMBOL(rdma_create_user_id);
1036
1037 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
1038 {
1039         struct ib_qp_attr qp_attr;
1040         int qp_attr_mask, ret;
1041
1042         qp_attr.qp_state = IB_QPS_INIT;
1043         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1044         if (ret)
1045                 return ret;
1046
1047         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
1048         if (ret)
1049                 return ret;
1050
1051         qp_attr.qp_state = IB_QPS_RTR;
1052         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
1053         if (ret)
1054                 return ret;
1055
1056         qp_attr.qp_state = IB_QPS_RTS;
1057         qp_attr.sq_psn = 0;
1058         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
1059
1060         return ret;
1061 }
1062
1063 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
1064 {
1065         struct ib_qp_attr qp_attr;
1066         int qp_attr_mask, ret;
1067
1068         qp_attr.qp_state = IB_QPS_INIT;
1069         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1070         if (ret)
1071                 return ret;
1072
1073         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
1074 }
1075
1076 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
1077                    struct ib_qp_init_attr *qp_init_attr)
1078 {
1079         struct rdma_id_private *id_priv;
1080         struct ib_qp *qp;
1081         int ret;
1082
1083         id_priv = container_of(id, struct rdma_id_private, id);
1084         if (id->device != pd->device) {
1085                 ret = -EINVAL;
1086                 goto out_err;
1087         }
1088
1089         qp_init_attr->port_num = id->port_num;
1090         qp = ib_create_qp(pd, qp_init_attr);
1091         if (IS_ERR(qp)) {
1092                 ret = PTR_ERR(qp);
1093                 goto out_err;
1094         }
1095
1096         if (id->qp_type == IB_QPT_UD)
1097                 ret = cma_init_ud_qp(id_priv, qp);
1098         else
1099                 ret = cma_init_conn_qp(id_priv, qp);
1100         if (ret)
1101                 goto out_destroy;
1102
1103         id->qp = qp;
1104         id_priv->qp_num = qp->qp_num;
1105         id_priv->srq = (qp->srq != NULL);
1106         trace_cm_qp_create(id_priv, pd, qp_init_attr, 0);
1107         return 0;
1108 out_destroy:
1109         ib_destroy_qp(qp);
1110 out_err:
1111         trace_cm_qp_create(id_priv, pd, qp_init_attr, ret);
1112         return ret;
1113 }
1114 EXPORT_SYMBOL(rdma_create_qp);
1115
1116 void rdma_destroy_qp(struct rdma_cm_id *id)
1117 {
1118         struct rdma_id_private *id_priv;
1119
1120         id_priv = container_of(id, struct rdma_id_private, id);
1121         trace_cm_qp_destroy(id_priv);
1122         mutex_lock(&id_priv->qp_mutex);
1123         ib_destroy_qp(id_priv->id.qp);
1124         id_priv->id.qp = NULL;
1125         mutex_unlock(&id_priv->qp_mutex);
1126 }
1127 EXPORT_SYMBOL(rdma_destroy_qp);
1128
1129 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
1130                              struct rdma_conn_param *conn_param)
1131 {
1132         struct ib_qp_attr qp_attr;
1133         int qp_attr_mask, ret;
1134
1135         mutex_lock(&id_priv->qp_mutex);
1136         if (!id_priv->id.qp) {
1137                 ret = 0;
1138                 goto out;
1139         }
1140
1141         /* Need to update QP attributes from default values. */
1142         qp_attr.qp_state = IB_QPS_INIT;
1143         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1144         if (ret)
1145                 goto out;
1146
1147         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1148         if (ret)
1149                 goto out;
1150
1151         qp_attr.qp_state = IB_QPS_RTR;
1152         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1153         if (ret)
1154                 goto out;
1155
1156         BUG_ON(id_priv->cma_dev->device != id_priv->id.device);
1157
1158         if (conn_param)
1159                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
1160         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1161 out:
1162         mutex_unlock(&id_priv->qp_mutex);
1163         return ret;
1164 }
1165
1166 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
1167                              struct rdma_conn_param *conn_param)
1168 {
1169         struct ib_qp_attr qp_attr;
1170         int qp_attr_mask, ret;
1171
1172         mutex_lock(&id_priv->qp_mutex);
1173         if (!id_priv->id.qp) {
1174                 ret = 0;
1175                 goto out;
1176         }
1177
1178         qp_attr.qp_state = IB_QPS_RTS;
1179         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1180         if (ret)
1181                 goto out;
1182
1183         if (conn_param)
1184                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
1185         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1186 out:
1187         mutex_unlock(&id_priv->qp_mutex);
1188         return ret;
1189 }
1190
1191 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
1192 {
1193         struct ib_qp_attr qp_attr;
1194         int ret;
1195
1196         mutex_lock(&id_priv->qp_mutex);
1197         if (!id_priv->id.qp) {
1198                 ret = 0;
1199                 goto out;
1200         }
1201
1202         qp_attr.qp_state = IB_QPS_ERR;
1203         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
1204 out:
1205         mutex_unlock(&id_priv->qp_mutex);
1206         return ret;
1207 }
1208
1209 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
1210                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
1211 {
1212         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1213         int ret;
1214         u16 pkey;
1215
1216         if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num))
1217                 pkey = 0xffff;
1218         else
1219                 pkey = ib_addr_get_pkey(dev_addr);
1220
1221         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
1222                                   pkey, &qp_attr->pkey_index);
1223         if (ret)
1224                 return ret;
1225
1226         qp_attr->port_num = id_priv->id.port_num;
1227         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
1228
1229         if (id_priv->id.qp_type == IB_QPT_UD) {
1230                 ret = cma_set_default_qkey(id_priv);
1231                 if (ret)
1232                         return ret;
1233
1234                 qp_attr->qkey = id_priv->qkey;
1235                 *qp_attr_mask |= IB_QP_QKEY;
1236         } else {
1237                 qp_attr->qp_access_flags = 0;
1238                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
1239         }
1240         return 0;
1241 }
1242
1243 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
1244                        int *qp_attr_mask)
1245 {
1246         struct rdma_id_private *id_priv;
1247         int ret = 0;
1248
1249         id_priv = container_of(id, struct rdma_id_private, id);
1250         if (rdma_cap_ib_cm(id->device, id->port_num)) {
1251                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
1252                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
1253                 else
1254                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
1255                                                  qp_attr_mask);
1256
1257                 if (qp_attr->qp_state == IB_QPS_RTR)
1258                         qp_attr->rq_psn = id_priv->seq_num;
1259         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
1260                 if (!id_priv->cm_id.iw) {
1261                         qp_attr->qp_access_flags = 0;
1262                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
1263                 } else
1264                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
1265                                                  qp_attr_mask);
1266                 qp_attr->port_num = id_priv->id.port_num;
1267                 *qp_attr_mask |= IB_QP_PORT;
1268         } else {
1269                 ret = -ENOSYS;
1270         }
1271
1272         if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set)
1273                 qp_attr->timeout = id_priv->timeout;
1274
1275         if ((*qp_attr_mask & IB_QP_MIN_RNR_TIMER) && id_priv->min_rnr_timer_set)
1276                 qp_attr->min_rnr_timer = id_priv->min_rnr_timer;
1277
1278         return ret;
1279 }
1280 EXPORT_SYMBOL(rdma_init_qp_attr);
1281
1282 static inline bool cma_zero_addr(const struct sockaddr *addr)
1283 {
1284         switch (addr->sa_family) {
1285         case AF_INET:
1286                 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
1287         case AF_INET6:
1288                 return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr);
1289         case AF_IB:
1290                 return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr);
1291         default:
1292                 return false;
1293         }
1294 }
1295
1296 static inline bool cma_loopback_addr(const struct sockaddr *addr)
1297 {
1298         switch (addr->sa_family) {
1299         case AF_INET:
1300                 return ipv4_is_loopback(
1301                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
1302         case AF_INET6:
1303                 return ipv6_addr_loopback(
1304                         &((struct sockaddr_in6 *)addr)->sin6_addr);
1305         case AF_IB:
1306                 return ib_addr_loopback(
1307                         &((struct sockaddr_ib *)addr)->sib_addr);
1308         default:
1309                 return false;
1310         }
1311 }
1312
1313 static inline bool cma_any_addr(const struct sockaddr *addr)
1314 {
1315         return cma_zero_addr(addr) || cma_loopback_addr(addr);
1316 }
1317
1318 static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst)
1319 {
1320         if (src->sa_family != dst->sa_family)
1321                 return -1;
1322
1323         switch (src->sa_family) {
1324         case AF_INET:
1325                 return ((struct sockaddr_in *)src)->sin_addr.s_addr !=
1326                        ((struct sockaddr_in *)dst)->sin_addr.s_addr;
1327         case AF_INET6: {
1328                 struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src;
1329                 struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst;
1330                 bool link_local;
1331
1332                 if (ipv6_addr_cmp(&src_addr6->sin6_addr,
1333                                           &dst_addr6->sin6_addr))
1334                         return 1;
1335                 link_local = ipv6_addr_type(&dst_addr6->sin6_addr) &
1336                              IPV6_ADDR_LINKLOCAL;
1337                 /* Link local must match their scope_ids */
1338                 return link_local ? (src_addr6->sin6_scope_id !=
1339                                      dst_addr6->sin6_scope_id) :
1340                                     0;
1341         }
1342
1343         default:
1344                 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
1345                                    &((struct sockaddr_ib *) dst)->sib_addr);
1346         }
1347 }
1348
1349 static __be16 cma_port(const struct sockaddr *addr)
1350 {
1351         struct sockaddr_ib *sib;
1352
1353         switch (addr->sa_family) {
1354         case AF_INET:
1355                 return ((struct sockaddr_in *) addr)->sin_port;
1356         case AF_INET6:
1357                 return ((struct sockaddr_in6 *) addr)->sin6_port;
1358         case AF_IB:
1359                 sib = (struct sockaddr_ib *) addr;
1360                 return htons((u16) (be64_to_cpu(sib->sib_sid) &
1361                                     be64_to_cpu(sib->sib_sid_mask)));
1362         default:
1363                 return 0;
1364         }
1365 }
1366
1367 static inline int cma_any_port(const struct sockaddr *addr)
1368 {
1369         return !cma_port(addr);
1370 }
1371
1372 static void cma_save_ib_info(struct sockaddr *src_addr,
1373                              struct sockaddr *dst_addr,
1374                              const struct rdma_cm_id *listen_id,
1375                              const struct sa_path_rec *path)
1376 {
1377         struct sockaddr_ib *listen_ib, *ib;
1378
1379         listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr;
1380         if (src_addr) {
1381                 ib = (struct sockaddr_ib *)src_addr;
1382                 ib->sib_family = AF_IB;
1383                 if (path) {
1384                         ib->sib_pkey = path->pkey;
1385                         ib->sib_flowinfo = path->flow_label;
1386                         memcpy(&ib->sib_addr, &path->sgid, 16);
1387                         ib->sib_sid = path->service_id;
1388                         ib->sib_scope_id = 0;
1389                 } else {
1390                         ib->sib_pkey = listen_ib->sib_pkey;
1391                         ib->sib_flowinfo = listen_ib->sib_flowinfo;
1392                         ib->sib_addr = listen_ib->sib_addr;
1393                         ib->sib_sid = listen_ib->sib_sid;
1394                         ib->sib_scope_id = listen_ib->sib_scope_id;
1395                 }
1396                 ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
1397         }
1398         if (dst_addr) {
1399                 ib = (struct sockaddr_ib *)dst_addr;
1400                 ib->sib_family = AF_IB;
1401                 if (path) {
1402                         ib->sib_pkey = path->pkey;
1403                         ib->sib_flowinfo = path->flow_label;
1404                         memcpy(&ib->sib_addr, &path->dgid, 16);
1405                 }
1406         }
1407 }
1408
1409 static void cma_save_ip4_info(struct sockaddr_in *src_addr,
1410                               struct sockaddr_in *dst_addr,
1411                               struct cma_hdr *hdr,
1412                               __be16 local_port)
1413 {
1414         if (src_addr) {
1415                 *src_addr = (struct sockaddr_in) {
1416                         .sin_family = AF_INET,
1417                         .sin_addr.s_addr = hdr->dst_addr.ip4.addr,
1418                         .sin_port = local_port,
1419                 };
1420         }
1421
1422         if (dst_addr) {
1423                 *dst_addr = (struct sockaddr_in) {
1424                         .sin_family = AF_INET,
1425                         .sin_addr.s_addr = hdr->src_addr.ip4.addr,
1426                         .sin_port = hdr->port,
1427                 };
1428         }
1429 }
1430
1431 static void cma_save_ip6_info(struct sockaddr_in6 *src_addr,
1432                               struct sockaddr_in6 *dst_addr,
1433                               struct cma_hdr *hdr,
1434                               __be16 local_port)
1435 {
1436         if (src_addr) {
1437                 *src_addr = (struct sockaddr_in6) {
1438                         .sin6_family = AF_INET6,
1439                         .sin6_addr = hdr->dst_addr.ip6,
1440                         .sin6_port = local_port,
1441                 };
1442         }
1443
1444         if (dst_addr) {
1445                 *dst_addr = (struct sockaddr_in6) {
1446                         .sin6_family = AF_INET6,
1447                         .sin6_addr = hdr->src_addr.ip6,
1448                         .sin6_port = hdr->port,
1449                 };
1450         }
1451 }
1452
1453 static u16 cma_port_from_service_id(__be64 service_id)
1454 {
1455         return (u16)be64_to_cpu(service_id);
1456 }
1457
1458 static int cma_save_ip_info(struct sockaddr *src_addr,
1459                             struct sockaddr *dst_addr,
1460                             const struct ib_cm_event *ib_event,
1461                             __be64 service_id)
1462 {
1463         struct cma_hdr *hdr;
1464         __be16 port;
1465
1466         hdr = ib_event->private_data;
1467         if (hdr->cma_version != CMA_VERSION)
1468                 return -EINVAL;
1469
1470         port = htons(cma_port_from_service_id(service_id));
1471
1472         switch (cma_get_ip_ver(hdr)) {
1473         case 4:
1474                 cma_save_ip4_info((struct sockaddr_in *)src_addr,
1475                                   (struct sockaddr_in *)dst_addr, hdr, port);
1476                 break;
1477         case 6:
1478                 cma_save_ip6_info((struct sockaddr_in6 *)src_addr,
1479                                   (struct sockaddr_in6 *)dst_addr, hdr, port);
1480                 break;
1481         default:
1482                 return -EAFNOSUPPORT;
1483         }
1484
1485         return 0;
1486 }
1487
1488 static int cma_save_net_info(struct sockaddr *src_addr,
1489                              struct sockaddr *dst_addr,
1490                              const struct rdma_cm_id *listen_id,
1491                              const struct ib_cm_event *ib_event,
1492                              sa_family_t sa_family, __be64 service_id)
1493 {
1494         if (sa_family == AF_IB) {
1495                 if (ib_event->event == IB_CM_REQ_RECEIVED)
1496                         cma_save_ib_info(src_addr, dst_addr, listen_id,
1497                                          ib_event->param.req_rcvd.primary_path);
1498                 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1499                         cma_save_ib_info(src_addr, dst_addr, listen_id, NULL);
1500                 return 0;
1501         }
1502
1503         return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id);
1504 }
1505
1506 static int cma_save_req_info(const struct ib_cm_event *ib_event,
1507                              struct cma_req_info *req)
1508 {
1509         const struct ib_cm_req_event_param *req_param =
1510                 &ib_event->param.req_rcvd;
1511         const struct ib_cm_sidr_req_event_param *sidr_param =
1512                 &ib_event->param.sidr_req_rcvd;
1513
1514         switch (ib_event->event) {
1515         case IB_CM_REQ_RECEIVED:
1516                 req->device     = req_param->listen_id->device;
1517                 req->port       = req_param->port;
1518                 memcpy(&req->local_gid, &req_param->primary_path->sgid,
1519                        sizeof(req->local_gid));
1520                 req->has_gid    = true;
1521                 req->service_id = req_param->primary_path->service_id;
1522                 req->pkey       = be16_to_cpu(req_param->primary_path->pkey);
1523                 if (req->pkey != req_param->bth_pkey)
1524                         pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
1525                                             "RDMA CMA: in the future this may cause the request to be dropped\n",
1526                                             req_param->bth_pkey, req->pkey);
1527                 break;
1528         case IB_CM_SIDR_REQ_RECEIVED:
1529                 req->device     = sidr_param->listen_id->device;
1530                 req->port       = sidr_param->port;
1531                 req->has_gid    = false;
1532                 req->service_id = sidr_param->service_id;
1533                 req->pkey       = sidr_param->pkey;
1534                 if (req->pkey != sidr_param->bth_pkey)
1535                         pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
1536                                             "RDMA CMA: in the future this may cause the request to be dropped\n",
1537                                             sidr_param->bth_pkey, req->pkey);
1538                 break;
1539         default:
1540                 return -EINVAL;
1541         }
1542
1543         return 0;
1544 }
1545
1546 static bool validate_ipv4_net_dev(struct net_device *net_dev,
1547                                   const struct sockaddr_in *dst_addr,
1548                                   const struct sockaddr_in *src_addr)
1549 {
1550         __be32 daddr = dst_addr->sin_addr.s_addr,
1551                saddr = src_addr->sin_addr.s_addr;
1552         struct fib_result res;
1553         struct flowi4 fl4;
1554         int err;
1555         bool ret;
1556
1557         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1558             ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) ||
1559             ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) ||
1560             ipv4_is_loopback(saddr))
1561                 return false;
1562
1563         memset(&fl4, 0, sizeof(fl4));
1564         fl4.flowi4_oif = net_dev->ifindex;
1565         fl4.daddr = daddr;
1566         fl4.saddr = saddr;
1567
1568         rcu_read_lock();
1569         err = fib_lookup(dev_net(net_dev), &fl4, &res, 0);
1570         ret = err == 0 && FIB_RES_DEV(res) == net_dev;
1571         rcu_read_unlock();
1572
1573         return ret;
1574 }
1575
1576 static bool validate_ipv6_net_dev(struct net_device *net_dev,
1577                                   const struct sockaddr_in6 *dst_addr,
1578                                   const struct sockaddr_in6 *src_addr)
1579 {
1580 #if IS_ENABLED(CONFIG_IPV6)
1581         const int strict = ipv6_addr_type(&dst_addr->sin6_addr) &
1582                            IPV6_ADDR_LINKLOCAL;
1583         struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr,
1584                                          &src_addr->sin6_addr, net_dev->ifindex,
1585                                          NULL, strict);
1586         bool ret;
1587
1588         if (!rt)
1589                 return false;
1590
1591         ret = rt->rt6i_idev->dev == net_dev;
1592         ip6_rt_put(rt);
1593
1594         return ret;
1595 #else
1596         return false;
1597 #endif
1598 }
1599
1600 static bool validate_net_dev(struct net_device *net_dev,
1601                              const struct sockaddr *daddr,
1602                              const struct sockaddr *saddr)
1603 {
1604         const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr;
1605         const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr;
1606         const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1607         const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr;
1608
1609         switch (daddr->sa_family) {
1610         case AF_INET:
1611                 return saddr->sa_family == AF_INET &&
1612                        validate_ipv4_net_dev(net_dev, daddr4, saddr4);
1613
1614         case AF_INET6:
1615                 return saddr->sa_family == AF_INET6 &&
1616                        validate_ipv6_net_dev(net_dev, daddr6, saddr6);
1617
1618         default:
1619                 return false;
1620         }
1621 }
1622
1623 static struct net_device *
1624 roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
1625 {
1626         const struct ib_gid_attr *sgid_attr = NULL;
1627         struct net_device *ndev;
1628
1629         if (ib_event->event == IB_CM_REQ_RECEIVED)
1630                 sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr;
1631         else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1632                 sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr;
1633
1634         if (!sgid_attr)
1635                 return NULL;
1636
1637         rcu_read_lock();
1638         ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr);
1639         if (IS_ERR(ndev))
1640                 ndev = NULL;
1641         else
1642                 dev_hold(ndev);
1643         rcu_read_unlock();
1644         return ndev;
1645 }
1646
1647 static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event,
1648                                           struct cma_req_info *req)
1649 {
1650         struct sockaddr *listen_addr =
1651                         (struct sockaddr *)&req->listen_addr_storage;
1652         struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage;
1653         struct net_device *net_dev;
1654         const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL;
1655         int err;
1656
1657         err = cma_save_ip_info(listen_addr, src_addr, ib_event,
1658                                req->service_id);
1659         if (err)
1660                 return ERR_PTR(err);
1661
1662         if (rdma_protocol_roce(req->device, req->port))
1663                 net_dev = roce_get_net_dev_by_cm_event(ib_event);
1664         else
1665                 net_dev = ib_get_net_dev_by_params(req->device, req->port,
1666                                                    req->pkey,
1667                                                    gid, listen_addr);
1668         if (!net_dev)
1669                 return ERR_PTR(-ENODEV);
1670
1671         return net_dev;
1672 }
1673
1674 static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id)
1675 {
1676         return (be64_to_cpu(service_id) >> 16) & 0xffff;
1677 }
1678
1679 static bool cma_match_private_data(struct rdma_id_private *id_priv,
1680                                    const struct cma_hdr *hdr)
1681 {
1682         struct sockaddr *addr = cma_src_addr(id_priv);
1683         __be32 ip4_addr;
1684         struct in6_addr ip6_addr;
1685
1686         if (cma_any_addr(addr) && !id_priv->afonly)
1687                 return true;
1688
1689         switch (addr->sa_family) {
1690         case AF_INET:
1691                 ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
1692                 if (cma_get_ip_ver(hdr) != 4)
1693                         return false;
1694                 if (!cma_any_addr(addr) &&
1695                     hdr->dst_addr.ip4.addr != ip4_addr)
1696                         return false;
1697                 break;
1698         case AF_INET6:
1699                 ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
1700                 if (cma_get_ip_ver(hdr) != 6)
1701                         return false;
1702                 if (!cma_any_addr(addr) &&
1703                     memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
1704                         return false;
1705                 break;
1706         case AF_IB:
1707                 return true;
1708         default:
1709                 return false;
1710         }
1711
1712         return true;
1713 }
1714
1715 static bool cma_protocol_roce(const struct rdma_cm_id *id)
1716 {
1717         struct ib_device *device = id->device;
1718         const u32 port_num = id->port_num ?: rdma_start_port(device);
1719
1720         return rdma_protocol_roce(device, port_num);
1721 }
1722
1723 static bool cma_is_req_ipv6_ll(const struct cma_req_info *req)
1724 {
1725         const struct sockaddr *daddr =
1726                         (const struct sockaddr *)&req->listen_addr_storage;
1727         const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1728
1729         /* Returns true if the req is for IPv6 link local */
1730         return (daddr->sa_family == AF_INET6 &&
1731                 (ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL));
1732 }
1733
1734 static bool cma_match_net_dev(const struct rdma_cm_id *id,
1735                               const struct net_device *net_dev,
1736                               const struct cma_req_info *req)
1737 {
1738         const struct rdma_addr *addr = &id->route.addr;
1739
1740         if (!net_dev)
1741                 /* This request is an AF_IB request */
1742                 return (!id->port_num || id->port_num == req->port) &&
1743                        (addr->src_addr.ss_family == AF_IB);
1744
1745         /*
1746          * If the request is not for IPv6 link local, allow matching
1747          * request to any netdevice of the one or multiport rdma device.
1748          */
1749         if (!cma_is_req_ipv6_ll(req))
1750                 return true;
1751         /*
1752          * Net namespaces must match, and if the listner is listening
1753          * on a specific netdevice than netdevice must match as well.
1754          */
1755         if (net_eq(dev_net(net_dev), addr->dev_addr.net) &&
1756             (!!addr->dev_addr.bound_dev_if ==
1757              (addr->dev_addr.bound_dev_if == net_dev->ifindex)))
1758                 return true;
1759         else
1760                 return false;
1761 }
1762
1763 static struct rdma_id_private *cma_find_listener(
1764                 const struct rdma_bind_list *bind_list,
1765                 const struct ib_cm_id *cm_id,
1766                 const struct ib_cm_event *ib_event,
1767                 const struct cma_req_info *req,
1768                 const struct net_device *net_dev)
1769 {
1770         struct rdma_id_private *id_priv, *id_priv_dev;
1771
1772         lockdep_assert_held(&lock);
1773
1774         if (!bind_list)
1775                 return ERR_PTR(-EINVAL);
1776
1777         hlist_for_each_entry(id_priv, &bind_list->owners, node) {
1778                 if (cma_match_private_data(id_priv, ib_event->private_data)) {
1779                         if (id_priv->id.device == cm_id->device &&
1780                             cma_match_net_dev(&id_priv->id, net_dev, req))
1781                                 return id_priv;
1782                         list_for_each_entry(id_priv_dev,
1783                                             &id_priv->listen_list,
1784                                             listen_item) {
1785                                 if (id_priv_dev->id.device == cm_id->device &&
1786                                     cma_match_net_dev(&id_priv_dev->id,
1787                                                       net_dev, req))
1788                                         return id_priv_dev;
1789                         }
1790                 }
1791         }
1792
1793         return ERR_PTR(-EINVAL);
1794 }
1795
1796 static struct rdma_id_private *
1797 cma_ib_id_from_event(struct ib_cm_id *cm_id,
1798                      const struct ib_cm_event *ib_event,
1799                      struct cma_req_info *req,
1800                      struct net_device **net_dev)
1801 {
1802         struct rdma_bind_list *bind_list;
1803         struct rdma_id_private *id_priv;
1804         int err;
1805
1806         err = cma_save_req_info(ib_event, req);
1807         if (err)
1808                 return ERR_PTR(err);
1809
1810         *net_dev = cma_get_net_dev(ib_event, req);
1811         if (IS_ERR(*net_dev)) {
1812                 if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) {
1813                         /* Assuming the protocol is AF_IB */
1814                         *net_dev = NULL;
1815                 } else {
1816                         return ERR_CAST(*net_dev);
1817                 }
1818         }
1819
1820         mutex_lock(&lock);
1821         /*
1822          * Net namespace might be getting deleted while route lookup,
1823          * cm_id lookup is in progress. Therefore, perform netdevice
1824          * validation, cm_id lookup under rcu lock.
1825          * RCU lock along with netdevice state check, synchronizes with
1826          * netdevice migrating to different net namespace and also avoids
1827          * case where net namespace doesn't get deleted while lookup is in
1828          * progress.
1829          * If the device state is not IFF_UP, its properties such as ifindex
1830          * and nd_net cannot be trusted to remain valid without rcu lock.
1831          * net/core/dev.c change_net_namespace() ensures to synchronize with
1832          * ongoing operations on net device after device is closed using
1833          * synchronize_net().
1834          */
1835         rcu_read_lock();
1836         if (*net_dev) {
1837                 /*
1838                  * If netdevice is down, it is likely that it is administratively
1839                  * down or it might be migrating to different namespace.
1840                  * In that case avoid further processing, as the net namespace
1841                  * or ifindex may change.
1842                  */
1843                 if (((*net_dev)->flags & IFF_UP) == 0) {
1844                         id_priv = ERR_PTR(-EHOSTUNREACH);
1845                         goto err;
1846                 }
1847
1848                 if (!validate_net_dev(*net_dev,
1849                                  (struct sockaddr *)&req->src_addr_storage,
1850                                  (struct sockaddr *)&req->listen_addr_storage)) {
1851                         id_priv = ERR_PTR(-EHOSTUNREACH);
1852                         goto err;
1853                 }
1854         }
1855
1856         bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net,
1857                                 rdma_ps_from_service_id(req->service_id),
1858                                 cma_port_from_service_id(req->service_id));
1859         id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev);
1860 err:
1861         rcu_read_unlock();
1862         mutex_unlock(&lock);
1863         if (IS_ERR(id_priv) && *net_dev) {
1864                 dev_put(*net_dev);
1865                 *net_dev = NULL;
1866         }
1867         return id_priv;
1868 }
1869
1870 static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv)
1871 {
1872         return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr);
1873 }
1874
1875 static void cma_cancel_route(struct rdma_id_private *id_priv)
1876 {
1877         if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) {
1878                 if (id_priv->query)
1879                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
1880         }
1881 }
1882
1883 static void _cma_cancel_listens(struct rdma_id_private *id_priv)
1884 {
1885         struct rdma_id_private *dev_id_priv;
1886
1887         lockdep_assert_held(&lock);
1888
1889         /*
1890          * Remove from listen_any_list to prevent added devices from spawning
1891          * additional listen requests.
1892          */
1893         list_del_init(&id_priv->listen_any_item);
1894
1895         while (!list_empty(&id_priv->listen_list)) {
1896                 dev_id_priv =
1897                         list_first_entry(&id_priv->listen_list,
1898                                          struct rdma_id_private, listen_item);
1899                 /* sync with device removal to avoid duplicate destruction */
1900                 list_del_init(&dev_id_priv->device_item);
1901                 list_del_init(&dev_id_priv->listen_item);
1902                 mutex_unlock(&lock);
1903
1904                 rdma_destroy_id(&dev_id_priv->id);
1905                 mutex_lock(&lock);
1906         }
1907 }
1908
1909 static void cma_cancel_listens(struct rdma_id_private *id_priv)
1910 {
1911         mutex_lock(&lock);
1912         _cma_cancel_listens(id_priv);
1913         mutex_unlock(&lock);
1914 }
1915
1916 static void cma_cancel_operation(struct rdma_id_private *id_priv,
1917                                  enum rdma_cm_state state)
1918 {
1919         switch (state) {
1920         case RDMA_CM_ADDR_QUERY:
1921                 /*
1922                  * We can avoid doing the rdma_addr_cancel() based on state,
1923                  * only RDMA_CM_ADDR_QUERY has a work that could still execute.
1924                  * Notice that the addr_handler work could still be exiting
1925                  * outside this state, however due to the interaction with the
1926                  * handler_mutex the work is guaranteed not to touch id_priv
1927                  * during exit.
1928                  */
1929                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
1930                 break;
1931         case RDMA_CM_ROUTE_QUERY:
1932                 cma_cancel_route(id_priv);
1933                 break;
1934         case RDMA_CM_LISTEN:
1935                 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
1936                         cma_cancel_listens(id_priv);
1937                 break;
1938         default:
1939                 break;
1940         }
1941 }
1942
1943 static void cma_release_port(struct rdma_id_private *id_priv)
1944 {
1945         struct rdma_bind_list *bind_list = id_priv->bind_list;
1946         struct net *net = id_priv->id.route.addr.dev_addr.net;
1947
1948         if (!bind_list)
1949                 return;
1950
1951         mutex_lock(&lock);
1952         hlist_del(&id_priv->node);
1953         if (hlist_empty(&bind_list->owners)) {
1954                 cma_ps_remove(net, bind_list->ps, bind_list->port);
1955                 kfree(bind_list);
1956         }
1957         mutex_unlock(&lock);
1958 }
1959
1960 static void destroy_mc(struct rdma_id_private *id_priv,
1961                        struct cma_multicast *mc)
1962 {
1963         bool send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
1964
1965         if (rdma_cap_ib_mcast(id_priv->id.device, id_priv->id.port_num))
1966                 ib_sa_free_multicast(mc->sa_mc);
1967
1968         if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) {
1969                 struct rdma_dev_addr *dev_addr =
1970                         &id_priv->id.route.addr.dev_addr;
1971                 struct net_device *ndev = NULL;
1972
1973                 if (dev_addr->bound_dev_if)
1974                         ndev = dev_get_by_index(dev_addr->net,
1975                                                 dev_addr->bound_dev_if);
1976                 if (ndev && !send_only) {
1977                         enum ib_gid_type gid_type;
1978                         union ib_gid mgid;
1979
1980                         gid_type = id_priv->cma_dev->default_gid_type
1981                                            [id_priv->id.port_num -
1982                                             rdma_start_port(
1983                                                     id_priv->cma_dev->device)];
1984                         cma_iboe_set_mgid((struct sockaddr *)&mc->addr, &mgid,
1985                                           gid_type);
1986                         cma_igmp_send(ndev, &mgid, false);
1987                 }
1988                 dev_put(ndev);
1989
1990                 cancel_work_sync(&mc->iboe_join.work);
1991         }
1992         kfree(mc);
1993 }
1994
1995 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
1996 {
1997         struct cma_multicast *mc;
1998
1999         while (!list_empty(&id_priv->mc_list)) {
2000                 mc = list_first_entry(&id_priv->mc_list, struct cma_multicast,
2001                                       list);
2002                 list_del(&mc->list);
2003                 destroy_mc(id_priv, mc);
2004         }
2005 }
2006
2007 static void _destroy_id(struct rdma_id_private *id_priv,
2008                         enum rdma_cm_state state)
2009 {
2010         cma_cancel_operation(id_priv, state);
2011
2012         rdma_restrack_del(&id_priv->res);
2013         cma_remove_id_from_tree(id_priv);
2014         if (id_priv->cma_dev) {
2015                 if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
2016                         if (id_priv->cm_id.ib)
2017                                 ib_destroy_cm_id(id_priv->cm_id.ib);
2018                 } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) {
2019                         if (id_priv->cm_id.iw)
2020                                 iw_destroy_cm_id(id_priv->cm_id.iw);
2021                 }
2022                 cma_leave_mc_groups(id_priv);
2023                 cma_release_dev(id_priv);
2024         }
2025
2026         cma_release_port(id_priv);
2027         cma_id_put(id_priv);
2028         wait_for_completion(&id_priv->comp);
2029
2030         if (id_priv->internal_id)
2031                 cma_id_put(id_priv->id.context);
2032
2033         kfree(id_priv->id.route.path_rec);
2034         kfree(id_priv->id.route.path_rec_inbound);
2035         kfree(id_priv->id.route.path_rec_outbound);
2036
2037         put_net(id_priv->id.route.addr.dev_addr.net);
2038         kfree(id_priv);
2039 }
2040
2041 /*
2042  * destroy an ID from within the handler_mutex. This ensures that no other
2043  * handlers can start running concurrently.
2044  */
2045 static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
2046         __releases(&idprv->handler_mutex)
2047 {
2048         enum rdma_cm_state state;
2049         unsigned long flags;
2050
2051         trace_cm_id_destroy(id_priv);
2052
2053         /*
2054          * Setting the state to destroyed under the handler mutex provides a
2055          * fence against calling handler callbacks. If this is invoked due to
2056          * the failure of a handler callback then it guarentees that no future
2057          * handlers will be called.
2058          */
2059         lockdep_assert_held(&id_priv->handler_mutex);
2060         spin_lock_irqsave(&id_priv->lock, flags);
2061         state = id_priv->state;
2062         id_priv->state = RDMA_CM_DESTROYING;
2063         spin_unlock_irqrestore(&id_priv->lock, flags);
2064         mutex_unlock(&id_priv->handler_mutex);
2065         _destroy_id(id_priv, state);
2066 }
2067
2068 void rdma_destroy_id(struct rdma_cm_id *id)
2069 {
2070         struct rdma_id_private *id_priv =
2071                 container_of(id, struct rdma_id_private, id);
2072
2073         mutex_lock(&id_priv->handler_mutex);
2074         destroy_id_handler_unlock(id_priv);
2075 }
2076 EXPORT_SYMBOL(rdma_destroy_id);
2077
2078 static int cma_rep_recv(struct rdma_id_private *id_priv)
2079 {
2080         int ret;
2081
2082         ret = cma_modify_qp_rtr(id_priv, NULL);
2083         if (ret)
2084                 goto reject;
2085
2086         ret = cma_modify_qp_rts(id_priv, NULL);
2087         if (ret)
2088                 goto reject;
2089
2090         trace_cm_send_rtu(id_priv);
2091         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
2092         if (ret)
2093                 goto reject;
2094
2095         return 0;
2096 reject:
2097         pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret);
2098         cma_modify_qp_err(id_priv);
2099         trace_cm_send_rej(id_priv);
2100         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
2101                        NULL, 0, NULL, 0);
2102         return ret;
2103 }
2104
2105 static void cma_set_rep_event_data(struct rdma_cm_event *event,
2106                                    const struct ib_cm_rep_event_param *rep_data,
2107                                    void *private_data)
2108 {
2109         event->param.conn.private_data = private_data;
2110         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
2111         event->param.conn.responder_resources = rep_data->responder_resources;
2112         event->param.conn.initiator_depth = rep_data->initiator_depth;
2113         event->param.conn.flow_control = rep_data->flow_control;
2114         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
2115         event->param.conn.srq = rep_data->srq;
2116         event->param.conn.qp_num = rep_data->remote_qpn;
2117
2118         event->ece.vendor_id = rep_data->ece.vendor_id;
2119         event->ece.attr_mod = rep_data->ece.attr_mod;
2120 }
2121
2122 static int cma_cm_event_handler(struct rdma_id_private *id_priv,
2123                                 struct rdma_cm_event *event)
2124 {
2125         int ret;
2126
2127         lockdep_assert_held(&id_priv->handler_mutex);
2128
2129         trace_cm_event_handler(id_priv, event);
2130         ret = id_priv->id.event_handler(&id_priv->id, event);
2131         trace_cm_event_done(id_priv, event, ret);
2132         return ret;
2133 }
2134
2135 static int cma_ib_handler(struct ib_cm_id *cm_id,
2136                           const struct ib_cm_event *ib_event)
2137 {
2138         struct rdma_id_private *id_priv = cm_id->context;
2139         struct rdma_cm_event event = {};
2140         enum rdma_cm_state state;
2141         int ret;
2142
2143         mutex_lock(&id_priv->handler_mutex);
2144         state = READ_ONCE(id_priv->state);
2145         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
2146              state != RDMA_CM_CONNECT) ||
2147             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
2148              state != RDMA_CM_DISCONNECT))
2149                 goto out;
2150
2151         switch (ib_event->event) {
2152         case IB_CM_REQ_ERROR:
2153         case IB_CM_REP_ERROR:
2154                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2155                 event.status = -ETIMEDOUT;
2156                 break;
2157         case IB_CM_REP_RECEIVED:
2158                 if (state == RDMA_CM_CONNECT &&
2159                     (id_priv->id.qp_type != IB_QPT_UD)) {
2160                         trace_cm_send_mra(id_priv);
2161                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2162                 }
2163                 if (id_priv->id.qp) {
2164                         event.status = cma_rep_recv(id_priv);
2165                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
2166                                                      RDMA_CM_EVENT_ESTABLISHED;
2167                 } else {
2168                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
2169                 }
2170                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
2171                                        ib_event->private_data);
2172                 break;
2173         case IB_CM_RTU_RECEIVED:
2174         case IB_CM_USER_ESTABLISHED:
2175                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2176                 break;
2177         case IB_CM_DREQ_ERROR:
2178                 event.status = -ETIMEDOUT;
2179                 fallthrough;
2180         case IB_CM_DREQ_RECEIVED:
2181         case IB_CM_DREP_RECEIVED:
2182                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
2183                                    RDMA_CM_DISCONNECT))
2184                         goto out;
2185                 event.event = RDMA_CM_EVENT_DISCONNECTED;
2186                 break;
2187         case IB_CM_TIMEWAIT_EXIT:
2188                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
2189                 break;
2190         case IB_CM_MRA_RECEIVED:
2191                 /* ignore event */
2192                 goto out;
2193         case IB_CM_REJ_RECEIVED:
2194                 pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id,
2195                                                                                 ib_event->param.rej_rcvd.reason));
2196                 cma_modify_qp_err(id_priv);
2197                 event.status = ib_event->param.rej_rcvd.reason;
2198                 event.event = RDMA_CM_EVENT_REJECTED;
2199                 event.param.conn.private_data = ib_event->private_data;
2200                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
2201                 break;
2202         default:
2203                 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
2204                        ib_event->event);
2205                 goto out;
2206         }
2207
2208         ret = cma_cm_event_handler(id_priv, &event);
2209         if (ret) {
2210                 /* Destroy the CM ID by returning a non-zero value. */
2211                 id_priv->cm_id.ib = NULL;
2212                 destroy_id_handler_unlock(id_priv);
2213                 return ret;
2214         }
2215 out:
2216         mutex_unlock(&id_priv->handler_mutex);
2217         return 0;
2218 }
2219
2220 static struct rdma_id_private *
2221 cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,
2222                    const struct ib_cm_event *ib_event,
2223                    struct net_device *net_dev)
2224 {
2225         struct rdma_id_private *listen_id_priv;
2226         struct rdma_id_private *id_priv;
2227         struct rdma_cm_id *id;
2228         struct rdma_route *rt;
2229         const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2230         struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path;
2231         const __be64 service_id =
2232                 ib_event->param.req_rcvd.primary_path->service_id;
2233         int ret;
2234
2235         listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2236         id_priv = __rdma_create_id(listen_id->route.addr.dev_addr.net,
2237                                    listen_id->event_handler, listen_id->context,
2238                                    listen_id->ps,
2239                                    ib_event->param.req_rcvd.qp_type,
2240                                    listen_id_priv);
2241         if (IS_ERR(id_priv))
2242                 return NULL;
2243
2244         id = &id_priv->id;
2245         if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2246                               (struct sockaddr *)&id->route.addr.dst_addr,
2247                               listen_id, ib_event, ss_family, service_id))
2248                 goto err;
2249
2250         rt = &id->route;
2251         rt->num_pri_alt_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
2252         rt->path_rec = kmalloc_array(rt->num_pri_alt_paths,
2253                                      sizeof(*rt->path_rec), GFP_KERNEL);
2254         if (!rt->path_rec)
2255                 goto err;
2256
2257         rt->path_rec[0] = *path;
2258         if (rt->num_pri_alt_paths == 2)
2259                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
2260
2261         if (net_dev) {
2262                 rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev);
2263         } else {
2264                 if (!cma_protocol_roce(listen_id) &&
2265                     cma_any_addr(cma_src_addr(id_priv))) {
2266                         rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
2267                         rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
2268                         ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
2269                 } else if (!cma_any_addr(cma_src_addr(id_priv))) {
2270                         ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
2271                         if (ret)
2272                                 goto err;
2273                 }
2274         }
2275         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
2276
2277         id_priv->state = RDMA_CM_CONNECT;
2278         return id_priv;
2279
2280 err:
2281         rdma_destroy_id(id);
2282         return NULL;
2283 }
2284
2285 static struct rdma_id_private *
2286 cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,
2287                   const struct ib_cm_event *ib_event,
2288                   struct net_device *net_dev)
2289 {
2290         const struct rdma_id_private *listen_id_priv;
2291         struct rdma_id_private *id_priv;
2292         struct rdma_cm_id *id;
2293         const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2294         struct net *net = listen_id->route.addr.dev_addr.net;
2295         int ret;
2296
2297         listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2298         id_priv = __rdma_create_id(net, listen_id->event_handler,
2299                                    listen_id->context, listen_id->ps, IB_QPT_UD,
2300                                    listen_id_priv);
2301         if (IS_ERR(id_priv))
2302                 return NULL;
2303
2304         id = &id_priv->id;
2305         if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2306                               (struct sockaddr *)&id->route.addr.dst_addr,
2307                               listen_id, ib_event, ss_family,
2308                               ib_event->param.sidr_req_rcvd.service_id))
2309                 goto err;
2310
2311         if (net_dev) {
2312                 rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev);
2313         } else {
2314                 if (!cma_any_addr(cma_src_addr(id_priv))) {
2315                         ret = cma_translate_addr(cma_src_addr(id_priv),
2316                                                  &id->route.addr.dev_addr);
2317                         if (ret)
2318                                 goto err;
2319                 }
2320         }
2321
2322         id_priv->state = RDMA_CM_CONNECT;
2323         return id_priv;
2324 err:
2325         rdma_destroy_id(id);
2326         return NULL;
2327 }
2328
2329 static void cma_set_req_event_data(struct rdma_cm_event *event,
2330                                    const struct ib_cm_req_event_param *req_data,
2331                                    void *private_data, int offset)
2332 {
2333         event->param.conn.private_data = private_data + offset;
2334         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
2335         event->param.conn.responder_resources = req_data->responder_resources;
2336         event->param.conn.initiator_depth = req_data->initiator_depth;
2337         event->param.conn.flow_control = req_data->flow_control;
2338         event->param.conn.retry_count = req_data->retry_count;
2339         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
2340         event->param.conn.srq = req_data->srq;
2341         event->param.conn.qp_num = req_data->remote_qpn;
2342
2343         event->ece.vendor_id = req_data->ece.vendor_id;
2344         event->ece.attr_mod = req_data->ece.attr_mod;
2345 }
2346
2347 static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,
2348                                     const struct ib_cm_event *ib_event)
2349 {
2350         return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
2351                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
2352                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
2353                  (id->qp_type == IB_QPT_UD)) ||
2354                 (!id->qp_type));
2355 }
2356
2357 static int cma_ib_req_handler(struct ib_cm_id *cm_id,
2358                               const struct ib_cm_event *ib_event)
2359 {
2360         struct rdma_id_private *listen_id, *conn_id = NULL;
2361         struct rdma_cm_event event = {};
2362         struct cma_req_info req = {};
2363         struct net_device *net_dev;
2364         u8 offset;
2365         int ret;
2366
2367         listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev);
2368         if (IS_ERR(listen_id))
2369                 return PTR_ERR(listen_id);
2370
2371         trace_cm_req_handler(listen_id, ib_event->event);
2372         if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) {
2373                 ret = -EINVAL;
2374                 goto net_dev_put;
2375         }
2376
2377         mutex_lock(&listen_id->handler_mutex);
2378         if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) {
2379                 ret = -ECONNABORTED;
2380                 goto err_unlock;
2381         }
2382
2383         offset = cma_user_data_offset(listen_id);
2384         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2385         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
2386                 conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev);
2387                 event.param.ud.private_data = ib_event->private_data + offset;
2388                 event.param.ud.private_data_len =
2389                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
2390         } else {
2391                 conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev);
2392                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
2393                                        ib_event->private_data, offset);
2394         }
2395         if (!conn_id) {
2396                 ret = -ENOMEM;
2397                 goto err_unlock;
2398         }
2399
2400         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2401         ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
2402         if (ret) {
2403                 destroy_id_handler_unlock(conn_id);
2404                 goto err_unlock;
2405         }
2406
2407         conn_id->cm_id.ib = cm_id;
2408         cm_id->context = conn_id;
2409         cm_id->cm_handler = cma_ib_handler;
2410
2411         ret = cma_cm_event_handler(conn_id, &event);
2412         if (ret) {
2413                 /* Destroy the CM ID by returning a non-zero value. */
2414                 conn_id->cm_id.ib = NULL;
2415                 mutex_unlock(&listen_id->handler_mutex);
2416                 destroy_id_handler_unlock(conn_id);
2417                 goto net_dev_put;
2418         }
2419
2420         if (READ_ONCE(conn_id->state) == RDMA_CM_CONNECT &&
2421             conn_id->id.qp_type != IB_QPT_UD) {
2422                 trace_cm_send_mra(cm_id->context);
2423                 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2424         }
2425         mutex_unlock(&conn_id->handler_mutex);
2426
2427 err_unlock:
2428         mutex_unlock(&listen_id->handler_mutex);
2429
2430 net_dev_put:
2431         dev_put(net_dev);
2432
2433         return ret;
2434 }
2435
2436 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr)
2437 {
2438         if (addr->sa_family == AF_IB)
2439                 return ((struct sockaddr_ib *) addr)->sib_sid;
2440
2441         return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr)));
2442 }
2443 EXPORT_SYMBOL(rdma_get_service_id);
2444
2445 void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid,
2446                     union ib_gid *dgid)
2447 {
2448         struct rdma_addr *addr = &cm_id->route.addr;
2449
2450         if (!cm_id->device) {
2451                 if (sgid)
2452                         memset(sgid, 0, sizeof(*sgid));
2453                 if (dgid)
2454                         memset(dgid, 0, sizeof(*dgid));
2455                 return;
2456         }
2457
2458         if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) {
2459                 if (sgid)
2460                         rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid);
2461                 if (dgid)
2462                         rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid);
2463         } else {
2464                 if (sgid)
2465                         rdma_addr_get_sgid(&addr->dev_addr, sgid);
2466                 if (dgid)
2467                         rdma_addr_get_dgid(&addr->dev_addr, dgid);
2468         }
2469 }
2470 EXPORT_SYMBOL(rdma_read_gids);
2471
2472 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
2473 {
2474         struct rdma_id_private *id_priv = iw_id->context;
2475         struct rdma_cm_event event = {};
2476         int ret = 0;
2477         struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2478         struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2479
2480         mutex_lock(&id_priv->handler_mutex);
2481         if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
2482                 goto out;
2483
2484         switch (iw_event->event) {
2485         case IW_CM_EVENT_CLOSE:
2486                 event.event = RDMA_CM_EVENT_DISCONNECTED;
2487                 break;
2488         case IW_CM_EVENT_CONNECT_REPLY:
2489                 memcpy(cma_src_addr(id_priv), laddr,
2490                        rdma_addr_size(laddr));
2491                 memcpy(cma_dst_addr(id_priv), raddr,
2492                        rdma_addr_size(raddr));
2493                 switch (iw_event->status) {
2494                 case 0:
2495                         event.event = RDMA_CM_EVENT_ESTABLISHED;
2496                         event.param.conn.initiator_depth = iw_event->ird;
2497                         event.param.conn.responder_resources = iw_event->ord;
2498                         break;
2499                 case -ECONNRESET:
2500                 case -ECONNREFUSED:
2501                         event.event = RDMA_CM_EVENT_REJECTED;
2502                         break;
2503                 case -ETIMEDOUT:
2504                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2505                         break;
2506                 default:
2507                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
2508                         break;
2509                 }
2510                 break;
2511         case IW_CM_EVENT_ESTABLISHED:
2512                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2513                 event.param.conn.initiator_depth = iw_event->ird;
2514                 event.param.conn.responder_resources = iw_event->ord;
2515                 break;
2516         default:
2517                 goto out;
2518         }
2519
2520         event.status = iw_event->status;
2521         event.param.conn.private_data = iw_event->private_data;
2522         event.param.conn.private_data_len = iw_event->private_data_len;
2523         ret = cma_cm_event_handler(id_priv, &event);
2524         if (ret) {
2525                 /* Destroy the CM ID by returning a non-zero value. */
2526                 id_priv->cm_id.iw = NULL;
2527                 destroy_id_handler_unlock(id_priv);
2528                 return ret;
2529         }
2530
2531 out:
2532         mutex_unlock(&id_priv->handler_mutex);
2533         return ret;
2534 }
2535
2536 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
2537                                struct iw_cm_event *iw_event)
2538 {
2539         struct rdma_id_private *listen_id, *conn_id;
2540         struct rdma_cm_event event = {};
2541         int ret = -ECONNABORTED;
2542         struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2543         struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2544
2545         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2546         event.param.conn.private_data = iw_event->private_data;
2547         event.param.conn.private_data_len = iw_event->private_data_len;
2548         event.param.conn.initiator_depth = iw_event->ird;
2549         event.param.conn.responder_resources = iw_event->ord;
2550
2551         listen_id = cm_id->context;
2552
2553         mutex_lock(&listen_id->handler_mutex);
2554         if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN)
2555                 goto out;
2556
2557         /* Create a new RDMA id for the new IW CM ID */
2558         conn_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
2559                                    listen_id->id.event_handler,
2560                                    listen_id->id.context, RDMA_PS_TCP,
2561                                    IB_QPT_RC, listen_id);
2562         if (IS_ERR(conn_id)) {
2563                 ret = -ENOMEM;
2564                 goto out;
2565         }
2566         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2567         conn_id->state = RDMA_CM_CONNECT;
2568
2569         ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
2570         if (ret) {
2571                 mutex_unlock(&listen_id->handler_mutex);
2572                 destroy_id_handler_unlock(conn_id);
2573                 return ret;
2574         }
2575
2576         ret = cma_iw_acquire_dev(conn_id, listen_id);
2577         if (ret) {
2578                 mutex_unlock(&listen_id->handler_mutex);
2579                 destroy_id_handler_unlock(conn_id);
2580                 return ret;
2581         }
2582
2583         conn_id->cm_id.iw = cm_id;
2584         cm_id->context = conn_id;
2585         cm_id->cm_handler = cma_iw_handler;
2586
2587         memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
2588         memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
2589
2590         ret = cma_cm_event_handler(conn_id, &event);
2591         if (ret) {
2592                 /* User wants to destroy the CM ID */
2593                 conn_id->cm_id.iw = NULL;
2594                 mutex_unlock(&listen_id->handler_mutex);
2595                 destroy_id_handler_unlock(conn_id);
2596                 return ret;
2597         }
2598
2599         mutex_unlock(&conn_id->handler_mutex);
2600
2601 out:
2602         mutex_unlock(&listen_id->handler_mutex);
2603         return ret;
2604 }
2605
2606 static int cma_ib_listen(struct rdma_id_private *id_priv)
2607 {
2608         struct sockaddr *addr;
2609         struct ib_cm_id *id;
2610         __be64 svc_id;
2611
2612         addr = cma_src_addr(id_priv);
2613         svc_id = rdma_get_service_id(&id_priv->id, addr);
2614         id = ib_cm_insert_listen(id_priv->id.device,
2615                                  cma_ib_req_handler, svc_id);
2616         if (IS_ERR(id))
2617                 return PTR_ERR(id);
2618         id_priv->cm_id.ib = id;
2619
2620         return 0;
2621 }
2622
2623 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
2624 {
2625         int ret;
2626         struct iw_cm_id *id;
2627
2628         id = iw_create_cm_id(id_priv->id.device,
2629                              iw_conn_req_handler,
2630                              id_priv);
2631         if (IS_ERR(id))
2632                 return PTR_ERR(id);
2633
2634         mutex_lock(&id_priv->qp_mutex);
2635         id->tos = id_priv->tos;
2636         id->tos_set = id_priv->tos_set;
2637         mutex_unlock(&id_priv->qp_mutex);
2638         id->afonly = id_priv->afonly;
2639         id_priv->cm_id.iw = id;
2640
2641         memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv),
2642                rdma_addr_size(cma_src_addr(id_priv)));
2643
2644         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
2645
2646         if (ret) {
2647                 iw_destroy_cm_id(id_priv->cm_id.iw);
2648                 id_priv->cm_id.iw = NULL;
2649         }
2650
2651         return ret;
2652 }
2653
2654 static int cma_listen_handler(struct rdma_cm_id *id,
2655                               struct rdma_cm_event *event)
2656 {
2657         struct rdma_id_private *id_priv = id->context;
2658
2659         /* Listening IDs are always destroyed on removal */
2660         if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
2661                 return -1;
2662
2663         id->context = id_priv->id.context;
2664         id->event_handler = id_priv->id.event_handler;
2665         trace_cm_event_handler(id_priv, event);
2666         return id_priv->id.event_handler(id, event);
2667 }
2668
2669 static int cma_listen_on_dev(struct rdma_id_private *id_priv,
2670                              struct cma_device *cma_dev,
2671                              struct rdma_id_private **to_destroy)
2672 {
2673         struct rdma_id_private *dev_id_priv;
2674         struct net *net = id_priv->id.route.addr.dev_addr.net;
2675         int ret;
2676
2677         lockdep_assert_held(&lock);
2678
2679         *to_destroy = NULL;
2680         if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
2681                 return 0;
2682
2683         dev_id_priv =
2684                 __rdma_create_id(net, cma_listen_handler, id_priv,
2685                                  id_priv->id.ps, id_priv->id.qp_type, id_priv);
2686         if (IS_ERR(dev_id_priv))
2687                 return PTR_ERR(dev_id_priv);
2688
2689         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
2690         memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
2691                rdma_addr_size(cma_src_addr(id_priv)));
2692
2693         _cma_attach_to_dev(dev_id_priv, cma_dev);
2694         rdma_restrack_add(&dev_id_priv->res);
2695         cma_id_get(id_priv);
2696         dev_id_priv->internal_id = 1;
2697         dev_id_priv->afonly = id_priv->afonly;
2698         mutex_lock(&id_priv->qp_mutex);
2699         dev_id_priv->tos_set = id_priv->tos_set;
2700         dev_id_priv->tos = id_priv->tos;
2701         mutex_unlock(&id_priv->qp_mutex);
2702
2703         ret = rdma_listen(&dev_id_priv->id, id_priv->backlog);
2704         if (ret)
2705                 goto err_listen;
2706         list_add_tail(&dev_id_priv->listen_item, &id_priv->listen_list);
2707         return 0;
2708 err_listen:
2709         /* Caller must destroy this after releasing lock */
2710         *to_destroy = dev_id_priv;
2711         dev_warn(&cma_dev->device->dev, "RDMA CMA: %s, error %d\n", __func__, ret);
2712         return ret;
2713 }
2714
2715 static int cma_listen_on_all(struct rdma_id_private *id_priv)
2716 {
2717         struct rdma_id_private *to_destroy;
2718         struct cma_device *cma_dev;
2719         int ret;
2720
2721         mutex_lock(&lock);
2722         list_add_tail(&id_priv->listen_any_item, &listen_any_list);
2723         list_for_each_entry(cma_dev, &dev_list, list) {
2724                 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
2725                 if (ret) {
2726                         /* Prevent racing with cma_process_remove() */
2727                         if (to_destroy)
2728                                 list_del_init(&to_destroy->device_item);
2729                         goto err_listen;
2730                 }
2731         }
2732         mutex_unlock(&lock);
2733         return 0;
2734
2735 err_listen:
2736         _cma_cancel_listens(id_priv);
2737         mutex_unlock(&lock);
2738         if (to_destroy)
2739                 rdma_destroy_id(&to_destroy->id);
2740         return ret;
2741 }
2742
2743 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
2744 {
2745         struct rdma_id_private *id_priv;
2746
2747         id_priv = container_of(id, struct rdma_id_private, id);
2748         mutex_lock(&id_priv->qp_mutex);
2749         id_priv->tos = (u8) tos;
2750         id_priv->tos_set = true;
2751         mutex_unlock(&id_priv->qp_mutex);
2752 }
2753 EXPORT_SYMBOL(rdma_set_service_type);
2754
2755 /**
2756  * rdma_set_ack_timeout() - Set the ack timeout of QP associated
2757  *                          with a connection identifier.
2758  * @id: Communication identifier to associated with service type.
2759  * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec.
2760  *
2761  * This function should be called before rdma_connect() on active side,
2762  * and on passive side before rdma_accept(). It is applicable to primary
2763  * path only. The timeout will affect the local side of the QP, it is not
2764  * negotiated with remote side and zero disables the timer. In case it is
2765  * set before rdma_resolve_route, the value will also be used to determine
2766  * PacketLifeTime for RoCE.
2767  *
2768  * Return: 0 for success
2769  */
2770 int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
2771 {
2772         struct rdma_id_private *id_priv;
2773
2774         if (id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_INI)
2775                 return -EINVAL;
2776
2777         id_priv = container_of(id, struct rdma_id_private, id);
2778         mutex_lock(&id_priv->qp_mutex);
2779         id_priv->timeout = timeout;
2780         id_priv->timeout_set = true;
2781         mutex_unlock(&id_priv->qp_mutex);
2782
2783         return 0;
2784 }
2785 EXPORT_SYMBOL(rdma_set_ack_timeout);
2786
2787 /**
2788  * rdma_set_min_rnr_timer() - Set the minimum RNR Retry timer of the
2789  *                            QP associated with a connection identifier.
2790  * @id: Communication identifier to associated with service type.
2791  * @min_rnr_timer: 5-bit value encoded as Table 45: "Encoding for RNR NAK
2792  *                 Timer Field" in the IBTA specification.
2793  *
2794  * This function should be called before rdma_connect() on active
2795  * side, and on passive side before rdma_accept(). The timer value
2796  * will be associated with the local QP. When it receives a send it is
2797  * not read to handle, typically if the receive queue is empty, an RNR
2798  * Retry NAK is returned to the requester with the min_rnr_timer
2799  * encoded. The requester will then wait at least the time specified
2800  * in the NAK before retrying. The default is zero, which translates
2801  * to a minimum RNR Timer value of 655 ms.
2802  *
2803  * Return: 0 for success
2804  */
2805 int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer)
2806 {
2807         struct rdma_id_private *id_priv;
2808
2809         /* It is a five-bit value */
2810         if (min_rnr_timer & 0xe0)
2811                 return -EINVAL;
2812
2813         if (WARN_ON(id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_TGT))
2814                 return -EINVAL;
2815
2816         id_priv = container_of(id, struct rdma_id_private, id);
2817         mutex_lock(&id_priv->qp_mutex);
2818         id_priv->min_rnr_timer = min_rnr_timer;
2819         id_priv->min_rnr_timer_set = true;
2820         mutex_unlock(&id_priv->qp_mutex);
2821
2822         return 0;
2823 }
2824 EXPORT_SYMBOL(rdma_set_min_rnr_timer);
2825
2826 static int route_set_path_rec_inbound(struct cma_work *work,
2827                                       struct sa_path_rec *path_rec)
2828 {
2829         struct rdma_route *route = &work->id->id.route;
2830
2831         if (!route->path_rec_inbound) {
2832                 route->path_rec_inbound =
2833                         kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL);
2834                 if (!route->path_rec_inbound)
2835                         return -ENOMEM;
2836         }
2837
2838         *route->path_rec_inbound = *path_rec;
2839         return 0;
2840 }
2841
2842 static int route_set_path_rec_outbound(struct cma_work *work,
2843                                        struct sa_path_rec *path_rec)
2844 {
2845         struct rdma_route *route = &work->id->id.route;
2846
2847         if (!route->path_rec_outbound) {
2848                 route->path_rec_outbound =
2849                         kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL);
2850                 if (!route->path_rec_outbound)
2851                         return -ENOMEM;
2852         }
2853
2854         *route->path_rec_outbound = *path_rec;
2855         return 0;
2856 }
2857
2858 static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2859                               unsigned int num_prs, void *context)
2860 {
2861         struct cma_work *work = context;
2862         struct rdma_route *route;
2863         int i;
2864
2865         route = &work->id->id.route;
2866
2867         if (status)
2868                 goto fail;
2869
2870         for (i = 0; i < num_prs; i++) {
2871                 if (!path_rec[i].flags || (path_rec[i].flags & IB_PATH_GMP))
2872                         *route->path_rec = path_rec[i];
2873                 else if (path_rec[i].flags & IB_PATH_INBOUND)
2874                         status = route_set_path_rec_inbound(work, &path_rec[i]);
2875                 else if (path_rec[i].flags & IB_PATH_OUTBOUND)
2876                         status = route_set_path_rec_outbound(work,
2877                                                              &path_rec[i]);
2878                 else
2879                         status = -EINVAL;
2880
2881                 if (status)
2882                         goto fail;
2883         }
2884
2885         route->num_pri_alt_paths = 1;
2886         queue_work(cma_wq, &work->work);
2887         return;
2888
2889 fail:
2890         work->old_state = RDMA_CM_ROUTE_QUERY;
2891         work->new_state = RDMA_CM_ADDR_RESOLVED;
2892         work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
2893         work->event.status = status;
2894         pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",
2895                              status);
2896         queue_work(cma_wq, &work->work);
2897 }
2898
2899 static int cma_query_ib_route(struct rdma_id_private *id_priv,
2900                               unsigned long timeout_ms, struct cma_work *work)
2901 {
2902         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2903         struct sa_path_rec path_rec;
2904         ib_sa_comp_mask comp_mask;
2905         struct sockaddr_in6 *sin6;
2906         struct sockaddr_ib *sib;
2907
2908         memset(&path_rec, 0, sizeof path_rec);
2909
2910         if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num))
2911                 path_rec.rec_type = SA_PATH_REC_TYPE_OPA;
2912         else
2913                 path_rec.rec_type = SA_PATH_REC_TYPE_IB;
2914         rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
2915         rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
2916         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2917         path_rec.numb_path = 1;
2918         path_rec.reversible = 1;
2919         path_rec.service_id = rdma_get_service_id(&id_priv->id,
2920                                                   cma_dst_addr(id_priv));
2921
2922         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
2923                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
2924                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
2925
2926         switch (cma_family(id_priv)) {
2927         case AF_INET:
2928                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
2929                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
2930                 break;
2931         case AF_INET6:
2932                 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2933                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
2934                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2935                 break;
2936         case AF_IB:
2937                 sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2938                 path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20);
2939                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2940                 break;
2941         }
2942
2943         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
2944                                                id_priv->id.port_num, &path_rec,
2945                                                comp_mask, timeout_ms,
2946                                                GFP_KERNEL, cma_query_handler,
2947                                                work, &id_priv->query);
2948
2949         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
2950 }
2951
2952 static void cma_iboe_join_work_handler(struct work_struct *work)
2953 {
2954         struct cma_multicast *mc =
2955                 container_of(work, struct cma_multicast, iboe_join.work);
2956         struct rdma_cm_event *event = &mc->iboe_join.event;
2957         struct rdma_id_private *id_priv = mc->id_priv;
2958         int ret;
2959
2960         mutex_lock(&id_priv->handler_mutex);
2961         if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
2962             READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
2963                 goto out_unlock;
2964
2965         ret = cma_cm_event_handler(id_priv, event);
2966         WARN_ON(ret);
2967
2968 out_unlock:
2969         mutex_unlock(&id_priv->handler_mutex);
2970         if (event->event == RDMA_CM_EVENT_MULTICAST_JOIN)
2971                 rdma_destroy_ah_attr(&event->param.ud.ah_attr);
2972 }
2973
2974 static void cma_work_handler(struct work_struct *_work)
2975 {
2976         struct cma_work *work = container_of(_work, struct cma_work, work);
2977         struct rdma_id_private *id_priv = work->id;
2978
2979         mutex_lock(&id_priv->handler_mutex);
2980         if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
2981             READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
2982                 goto out_unlock;
2983         if (work->old_state != 0 || work->new_state != 0) {
2984                 if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
2985                         goto out_unlock;
2986         }
2987
2988         if (cma_cm_event_handler(id_priv, &work->event)) {
2989                 cma_id_put(id_priv);
2990                 destroy_id_handler_unlock(id_priv);
2991                 goto out_free;
2992         }
2993
2994 out_unlock:
2995         mutex_unlock(&id_priv->handler_mutex);
2996         cma_id_put(id_priv);
2997 out_free:
2998         if (work->event.event == RDMA_CM_EVENT_MULTICAST_JOIN)
2999                 rdma_destroy_ah_attr(&work->event.param.ud.ah_attr);
3000         kfree(work);
3001 }
3002
3003 static void cma_init_resolve_route_work(struct cma_work *work,
3004                                         struct rdma_id_private *id_priv)
3005 {
3006         work->id = id_priv;
3007         INIT_WORK(&work->work, cma_work_handler);
3008         work->old_state = RDMA_CM_ROUTE_QUERY;
3009         work->new_state = RDMA_CM_ROUTE_RESOLVED;
3010         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
3011 }
3012
3013 static void enqueue_resolve_addr_work(struct cma_work *work,
3014                                       struct rdma_id_private *id_priv)
3015 {
3016         /* Balances with cma_id_put() in cma_work_handler */
3017         cma_id_get(id_priv);
3018
3019         work->id = id_priv;
3020         INIT_WORK(&work->work, cma_work_handler);
3021         work->old_state = RDMA_CM_ADDR_QUERY;
3022         work->new_state = RDMA_CM_ADDR_RESOLVED;
3023         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3024
3025         queue_work(cma_wq, &work->work);
3026 }
3027
3028 static int cma_resolve_ib_route(struct rdma_id_private *id_priv,
3029                                 unsigned long timeout_ms)
3030 {
3031         struct rdma_route *route = &id_priv->id.route;
3032         struct cma_work *work;
3033         int ret;
3034
3035         work = kzalloc(sizeof *work, GFP_KERNEL);
3036         if (!work)
3037                 return -ENOMEM;
3038
3039         cma_init_resolve_route_work(work, id_priv);
3040
3041         if (!route->path_rec)
3042                 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
3043         if (!route->path_rec) {
3044                 ret = -ENOMEM;
3045                 goto err1;
3046         }
3047
3048         ret = cma_query_ib_route(id_priv, timeout_ms, work);
3049         if (ret)
3050                 goto err2;
3051
3052         return 0;
3053 err2:
3054         kfree(route->path_rec);
3055         route->path_rec = NULL;
3056 err1:
3057         kfree(work);
3058         return ret;
3059 }
3060
3061 static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
3062                                            unsigned long supported_gids,
3063                                            enum ib_gid_type default_gid)
3064 {
3065         if ((network_type == RDMA_NETWORK_IPV4 ||
3066              network_type == RDMA_NETWORK_IPV6) &&
3067             test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
3068                 return IB_GID_TYPE_ROCE_UDP_ENCAP;
3069
3070         return default_gid;
3071 }
3072
3073 /*
3074  * cma_iboe_set_path_rec_l2_fields() is helper function which sets
3075  * path record type based on GID type.
3076  * It also sets up other L2 fields which includes destination mac address
3077  * netdev ifindex, of the path record.
3078  * It returns the netdev of the bound interface for this path record entry.
3079  */
3080 static struct net_device *
3081 cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv)
3082 {
3083         struct rdma_route *route = &id_priv->id.route;
3084         enum ib_gid_type gid_type = IB_GID_TYPE_ROCE;
3085         struct rdma_addr *addr = &route->addr;
3086         unsigned long supported_gids;
3087         struct net_device *ndev;
3088
3089         if (!addr->dev_addr.bound_dev_if)
3090                 return NULL;
3091
3092         ndev = dev_get_by_index(addr->dev_addr.net,
3093                                 addr->dev_addr.bound_dev_if);
3094         if (!ndev)
3095                 return NULL;
3096
3097         supported_gids = roce_gid_type_mask_support(id_priv->id.device,
3098                                                     id_priv->id.port_num);
3099         gid_type = cma_route_gid_type(addr->dev_addr.network,
3100                                       supported_gids,
3101                                       id_priv->gid_type);
3102         /* Use the hint from IP Stack to select GID Type */
3103         if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
3104                 gid_type = ib_network_to_gid_type(addr->dev_addr.network);
3105         route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
3106
3107         route->path_rec->roce.route_resolved = true;
3108         sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
3109         return ndev;
3110 }
3111
3112 int rdma_set_ib_path(struct rdma_cm_id *id,
3113                      struct sa_path_rec *path_rec)
3114 {
3115         struct rdma_id_private *id_priv;
3116         struct net_device *ndev;
3117         int ret;
3118
3119         id_priv = container_of(id, struct rdma_id_private, id);
3120         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3121                            RDMA_CM_ROUTE_RESOLVED))
3122                 return -EINVAL;
3123
3124         id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec),
3125                                      GFP_KERNEL);
3126         if (!id->route.path_rec) {
3127                 ret = -ENOMEM;
3128                 goto err;
3129         }
3130
3131         if (rdma_protocol_roce(id->device, id->port_num)) {
3132                 ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
3133                 if (!ndev) {
3134                         ret = -ENODEV;
3135                         goto err_free;
3136                 }
3137                 dev_put(ndev);
3138         }
3139
3140         id->route.num_pri_alt_paths = 1;
3141         return 0;
3142
3143 err_free:
3144         kfree(id->route.path_rec);
3145         id->route.path_rec = NULL;
3146 err:
3147         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
3148         return ret;
3149 }
3150 EXPORT_SYMBOL(rdma_set_ib_path);
3151
3152 static int cma_resolve_iw_route(struct rdma_id_private *id_priv)
3153 {
3154         struct cma_work *work;
3155
3156         work = kzalloc(sizeof *work, GFP_KERNEL);
3157         if (!work)
3158                 return -ENOMEM;
3159
3160         cma_init_resolve_route_work(work, id_priv);
3161         queue_work(cma_wq, &work->work);
3162         return 0;
3163 }
3164
3165 static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio)
3166 {
3167         struct net_device *dev;
3168
3169         dev = vlan_dev_real_dev(vlan_ndev);
3170         if (dev->num_tc)
3171                 return netdev_get_prio_tc_map(dev, prio);
3172
3173         return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) &
3174                 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
3175 }
3176
3177 struct iboe_prio_tc_map {
3178         int input_prio;
3179         int output_tc;
3180         bool found;
3181 };
3182
3183 static int get_lower_vlan_dev_tc(struct net_device *dev,
3184                                  struct netdev_nested_priv *priv)
3185 {
3186         struct iboe_prio_tc_map *map = (struct iboe_prio_tc_map *)priv->data;
3187
3188         if (is_vlan_dev(dev))
3189                 map->output_tc = get_vlan_ndev_tc(dev, map->input_prio);
3190         else if (dev->num_tc)
3191                 map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio);
3192         else
3193                 map->output_tc = 0;
3194         /* We are interested only in first level VLAN device, so always
3195          * return 1 to stop iterating over next level devices.
3196          */
3197         map->found = true;
3198         return 1;
3199 }
3200
3201 static int iboe_tos_to_sl(struct net_device *ndev, int tos)
3202 {
3203         struct iboe_prio_tc_map prio_tc_map = {};
3204         int prio = rt_tos2priority(tos);
3205         struct netdev_nested_priv priv;
3206
3207         /* If VLAN device, get it directly from the VLAN netdev */
3208         if (is_vlan_dev(ndev))
3209                 return get_vlan_ndev_tc(ndev, prio);
3210
3211         prio_tc_map.input_prio = prio;
3212         priv.data = (void *)&prio_tc_map;
3213         rcu_read_lock();
3214         netdev_walk_all_lower_dev_rcu(ndev,
3215                                       get_lower_vlan_dev_tc,
3216                                       &priv);
3217         rcu_read_unlock();
3218         /* If map is found from lower device, use it; Otherwise
3219          * continue with the current netdevice to get priority to tc map.
3220          */
3221         if (prio_tc_map.found)
3222                 return prio_tc_map.output_tc;
3223         else if (ndev->num_tc)
3224                 return netdev_get_prio_tc_map(ndev, prio);
3225         else
3226                 return 0;
3227 }
3228
3229 static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv)
3230 {
3231         struct sockaddr_in6 *addr6;
3232         u16 dport, sport;
3233         u32 hash, fl;
3234
3235         addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv);
3236         fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK;
3237         if ((cma_family(id_priv) != AF_INET6) || !fl) {
3238                 dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv)));
3239                 sport = be16_to_cpu(cma_port(cma_src_addr(id_priv)));
3240                 hash = (u32)sport * 31 + dport;
3241                 fl = hash & IB_GRH_FLOWLABEL_MASK;
3242         }
3243
3244         return cpu_to_be32(fl);
3245 }
3246
3247 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
3248 {
3249         struct rdma_route *route = &id_priv->id.route;
3250         struct rdma_addr *addr = &route->addr;
3251         struct cma_work *work;
3252         int ret;
3253         struct net_device *ndev;
3254
3255         u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -
3256                                         rdma_start_port(id_priv->cma_dev->device)];
3257         u8 tos;
3258
3259         mutex_lock(&id_priv->qp_mutex);
3260         tos = id_priv->tos_set ? id_priv->tos : default_roce_tos;
3261         mutex_unlock(&id_priv->qp_mutex);
3262
3263         work = kzalloc(sizeof *work, GFP_KERNEL);
3264         if (!work)
3265                 return -ENOMEM;
3266
3267         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
3268         if (!route->path_rec) {
3269                 ret = -ENOMEM;
3270                 goto err1;
3271         }
3272
3273         route->num_pri_alt_paths = 1;
3274
3275         ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
3276         if (!ndev) {
3277                 ret = -ENODEV;
3278                 goto err2;
3279         }
3280
3281         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
3282                     &route->path_rec->sgid);
3283         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
3284                     &route->path_rec->dgid);
3285
3286         if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
3287                 /* TODO: get the hoplimit from the inet/inet6 device */
3288                 route->path_rec->hop_limit = addr->dev_addr.hoplimit;
3289         else
3290                 route->path_rec->hop_limit = 1;
3291         route->path_rec->reversible = 1;
3292         route->path_rec->pkey = cpu_to_be16(0xffff);
3293         route->path_rec->mtu_selector = IB_SA_EQ;
3294         route->path_rec->sl = iboe_tos_to_sl(ndev, tos);
3295         route->path_rec->traffic_class = tos;
3296         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
3297         route->path_rec->rate_selector = IB_SA_EQ;
3298         route->path_rec->rate = IB_RATE_PORT_CURRENT;
3299         dev_put(ndev);
3300         route->path_rec->packet_life_time_selector = IB_SA_EQ;
3301         /* In case ACK timeout is set, use this value to calculate
3302          * PacketLifeTime.  As per IBTA 12.7.34,
3303          * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay).
3304          * Assuming a negligible local ACK delay, we can use
3305          * PacketLifeTime = local ACK timeout/2
3306          * as a reasonable approximation for RoCE networks.
3307          */
3308         mutex_lock(&id_priv->qp_mutex);
3309         if (id_priv->timeout_set && id_priv->timeout)
3310                 route->path_rec->packet_life_time = id_priv->timeout - 1;
3311         else
3312                 route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
3313         mutex_unlock(&id_priv->qp_mutex);
3314
3315         if (!route->path_rec->mtu) {
3316                 ret = -EINVAL;
3317                 goto err2;
3318         }
3319
3320         if (rdma_protocol_roce_udp_encap(id_priv->id.device,
3321                                          id_priv->id.port_num))
3322                 route->path_rec->flow_label =
3323                         cma_get_roce_udp_flow_label(id_priv);
3324
3325         cma_init_resolve_route_work(work, id_priv);
3326         queue_work(cma_wq, &work->work);
3327
3328         return 0;
3329
3330 err2:
3331         kfree(route->path_rec);
3332         route->path_rec = NULL;
3333         route->num_pri_alt_paths = 0;
3334 err1:
3335         kfree(work);
3336         return ret;
3337 }
3338
3339 int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
3340 {
3341         struct rdma_id_private *id_priv;
3342         int ret;
3343
3344         if (!timeout_ms)
3345                 return -EINVAL;
3346
3347         id_priv = container_of(id, struct rdma_id_private, id);
3348         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
3349                 return -EINVAL;
3350
3351         cma_id_get(id_priv);
3352         if (rdma_cap_ib_sa(id->device, id->port_num))
3353                 ret = cma_resolve_ib_route(id_priv, timeout_ms);
3354         else if (rdma_protocol_roce(id->device, id->port_num)) {
3355                 ret = cma_resolve_iboe_route(id_priv);
3356                 if (!ret)
3357                         cma_add_id_to_tree(id_priv);
3358         }
3359         else if (rdma_protocol_iwarp(id->device, id->port_num))
3360                 ret = cma_resolve_iw_route(id_priv);
3361         else
3362                 ret = -ENOSYS;
3363
3364         if (ret)
3365                 goto err;
3366
3367         return 0;
3368 err:
3369         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
3370         cma_id_put(id_priv);
3371         return ret;
3372 }
3373 EXPORT_SYMBOL(rdma_resolve_route);
3374
3375 static void cma_set_loopback(struct sockaddr *addr)
3376 {
3377         switch (addr->sa_family) {
3378         case AF_INET:
3379                 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
3380                 break;
3381         case AF_INET6:
3382                 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
3383                               0, 0, 0, htonl(1));
3384                 break;
3385         default:
3386                 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
3387                             0, 0, 0, htonl(1));
3388                 break;
3389         }
3390 }
3391
3392 static int cma_bind_loopback(struct rdma_id_private *id_priv)
3393 {
3394         struct cma_device *cma_dev, *cur_dev;
3395         union ib_gid gid;
3396         enum ib_port_state port_state;
3397         unsigned int p;
3398         u16 pkey;
3399         int ret;
3400
3401         cma_dev = NULL;
3402         mutex_lock(&lock);
3403         list_for_each_entry(cur_dev, &dev_list, list) {
3404                 if (cma_family(id_priv) == AF_IB &&
3405                     !rdma_cap_ib_cm(cur_dev->device, 1))
3406                         continue;
3407
3408                 if (!cma_dev)
3409                         cma_dev = cur_dev;
3410
3411                 rdma_for_each_port (cur_dev->device, p) {
3412                         if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) &&
3413                             port_state == IB_PORT_ACTIVE) {
3414                                 cma_dev = cur_dev;
3415                                 goto port_found;
3416                         }
3417                 }
3418         }
3419
3420         if (!cma_dev) {
3421                 ret = -ENODEV;
3422                 goto out;
3423         }
3424
3425         p = 1;
3426
3427 port_found:
3428         ret = rdma_query_gid(cma_dev->device, p, 0, &gid);
3429         if (ret)
3430                 goto out;
3431
3432         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
3433         if (ret)
3434                 goto out;
3435
3436         id_priv->id.route.addr.dev_addr.dev_type =
3437                 (rdma_protocol_ib(cma_dev->device, p)) ?
3438                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
3439
3440         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3441         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
3442         id_priv->id.port_num = p;
3443         cma_attach_to_dev(id_priv, cma_dev);
3444         rdma_restrack_add(&id_priv->res);
3445         cma_set_loopback(cma_src_addr(id_priv));
3446 out:
3447         mutex_unlock(&lock);
3448         return ret;
3449 }
3450
3451 static void addr_handler(int status, struct sockaddr *src_addr,
3452                          struct rdma_dev_addr *dev_addr, void *context)
3453 {
3454         struct rdma_id_private *id_priv = context;
3455         struct rdma_cm_event event = {};
3456         struct sockaddr *addr;
3457         struct sockaddr_storage old_addr;
3458
3459         mutex_lock(&id_priv->handler_mutex);
3460         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
3461                            RDMA_CM_ADDR_RESOLVED))
3462                 goto out;
3463
3464         /*
3465          * Store the previous src address, so that if we fail to acquire
3466          * matching rdma device, old address can be restored back, which helps
3467          * to cancel the cma listen operation correctly.
3468          */
3469         addr = cma_src_addr(id_priv);
3470         memcpy(&old_addr, addr, rdma_addr_size(addr));
3471         memcpy(addr, src_addr, rdma_addr_size(src_addr));
3472         if (!status && !id_priv->cma_dev) {
3473                 status = cma_acquire_dev_by_src_ip(id_priv);
3474                 if (status)
3475                         pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n",
3476                                              status);
3477                 rdma_restrack_add(&id_priv->res);
3478         } else if (status) {
3479                 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status);
3480         }
3481
3482         if (status) {
3483                 memcpy(addr, &old_addr,
3484                        rdma_addr_size((struct sockaddr *)&old_addr));
3485                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3486                                    RDMA_CM_ADDR_BOUND))
3487                         goto out;
3488                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
3489                 event.status = status;
3490         } else
3491                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3492
3493         if (cma_cm_event_handler(id_priv, &event)) {
3494                 destroy_id_handler_unlock(id_priv);
3495                 return;
3496         }
3497 out:
3498         mutex_unlock(&id_priv->handler_mutex);
3499 }
3500
3501 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
3502 {
3503         struct cma_work *work;
3504         union ib_gid gid;
3505         int ret;
3506
3507         work = kzalloc(sizeof *work, GFP_KERNEL);
3508         if (!work)
3509                 return -ENOMEM;
3510
3511         if (!id_priv->cma_dev) {
3512                 ret = cma_bind_loopback(id_priv);
3513                 if (ret)
3514                         goto err;
3515         }
3516
3517         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3518         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
3519
3520         enqueue_resolve_addr_work(work, id_priv);
3521         return 0;
3522 err:
3523         kfree(work);
3524         return ret;
3525 }
3526
3527 static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
3528 {
3529         struct cma_work *work;
3530         int ret;
3531
3532         work = kzalloc(sizeof *work, GFP_KERNEL);
3533         if (!work)
3534                 return -ENOMEM;
3535
3536         if (!id_priv->cma_dev) {
3537                 ret = cma_resolve_ib_dev(id_priv);
3538                 if (ret)
3539                         goto err;
3540         }
3541
3542         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
3543                 &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
3544
3545         enqueue_resolve_addr_work(work, id_priv);
3546         return 0;
3547 err:
3548         kfree(work);
3549         return ret;
3550 }
3551
3552 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
3553 {
3554         struct rdma_id_private *id_priv;
3555         unsigned long flags;
3556         int ret;
3557
3558         id_priv = container_of(id, struct rdma_id_private, id);
3559         spin_lock_irqsave(&id_priv->lock, flags);
3560         if ((reuse && id_priv->state != RDMA_CM_LISTEN) ||
3561             id_priv->state == RDMA_CM_IDLE) {
3562                 id_priv->reuseaddr = reuse;
3563                 ret = 0;
3564         } else {
3565                 ret = -EINVAL;
3566         }
3567         spin_unlock_irqrestore(&id_priv->lock, flags);
3568         return ret;
3569 }
3570 EXPORT_SYMBOL(rdma_set_reuseaddr);
3571
3572 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
3573 {
3574         struct rdma_id_private *id_priv;
3575         unsigned long flags;
3576         int ret;
3577
3578         id_priv = container_of(id, struct rdma_id_private, id);
3579         spin_lock_irqsave(&id_priv->lock, flags);
3580         if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
3581                 id_priv->options |= (1 << CMA_OPTION_AFONLY);
3582                 id_priv->afonly = afonly;
3583                 ret = 0;
3584         } else {
3585                 ret = -EINVAL;
3586         }
3587         spin_unlock_irqrestore(&id_priv->lock, flags);
3588         return ret;
3589 }
3590 EXPORT_SYMBOL(rdma_set_afonly);
3591
3592 static void cma_bind_port(struct rdma_bind_list *bind_list,
3593                           struct rdma_id_private *id_priv)
3594 {
3595         struct sockaddr *addr;
3596         struct sockaddr_ib *sib;
3597         u64 sid, mask;
3598         __be16 port;
3599
3600         lockdep_assert_held(&lock);
3601
3602         addr = cma_src_addr(id_priv);
3603         port = htons(bind_list->port);
3604
3605         switch (addr->sa_family) {
3606         case AF_INET:
3607                 ((struct sockaddr_in *) addr)->sin_port = port;
3608                 break;
3609         case AF_INET6:
3610                 ((struct sockaddr_in6 *) addr)->sin6_port = port;
3611                 break;
3612         case AF_IB:
3613                 sib = (struct sockaddr_ib *) addr;
3614                 sid = be64_to_cpu(sib->sib_sid);
3615                 mask = be64_to_cpu(sib->sib_sid_mask);
3616                 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
3617                 sib->sib_sid_mask = cpu_to_be64(~0ULL);
3618                 break;
3619         }
3620         id_priv->bind_list = bind_list;
3621         hlist_add_head(&id_priv->node, &bind_list->owners);
3622 }
3623
3624 static int cma_alloc_port(enum rdma_ucm_port_space ps,
3625                           struct rdma_id_private *id_priv, unsigned short snum)
3626 {
3627         struct rdma_bind_list *bind_list;
3628         int ret;
3629
3630         lockdep_assert_held(&lock);
3631
3632         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
3633         if (!bind_list)
3634                 return -ENOMEM;
3635
3636         ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list,
3637                            snum);
3638         if (ret < 0)
3639                 goto err;
3640
3641         bind_list->ps = ps;
3642         bind_list->port = snum;
3643         cma_bind_port(bind_list, id_priv);
3644         return 0;
3645 err:
3646         kfree(bind_list);
3647         return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
3648 }
3649
3650 static int cma_port_is_unique(struct rdma_bind_list *bind_list,
3651                               struct rdma_id_private *id_priv)
3652 {
3653         struct rdma_id_private *cur_id;
3654         struct sockaddr  *daddr = cma_dst_addr(id_priv);
3655         struct sockaddr  *saddr = cma_src_addr(id_priv);
3656         __be16 dport = cma_port(daddr);
3657
3658         lockdep_assert_held(&lock);
3659
3660         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3661                 struct sockaddr  *cur_daddr = cma_dst_addr(cur_id);
3662                 struct sockaddr  *cur_saddr = cma_src_addr(cur_id);
3663                 __be16 cur_dport = cma_port(cur_daddr);
3664
3665                 if (id_priv == cur_id)
3666                         continue;
3667
3668                 /* different dest port -> unique */
3669                 if (!cma_any_port(daddr) &&
3670                     !cma_any_port(cur_daddr) &&
3671                     (dport != cur_dport))
3672                         continue;
3673
3674                 /* different src address -> unique */
3675                 if (!cma_any_addr(saddr) &&
3676                     !cma_any_addr(cur_saddr) &&
3677                     cma_addr_cmp(saddr, cur_saddr))
3678                         continue;
3679
3680                 /* different dst address -> unique */
3681                 if (!cma_any_addr(daddr) &&
3682                     !cma_any_addr(cur_daddr) &&
3683                     cma_addr_cmp(daddr, cur_daddr))
3684                         continue;
3685
3686                 return -EADDRNOTAVAIL;
3687         }
3688         return 0;
3689 }
3690
3691 static int cma_alloc_any_port(enum rdma_ucm_port_space ps,
3692                               struct rdma_id_private *id_priv)
3693 {
3694         static unsigned int last_used_port;
3695         int low, high, remaining;
3696         unsigned int rover;
3697         struct net *net = id_priv->id.route.addr.dev_addr.net;
3698
3699         lockdep_assert_held(&lock);
3700
3701         inet_get_local_port_range(net, &low, &high);
3702         remaining = (high - low) + 1;
3703         rover = get_random_u32_inclusive(low, remaining + low - 1);
3704 retry:
3705         if (last_used_port != rover) {
3706                 struct rdma_bind_list *bind_list;
3707                 int ret;
3708
3709                 bind_list = cma_ps_find(net, ps, (unsigned short)rover);
3710
3711                 if (!bind_list) {
3712                         ret = cma_alloc_port(ps, id_priv, rover);
3713                 } else {
3714                         ret = cma_port_is_unique(bind_list, id_priv);
3715                         if (!ret)
3716                                 cma_bind_port(bind_list, id_priv);
3717                 }
3718                 /*
3719                  * Remember previously used port number in order to avoid
3720                  * re-using same port immediately after it is closed.
3721                  */
3722                 if (!ret)
3723                         last_used_port = rover;
3724                 if (ret != -EADDRNOTAVAIL)
3725                         return ret;
3726         }
3727         if (--remaining) {
3728                 rover++;
3729                 if ((rover < low) || (rover > high))
3730                         rover = low;
3731                 goto retry;
3732         }
3733         return -EADDRNOTAVAIL;
3734 }
3735
3736 /*
3737  * Check that the requested port is available.  This is called when trying to
3738  * bind to a specific port, or when trying to listen on a bound port.  In
3739  * the latter case, the provided id_priv may already be on the bind_list, but
3740  * we still need to check that it's okay to start listening.
3741  */
3742 static int cma_check_port(struct rdma_bind_list *bind_list,
3743                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
3744 {
3745         struct rdma_id_private *cur_id;
3746         struct sockaddr *addr, *cur_addr;
3747
3748         lockdep_assert_held(&lock);
3749
3750         addr = cma_src_addr(id_priv);
3751         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3752                 if (id_priv == cur_id)
3753                         continue;
3754
3755                 if (reuseaddr && cur_id->reuseaddr)
3756                         continue;
3757
3758                 cur_addr = cma_src_addr(cur_id);
3759                 if (id_priv->afonly && cur_id->afonly &&
3760                     (addr->sa_family != cur_addr->sa_family))
3761                         continue;
3762
3763                 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
3764                         return -EADDRNOTAVAIL;
3765
3766                 if (!cma_addr_cmp(addr, cur_addr))
3767                         return -EADDRINUSE;
3768         }
3769         return 0;
3770 }
3771
3772 static int cma_use_port(enum rdma_ucm_port_space ps,
3773                         struct rdma_id_private *id_priv)
3774 {
3775         struct rdma_bind_list *bind_list;
3776         unsigned short snum;
3777         int ret;
3778
3779         lockdep_assert_held(&lock);
3780
3781         snum = ntohs(cma_port(cma_src_addr(id_priv)));
3782         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
3783                 return -EACCES;
3784
3785         bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum);
3786         if (!bind_list) {
3787                 ret = cma_alloc_port(ps, id_priv, snum);
3788         } else {
3789                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
3790                 if (!ret)
3791                         cma_bind_port(bind_list, id_priv);
3792         }
3793         return ret;
3794 }
3795
3796 static enum rdma_ucm_port_space
3797 cma_select_inet_ps(struct rdma_id_private *id_priv)
3798 {
3799         switch (id_priv->id.ps) {
3800         case RDMA_PS_TCP:
3801         case RDMA_PS_UDP:
3802         case RDMA_PS_IPOIB:
3803         case RDMA_PS_IB:
3804                 return id_priv->id.ps;
3805         default:
3806
3807                 return 0;
3808         }
3809 }
3810
3811 static enum rdma_ucm_port_space
3812 cma_select_ib_ps(struct rdma_id_private *id_priv)
3813 {
3814         enum rdma_ucm_port_space ps = 0;
3815         struct sockaddr_ib *sib;
3816         u64 sid_ps, mask, sid;
3817
3818         sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
3819         mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
3820         sid = be64_to_cpu(sib->sib_sid) & mask;
3821
3822         if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
3823                 sid_ps = RDMA_IB_IP_PS_IB;
3824                 ps = RDMA_PS_IB;
3825         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
3826                    (sid == (RDMA_IB_IP_PS_TCP & mask))) {
3827                 sid_ps = RDMA_IB_IP_PS_TCP;
3828                 ps = RDMA_PS_TCP;
3829         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
3830                    (sid == (RDMA_IB_IP_PS_UDP & mask))) {
3831                 sid_ps = RDMA_IB_IP_PS_UDP;
3832                 ps = RDMA_PS_UDP;
3833         }
3834
3835         if (ps) {
3836                 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
3837                 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
3838                                                 be64_to_cpu(sib->sib_sid_mask));
3839         }
3840         return ps;
3841 }
3842
3843 static int cma_get_port(struct rdma_id_private *id_priv)
3844 {
3845         enum rdma_ucm_port_space ps;
3846         int ret;
3847
3848         if (cma_family(id_priv) != AF_IB)
3849                 ps = cma_select_inet_ps(id_priv);
3850         else
3851                 ps = cma_select_ib_ps(id_priv);
3852         if (!ps)
3853                 return -EPROTONOSUPPORT;
3854
3855         mutex_lock(&lock);
3856         if (cma_any_port(cma_src_addr(id_priv)))
3857                 ret = cma_alloc_any_port(ps, id_priv);
3858         else
3859                 ret = cma_use_port(ps, id_priv);
3860         mutex_unlock(&lock);
3861
3862         return ret;
3863 }
3864
3865 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
3866                                struct sockaddr *addr)
3867 {
3868 #if IS_ENABLED(CONFIG_IPV6)
3869         struct sockaddr_in6 *sin6;
3870
3871         if (addr->sa_family != AF_INET6)
3872                 return 0;
3873
3874         sin6 = (struct sockaddr_in6 *) addr;
3875
3876         if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL))
3877                 return 0;
3878
3879         if (!sin6->sin6_scope_id)
3880                         return -EINVAL;
3881
3882         dev_addr->bound_dev_if = sin6->sin6_scope_id;
3883 #endif
3884         return 0;
3885 }
3886
3887 int rdma_listen(struct rdma_cm_id *id, int backlog)
3888 {
3889         struct rdma_id_private *id_priv =
3890                 container_of(id, struct rdma_id_private, id);
3891         int ret;
3892
3893         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) {
3894                 struct sockaddr_in any_in = {
3895                         .sin_family = AF_INET,
3896                         .sin_addr.s_addr = htonl(INADDR_ANY),
3897                 };
3898
3899                 /* For a well behaved ULP state will be RDMA_CM_IDLE */
3900                 ret = rdma_bind_addr(id, (struct sockaddr *)&any_in);
3901                 if (ret)
3902                         return ret;
3903                 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
3904                                            RDMA_CM_LISTEN)))
3905                         return -EINVAL;
3906         }
3907
3908         /*
3909          * Once the ID reaches RDMA_CM_LISTEN it is not allowed to be reusable
3910          * any more, and has to be unique in the bind list.
3911          */
3912         if (id_priv->reuseaddr) {
3913                 mutex_lock(&lock);
3914                 ret = cma_check_port(id_priv->bind_list, id_priv, 0);
3915                 if (!ret)
3916                         id_priv->reuseaddr = 0;
3917                 mutex_unlock(&lock);
3918                 if (ret)
3919                         goto err;
3920         }
3921
3922         id_priv->backlog = backlog;
3923         if (id_priv->cma_dev) {
3924                 if (rdma_cap_ib_cm(id->device, 1)) {
3925                         ret = cma_ib_listen(id_priv);
3926                         if (ret)
3927                                 goto err;
3928                 } else if (rdma_cap_iw_cm(id->device, 1)) {
3929                         ret = cma_iw_listen(id_priv, backlog);
3930                         if (ret)
3931                                 goto err;
3932                 } else {
3933                         ret = -ENOSYS;
3934                         goto err;
3935                 }
3936         } else {
3937                 ret = cma_listen_on_all(id_priv);
3938                 if (ret)
3939                         goto err;
3940         }
3941
3942         return 0;
3943 err:
3944         id_priv->backlog = 0;
3945         /*
3946          * All the failure paths that lead here will not allow the req_handler's
3947          * to have run.
3948          */
3949         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
3950         return ret;
3951 }
3952 EXPORT_SYMBOL(rdma_listen);
3953
3954 static int rdma_bind_addr_dst(struct rdma_id_private *id_priv,
3955                               struct sockaddr *addr, const struct sockaddr *daddr)
3956 {
3957         struct sockaddr *id_daddr;
3958         int ret;
3959
3960         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
3961             addr->sa_family != AF_IB)
3962                 return -EAFNOSUPPORT;
3963
3964         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
3965                 return -EINVAL;
3966
3967         ret = cma_check_linklocal(&id_priv->id.route.addr.dev_addr, addr);
3968         if (ret)
3969                 goto err1;
3970
3971         memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
3972         if (!cma_any_addr(addr)) {
3973                 ret = cma_translate_addr(addr, &id_priv->id.route.addr.dev_addr);
3974                 if (ret)
3975                         goto err1;
3976
3977                 ret = cma_acquire_dev_by_src_ip(id_priv);
3978                 if (ret)
3979                         goto err1;
3980         }
3981
3982         if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
3983                 if (addr->sa_family == AF_INET)
3984                         id_priv->afonly = 1;
3985 #if IS_ENABLED(CONFIG_IPV6)
3986                 else if (addr->sa_family == AF_INET6) {
3987                         struct net *net = id_priv->id.route.addr.dev_addr.net;
3988
3989                         id_priv->afonly = net->ipv6.sysctl.bindv6only;
3990                 }
3991 #endif
3992         }
3993         id_daddr = cma_dst_addr(id_priv);
3994         if (daddr != id_daddr)
3995                 memcpy(id_daddr, daddr, rdma_addr_size(addr));
3996         id_daddr->sa_family = addr->sa_family;
3997
3998         ret = cma_get_port(id_priv);
3999         if (ret)
4000                 goto err2;
4001
4002         if (!cma_any_addr(addr))
4003                 rdma_restrack_add(&id_priv->res);
4004         return 0;
4005 err2:
4006         if (id_priv->cma_dev)
4007                 cma_release_dev(id_priv);
4008 err1:
4009         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
4010         return ret;
4011 }
4012
4013 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4014                          const struct sockaddr *dst_addr)
4015 {
4016         struct rdma_id_private *id_priv =
4017                 container_of(id, struct rdma_id_private, id);
4018         struct sockaddr_storage zero_sock = {};
4019
4020         if (src_addr && src_addr->sa_family)
4021                 return rdma_bind_addr_dst(id_priv, src_addr, dst_addr);
4022
4023         /*
4024          * When the src_addr is not specified, automatically supply an any addr
4025          */
4026         zero_sock.ss_family = dst_addr->sa_family;
4027         if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
4028                 struct sockaddr_in6 *src_addr6 =
4029                         (struct sockaddr_in6 *)&zero_sock;
4030                 struct sockaddr_in6 *dst_addr6 =
4031                         (struct sockaddr_in6 *)dst_addr;
4032
4033                 src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
4034                 if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
4035                         id->route.addr.dev_addr.bound_dev_if =
4036                                 dst_addr6->sin6_scope_id;
4037         } else if (dst_addr->sa_family == AF_IB) {
4038                 ((struct sockaddr_ib *)&zero_sock)->sib_pkey =
4039                         ((struct sockaddr_ib *)dst_addr)->sib_pkey;
4040         }
4041         return rdma_bind_addr_dst(id_priv, (struct sockaddr *)&zero_sock, dst_addr);
4042 }
4043
4044 /*
4045  * If required, resolve the source address for bind and leave the id_priv in
4046  * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
4047  * calls made by ULP, a previously bound ID will not be re-bound and src_addr is
4048  * ignored.
4049  */
4050 static int resolve_prepare_src(struct rdma_id_private *id_priv,
4051                                struct sockaddr *src_addr,
4052                                const struct sockaddr *dst_addr)
4053 {
4054         int ret;
4055
4056         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
4057                 /* For a well behaved ULP state will be RDMA_CM_IDLE */
4058                 ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
4059                 if (ret)
4060                         return ret;
4061                 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
4062                                            RDMA_CM_ADDR_QUERY)))
4063                         return -EINVAL;
4064
4065         } else {
4066                 memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
4067         }
4068
4069         if (cma_family(id_priv) != dst_addr->sa_family) {
4070                 ret = -EINVAL;
4071                 goto err_state;
4072         }
4073         return 0;
4074
4075 err_state:
4076         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4077         return ret;
4078 }
4079
4080 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4081                       const struct sockaddr *dst_addr, unsigned long timeout_ms)
4082 {
4083         struct rdma_id_private *id_priv =
4084                 container_of(id, struct rdma_id_private, id);
4085         int ret;
4086
4087         ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
4088         if (ret)
4089                 return ret;
4090
4091         if (cma_any_addr(dst_addr)) {
4092                 ret = cma_resolve_loopback(id_priv);
4093         } else {
4094                 if (dst_addr->sa_family == AF_IB) {
4095                         ret = cma_resolve_ib_addr(id_priv);
4096                 } else {
4097                         /*
4098                          * The FSM can return back to RDMA_CM_ADDR_BOUND after
4099                          * rdma_resolve_ip() is called, eg through the error
4100                          * path in addr_handler(). If this happens the existing
4101                          * request must be canceled before issuing a new one.
4102                          * Since canceling a request is a bit slow and this
4103                          * oddball path is rare, keep track once a request has
4104                          * been issued. The track turns out to be a permanent
4105                          * state since this is the only cancel as it is
4106                          * immediately before rdma_resolve_ip().
4107                          */
4108                         if (id_priv->used_resolve_ip)
4109                                 rdma_addr_cancel(&id->route.addr.dev_addr);
4110                         else
4111                                 id_priv->used_resolve_ip = 1;
4112                         ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
4113                                               &id->route.addr.dev_addr,
4114                                               timeout_ms, addr_handler,
4115                                               false, id_priv);
4116                 }
4117         }
4118         if (ret)
4119                 goto err;
4120
4121         return 0;
4122 err:
4123         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4124         return ret;
4125 }
4126 EXPORT_SYMBOL(rdma_resolve_addr);
4127
4128 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
4129 {
4130         struct rdma_id_private *id_priv =
4131                 container_of(id, struct rdma_id_private, id);
4132
4133         return rdma_bind_addr_dst(id_priv, addr, cma_dst_addr(id_priv));
4134 }
4135 EXPORT_SYMBOL(rdma_bind_addr);
4136
4137 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
4138 {
4139         struct cma_hdr *cma_hdr;
4140
4141         cma_hdr = hdr;
4142         cma_hdr->cma_version = CMA_VERSION;
4143         if (cma_family(id_priv) == AF_INET) {
4144                 struct sockaddr_in *src4, *dst4;
4145
4146                 src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
4147                 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
4148
4149                 cma_set_ip_ver(cma_hdr, 4);
4150                 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
4151                 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
4152                 cma_hdr->port = src4->sin_port;
4153         } else if (cma_family(id_priv) == AF_INET6) {
4154                 struct sockaddr_in6 *src6, *dst6;
4155
4156                 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
4157                 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
4158
4159                 cma_set_ip_ver(cma_hdr, 6);
4160                 cma_hdr->src_addr.ip6 = src6->sin6_addr;
4161                 cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
4162                 cma_hdr->port = src6->sin6_port;
4163         }
4164         return 0;
4165 }
4166
4167 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
4168                                 const struct ib_cm_event *ib_event)
4169 {
4170         struct rdma_id_private *id_priv = cm_id->context;
4171         struct rdma_cm_event event = {};
4172         const struct ib_cm_sidr_rep_event_param *rep =
4173                                 &ib_event->param.sidr_rep_rcvd;
4174         int ret;
4175
4176         mutex_lock(&id_priv->handler_mutex);
4177         if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
4178                 goto out;
4179
4180         switch (ib_event->event) {
4181         case IB_CM_SIDR_REQ_ERROR:
4182                 event.event = RDMA_CM_EVENT_UNREACHABLE;
4183                 event.status = -ETIMEDOUT;
4184                 break;
4185         case IB_CM_SIDR_REP_RECEIVED:
4186                 event.param.ud.private_data = ib_event->private_data;
4187                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
4188                 if (rep->status != IB_SIDR_SUCCESS) {
4189                         event.event = RDMA_CM_EVENT_UNREACHABLE;
4190                         event.status = ib_event->param.sidr_rep_rcvd.status;
4191                         pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n",
4192                                              event.status);
4193                         break;
4194                 }
4195                 ret = cma_set_qkey(id_priv, rep->qkey);
4196                 if (ret) {
4197                         pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret);
4198                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
4199                         event.status = ret;
4200                         break;
4201                 }
4202                 ib_init_ah_attr_from_path(id_priv->id.device,
4203                                           id_priv->id.port_num,
4204                                           id_priv->id.route.path_rec,
4205                                           &event.param.ud.ah_attr,
4206                                           rep->sgid_attr);
4207                 event.param.ud.qp_num = rep->qpn;
4208                 event.param.ud.qkey = rep->qkey;
4209                 event.event = RDMA_CM_EVENT_ESTABLISHED;
4210                 event.status = 0;
4211                 break;
4212         default:
4213                 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
4214                        ib_event->event);
4215                 goto out;
4216         }
4217
4218         ret = cma_cm_event_handler(id_priv, &event);
4219
4220         rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4221         if (ret) {
4222                 /* Destroy the CM ID by returning a non-zero value. */
4223                 id_priv->cm_id.ib = NULL;
4224                 destroy_id_handler_unlock(id_priv);
4225                 return ret;
4226         }
4227 out:
4228         mutex_unlock(&id_priv->handler_mutex);
4229         return 0;
4230 }
4231
4232 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
4233                               struct rdma_conn_param *conn_param)
4234 {
4235         struct ib_cm_sidr_req_param req;
4236         struct ib_cm_id *id;
4237         void *private_data;
4238         u8 offset;
4239         int ret;
4240
4241         memset(&req, 0, sizeof req);
4242         offset = cma_user_data_offset(id_priv);
4243         if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4244                 return -EINVAL;
4245
4246         if (req.private_data_len) {
4247                 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
4248                 if (!private_data)
4249                         return -ENOMEM;
4250         } else {
4251                 private_data = NULL;
4252         }
4253
4254         if (conn_param->private_data && conn_param->private_data_len)
4255                 memcpy(private_data + offset, conn_param->private_data,
4256                        conn_param->private_data_len);
4257
4258         if (private_data) {
4259                 ret = cma_format_hdr(private_data, id_priv);
4260                 if (ret)
4261                         goto out;
4262                 req.private_data = private_data;
4263         }
4264
4265         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
4266                              id_priv);
4267         if (IS_ERR(id)) {
4268                 ret = PTR_ERR(id);
4269                 goto out;
4270         }
4271         id_priv->cm_id.ib = id;
4272
4273         req.path = id_priv->id.route.path_rec;
4274         req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
4275         req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
4276         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
4277         req.max_cm_retries = CMA_MAX_CM_RETRIES;
4278
4279         trace_cm_send_sidr_req(id_priv);
4280         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
4281         if (ret) {
4282                 ib_destroy_cm_id(id_priv->cm_id.ib);
4283                 id_priv->cm_id.ib = NULL;
4284         }
4285 out:
4286         kfree(private_data);
4287         return ret;
4288 }
4289
4290 static int cma_connect_ib(struct rdma_id_private *id_priv,
4291                           struct rdma_conn_param *conn_param)
4292 {
4293         struct ib_cm_req_param req;
4294         struct rdma_route *route;
4295         void *private_data;
4296         struct ib_cm_id *id;
4297         u8 offset;
4298         int ret;
4299
4300         memset(&req, 0, sizeof req);
4301         offset = cma_user_data_offset(id_priv);
4302         if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4303                 return -EINVAL;
4304
4305         if (req.private_data_len) {
4306                 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
4307                 if (!private_data)
4308                         return -ENOMEM;
4309         } else {
4310                 private_data = NULL;
4311         }
4312
4313         if (conn_param->private_data && conn_param->private_data_len)
4314                 memcpy(private_data + offset, conn_param->private_data,
4315                        conn_param->private_data_len);
4316
4317         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
4318         if (IS_ERR(id)) {
4319                 ret = PTR_ERR(id);
4320                 goto out;
4321         }
4322         id_priv->cm_id.ib = id;
4323
4324         route = &id_priv->id.route;
4325         if (private_data) {
4326                 ret = cma_format_hdr(private_data, id_priv);
4327                 if (ret)
4328                         goto out;
4329                 req.private_data = private_data;
4330         }
4331
4332         req.primary_path = &route->path_rec[0];
4333         req.primary_path_inbound = route->path_rec_inbound;
4334         req.primary_path_outbound = route->path_rec_outbound;
4335         if (route->num_pri_alt_paths == 2)
4336                 req.alternate_path = &route->path_rec[1];
4337
4338         req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
4339         /* Alternate path SGID attribute currently unsupported */
4340         req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
4341         req.qp_num = id_priv->qp_num;
4342         req.qp_type = id_priv->id.qp_type;
4343         req.starting_psn = id_priv->seq_num;
4344         req.responder_resources = conn_param->responder_resources;
4345         req.initiator_depth = conn_param->initiator_depth;
4346         req.flow_control = conn_param->flow_control;
4347         req.retry_count = min_t(u8, 7, conn_param->retry_count);
4348         req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4349         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4350         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4351         req.max_cm_retries = CMA_MAX_CM_RETRIES;
4352         req.srq = id_priv->srq ? 1 : 0;
4353         req.ece.vendor_id = id_priv->ece.vendor_id;
4354         req.ece.attr_mod = id_priv->ece.attr_mod;
4355
4356         trace_cm_send_req(id_priv);
4357         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
4358 out:
4359         if (ret && !IS_ERR(id)) {
4360                 ib_destroy_cm_id(id);
4361                 id_priv->cm_id.ib = NULL;
4362         }
4363
4364         kfree(private_data);
4365         return ret;
4366 }
4367
4368 static int cma_connect_iw(struct rdma_id_private *id_priv,
4369                           struct rdma_conn_param *conn_param)
4370 {
4371         struct iw_cm_id *cm_id;
4372         int ret;
4373         struct iw_cm_conn_param iw_param;
4374
4375         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
4376         if (IS_ERR(cm_id))
4377                 return PTR_ERR(cm_id);
4378
4379         mutex_lock(&id_priv->qp_mutex);
4380         cm_id->tos = id_priv->tos;
4381         cm_id->tos_set = id_priv->tos_set;
4382         mutex_unlock(&id_priv->qp_mutex);
4383
4384         id_priv->cm_id.iw = cm_id;
4385
4386         memcpy(&cm_id->local_addr, cma_src_addr(id_priv),
4387                rdma_addr_size(cma_src_addr(id_priv)));
4388         memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv),
4389                rdma_addr_size(cma_dst_addr(id_priv)));
4390
4391         ret = cma_modify_qp_rtr(id_priv, conn_param);
4392         if (ret)
4393                 goto out;
4394
4395         if (conn_param) {
4396                 iw_param.ord = conn_param->initiator_depth;
4397                 iw_param.ird = conn_param->responder_resources;
4398                 iw_param.private_data = conn_param->private_data;
4399                 iw_param.private_data_len = conn_param->private_data_len;
4400                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
4401         } else {
4402                 memset(&iw_param, 0, sizeof iw_param);
4403                 iw_param.qpn = id_priv->qp_num;
4404         }
4405         ret = iw_cm_connect(cm_id, &iw_param);
4406 out:
4407         if (ret) {
4408                 iw_destroy_cm_id(cm_id);
4409                 id_priv->cm_id.iw = NULL;
4410         }
4411         return ret;
4412 }
4413
4414 /**
4415  * rdma_connect_locked - Initiate an active connection request.
4416  * @id: Connection identifier to connect.
4417  * @conn_param: Connection information used for connected QPs.
4418  *
4419  * Same as rdma_connect() but can only be called from the
4420  * RDMA_CM_EVENT_ROUTE_RESOLVED handler callback.
4421  */
4422 int rdma_connect_locked(struct rdma_cm_id *id,
4423                         struct rdma_conn_param *conn_param)
4424 {
4425         struct rdma_id_private *id_priv =
4426                 container_of(id, struct rdma_id_private, id);
4427         int ret;
4428
4429         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
4430                 return -EINVAL;
4431
4432         if (!id->qp) {
4433                 id_priv->qp_num = conn_param->qp_num;
4434                 id_priv->srq = conn_param->srq;
4435         }
4436
4437         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4438                 if (id->qp_type == IB_QPT_UD)
4439                         ret = cma_resolve_ib_udp(id_priv, conn_param);
4440                 else
4441                         ret = cma_connect_ib(id_priv, conn_param);
4442         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4443                 ret = cma_connect_iw(id_priv, conn_param);
4444         } else {
4445                 ret = -ENOSYS;
4446         }
4447         if (ret)
4448                 goto err_state;
4449         return 0;
4450 err_state:
4451         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
4452         return ret;
4453 }
4454 EXPORT_SYMBOL(rdma_connect_locked);
4455
4456 /**
4457  * rdma_connect - Initiate an active connection request.
4458  * @id: Connection identifier to connect.
4459  * @conn_param: Connection information used for connected QPs.
4460  *
4461  * Users must have resolved a route for the rdma_cm_id to connect with by having
4462  * called rdma_resolve_route before calling this routine.
4463  *
4464  * This call will either connect to a remote QP or obtain remote QP information
4465  * for unconnected rdma_cm_id's.  The actual operation is based on the
4466  * rdma_cm_id's port space.
4467  */
4468 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4469 {
4470         struct rdma_id_private *id_priv =
4471                 container_of(id, struct rdma_id_private, id);
4472         int ret;
4473
4474         mutex_lock(&id_priv->handler_mutex);
4475         ret = rdma_connect_locked(id, conn_param);
4476         mutex_unlock(&id_priv->handler_mutex);
4477         return ret;
4478 }
4479 EXPORT_SYMBOL(rdma_connect);
4480
4481 /**
4482  * rdma_connect_ece - Initiate an active connection request with ECE data.
4483  * @id: Connection identifier to connect.
4484  * @conn_param: Connection information used for connected QPs.
4485  * @ece: ECE parameters
4486  *
4487  * See rdma_connect() explanation.
4488  */
4489 int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4490                      struct rdma_ucm_ece *ece)
4491 {
4492         struct rdma_id_private *id_priv =
4493                 container_of(id, struct rdma_id_private, id);
4494
4495         id_priv->ece.vendor_id = ece->vendor_id;
4496         id_priv->ece.attr_mod = ece->attr_mod;
4497
4498         return rdma_connect(id, conn_param);
4499 }
4500 EXPORT_SYMBOL(rdma_connect_ece);
4501
4502 static int cma_accept_ib(struct rdma_id_private *id_priv,
4503                          struct rdma_conn_param *conn_param)
4504 {
4505         struct ib_cm_rep_param rep;
4506         int ret;
4507
4508         ret = cma_modify_qp_rtr(id_priv, conn_param);
4509         if (ret)
4510                 goto out;
4511
4512         ret = cma_modify_qp_rts(id_priv, conn_param);
4513         if (ret)
4514                 goto out;
4515
4516         memset(&rep, 0, sizeof rep);
4517         rep.qp_num = id_priv->qp_num;
4518         rep.starting_psn = id_priv->seq_num;
4519         rep.private_data = conn_param->private_data;
4520         rep.private_data_len = conn_param->private_data_len;
4521         rep.responder_resources = conn_param->responder_resources;
4522         rep.initiator_depth = conn_param->initiator_depth;
4523         rep.failover_accepted = 0;
4524         rep.flow_control = conn_param->flow_control;
4525         rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4526         rep.srq = id_priv->srq ? 1 : 0;
4527         rep.ece.vendor_id = id_priv->ece.vendor_id;
4528         rep.ece.attr_mod = id_priv->ece.attr_mod;
4529
4530         trace_cm_send_rep(id_priv);
4531         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
4532 out:
4533         return ret;
4534 }
4535
4536 static int cma_accept_iw(struct rdma_id_private *id_priv,
4537                   struct rdma_conn_param *conn_param)
4538 {
4539         struct iw_cm_conn_param iw_param;
4540         int ret;
4541
4542         if (!conn_param)
4543                 return -EINVAL;
4544
4545         ret = cma_modify_qp_rtr(id_priv, conn_param);
4546         if (ret)
4547                 return ret;
4548
4549         iw_param.ord = conn_param->initiator_depth;
4550         iw_param.ird = conn_param->responder_resources;
4551         iw_param.private_data = conn_param->private_data;
4552         iw_param.private_data_len = conn_param->private_data_len;
4553         if (id_priv->id.qp)
4554                 iw_param.qpn = id_priv->qp_num;
4555         else
4556                 iw_param.qpn = conn_param->qp_num;
4557
4558         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
4559 }
4560
4561 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
4562                              enum ib_cm_sidr_status status, u32 qkey,
4563                              const void *private_data, int private_data_len)
4564 {
4565         struct ib_cm_sidr_rep_param rep;
4566         int ret;
4567
4568         memset(&rep, 0, sizeof rep);
4569         rep.status = status;
4570         if (status == IB_SIDR_SUCCESS) {
4571                 if (qkey)
4572                         ret = cma_set_qkey(id_priv, qkey);
4573                 else
4574                         ret = cma_set_default_qkey(id_priv);
4575                 if (ret)
4576                         return ret;
4577                 rep.qp_num = id_priv->qp_num;
4578                 rep.qkey = id_priv->qkey;
4579
4580                 rep.ece.vendor_id = id_priv->ece.vendor_id;
4581                 rep.ece.attr_mod = id_priv->ece.attr_mod;
4582         }
4583
4584         rep.private_data = private_data;
4585         rep.private_data_len = private_data_len;
4586
4587         trace_cm_send_sidr_rep(id_priv);
4588         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
4589 }
4590
4591 /**
4592  * rdma_accept - Called to accept a connection request or response.
4593  * @id: Connection identifier associated with the request.
4594  * @conn_param: Information needed to establish the connection.  This must be
4595  *   provided if accepting a connection request.  If accepting a connection
4596  *   response, this parameter must be NULL.
4597  *
4598  * Typically, this routine is only called by the listener to accept a connection
4599  * request.  It must also be called on the active side of a connection if the
4600  * user is performing their own QP transitions.
4601  *
4602  * In the case of error, a reject message is sent to the remote side and the
4603  * state of the qp associated with the id is modified to error, such that any
4604  * previously posted receive buffers would be flushed.
4605  *
4606  * This function is for use by kernel ULPs and must be called from under the
4607  * handler callback.
4608  */
4609 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4610 {
4611         struct rdma_id_private *id_priv =
4612                 container_of(id, struct rdma_id_private, id);
4613         int ret;
4614
4615         lockdep_assert_held(&id_priv->handler_mutex);
4616
4617         if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
4618                 return -EINVAL;
4619
4620         if (!id->qp && conn_param) {
4621                 id_priv->qp_num = conn_param->qp_num;
4622                 id_priv->srq = conn_param->srq;
4623         }
4624
4625         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4626                 if (id->qp_type == IB_QPT_UD) {
4627                         if (conn_param)
4628                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4629                                                         conn_param->qkey,
4630                                                         conn_param->private_data,
4631                                                         conn_param->private_data_len);
4632                         else
4633                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4634                                                         0, NULL, 0);
4635                 } else {
4636                         if (conn_param)
4637                                 ret = cma_accept_ib(id_priv, conn_param);
4638                         else
4639                                 ret = cma_rep_recv(id_priv);
4640                 }
4641         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4642                 ret = cma_accept_iw(id_priv, conn_param);
4643         } else {
4644                 ret = -ENOSYS;
4645         }
4646         if (ret)
4647                 goto reject;
4648
4649         return 0;
4650 reject:
4651         cma_modify_qp_err(id_priv);
4652         rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
4653         return ret;
4654 }
4655 EXPORT_SYMBOL(rdma_accept);
4656
4657 int rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4658                     struct rdma_ucm_ece *ece)
4659 {
4660         struct rdma_id_private *id_priv =
4661                 container_of(id, struct rdma_id_private, id);
4662
4663         id_priv->ece.vendor_id = ece->vendor_id;
4664         id_priv->ece.attr_mod = ece->attr_mod;
4665
4666         return rdma_accept(id, conn_param);
4667 }
4668 EXPORT_SYMBOL(rdma_accept_ece);
4669
4670 void rdma_lock_handler(struct rdma_cm_id *id)
4671 {
4672         struct rdma_id_private *id_priv =
4673                 container_of(id, struct rdma_id_private, id);
4674
4675         mutex_lock(&id_priv->handler_mutex);
4676 }
4677 EXPORT_SYMBOL(rdma_lock_handler);
4678
4679 void rdma_unlock_handler(struct rdma_cm_id *id)
4680 {
4681         struct rdma_id_private *id_priv =
4682                 container_of(id, struct rdma_id_private, id);
4683
4684         mutex_unlock(&id_priv->handler_mutex);
4685 }
4686 EXPORT_SYMBOL(rdma_unlock_handler);
4687
4688 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
4689 {
4690         struct rdma_id_private *id_priv;
4691         int ret;
4692
4693         id_priv = container_of(id, struct rdma_id_private, id);
4694         if (!id_priv->cm_id.ib)
4695                 return -EINVAL;
4696
4697         switch (id->device->node_type) {
4698         case RDMA_NODE_IB_CA:
4699                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
4700                 break;
4701         default:
4702                 ret = 0;
4703                 break;
4704         }
4705         return ret;
4706 }
4707 EXPORT_SYMBOL(rdma_notify);
4708
4709 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
4710                 u8 private_data_len, u8 reason)
4711 {
4712         struct rdma_id_private *id_priv;
4713         int ret;
4714
4715         id_priv = container_of(id, struct rdma_id_private, id);
4716         if (!id_priv->cm_id.ib)
4717                 return -EINVAL;
4718
4719         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4720                 if (id->qp_type == IB_QPT_UD) {
4721                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,
4722                                                 private_data, private_data_len);
4723                 } else {
4724                         trace_cm_send_rej(id_priv);
4725                         ret = ib_send_cm_rej(id_priv->cm_id.ib, reason, NULL, 0,
4726                                              private_data, private_data_len);
4727                 }
4728         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4729                 ret = iw_cm_reject(id_priv->cm_id.iw,
4730                                    private_data, private_data_len);
4731         } else {
4732                 ret = -ENOSYS;
4733         }
4734
4735         return ret;
4736 }
4737 EXPORT_SYMBOL(rdma_reject);
4738
4739 int rdma_disconnect(struct rdma_cm_id *id)
4740 {
4741         struct rdma_id_private *id_priv;
4742         int ret;
4743
4744         id_priv = container_of(id, struct rdma_id_private, id);
4745         if (!id_priv->cm_id.ib)
4746                 return -EINVAL;
4747
4748         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4749                 ret = cma_modify_qp_err(id_priv);
4750                 if (ret)
4751                         goto out;
4752                 /* Initiate or respond to a disconnect. */
4753                 trace_cm_disconnect(id_priv);
4754                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
4755                         if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
4756                                 trace_cm_sent_drep(id_priv);
4757                 } else {
4758                         trace_cm_sent_dreq(id_priv);
4759                 }
4760         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4761                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
4762         } else
4763                 ret = -EINVAL;
4764
4765 out:
4766         return ret;
4767 }
4768 EXPORT_SYMBOL(rdma_disconnect);
4769
4770 static void cma_make_mc_event(int status, struct rdma_id_private *id_priv,
4771                               struct ib_sa_multicast *multicast,
4772                               struct rdma_cm_event *event,
4773                               struct cma_multicast *mc)
4774 {
4775         struct rdma_dev_addr *dev_addr;
4776         enum ib_gid_type gid_type;
4777         struct net_device *ndev;
4778
4779         if (status)
4780                 pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",
4781                                      status);
4782
4783         event->status = status;
4784         event->param.ud.private_data = mc->context;
4785         if (status) {
4786                 event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4787                 return;
4788         }
4789
4790         dev_addr = &id_priv->id.route.addr.dev_addr;
4791         ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4792         gid_type =
4793                 id_priv->cma_dev
4794                         ->default_gid_type[id_priv->id.port_num -
4795                                            rdma_start_port(
4796                                                    id_priv->cma_dev->device)];
4797
4798         event->event = RDMA_CM_EVENT_MULTICAST_JOIN;
4799         if (ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num,
4800                                      &multicast->rec, ndev, gid_type,
4801                                      &event->param.ud.ah_attr)) {
4802                 event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4803                 goto out;
4804         }
4805
4806         event->param.ud.qp_num = 0xFFFFFF;
4807         event->param.ud.qkey = id_priv->qkey;
4808
4809 out:
4810         dev_put(ndev);
4811 }
4812
4813 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
4814 {
4815         struct cma_multicast *mc = multicast->context;
4816         struct rdma_id_private *id_priv = mc->id_priv;
4817         struct rdma_cm_event event = {};
4818         int ret = 0;
4819
4820         mutex_lock(&id_priv->handler_mutex);
4821         if (READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL ||
4822             READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING)
4823                 goto out;
4824
4825         ret = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey));
4826         if (!ret) {
4827                 cma_make_mc_event(status, id_priv, multicast, &event, mc);
4828                 ret = cma_cm_event_handler(id_priv, &event);
4829         }
4830         rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4831         WARN_ON(ret);
4832
4833 out:
4834         mutex_unlock(&id_priv->handler_mutex);
4835         return 0;
4836 }
4837
4838 static void cma_set_mgid(struct rdma_id_private *id_priv,
4839                          struct sockaddr *addr, union ib_gid *mgid)
4840 {
4841         unsigned char mc_map[MAX_ADDR_LEN];
4842         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4843         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
4844         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
4845
4846         if (cma_any_addr(addr)) {
4847                 memset(mgid, 0, sizeof *mgid);
4848         } else if ((addr->sa_family == AF_INET6) &&
4849                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
4850                                                                  0xFF10A01B)) {
4851                 /* IPv6 address is an SA assigned MGID. */
4852                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4853         } else if (addr->sa_family == AF_IB) {
4854                 memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid);
4855         } else if (addr->sa_family == AF_INET6) {
4856                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
4857                 if (id_priv->id.ps == RDMA_PS_UDP)
4858                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
4859                 *mgid = *(union ib_gid *) (mc_map + 4);
4860         } else {
4861                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
4862                 if (id_priv->id.ps == RDMA_PS_UDP)
4863                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
4864                 *mgid = *(union ib_gid *) (mc_map + 4);
4865         }
4866 }
4867
4868 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
4869                                  struct cma_multicast *mc)
4870 {
4871         struct ib_sa_mcmember_rec rec;
4872         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4873         ib_sa_comp_mask comp_mask;
4874         int ret;
4875
4876         ib_addr_get_mgid(dev_addr, &rec.mgid);
4877         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
4878                                      &rec.mgid, &rec);
4879         if (ret)
4880                 return ret;
4881
4882         if (!id_priv->qkey) {
4883                 ret = cma_set_default_qkey(id_priv);
4884                 if (ret)
4885                         return ret;
4886         }
4887
4888         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
4889         rec.qkey = cpu_to_be32(id_priv->qkey);
4890         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
4891         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
4892         rec.join_state = mc->join_state;
4893
4894         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
4895                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
4896                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
4897                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
4898                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
4899
4900         if (id_priv->id.ps == RDMA_PS_IPOIB)
4901                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
4902                              IB_SA_MCMEMBER_REC_RATE_SELECTOR |
4903                              IB_SA_MCMEMBER_REC_MTU_SELECTOR |
4904                              IB_SA_MCMEMBER_REC_MTU |
4905                              IB_SA_MCMEMBER_REC_HOP_LIMIT;
4906
4907         mc->sa_mc = ib_sa_join_multicast(&sa_client, id_priv->id.device,
4908                                          id_priv->id.port_num, &rec, comp_mask,
4909                                          GFP_KERNEL, cma_ib_mc_handler, mc);
4910         return PTR_ERR_OR_ZERO(mc->sa_mc);
4911 }
4912
4913 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
4914                               enum ib_gid_type gid_type)
4915 {
4916         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
4917         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
4918
4919         if (cma_any_addr(addr)) {
4920                 memset(mgid, 0, sizeof *mgid);
4921         } else if (addr->sa_family == AF_INET6) {
4922                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4923         } else {
4924                 mgid->raw[0] =
4925                         (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff;
4926                 mgid->raw[1] =
4927                         (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e;
4928                 mgid->raw[2] = 0;
4929                 mgid->raw[3] = 0;
4930                 mgid->raw[4] = 0;
4931                 mgid->raw[5] = 0;
4932                 mgid->raw[6] = 0;
4933                 mgid->raw[7] = 0;
4934                 mgid->raw[8] = 0;
4935                 mgid->raw[9] = 0;
4936                 mgid->raw[10] = 0xff;
4937                 mgid->raw[11] = 0xff;
4938                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
4939         }
4940 }
4941
4942 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
4943                                    struct cma_multicast *mc)
4944 {
4945         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4946         int err = 0;
4947         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
4948         struct net_device *ndev = NULL;
4949         struct ib_sa_multicast ib;
4950         enum ib_gid_type gid_type;
4951         bool send_only;
4952
4953         send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
4954
4955         if (cma_zero_addr(addr))
4956                 return -EINVAL;
4957
4958         gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
4959                    rdma_start_port(id_priv->cma_dev->device)];
4960         cma_iboe_set_mgid(addr, &ib.rec.mgid, gid_type);
4961
4962         ib.rec.pkey = cpu_to_be16(0xffff);
4963         if (dev_addr->bound_dev_if)
4964                 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4965         if (!ndev)
4966                 return -ENODEV;
4967
4968         ib.rec.rate = IB_RATE_PORT_CURRENT;
4969         ib.rec.hop_limit = 1;
4970         ib.rec.mtu = iboe_get_mtu(ndev->mtu);
4971
4972         if (addr->sa_family == AF_INET) {
4973                 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
4974                         ib.rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
4975                         if (!send_only) {
4976                                 err = cma_igmp_send(ndev, &ib.rec.mgid,
4977                                                     true);
4978                         }
4979                 }
4980         } else {
4981                 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4982                         err = -ENOTSUPP;
4983         }
4984         dev_put(ndev);
4985         if (err || !ib.rec.mtu)
4986                 return err ?: -EINVAL;
4987
4988         if (!id_priv->qkey)
4989                 cma_set_default_qkey(id_priv);
4990
4991         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
4992                     &ib.rec.port_gid);
4993         INIT_WORK(&mc->iboe_join.work, cma_iboe_join_work_handler);
4994         cma_make_mc_event(0, id_priv, &ib, &mc->iboe_join.event, mc);
4995         queue_work(cma_wq, &mc->iboe_join.work);
4996         return 0;
4997 }
4998
4999 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
5000                         u8 join_state, void *context)
5001 {
5002         struct rdma_id_private *id_priv =
5003                 container_of(id, struct rdma_id_private, id);
5004         struct cma_multicast *mc;
5005         int ret;
5006
5007         /* Not supported for kernel QPs */
5008         if (WARN_ON(id->qp))
5009                 return -EINVAL;
5010
5011         /* ULP is calling this wrong. */
5012         if (!id->device || (READ_ONCE(id_priv->state) != RDMA_CM_ADDR_BOUND &&
5013                             READ_ONCE(id_priv->state) != RDMA_CM_ADDR_RESOLVED))
5014                 return -EINVAL;
5015
5016         if (id_priv->id.qp_type != IB_QPT_UD)
5017                 return -EINVAL;
5018
5019         mc = kzalloc(sizeof(*mc), GFP_KERNEL);
5020         if (!mc)
5021                 return -ENOMEM;
5022
5023         memcpy(&mc->addr, addr, rdma_addr_size(addr));
5024         mc->context = context;
5025         mc->id_priv = id_priv;
5026         mc->join_state = join_state;
5027
5028         if (rdma_protocol_roce(id->device, id->port_num)) {
5029                 ret = cma_iboe_join_multicast(id_priv, mc);
5030                 if (ret)
5031                         goto out_err;
5032         } else if (rdma_cap_ib_mcast(id->device, id->port_num)) {
5033                 ret = cma_join_ib_multicast(id_priv, mc);
5034                 if (ret)
5035                         goto out_err;
5036         } else {
5037                 ret = -ENOSYS;
5038                 goto out_err;
5039         }
5040
5041         spin_lock(&id_priv->lock);
5042         list_add(&mc->list, &id_priv->mc_list);
5043         spin_unlock(&id_priv->lock);
5044
5045         return 0;
5046 out_err:
5047         kfree(mc);
5048         return ret;
5049 }
5050 EXPORT_SYMBOL(rdma_join_multicast);
5051
5052 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
5053 {
5054         struct rdma_id_private *id_priv;
5055         struct cma_multicast *mc;
5056
5057         id_priv = container_of(id, struct rdma_id_private, id);
5058         spin_lock_irq(&id_priv->lock);
5059         list_for_each_entry(mc, &id_priv->mc_list, list) {
5060                 if (memcmp(&mc->addr, addr, rdma_addr_size(addr)) != 0)
5061                         continue;
5062                 list_del(&mc->list);
5063                 spin_unlock_irq(&id_priv->lock);
5064
5065                 WARN_ON(id_priv->cma_dev->device != id->device);
5066                 destroy_mc(id_priv, mc);
5067                 return;
5068         }
5069         spin_unlock_irq(&id_priv->lock);
5070 }
5071 EXPORT_SYMBOL(rdma_leave_multicast);
5072
5073 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
5074 {
5075         struct rdma_dev_addr *dev_addr;
5076         struct cma_work *work;
5077
5078         dev_addr = &id_priv->id.route.addr.dev_addr;
5079
5080         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
5081             (net_eq(dev_net(ndev), dev_addr->net)) &&
5082             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
5083                 pr_info("RDMA CM addr change for ndev %s used by id %p\n",
5084                         ndev->name, &id_priv->id);
5085                 work = kzalloc(sizeof *work, GFP_KERNEL);
5086                 if (!work)
5087                         return -ENOMEM;
5088
5089                 INIT_WORK(&work->work, cma_work_handler);
5090                 work->id = id_priv;
5091                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
5092                 cma_id_get(id_priv);
5093                 queue_work(cma_wq, &work->work);
5094         }
5095
5096         return 0;
5097 }
5098
5099 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
5100                                void *ptr)
5101 {
5102         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
5103         struct cma_device *cma_dev;
5104         struct rdma_id_private *id_priv;
5105         int ret = NOTIFY_DONE;
5106
5107         if (event != NETDEV_BONDING_FAILOVER)
5108                 return NOTIFY_DONE;
5109
5110         if (!netif_is_bond_master(ndev))
5111                 return NOTIFY_DONE;
5112
5113         mutex_lock(&lock);
5114         list_for_each_entry(cma_dev, &dev_list, list)
5115                 list_for_each_entry(id_priv, &cma_dev->id_list, device_item) {
5116                         ret = cma_netdev_change(ndev, id_priv);
5117                         if (ret)
5118                                 goto out;
5119                 }
5120
5121 out:
5122         mutex_unlock(&lock);
5123         return ret;
5124 }
5125
5126 static void cma_netevent_work_handler(struct work_struct *_work)
5127 {
5128         struct rdma_id_private *id_priv =
5129                 container_of(_work, struct rdma_id_private, id.net_work);
5130         struct rdma_cm_event event = {};
5131
5132         mutex_lock(&id_priv->handler_mutex);
5133
5134         if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
5135             READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
5136                 goto out_unlock;
5137
5138         event.event = RDMA_CM_EVENT_UNREACHABLE;
5139         event.status = -ETIMEDOUT;
5140
5141         if (cma_cm_event_handler(id_priv, &event)) {
5142                 __acquire(&id_priv->handler_mutex);
5143                 id_priv->cm_id.ib = NULL;
5144                 cma_id_put(id_priv);
5145                 destroy_id_handler_unlock(id_priv);
5146                 return;
5147         }
5148
5149 out_unlock:
5150         mutex_unlock(&id_priv->handler_mutex);
5151         cma_id_put(id_priv);
5152 }
5153
5154 static int cma_netevent_callback(struct notifier_block *self,
5155                                  unsigned long event, void *ctx)
5156 {
5157         struct id_table_entry *ips_node = NULL;
5158         struct rdma_id_private *current_id;
5159         struct neighbour *neigh = ctx;
5160         unsigned long flags;
5161
5162         if (event != NETEVENT_NEIGH_UPDATE)
5163                 return NOTIFY_DONE;
5164
5165         spin_lock_irqsave(&id_table_lock, flags);
5166         if (neigh->tbl->family == AF_INET6) {
5167                 struct sockaddr_in6 neigh_sock_6;
5168
5169                 neigh_sock_6.sin6_family = AF_INET6;
5170                 neigh_sock_6.sin6_addr = *(struct in6_addr *)neigh->primary_key;
5171                 ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex,
5172                                              (struct sockaddr *)&neigh_sock_6);
5173         } else if (neigh->tbl->family == AF_INET) {
5174                 struct sockaddr_in neigh_sock_4;
5175
5176                 neigh_sock_4.sin_family = AF_INET;
5177                 neigh_sock_4.sin_addr.s_addr = *(__be32 *)(neigh->primary_key);
5178                 ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex,
5179                                              (struct sockaddr *)&neigh_sock_4);
5180         } else
5181                 goto out;
5182
5183         if (!ips_node)
5184                 goto out;
5185
5186         list_for_each_entry(current_id, &ips_node->id_list, id_list_entry) {
5187                 if (!memcmp(current_id->id.route.addr.dev_addr.dst_dev_addr,
5188                            neigh->ha, ETH_ALEN))
5189                         continue;
5190                 INIT_WORK(&current_id->id.net_work, cma_netevent_work_handler);
5191                 cma_id_get(current_id);
5192                 queue_work(cma_wq, &current_id->id.net_work);
5193         }
5194 out:
5195         spin_unlock_irqrestore(&id_table_lock, flags);
5196         return NOTIFY_DONE;
5197 }
5198
5199 static struct notifier_block cma_nb = {
5200         .notifier_call = cma_netdev_callback
5201 };
5202
5203 static struct notifier_block cma_netevent_cb = {
5204         .notifier_call = cma_netevent_callback
5205 };
5206
5207 static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
5208 {
5209         struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
5210         enum rdma_cm_state state;
5211         unsigned long flags;
5212
5213         mutex_lock(&id_priv->handler_mutex);
5214         /* Record that we want to remove the device */
5215         spin_lock_irqsave(&id_priv->lock, flags);
5216         state = id_priv->state;
5217         if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) {
5218                 spin_unlock_irqrestore(&id_priv->lock, flags);
5219                 mutex_unlock(&id_priv->handler_mutex);
5220                 cma_id_put(id_priv);
5221                 return;
5222         }
5223         id_priv->state = RDMA_CM_DEVICE_REMOVAL;
5224         spin_unlock_irqrestore(&id_priv->lock, flags);
5225
5226         if (cma_cm_event_handler(id_priv, &event)) {
5227                 /*
5228                  * At this point the ULP promises it won't call
5229                  * rdma_destroy_id() concurrently
5230                  */
5231                 cma_id_put(id_priv);
5232                 mutex_unlock(&id_priv->handler_mutex);
5233                 trace_cm_id_destroy(id_priv);
5234                 _destroy_id(id_priv, state);
5235                 return;
5236         }
5237         mutex_unlock(&id_priv->handler_mutex);
5238
5239         /*
5240          * If this races with destroy then the thread that first assigns state
5241          * to a destroying does the cancel.
5242          */
5243         cma_cancel_operation(id_priv, state);
5244         cma_id_put(id_priv);
5245 }
5246
5247 static void cma_process_remove(struct cma_device *cma_dev)
5248 {
5249         mutex_lock(&lock);
5250         while (!list_empty(&cma_dev->id_list)) {
5251                 struct rdma_id_private *id_priv = list_first_entry(
5252                         &cma_dev->id_list, struct rdma_id_private, device_item);
5253
5254                 list_del_init(&id_priv->listen_item);
5255                 list_del_init(&id_priv->device_item);
5256                 cma_id_get(id_priv);
5257                 mutex_unlock(&lock);
5258
5259                 cma_send_device_removal_put(id_priv);
5260
5261                 mutex_lock(&lock);
5262         }
5263         mutex_unlock(&lock);
5264
5265         cma_dev_put(cma_dev);
5266         wait_for_completion(&cma_dev->comp);
5267 }
5268
5269 static bool cma_supported(struct ib_device *device)
5270 {
5271         u32 i;
5272
5273         rdma_for_each_port(device, i) {
5274                 if (rdma_cap_ib_cm(device, i) || rdma_cap_iw_cm(device, i))
5275                         return true;
5276         }
5277         return false;
5278 }
5279
5280 static int cma_add_one(struct ib_device *device)
5281 {
5282         struct rdma_id_private *to_destroy;
5283         struct cma_device *cma_dev;
5284         struct rdma_id_private *id_priv;
5285         unsigned long supported_gids = 0;
5286         int ret;
5287         u32 i;
5288
5289         if (!cma_supported(device))
5290                 return -EOPNOTSUPP;
5291
5292         cma_dev = kmalloc(sizeof(*cma_dev), GFP_KERNEL);
5293         if (!cma_dev)
5294                 return -ENOMEM;
5295
5296         cma_dev->device = device;
5297         cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
5298                                             sizeof(*cma_dev->default_gid_type),
5299                                             GFP_KERNEL);
5300         if (!cma_dev->default_gid_type) {
5301                 ret = -ENOMEM;
5302                 goto free_cma_dev;
5303         }
5304
5305         cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
5306                                             sizeof(*cma_dev->default_roce_tos),
5307                                             GFP_KERNEL);
5308         if (!cma_dev->default_roce_tos) {
5309                 ret = -ENOMEM;
5310                 goto free_gid_type;
5311         }
5312
5313         rdma_for_each_port (device, i) {
5314                 supported_gids = roce_gid_type_mask_support(device, i);
5315                 WARN_ON(!supported_gids);
5316                 if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
5317                         cma_dev->default_gid_type[i - rdma_start_port(device)] =
5318                                 CMA_PREFERRED_ROCE_GID_TYPE;
5319                 else
5320                         cma_dev->default_gid_type[i - rdma_start_port(device)] =
5321                                 find_first_bit(&supported_gids, BITS_PER_LONG);
5322                 cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
5323         }
5324
5325         init_completion(&cma_dev->comp);
5326         refcount_set(&cma_dev->refcount, 1);
5327         INIT_LIST_HEAD(&cma_dev->id_list);
5328         ib_set_client_data(device, &cma_client, cma_dev);
5329
5330         mutex_lock(&lock);
5331         list_add_tail(&cma_dev->list, &dev_list);
5332         list_for_each_entry(id_priv, &listen_any_list, listen_any_item) {
5333                 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
5334                 if (ret)
5335                         goto free_listen;
5336         }
5337         mutex_unlock(&lock);
5338
5339         trace_cm_add_one(device);
5340         return 0;
5341
5342 free_listen:
5343         list_del(&cma_dev->list);
5344         mutex_unlock(&lock);
5345
5346         /* cma_process_remove() will delete to_destroy */
5347         cma_process_remove(cma_dev);
5348         kfree(cma_dev->default_roce_tos);
5349 free_gid_type:
5350         kfree(cma_dev->default_gid_type);
5351
5352 free_cma_dev:
5353         kfree(cma_dev);
5354         return ret;
5355 }
5356
5357 static void cma_remove_one(struct ib_device *device, void *client_data)
5358 {
5359         struct cma_device *cma_dev = client_data;
5360
5361         trace_cm_remove_one(device);
5362
5363         mutex_lock(&lock);
5364         list_del(&cma_dev->list);
5365         mutex_unlock(&lock);
5366
5367         cma_process_remove(cma_dev);
5368         kfree(cma_dev->default_roce_tos);
5369         kfree(cma_dev->default_gid_type);
5370         kfree(cma_dev);
5371 }
5372
5373 static int cma_init_net(struct net *net)
5374 {
5375         struct cma_pernet *pernet = cma_pernet(net);
5376
5377         xa_init(&pernet->tcp_ps);
5378         xa_init(&pernet->udp_ps);
5379         xa_init(&pernet->ipoib_ps);
5380         xa_init(&pernet->ib_ps);
5381
5382         return 0;
5383 }
5384
5385 static void cma_exit_net(struct net *net)
5386 {
5387         struct cma_pernet *pernet = cma_pernet(net);
5388
5389         WARN_ON(!xa_empty(&pernet->tcp_ps));
5390         WARN_ON(!xa_empty(&pernet->udp_ps));
5391         WARN_ON(!xa_empty(&pernet->ipoib_ps));
5392         WARN_ON(!xa_empty(&pernet->ib_ps));
5393 }
5394
5395 static struct pernet_operations cma_pernet_operations = {
5396         .init = cma_init_net,
5397         .exit = cma_exit_net,
5398         .id = &cma_pernet_id,
5399         .size = sizeof(struct cma_pernet),
5400 };
5401
5402 static int __init cma_init(void)
5403 {
5404         int ret;
5405
5406         /*
5407          * There is a rare lock ordering dependency in cma_netdev_callback()
5408          * that only happens when bonding is enabled. Teach lockdep that rtnl
5409          * must never be nested under lock so it can find these without having
5410          * to test with bonding.
5411          */
5412         if (IS_ENABLED(CONFIG_LOCKDEP)) {
5413                 rtnl_lock();
5414                 mutex_lock(&lock);
5415                 mutex_unlock(&lock);
5416                 rtnl_unlock();
5417         }
5418
5419         cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
5420         if (!cma_wq)
5421                 return -ENOMEM;
5422
5423         ret = register_pernet_subsys(&cma_pernet_operations);
5424         if (ret)
5425                 goto err_wq;
5426
5427         ib_sa_register_client(&sa_client);
5428         register_netdevice_notifier(&cma_nb);
5429         register_netevent_notifier(&cma_netevent_cb);
5430
5431         ret = ib_register_client(&cma_client);
5432         if (ret)
5433                 goto err;
5434
5435         ret = cma_configfs_init();
5436         if (ret)
5437                 goto err_ib;
5438
5439         return 0;
5440
5441 err_ib:
5442         ib_unregister_client(&cma_client);
5443 err:
5444         unregister_netevent_notifier(&cma_netevent_cb);
5445         unregister_netdevice_notifier(&cma_nb);
5446         ib_sa_unregister_client(&sa_client);
5447         unregister_pernet_subsys(&cma_pernet_operations);
5448 err_wq:
5449         destroy_workqueue(cma_wq);
5450         return ret;
5451 }
5452
5453 static void __exit cma_cleanup(void)
5454 {
5455         cma_configfs_exit();
5456         ib_unregister_client(&cma_client);
5457         unregister_netevent_notifier(&cma_netevent_cb);
5458         unregister_netdevice_notifier(&cma_nb);
5459         ib_sa_unregister_client(&sa_client);
5460         unregister_pernet_subsys(&cma_pernet_operations);
5461         destroy_workqueue(cma_wq);
5462 }
5463
5464 module_init(cma_init);
5465 module_exit(cma_cleanup);