drivers/infiniband/core/cm.c

   1 /*
   2  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
   3  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   4  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
   5  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/device.h>
  39 #include <linux/module.h>
  40 #include <linux/err.h>
  41 #include <linux/idr.h>
  42 #include <linux/interrupt.h>
  43 #include <linux/random.h>
  44 #include <linux/rbtree.h>
  45 #include <linux/spinlock.h>
  46 #include <linux/slab.h>
  47 #include <linux/sysfs.h>
  48 #include <linux/workqueue.h>
  49 #include <linux/kdev_t.h>
  50 #include <linux/etherdevice.h>
  51
  52 #include <rdma/ib_cache.h>
  53 #include <rdma/ib_cm.h>
  54 #include "cm_msgs.h"
  55
  56 MODULE_AUTHOR("Sean Hefty");
  57 MODULE_DESCRIPTION("InfiniBand CM");
  58 MODULE_LICENSE("Dual BSD/GPL");
  59
  60 static const char * const ibcm_rej_reason_strs[] = {
  61         [IB_CM_REJ_NO_QP]                       = "no QP",
  62         [IB_CM_REJ_NO_EEC]                      = "no EEC",
  63         [IB_CM_REJ_NO_RESOURCES]                = "no resources",
  64         [IB_CM_REJ_TIMEOUT]                     = "timeout",
  65         [IB_CM_REJ_UNSUPPORTED]                 = "unsupported",
  66         [IB_CM_REJ_INVALID_COMM_ID]             = "invalid comm ID",
  67         [IB_CM_REJ_INVALID_COMM_INSTANCE]       = "invalid comm instance",
  68         [IB_CM_REJ_INVALID_SERVICE_ID]          = "invalid service ID",
  69         [IB_CM_REJ_INVALID_TRANSPORT_TYPE]      = "invalid transport type",
  70         [IB_CM_REJ_STALE_CONN]                  = "stale conn",
  71         [IB_CM_REJ_RDC_NOT_EXIST]               = "RDC not exist",
  72         [IB_CM_REJ_INVALID_GID]                 = "invalid GID",
  73         [IB_CM_REJ_INVALID_LID]                 = "invalid LID",
  74         [IB_CM_REJ_INVALID_SL]                  = "invalid SL",
  75         [IB_CM_REJ_INVALID_TRAFFIC_CLASS]       = "invalid traffic class",
  76         [IB_CM_REJ_INVALID_HOP_LIMIT]           = "invalid hop limit",
  77         [IB_CM_REJ_INVALID_PACKET_RATE]         = "invalid packet rate",
  78         [IB_CM_REJ_INVALID_ALT_GID]             = "invalid alt GID",
  79         [IB_CM_REJ_INVALID_ALT_LID]             = "invalid alt LID",
  80         [IB_CM_REJ_INVALID_ALT_SL]              = "invalid alt SL",
  81         [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]   = "invalid alt traffic class",
  82         [IB_CM_REJ_INVALID_ALT_HOP_LIMIT]       = "invalid alt hop limit",
  83         [IB_CM_REJ_INVALID_ALT_PACKET_RATE]     = "invalid alt packet rate",
  84         [IB_CM_REJ_PORT_CM_REDIRECT]            = "port CM redirect",
  85         [IB_CM_REJ_PORT_REDIRECT]               = "port redirect",
  86         [IB_CM_REJ_INVALID_MTU]                 = "invalid MTU",
  87         [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
  88         [IB_CM_REJ_CONSUMER_DEFINED]            = "consumer defined",
  89         [IB_CM_REJ_INVALID_RNR_RETRY]           = "invalid RNR retry",
  90         [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]     = "duplicate local comm ID",
  91         [IB_CM_REJ_INVALID_CLASS_VERSION]       = "invalid class version",
  92         [IB_CM_REJ_INVALID_FLOW_LABEL]          = "invalid flow label",
  93         [IB_CM_REJ_INVALID_ALT_FLOW_LABEL]      = "invalid alt flow label",
  94 };
  95
  96 const char *__attribute_const__ ibcm_reject_msg(int reason)
  97 {
  98         size_t index = reason;
  99
 100         if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
 101             ibcm_rej_reason_strs[index])
 102                 return ibcm_rej_reason_strs[index];
 103         else
 104                 return "unrecognized reason";
 105 }
 106 EXPORT_SYMBOL(ibcm_reject_msg);
 107
 108 static void cm_add_one(struct ib_device *device);
 109 static void cm_remove_one(struct ib_device *device, void *client_data);
 110
 111 static struct ib_client cm_client = {
 112         .name   = "cm",
 113         .add    = cm_add_one,
 114         .remove = cm_remove_one
 115 };
 116
 117 static struct ib_cm {
 118         spinlock_t lock;
 119         struct list_head device_list;
 120         rwlock_t device_lock;
 121         struct rb_root listen_service_table;
 122         u64 listen_service_id;
 123         /* struct rb_root peer_service_table; todo: fix peer to peer */
 124         struct rb_root remote_qp_table;
 125         struct rb_root remote_id_table;
 126         struct rb_root remote_sidr_table;
 127         struct idr local_id_table;
 128         __be32 random_id_operand;
 129         struct list_head timewait_list;
 130         struct workqueue_struct *wq;
 131         /* Sync on cm change port state */
 132         spinlock_t state_lock;
 133 } cm;
 134
 135 /* Counter indexes ordered by attribute ID */
 136 enum {
 137         CM_REQ_COUNTER,
 138         CM_MRA_COUNTER,
 139         CM_REJ_COUNTER,
 140         CM_REP_COUNTER,
 141         CM_RTU_COUNTER,
 142         CM_DREQ_COUNTER,
 143         CM_DREP_COUNTER,
 144         CM_SIDR_REQ_COUNTER,
 145         CM_SIDR_REP_COUNTER,
 146         CM_LAP_COUNTER,
 147         CM_APR_COUNTER,
 148         CM_ATTR_COUNT,
 149         CM_ATTR_ID_OFFSET = 0x0010,
 150 };
 151
 152 enum {
 153         CM_XMIT,
 154         CM_XMIT_RETRIES,
 155         CM_RECV,
 156         CM_RECV_DUPLICATES,
 157         CM_COUNTER_GROUPS
 158 };
 159
 160 static char const counter_group_names[CM_COUNTER_GROUPS]
 161                                      [sizeof("cm_rx_duplicates")] = {
 162         "cm_tx_msgs", "cm_tx_retries",
 163         "cm_rx_msgs", "cm_rx_duplicates"
 164 };
 165
 166 struct cm_counter_group {
 167         struct kobject obj;
 168         atomic_long_t counter[CM_ATTR_COUNT];
 169 };
 170
 171 struct cm_counter_attribute {
 172         struct attribute attr;
 173         int index;
 174 };
 175
 176 #define CM_COUNTER_ATTR(_name, _index) \
 177 struct cm_counter_attribute cm_##_name##_counter_attr = { \
 178         .attr = { .name = __stringify(_name), .mode = 0444 }, \
 179         .index = _index \
 180 }
 181
 182 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
 183 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
 184 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
 185 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
 186 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
 187 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
 188 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
 189 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
 190 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
 191 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
 192 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
 193
 194 static struct attribute *cm_counter_default_attrs[] = {
 195         &cm_req_counter_attr.attr,
 196         &cm_mra_counter_attr.attr,
 197         &cm_rej_counter_attr.attr,
 198         &cm_rep_counter_attr.attr,
 199         &cm_rtu_counter_attr.attr,
 200         &cm_dreq_counter_attr.attr,
 201         &cm_drep_counter_attr.attr,
 202         &cm_sidr_req_counter_attr.attr,
 203         &cm_sidr_rep_counter_attr.attr,
 204         &cm_lap_counter_attr.attr,
 205         &cm_apr_counter_attr.attr,
 206         NULL
 207 };
 208
 209 struct cm_port {
 210         struct cm_device *cm_dev;
 211         struct ib_mad_agent *mad_agent;
 212         struct kobject port_obj;
 213         u8 port_num;
 214         struct list_head cm_priv_prim_list;
 215         struct list_head cm_priv_altr_list;
 216         struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
 217 };
 218
 219 struct cm_device {
 220         struct list_head list;
 221         struct ib_device *ib_device;
 222         struct device *device;
 223         u8 ack_delay;
 224         int going_down;
 225         struct cm_port *port[0];
 226 };
 227
 228 struct cm_av {
 229         struct cm_port *port;
 230         union ib_gid dgid;
 231         struct rdma_ah_attr ah_attr;
 232         u16 pkey_index;
 233         u8 timeout;
 234 };
 235
 236 struct cm_work {
 237         struct delayed_work work;
 238         struct list_head list;
 239         struct cm_port *port;
 240         struct ib_mad_recv_wc *mad_recv_wc;     /* Received MADs */
 241         __be32 local_id;                        /* Established / timewait */
 242         __be32 remote_id;
 243         struct ib_cm_event cm_event;
 244         struct sa_path_rec path[0];
 245 };
 246
 247 struct cm_timewait_info {
 248         struct cm_work work;                    /* Must be first. */
 249         struct list_head list;
 250         struct rb_node remote_qp_node;
 251         struct rb_node remote_id_node;
 252         __be64 remote_ca_guid;
 253         __be32 remote_qpn;
 254         u8 inserted_remote_qp;
 255         u8 inserted_remote_id;
 256 };
 257
 258 struct cm_id_private {
 259         struct ib_cm_id id;
 260
 261         struct rb_node service_node;
 262         struct rb_node sidr_id_node;
 263         spinlock_t lock;        /* Do not acquire inside cm.lock */
 264         struct completion comp;
 265         atomic_t refcount;
 266         /* Number of clients sharing this ib_cm_id. Only valid for listeners.
 267          * Protected by the cm.lock spinlock. */
 268         int listen_sharecount;
 269
 270         struct ib_mad_send_buf *msg;
 271         struct cm_timewait_info *timewait_info;
 272         /* todo: use alternate port on send failure */
 273         struct cm_av av;
 274         struct cm_av alt_av;
 275
 276         void *private_data;
 277         __be64 tid;
 278         __be32 local_qpn;
 279         __be32 remote_qpn;
 280         enum ib_qp_type qp_type;
 281         __be32 sq_psn;
 282         __be32 rq_psn;
 283         int timeout_ms;
 284         enum ib_mtu path_mtu;
 285         __be16 pkey;
 286         u8 private_data_len;
 287         u8 max_cm_retries;
 288         u8 peer_to_peer;
 289         u8 responder_resources;
 290         u8 initiator_depth;
 291         u8 retry_count;
 292         u8 rnr_retry_count;
 293         u8 service_timeout;
 294         u8 target_ack_delay;
 295
 296         struct list_head prim_list;
 297         struct list_head altr_list;
 298         /* Indicates that the send port mad is registered and av is set */
 299         int prim_send_port_not_ready;
 300         int altr_send_port_not_ready;
 301
 302         struct list_head work_list;
 303         atomic_t work_count;
 304 };
 305
 306 static void cm_work_handler(struct work_struct *work);
 307
 308 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
 309 {
 310         if (atomic_dec_and_test(&cm_id_priv->refcount))
 311                 complete(&cm_id_priv->comp);
 312 }
 313
 314 static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
 315                         struct ib_mad_send_buf **msg)
 316 {
 317         struct ib_mad_agent *mad_agent;
 318         struct ib_mad_send_buf *m;
 319         struct ib_ah *ah;
 320         struct cm_av *av;
 321         unsigned long flags, flags2;
 322         int ret = 0;
 323
 324         /* don't let the port to be released till the agent is down */
 325         spin_lock_irqsave(&cm.state_lock, flags2);
 326         spin_lock_irqsave(&cm.lock, flags);
 327         if (!cm_id_priv->prim_send_port_not_ready)
 328                 av = &cm_id_priv->av;
 329         else if (!cm_id_priv->altr_send_port_not_ready &&
 330                  (cm_id_priv->alt_av.port))
 331                 av = &cm_id_priv->alt_av;
 332         else {
 333                 pr_info("%s: not valid CM id\n", __func__);
 334                 ret = -ENODEV;
 335                 spin_unlock_irqrestore(&cm.lock, flags);
 336                 goto out;
 337         }
 338         spin_unlock_irqrestore(&cm.lock, flags);
 339         /* Make sure the port haven't released the mad yet */
 340         mad_agent = cm_id_priv->av.port->mad_agent;
 341         if (!mad_agent) {
 342                 pr_info("%s: not a valid MAD agent\n", __func__);
 343                 ret = -ENODEV;
 344                 goto out;
 345         }
 346         ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr);
 347         if (IS_ERR(ah)) {
 348                 ret = PTR_ERR(ah);
 349                 goto out;
 350         }
 351
 352         m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
 353                                av->pkey_index,
 354                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 355                                GFP_ATOMIC,
 356                                IB_MGMT_BASE_VERSION);
 357         if (IS_ERR(m)) {
 358                 rdma_destroy_ah(ah);
 359                 ret = PTR_ERR(m);
 360                 goto out;
 361         }
 362
 363         /* Timeout set by caller if response is expected. */
 364         m->ah = ah;
 365         m->retries = cm_id_priv->max_cm_retries;
 366
 367         atomic_inc(&cm_id_priv->refcount);
 368         m->context[0] = cm_id_priv;
 369         *msg = m;
 370
 371 out:
 372         spin_unlock_irqrestore(&cm.state_lock, flags2);
 373         return ret;
 374 }
 375
 376 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
 377                                                            struct ib_mad_recv_wc *mad_recv_wc)
 378 {
 379         return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
 380                                   0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 381                                   GFP_ATOMIC,
 382                                   IB_MGMT_BASE_VERSION);
 383 }
 384
 385 static int cm_create_response_msg_ah(struct cm_port *port,
 386                                      struct ib_mad_recv_wc *mad_recv_wc,
 387                                      struct ib_mad_send_buf *msg)
 388 {
 389         struct ib_ah *ah;
 390
 391         ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
 392                                   mad_recv_wc->recv_buf.grh, port->port_num);
 393         if (IS_ERR(ah))
 394                 return PTR_ERR(ah);
 395
 396         msg->ah = ah;
 397         return 0;
 398 }
 399
 400 static void cm_free_msg(struct ib_mad_send_buf *msg)
 401 {
 402         if (msg->ah)
 403                 rdma_destroy_ah(msg->ah);
 404         if (msg->context[0])
 405                 cm_deref_id(msg->context[0]);
 406         ib_free_send_mad(msg);
 407 }
 408
 409 static int cm_alloc_response_msg(struct cm_port *port,
 410                                  struct ib_mad_recv_wc *mad_recv_wc,
 411                                  struct ib_mad_send_buf **msg)
 412 {
 413         struct ib_mad_send_buf *m;
 414         int ret;
 415
 416         m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
 417         if (IS_ERR(m))
 418                 return PTR_ERR(m);
 419
 420         ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
 421         if (ret) {
 422                 cm_free_msg(m);
 423                 return ret;
 424         }
 425
 426         *msg = m;
 427         return 0;
 428 }
 429
 430 static void * cm_copy_private_data(const void *private_data,
 431                                    u8 private_data_len)
 432 {
 433         void *data;
 434
 435         if (!private_data || !private_data_len)
 436                 return NULL;
 437
 438         data = kmemdup(private_data, private_data_len, GFP_KERNEL);
 439         if (!data)
 440                 return ERR_PTR(-ENOMEM);
 441
 442         return data;
 443 }
 444
 445 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 446                                  void *private_data, u8 private_data_len)
 447 {
 448         if (cm_id_priv->private_data && cm_id_priv->private_data_len)
 449                 kfree(cm_id_priv->private_data);
 450
 451         cm_id_priv->private_data = private_data;
 452         cm_id_priv->private_data_len = private_data_len;
 453 }
 454
 455 static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
 456                               struct ib_grh *grh, struct cm_av *av)
 457 {
 458         struct rdma_ah_attr new_ah_attr;
 459         int ret;
 460
 461         av->port = port;
 462         av->pkey_index = wc->pkey_index;
 463
 464         /*
 465          * av->ah_attr might be initialized based on past wc during incoming
 466          * connect request or while sending out connect request. So initialize
 467          * a new ah_attr on stack. If initialization fails, old ah_attr is
 468          * used for sending any responses. If initialization is successful,
 469          * than new ah_attr is used by overwriting old one.
 470          */
 471         ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
 472                                       port->port_num, wc,
 473                                       grh, &new_ah_attr);
 474         if (ret)
 475                 return ret;
 476
 477         memcpy(&av->ah_attr, &new_ah_attr, sizeof(new_ah_attr));
 478         return 0;
 479 }
 480
 481 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 482                                    struct ib_grh *grh, struct cm_av *av)
 483 {
 484         av->port = port;
 485         av->pkey_index = wc->pkey_index;
 486         return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
 487                                        port->port_num, wc,
 488                                        grh, &av->ah_attr);
 489 }
 490
 491 static int add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,
 492                                   struct cm_av *av,
 493                                   struct cm_port *port)
 494 {
 495         unsigned long flags;
 496         int ret = 0;
 497
 498         spin_lock_irqsave(&cm.lock, flags);
 499
 500         if (&cm_id_priv->av == av)
 501                 list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
 502         else if (&cm_id_priv->alt_av == av)
 503                 list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
 504         else
 505                 ret = -EINVAL;
 506
 507         spin_unlock_irqrestore(&cm.lock, flags);
 508         return ret;
 509 }
 510
 511 static struct cm_port *get_cm_port_from_path(struct sa_path_rec *path)
 512 {
 513         struct cm_device *cm_dev;
 514         struct cm_port *port = NULL;
 515         unsigned long flags;
 516         u8 p;
 517         struct net_device *ndev = ib_get_ndev_from_path(path);
 518
 519         read_lock_irqsave(&cm.device_lock, flags);
 520         list_for_each_entry(cm_dev, &cm.device_list, list) {
 521                 if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid,
 522                                         sa_conv_pathrec_to_gid_type(path),
 523                                         ndev, &p, NULL)) {
 524                         port = cm_dev->port[p - 1];
 525                         break;
 526                 }
 527         }
 528         read_unlock_irqrestore(&cm.device_lock, flags);
 529
 530         if (ndev)
 531                 dev_put(ndev);
 532         return port;
 533 }
 534
 535 static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av,
 536                               struct cm_id_private *cm_id_priv)
 537 {
 538         struct rdma_ah_attr new_ah_attr;
 539         struct cm_device *cm_dev;
 540         struct cm_port *port;
 541         int ret;
 542
 543         port = get_cm_port_from_path(path);
 544         if (!port)
 545                 return -EINVAL;
 546         cm_dev = port->cm_dev;
 547
 548         ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
 549                                   be16_to_cpu(path->pkey), &av->pkey_index);
 550         if (ret)
 551                 return ret;
 552
 553         av->port = port;
 554
 555         /*
 556          * av->ah_attr might be initialized based on wc or during
 557          * request processing time. So initialize a new ah_attr on stack.
 558          * If initialization fails, old ah_attr is used for sending any
 559          * responses. If initialization is successful, than new ah_attr
 560          * is used by overwriting the old one.
 561          */
 562         ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
 563                                         &new_ah_attr);
 564         if (ret)
 565                 return ret;
 566
 567         av->timeout = path->packet_life_time + 1;
 568
 569         ret = add_cm_id_to_port_list(cm_id_priv, av, port);
 570         if (ret)
 571                 return ret;
 572         memcpy(&av->ah_attr, &new_ah_attr, sizeof(new_ah_attr));
 573         return 0;
 574 }
 575
 576 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
 577 {
 578         unsigned long flags;
 579         int id;
 580
 581         idr_preload(GFP_KERNEL);
 582         spin_lock_irqsave(&cm.lock, flags);
 583
 584         id = idr_alloc_cyclic(&cm.local_id_table, cm_id_priv, 0, 0, GFP_NOWAIT);
 585
 586         spin_unlock_irqrestore(&cm.lock, flags);
 587         idr_preload_end();
 588
 589         cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
 590         return id < 0 ? id : 0;
 591 }
 592
 593 static void cm_free_id(__be32 local_id)
 594 {
 595         spin_lock_irq(&cm.lock);
 596         idr_remove(&cm.local_id_table,
 597                    (__force int) (local_id ^ cm.random_id_operand));
 598         spin_unlock_irq(&cm.lock);
 599 }
 600
 601 static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
 602 {
 603         struct cm_id_private *cm_id_priv;
 604
 605         cm_id_priv = idr_find(&cm.local_id_table,
 606                               (__force int) (local_id ^ cm.random_id_operand));
 607         if (cm_id_priv) {
 608                 if (cm_id_priv->id.remote_id == remote_id)
 609                         atomic_inc(&cm_id_priv->refcount);
 610                 else
 611                         cm_id_priv = NULL;
 612         }
 613
 614         return cm_id_priv;
 615 }
 616
 617 static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id)
 618 {
 619         struct cm_id_private *cm_id_priv;
 620
 621         spin_lock_irq(&cm.lock);
 622         cm_id_priv = cm_get_id(local_id, remote_id);
 623         spin_unlock_irq(&cm.lock);
 624
 625         return cm_id_priv;
 626 }
 627
 628 /*
 629  * Trivial helpers to strip endian annotation and compare; the
 630  * endianness doesn't actually matter since we just need a stable
 631  * order for the RB tree.
 632  */
 633 static int be32_lt(__be32 a, __be32 b)
 634 {
 635         return (__force u32) a < (__force u32) b;
 636 }
 637
 638 static int be32_gt(__be32 a, __be32 b)
 639 {
 640         return (__force u32) a > (__force u32) b;
 641 }
 642
 643 static int be64_lt(__be64 a, __be64 b)
 644 {
 645         return (__force u64) a < (__force u64) b;
 646 }
 647
 648 static int be64_gt(__be64 a, __be64 b)
 649 {
 650         return (__force u64) a > (__force u64) b;
 651 }
 652
 653 static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)
 654 {
 655         struct rb_node **link = &cm.listen_service_table.rb_node;
 656         struct rb_node *parent = NULL;
 657         struct cm_id_private *cur_cm_id_priv;
 658         __be64 service_id = cm_id_priv->id.service_id;
 659         __be64 service_mask = cm_id_priv->id.service_mask;
 660
 661         while (*link) {
 662                 parent = *link;
 663                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 664                                           service_node);
 665                 if ((cur_cm_id_priv->id.service_mask & service_id) ==
 666                     (service_mask & cur_cm_id_priv->id.service_id) &&
 667                     (cm_id_priv->id.device == cur_cm_id_priv->id.device))
 668                         return cur_cm_id_priv;
 669
 670                 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
 671                         link = &(*link)->rb_left;
 672                 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
 673                         link = &(*link)->rb_right;
 674                 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
 675                         link = &(*link)->rb_left;
 676                 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
 677                         link = &(*link)->rb_right;
 678                 else
 679                         link = &(*link)->rb_right;
 680         }
 681         rb_link_node(&cm_id_priv->service_node, parent, link);
 682         rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
 683         return NULL;
 684 }
 685
 686 static struct cm_id_private * cm_find_listen(struct ib_device *device,
 687                                              __be64 service_id)
 688 {
 689         struct rb_node *node = cm.listen_service_table.rb_node;
 690         struct cm_id_private *cm_id_priv;
 691
 692         while (node) {
 693                 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
 694                 if ((cm_id_priv->id.service_mask & service_id) ==
 695                      cm_id_priv->id.service_id &&
 696                     (cm_id_priv->id.device == device))
 697                         return cm_id_priv;
 698
 699                 if (device < cm_id_priv->id.device)
 700                         node = node->rb_left;
 701                 else if (device > cm_id_priv->id.device)
 702                         node = node->rb_right;
 703                 else if (be64_lt(service_id, cm_id_priv->id.service_id))
 704                         node = node->rb_left;
 705                 else if (be64_gt(service_id, cm_id_priv->id.service_id))
 706                         node = node->rb_right;
 707                 else
 708                         node = node->rb_right;
 709         }
 710         return NULL;
 711 }
 712
 713 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
 714                                                      *timewait_info)
 715 {
 716         struct rb_node **link = &cm.remote_id_table.rb_node;
 717         struct rb_node *parent = NULL;
 718         struct cm_timewait_info *cur_timewait_info;
 719         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 720         __be32 remote_id = timewait_info->work.remote_id;
 721
 722         while (*link) {
 723                 parent = *link;
 724                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 725                                              remote_id_node);
 726                 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
 727                         link = &(*link)->rb_left;
 728                 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
 729                         link = &(*link)->rb_right;
 730                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 731                         link = &(*link)->rb_left;
 732                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 733                         link = &(*link)->rb_right;
 734                 else
 735                         return cur_timewait_info;
 736         }
 737         timewait_info->inserted_remote_id = 1;
 738         rb_link_node(&timewait_info->remote_id_node, parent, link);
 739         rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
 740         return NULL;
 741 }
 742
 743 static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
 744                                                    __be32 remote_id)
 745 {
 746         struct rb_node *node = cm.remote_id_table.rb_node;
 747         struct cm_timewait_info *timewait_info;
 748
 749         while (node) {
 750                 timewait_info = rb_entry(node, struct cm_timewait_info,
 751                                          remote_id_node);
 752                 if (be32_lt(remote_id, timewait_info->work.remote_id))
 753                         node = node->rb_left;
 754                 else if (be32_gt(remote_id, timewait_info->work.remote_id))
 755                         node = node->rb_right;
 756                 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
 757                         node = node->rb_left;
 758                 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
 759                         node = node->rb_right;
 760                 else
 761                         return timewait_info;
 762         }
 763         return NULL;
 764 }
 765
 766 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
 767                                                       *timewait_info)
 768 {
 769         struct rb_node **link = &cm.remote_qp_table.rb_node;
 770         struct rb_node *parent = NULL;
 771         struct cm_timewait_info *cur_timewait_info;
 772         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 773         __be32 remote_qpn = timewait_info->remote_qpn;
 774
 775         while (*link) {
 776                 parent = *link;
 777                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 778                                              remote_qp_node);
 779                 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
 780                         link = &(*link)->rb_left;
 781                 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
 782                         link = &(*link)->rb_right;
 783                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 784                         link = &(*link)->rb_left;
 785                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 786                         link = &(*link)->rb_right;
 787                 else
 788                         return cur_timewait_info;
 789         }
 790         timewait_info->inserted_remote_qp = 1;
 791         rb_link_node(&timewait_info->remote_qp_node, parent, link);
 792         rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 793         return NULL;
 794 }
 795
 796 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
 797                                                     *cm_id_priv)
 798 {
 799         struct rb_node **link = &cm.remote_sidr_table.rb_node;
 800         struct rb_node *parent = NULL;
 801         struct cm_id_private *cur_cm_id_priv;
 802         union ib_gid *port_gid = &cm_id_priv->av.dgid;
 803         __be32 remote_id = cm_id_priv->id.remote_id;
 804
 805         while (*link) {
 806                 parent = *link;
 807                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 808                                           sidr_id_node);
 809                 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
 810                         link = &(*link)->rb_left;
 811                 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
 812                         link = &(*link)->rb_right;
 813                 else {
 814                         int cmp;
 815                         cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
 816                                      sizeof *port_gid);
 817                         if (cmp < 0)
 818                                 link = &(*link)->rb_left;
 819                         else if (cmp > 0)
 820                                 link = &(*link)->rb_right;
 821                         else
 822                                 return cur_cm_id_priv;
 823                 }
 824         }
 825         rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
 826         rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
 827         return NULL;
 828 }
 829
 830 static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,
 831                                enum ib_cm_sidr_status status)
 832 {
 833         struct ib_cm_sidr_rep_param param;
 834
 835         memset(&param, 0, sizeof param);
 836         param.status = status;
 837         ib_send_cm_sidr_rep(&cm_id_priv->id, &param);
 838 }
 839
 840 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
 841                                  ib_cm_handler cm_handler,
 842                                  void *context)
 843 {
 844         struct cm_id_private *cm_id_priv;
 845         int ret;
 846
 847         cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
 848         if (!cm_id_priv)
 849                 return ERR_PTR(-ENOMEM);
 850
 851         cm_id_priv->id.state = IB_CM_IDLE;
 852         cm_id_priv->id.device = device;
 853         cm_id_priv->id.cm_handler = cm_handler;
 854         cm_id_priv->id.context = context;
 855         cm_id_priv->id.remote_cm_qpn = 1;
 856         ret = cm_alloc_id(cm_id_priv);
 857         if (ret)
 858                 goto error;
 859
 860         spin_lock_init(&cm_id_priv->lock);
 861         init_completion(&cm_id_priv->comp);
 862         INIT_LIST_HEAD(&cm_id_priv->work_list);
 863         INIT_LIST_HEAD(&cm_id_priv->prim_list);
 864         INIT_LIST_HEAD(&cm_id_priv->altr_list);
 865         atomic_set(&cm_id_priv->work_count, -1);
 866         atomic_set(&cm_id_priv->refcount, 1);
 867         return &cm_id_priv->id;
 868
 869 error:
 870         kfree(cm_id_priv);
 871         return ERR_PTR(-ENOMEM);
 872 }
 873 EXPORT_SYMBOL(ib_create_cm_id);
 874
 875 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
 876 {
 877         struct cm_work *work;
 878
 879         if (list_empty(&cm_id_priv->work_list))
 880                 return NULL;
 881
 882         work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
 883         list_del(&work->list);
 884         return work;
 885 }
 886
 887 static void cm_free_work(struct cm_work *work)
 888 {
 889         if (work->mad_recv_wc)
 890                 ib_free_recv_mad(work->mad_recv_wc);
 891         kfree(work);
 892 }
 893
 894 static inline int cm_convert_to_ms(int iba_time)
 895 {
 896         /* approximate conversion to ms from 4.096us x 2^iba_time */
 897         return 1 << max(iba_time - 8, 0);
 898 }
 899
 900 /*
 901  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
 902  * Because of how ack_timeout is stored, adding one doubles the timeout.
 903  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
 904  * increment it (round up) only if the other is within 50%.
 905  */
 906 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
 907 {
 908         int ack_timeout = packet_life_time + 1;
 909
 910         if (ack_timeout >= ca_ack_delay)
 911                 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
 912         else
 913                 ack_timeout = ca_ack_delay +
 914                               (ack_timeout >= (ca_ack_delay - 1));
 915
 916         return min(31, ack_timeout);
 917 }
 918
 919 static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
 920 {
 921         if (timewait_info->inserted_remote_id) {
 922                 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
 923                 timewait_info->inserted_remote_id = 0;
 924         }
 925
 926         if (timewait_info->inserted_remote_qp) {
 927                 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 928                 timewait_info->inserted_remote_qp = 0;
 929         }
 930 }
 931
 932 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
 933 {
 934         struct cm_timewait_info *timewait_info;
 935
 936         timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
 937         if (!timewait_info)
 938                 return ERR_PTR(-ENOMEM);
 939
 940         timewait_info->work.local_id = local_id;
 941         INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
 942         timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
 943         return timewait_info;
 944 }
 945
 946 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 947 {
 948         int wait_time;
 949         unsigned long flags;
 950         struct cm_device *cm_dev;
 951
 952         cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
 953         if (!cm_dev)
 954                 return;
 955
 956         spin_lock_irqsave(&cm.lock, flags);
 957         cm_cleanup_timewait(cm_id_priv->timewait_info);
 958         list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
 959         spin_unlock_irqrestore(&cm.lock, flags);
 960
 961         /*
 962          * The cm_id could be destroyed by the user before we exit timewait.
 963          * To protect against this, we search for the cm_id after exiting
 964          * timewait before notifying the user that we've exited timewait.
 965          */
 966         cm_id_priv->id.state = IB_CM_TIMEWAIT;
 967         wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
 968
 969         /* Check if the device started its remove_one */
 970         spin_lock_irqsave(&cm.lock, flags);
 971         if (!cm_dev->going_down)
 972                 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
 973                                    msecs_to_jiffies(wait_time));
 974         spin_unlock_irqrestore(&cm.lock, flags);
 975
 976         cm_id_priv->timewait_info = NULL;
 977 }
 978
 979 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
 980 {
 981         unsigned long flags;
 982
 983         cm_id_priv->id.state = IB_CM_IDLE;
 984         if (cm_id_priv->timewait_info) {
 985                 spin_lock_irqsave(&cm.lock, flags);
 986                 cm_cleanup_timewait(cm_id_priv->timewait_info);
 987                 spin_unlock_irqrestore(&cm.lock, flags);
 988                 kfree(cm_id_priv->timewait_info);
 989                 cm_id_priv->timewait_info = NULL;
 990         }
 991 }
 992
 993 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
 994 {
 995         struct cm_id_private *cm_id_priv;
 996         struct cm_work *work;
 997
 998         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
 999 retest:
1000         spin_lock_irq(&cm_id_priv->lock);
1001         switch (cm_id->state) {
1002         case IB_CM_LISTEN:
1003                 spin_unlock_irq(&cm_id_priv->lock);
1004
1005                 spin_lock_irq(&cm.lock);
1006                 if (--cm_id_priv->listen_sharecount > 0) {
1007                         /* The id is still shared. */
1008                         cm_deref_id(cm_id_priv);
1009                         spin_unlock_irq(&cm.lock);
1010                         return;
1011                 }
1012                 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1013                 spin_unlock_irq(&cm.lock);
1014                 break;
1015         case IB_CM_SIDR_REQ_SENT:
1016                 cm_id->state = IB_CM_IDLE;
1017                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1018                 spin_unlock_irq(&cm_id_priv->lock);
1019                 break;
1020         case IB_CM_SIDR_REQ_RCVD:
1021                 spin_unlock_irq(&cm_id_priv->lock);
1022                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);
1023                 spin_lock_irq(&cm.lock);
1024                 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1025                         rb_erase(&cm_id_priv->sidr_id_node,
1026                                  &cm.remote_sidr_table);
1027                 spin_unlock_irq(&cm.lock);
1028                 break;
1029         case IB_CM_REQ_SENT:
1030         case IB_CM_MRA_REQ_RCVD:
1031                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1032                 spin_unlock_irq(&cm_id_priv->lock);
1033                 ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
1034                                &cm_id_priv->id.device->node_guid,
1035                                sizeof cm_id_priv->id.device->node_guid,
1036                                NULL, 0);
1037                 break;
1038         case IB_CM_REQ_RCVD:
1039                 if (err == -ENOMEM) {
1040                         /* Do not reject to allow future retries. */
1041                         cm_reset_to_idle(cm_id_priv);
1042                         spin_unlock_irq(&cm_id_priv->lock);
1043                 } else {
1044                         spin_unlock_irq(&cm_id_priv->lock);
1045                         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
1046                                        NULL, 0, NULL, 0);
1047                 }
1048                 break;
1049         case IB_CM_REP_SENT:
1050         case IB_CM_MRA_REP_RCVD:
1051                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1052                 /* Fall through */
1053         case IB_CM_MRA_REQ_SENT:
1054         case IB_CM_REP_RCVD:
1055         case IB_CM_MRA_REP_SENT:
1056                 spin_unlock_irq(&cm_id_priv->lock);
1057                 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
1058                                NULL, 0, NULL, 0);
1059                 break;
1060         case IB_CM_ESTABLISHED:
1061                 spin_unlock_irq(&cm_id_priv->lock);
1062                 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT)
1063                         break;
1064                 ib_send_cm_dreq(cm_id, NULL, 0);
1065                 goto retest;
1066         case IB_CM_DREQ_SENT:
1067                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1068                 cm_enter_timewait(cm_id_priv);
1069                 spin_unlock_irq(&cm_id_priv->lock);
1070                 break;
1071         case IB_CM_DREQ_RCVD:
1072                 spin_unlock_irq(&cm_id_priv->lock);
1073                 ib_send_cm_drep(cm_id, NULL, 0);
1074                 break;
1075         default:
1076                 spin_unlock_irq(&cm_id_priv->lock);
1077                 break;
1078         }
1079
1080         spin_lock_irq(&cm.lock);
1081         if (!list_empty(&cm_id_priv->altr_list) &&
1082             (!cm_id_priv->altr_send_port_not_ready))
1083                 list_del(&cm_id_priv->altr_list);
1084         if (!list_empty(&cm_id_priv->prim_list) &&
1085             (!cm_id_priv->prim_send_port_not_ready))
1086                 list_del(&cm_id_priv->prim_list);
1087         spin_unlock_irq(&cm.lock);
1088
1089         cm_free_id(cm_id->local_id);
1090         cm_deref_id(cm_id_priv);
1091         wait_for_completion(&cm_id_priv->comp);
1092         while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1093                 cm_free_work(work);
1094         kfree(cm_id_priv->private_data);
1095         kfree(cm_id_priv);
1096 }
1097
1098 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1099 {
1100         cm_destroy_id(cm_id, 0);
1101 }
1102 EXPORT_SYMBOL(ib_destroy_cm_id);
1103
1104 /**
1105  * __ib_cm_listen - Initiates listening on the specified service ID for
1106  *   connection and service ID resolution requests.
1107  * @cm_id: Connection identifier associated with the listen request.
1108  * @service_id: Service identifier matched against incoming connection
1109  *   and service ID resolution requests.  The service ID should be specified
1110  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1111  *   assign a service ID to the caller.
1112  * @service_mask: Mask applied to service ID used to listen across a
1113  *   range of service IDs.  If set to 0, the service ID is matched
1114  *   exactly.  This parameter is ignored if %service_id is set to
1115  *   IB_CM_ASSIGN_SERVICE_ID.
1116  */
1117 static int __ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id,
1118                           __be64 service_mask)
1119 {
1120         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
1121         int ret = 0;
1122
1123         service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
1124         service_id &= service_mask;
1125         if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1126             (service_id != IB_CM_ASSIGN_SERVICE_ID))
1127                 return -EINVAL;
1128
1129         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1130         if (cm_id->state != IB_CM_IDLE)
1131                 return -EINVAL;
1132
1133         cm_id->state = IB_CM_LISTEN;
1134         ++cm_id_priv->listen_sharecount;
1135
1136         if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
1137                 cm_id->service_id = cpu_to_be64(cm.listen_service_id++);
1138                 cm_id->service_mask = ~cpu_to_be64(0);
1139         } else {
1140                 cm_id->service_id = service_id;
1141                 cm_id->service_mask = service_mask;
1142         }
1143         cur_cm_id_priv = cm_insert_listen(cm_id_priv);
1144
1145         if (cur_cm_id_priv) {
1146                 cm_id->state = IB_CM_IDLE;
1147                 --cm_id_priv->listen_sharecount;
1148                 ret = -EBUSY;
1149         }
1150         return ret;
1151 }
1152
1153 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)
1154 {
1155         unsigned long flags;
1156         int ret;
1157
1158         spin_lock_irqsave(&cm.lock, flags);
1159         ret = __ib_cm_listen(cm_id, service_id, service_mask);
1160         spin_unlock_irqrestore(&cm.lock, flags);
1161
1162         return ret;
1163 }
1164 EXPORT_SYMBOL(ib_cm_listen);
1165
1166 /**
1167  * Create a new listening ib_cm_id and listen on the given service ID.
1168  *
1169  * If there's an existing ID listening on that same device and service ID,
1170  * return it.
1171  *
1172  * @device: Device associated with the cm_id.  All related communication will
1173  * be associated with the specified device.
1174  * @cm_handler: Callback invoked to notify the user of CM events.
1175  * @service_id: Service identifier matched against incoming connection
1176  *   and service ID resolution requests.  The service ID should be specified
1177  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1178  *   assign a service ID to the caller.
1179  *
1180  * Callers should call ib_destroy_cm_id when done with the listener ID.
1181  */
1182 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1183                                      ib_cm_handler cm_handler,
1184                                      __be64 service_id)
1185 {
1186         struct cm_id_private *cm_id_priv;
1187         struct ib_cm_id *cm_id;
1188         unsigned long flags;
1189         int err = 0;
1190
1191         /* Create an ID in advance, since the creation may sleep */
1192         cm_id = ib_create_cm_id(device, cm_handler, NULL);
1193         if (IS_ERR(cm_id))
1194                 return cm_id;
1195
1196         spin_lock_irqsave(&cm.lock, flags);
1197
1198         if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1199                 goto new_id;
1200
1201         /* Find an existing ID */
1202         cm_id_priv = cm_find_listen(device, service_id);
1203         if (cm_id_priv) {
1204                 if (cm_id->cm_handler != cm_handler || cm_id->context) {
1205                         /* Sharing an ib_cm_id with different handlers is not
1206                          * supported */
1207                         spin_unlock_irqrestore(&cm.lock, flags);
1208                         return ERR_PTR(-EINVAL);
1209                 }
1210                 atomic_inc(&cm_id_priv->refcount);
1211                 ++cm_id_priv->listen_sharecount;
1212                 spin_unlock_irqrestore(&cm.lock, flags);
1213
1214                 ib_destroy_cm_id(cm_id);
1215                 cm_id = &cm_id_priv->id;
1216                 return cm_id;
1217         }
1218
1219 new_id:
1220         /* Use newly created ID */
1221         err = __ib_cm_listen(cm_id, service_id, 0);
1222
1223         spin_unlock_irqrestore(&cm.lock, flags);
1224
1225         if (err) {
1226                 ib_destroy_cm_id(cm_id);
1227                 return ERR_PTR(err);
1228         }
1229         return cm_id;
1230 }
1231 EXPORT_SYMBOL(ib_cm_insert_listen);
1232
1233 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv,
1234                           enum cm_msg_sequence msg_seq)
1235 {
1236         u64 hi_tid, low_tid;
1237
1238         hi_tid   = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1239         low_tid  = (u64) ((__force u32)cm_id_priv->id.local_id |
1240                           (msg_seq << 30));
1241         return cpu_to_be64(hi_tid | low_tid);
1242 }
1243
1244 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1245                               __be16 attr_id, __be64 tid)
1246 {
1247         hdr->base_version  = IB_MGMT_BASE_VERSION;
1248         hdr->mgmt_class    = IB_MGMT_CLASS_CM;
1249         hdr->class_version = IB_CM_CLASS_VERSION;
1250         hdr->method        = IB_MGMT_METHOD_SEND;
1251         hdr->attr_id       = attr_id;
1252         hdr->tid           = tid;
1253 }
1254
1255 static void cm_format_req(struct cm_req_msg *req_msg,
1256                           struct cm_id_private *cm_id_priv,
1257                           struct ib_cm_req_param *param)
1258 {
1259         struct sa_path_rec *pri_path = param->primary_path;
1260         struct sa_path_rec *alt_path = param->alternate_path;
1261         bool pri_ext = false;
1262
1263         if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1264                 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1265                                               pri_path->opa.slid);
1266
1267         cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1268                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ));
1269
1270         req_msg->local_comm_id = cm_id_priv->id.local_id;
1271         req_msg->service_id = param->service_id;
1272         req_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
1273         cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num));
1274         cm_req_set_init_depth(req_msg, param->initiator_depth);
1275         cm_req_set_remote_resp_timeout(req_msg,
1276                                        param->remote_cm_response_timeout);
1277         cm_req_set_qp_type(req_msg, param->qp_type);
1278         cm_req_set_flow_ctrl(req_msg, param->flow_control);
1279         cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn));
1280         cm_req_set_local_resp_timeout(req_msg,
1281                                       param->local_cm_response_timeout);
1282         req_msg->pkey = param->primary_path->pkey;
1283         cm_req_set_path_mtu(req_msg, param->primary_path->mtu);
1284         cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
1285
1286         if (param->qp_type != IB_QPT_XRC_INI) {
1287                 cm_req_set_resp_res(req_msg, param->responder_resources);
1288                 cm_req_set_retry_count(req_msg, param->retry_count);
1289                 cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count);
1290                 cm_req_set_srq(req_msg, param->srq);
1291         }
1292
1293         req_msg->primary_local_gid = pri_path->sgid;
1294         req_msg->primary_remote_gid = pri_path->dgid;
1295         if (pri_ext) {
1296                 req_msg->primary_local_gid.global.interface_id
1297                         = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1298                 req_msg->primary_remote_gid.global.interface_id
1299                         = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1300         }
1301         if (pri_path->hop_limit <= 1) {
1302                 req_msg->primary_local_lid = pri_ext ? 0 :
1303                         htons(ntohl(sa_path_get_slid(pri_path)));
1304                 req_msg->primary_remote_lid = pri_ext ? 0 :
1305                         htons(ntohl(sa_path_get_dlid(pri_path)));
1306         } else {
1307                 /* Work-around until there's a way to obtain remote LID info */
1308                 req_msg->primary_local_lid = IB_LID_PERMISSIVE;
1309                 req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
1310         }
1311         cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
1312         cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
1313         req_msg->primary_traffic_class = pri_path->traffic_class;
1314         req_msg->primary_hop_limit = pri_path->hop_limit;
1315         cm_req_set_primary_sl(req_msg, pri_path->sl);
1316         cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1));
1317         cm_req_set_primary_local_ack_timeout(req_msg,
1318                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1319                                pri_path->packet_life_time));
1320
1321         if (alt_path) {
1322                 bool alt_ext = false;
1323
1324                 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1325                         alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1326                                                       alt_path->opa.slid);
1327
1328                 req_msg->alt_local_gid = alt_path->sgid;
1329                 req_msg->alt_remote_gid = alt_path->dgid;
1330                 if (alt_ext) {
1331                         req_msg->alt_local_gid.global.interface_id
1332                                 = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1333                         req_msg->alt_remote_gid.global.interface_id
1334                                 = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1335                 }
1336                 if (alt_path->hop_limit <= 1) {
1337                         req_msg->alt_local_lid = alt_ext ? 0 :
1338                                 htons(ntohl(sa_path_get_slid(alt_path)));
1339                         req_msg->alt_remote_lid = alt_ext ? 0 :
1340                                 htons(ntohl(sa_path_get_dlid(alt_path)));
1341                 } else {
1342                         req_msg->alt_local_lid = IB_LID_PERMISSIVE;
1343                         req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
1344                 }
1345                 cm_req_set_alt_flow_label(req_msg,
1346                                           alt_path->flow_label);
1347                 cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
1348                 req_msg->alt_traffic_class = alt_path->traffic_class;
1349                 req_msg->alt_hop_limit = alt_path->hop_limit;
1350                 cm_req_set_alt_sl(req_msg, alt_path->sl);
1351                 cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1));
1352                 cm_req_set_alt_local_ack_timeout(req_msg,
1353                         cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1354                                        alt_path->packet_life_time));
1355         }
1356
1357         if (param->private_data && param->private_data_len)
1358                 memcpy(req_msg->private_data, param->private_data,
1359                        param->private_data_len);
1360 }
1361
1362 static int cm_validate_req_param(struct ib_cm_req_param *param)
1363 {
1364         /* peer-to-peer not supported */
1365         if (param->peer_to_peer)
1366                 return -EINVAL;
1367
1368         if (!param->primary_path)
1369                 return -EINVAL;
1370
1371         if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1372             param->qp_type != IB_QPT_XRC_INI)
1373                 return -EINVAL;
1374
1375         if (param->private_data &&
1376             param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1377                 return -EINVAL;
1378
1379         if (param->alternate_path &&
1380             (param->alternate_path->pkey != param->primary_path->pkey ||
1381              param->alternate_path->mtu != param->primary_path->mtu))
1382                 return -EINVAL;
1383
1384         return 0;
1385 }
1386
1387 int ib_send_cm_req(struct ib_cm_id *cm_id,
1388                    struct ib_cm_req_param *param)
1389 {
1390         struct cm_id_private *cm_id_priv;
1391         struct cm_req_msg *req_msg;
1392         unsigned long flags;
1393         int ret;
1394
1395         ret = cm_validate_req_param(param);
1396         if (ret)
1397                 return ret;
1398
1399         /* Verify that we're not in timewait. */
1400         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1401         spin_lock_irqsave(&cm_id_priv->lock, flags);
1402         if (cm_id->state != IB_CM_IDLE) {
1403                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1404                 ret = -EINVAL;
1405                 goto out;
1406         }
1407         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1408
1409         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1410                                                             id.local_id);
1411         if (IS_ERR(cm_id_priv->timewait_info)) {
1412                 ret = PTR_ERR(cm_id_priv->timewait_info);
1413                 goto out;
1414         }
1415
1416         ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av,
1417                                  cm_id_priv);
1418         if (ret)
1419                 goto error1;
1420         if (param->alternate_path) {
1421                 ret = cm_init_av_by_path(param->alternate_path,
1422                                          &cm_id_priv->alt_av, cm_id_priv);
1423                 if (ret)
1424                         goto error1;
1425         }
1426         cm_id->service_id = param->service_id;
1427         cm_id->service_mask = ~cpu_to_be64(0);
1428         cm_id_priv->timeout_ms = cm_convert_to_ms(
1429                                     param->primary_path->packet_life_time) * 2 +
1430                                  cm_convert_to_ms(
1431                                     param->remote_cm_response_timeout);
1432         cm_id_priv->max_cm_retries = param->max_cm_retries;
1433         cm_id_priv->initiator_depth = param->initiator_depth;
1434         cm_id_priv->responder_resources = param->responder_resources;
1435         cm_id_priv->retry_count = param->retry_count;
1436         cm_id_priv->path_mtu = param->primary_path->mtu;
1437         cm_id_priv->pkey = param->primary_path->pkey;
1438         cm_id_priv->qp_type = param->qp_type;
1439
1440         ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
1441         if (ret)
1442                 goto error1;
1443
1444         req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
1445         cm_format_req(req_msg, cm_id_priv, param);
1446         cm_id_priv->tid = req_msg->hdr.tid;
1447         cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
1448         cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
1449
1450         cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg);
1451         cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg);
1452
1453         spin_lock_irqsave(&cm_id_priv->lock, flags);
1454         ret = ib_post_send_mad(cm_id_priv->msg, NULL);
1455         if (ret) {
1456                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1457                 goto error2;
1458         }
1459         BUG_ON(cm_id->state != IB_CM_IDLE);
1460         cm_id->state = IB_CM_REQ_SENT;
1461         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1462         return 0;
1463
1464 error2: cm_free_msg(cm_id_priv->msg);
1465 error1: kfree(cm_id_priv->timewait_info);
1466 out:    return ret;
1467 }
1468 EXPORT_SYMBOL(ib_send_cm_req);
1469
1470 static int cm_issue_rej(struct cm_port *port,
1471                         struct ib_mad_recv_wc *mad_recv_wc,
1472                         enum ib_cm_rej_reason reason,
1473                         enum cm_msg_response msg_rejected,
1474                         void *ari, u8 ari_length)
1475 {
1476         struct ib_mad_send_buf *msg = NULL;
1477         struct cm_rej_msg *rej_msg, *rcv_msg;
1478         int ret;
1479
1480         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1481         if (ret)
1482                 return ret;
1483
1484         /* We just need common CM header information.  Cast to any message. */
1485         rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1486         rej_msg = (struct cm_rej_msg *) msg->mad;
1487
1488         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1489         rej_msg->remote_comm_id = rcv_msg->local_comm_id;
1490         rej_msg->local_comm_id = rcv_msg->remote_comm_id;
1491         cm_rej_set_msg_rejected(rej_msg, msg_rejected);
1492         rej_msg->reason = cpu_to_be16(reason);
1493
1494         if (ari && ari_length) {
1495                 cm_rej_set_reject_info_len(rej_msg, ari_length);
1496                 memcpy(rej_msg->ari, ari, ari_length);
1497         }
1498
1499         ret = ib_post_send_mad(msg, NULL);
1500         if (ret)
1501                 cm_free_msg(msg);
1502
1503         return ret;
1504 }
1505
1506 static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid,
1507                                     __be32 local_qpn, __be32 remote_qpn)
1508 {
1509         return (be64_to_cpu(local_ca_guid) > be64_to_cpu(remote_ca_guid) ||
1510                 ((local_ca_guid == remote_ca_guid) &&
1511                  (be32_to_cpu(local_qpn) > be32_to_cpu(remote_qpn))));
1512 }
1513
1514 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1515 {
1516         return ((req_msg->alt_local_lid) ||
1517                 (ib_is_opa_gid(&req_msg->alt_local_gid)));
1518 }
1519
1520 static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,
1521                                  struct sa_path_rec *path, union ib_gid *gid)
1522 {
1523         if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1524                 path->rec_type = SA_PATH_REC_TYPE_OPA;
1525         else
1526                 path->rec_type = SA_PATH_REC_TYPE_IB;
1527 }
1528
1529 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1530                                         struct sa_path_rec *primary_path,
1531                                         struct sa_path_rec *alt_path)
1532 {
1533         u32 lid;
1534
1535         if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1536                 sa_path_set_dlid(primary_path,
1537                                  ntohs(req_msg->primary_local_lid));
1538                 sa_path_set_slid(primary_path,
1539                                  ntohs(req_msg->primary_remote_lid));
1540         } else {
1541                 lid = opa_get_lid_from_gid(&req_msg->primary_local_gid);
1542                 sa_path_set_dlid(primary_path, lid);
1543
1544                 lid = opa_get_lid_from_gid(&req_msg->primary_remote_gid);
1545                 sa_path_set_slid(primary_path, lid);
1546         }
1547
1548         if (!cm_req_has_alt_path(req_msg))
1549                 return;
1550
1551         if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1552                 sa_path_set_dlid(alt_path, ntohs(req_msg->alt_local_lid));
1553                 sa_path_set_slid(alt_path, ntohs(req_msg->alt_remote_lid));
1554         } else {
1555                 lid = opa_get_lid_from_gid(&req_msg->alt_local_gid);
1556                 sa_path_set_dlid(alt_path, lid);
1557
1558                 lid = opa_get_lid_from_gid(&req_msg->alt_remote_gid);
1559                 sa_path_set_slid(alt_path, lid);
1560         }
1561 }
1562
1563 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1564                                      struct sa_path_rec *primary_path,
1565                                      struct sa_path_rec *alt_path)
1566 {
1567         primary_path->dgid = req_msg->primary_local_gid;
1568         primary_path->sgid = req_msg->primary_remote_gid;
1569         primary_path->flow_label = cm_req_get_primary_flow_label(req_msg);
1570         primary_path->hop_limit = req_msg->primary_hop_limit;
1571         primary_path->traffic_class = req_msg->primary_traffic_class;
1572         primary_path->reversible = 1;
1573         primary_path->pkey = req_msg->pkey;
1574         primary_path->sl = cm_req_get_primary_sl(req_msg);
1575         primary_path->mtu_selector = IB_SA_EQ;
1576         primary_path->mtu = cm_req_get_path_mtu(req_msg);
1577         primary_path->rate_selector = IB_SA_EQ;
1578         primary_path->rate = cm_req_get_primary_packet_rate(req_msg);
1579         primary_path->packet_life_time_selector = IB_SA_EQ;
1580         primary_path->packet_life_time =
1581                 cm_req_get_primary_local_ack_timeout(req_msg);
1582         primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1583         primary_path->service_id = req_msg->service_id;
1584         if (sa_path_is_roce(primary_path))
1585                 primary_path->roce.route_resolved = false;
1586
1587         if (cm_req_has_alt_path(req_msg)) {
1588                 alt_path->dgid = req_msg->alt_local_gid;
1589                 alt_path->sgid = req_msg->alt_remote_gid;
1590                 alt_path->flow_label = cm_req_get_alt_flow_label(req_msg);
1591                 alt_path->hop_limit = req_msg->alt_hop_limit;
1592                 alt_path->traffic_class = req_msg->alt_traffic_class;
1593                 alt_path->reversible = 1;
1594                 alt_path->pkey = req_msg->pkey;
1595                 alt_path->sl = cm_req_get_alt_sl(req_msg);
1596                 alt_path->mtu_selector = IB_SA_EQ;
1597                 alt_path->mtu = cm_req_get_path_mtu(req_msg);
1598                 alt_path->rate_selector = IB_SA_EQ;
1599                 alt_path->rate = cm_req_get_alt_packet_rate(req_msg);
1600                 alt_path->packet_life_time_selector = IB_SA_EQ;
1601                 alt_path->packet_life_time =
1602                         cm_req_get_alt_local_ack_timeout(req_msg);
1603                 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1604                 alt_path->service_id = req_msg->service_id;
1605
1606                 if (sa_path_is_roce(alt_path))
1607                         alt_path->roce.route_resolved = false;
1608         }
1609         cm_format_path_lid_from_req(req_msg, primary_path, alt_path);
1610 }
1611
1612 static u16 cm_get_bth_pkey(struct cm_work *work)
1613 {
1614         struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1615         u8 port_num = work->port->port_num;
1616         u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1617         u16 pkey;
1618         int ret;
1619
1620         ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1621         if (ret) {
1622                 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n",
1623                                      port_num, pkey_index, ret);
1624                 return 0;
1625         }
1626
1627         return pkey;
1628 }
1629
1630 /**
1631  * Convert OPA SGID to IB SGID
1632  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1633  * reject them as the local_gid will not match the sgid. Therefore,
1634  * change the pathrec's SGID to an IB SGID.
1635  *
1636  * @work: Work completion
1637  * @path: Path record
1638  */
1639 static void cm_opa_to_ib_sgid(struct cm_work *work,
1640                               struct sa_path_rec *path)
1641 {
1642         struct ib_device *dev = work->port->cm_dev->ib_device;
1643         u8 port_num = work->port->port_num;
1644
1645         if (rdma_cap_opa_ah(dev, port_num) &&
1646             (ib_is_opa_gid(&path->sgid))) {
1647                 union ib_gid sgid;
1648
1649                 if (ib_get_cached_gid(dev, port_num, 0, &sgid, NULL)) {
1650                         dev_warn(&dev->dev,
1651                                  "Error updating sgid in CM request\n");
1652                         return;
1653                 }
1654
1655                 path->sgid = sgid;
1656         }
1657 }
1658
1659 static void cm_format_req_event(struct cm_work *work,
1660                                 struct cm_id_private *cm_id_priv,
1661                                 struct ib_cm_id *listen_id)
1662 {
1663         struct cm_req_msg *req_msg;
1664         struct ib_cm_req_event_param *param;
1665
1666         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1667         param = &work->cm_event.param.req_rcvd;
1668         param->listen_id = listen_id;
1669         param->bth_pkey = cm_get_bth_pkey(work);
1670         param->port = cm_id_priv->av.port->port_num;
1671         param->primary_path = &work->path[0];
1672         cm_opa_to_ib_sgid(work, param->primary_path);
1673         if (cm_req_has_alt_path(req_msg)) {
1674                 param->alternate_path = &work->path[1];
1675                 cm_opa_to_ib_sgid(work, param->alternate_path);
1676         } else {
1677                 param->alternate_path = NULL;
1678         }
1679         param->remote_ca_guid = req_msg->local_ca_guid;
1680         param->remote_qkey = be32_to_cpu(req_msg->local_qkey);
1681         param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));
1682         param->qp_type = cm_req_get_qp_type(req_msg);
1683         param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg));
1684         param->responder_resources = cm_req_get_init_depth(req_msg);
1685         param->initiator_depth = cm_req_get_resp_res(req_msg);
1686         param->local_cm_response_timeout =
1687                                         cm_req_get_remote_resp_timeout(req_msg);
1688         param->flow_control = cm_req_get_flow_ctrl(req_msg);
1689         param->remote_cm_response_timeout =
1690                                         cm_req_get_local_resp_timeout(req_msg);
1691         param->retry_count = cm_req_get_retry_count(req_msg);
1692         param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
1693         param->srq = cm_req_get_srq(req_msg);
1694         work->cm_event.private_data = &req_msg->private_data;
1695 }
1696
1697 static void cm_process_work(struct cm_id_private *cm_id_priv,
1698                             struct cm_work *work)
1699 {
1700         int ret;
1701
1702         /* We will typically only have the current event to report. */
1703         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1704         cm_free_work(work);
1705
1706         while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1707                 spin_lock_irq(&cm_id_priv->lock);
1708                 work = cm_dequeue_work(cm_id_priv);
1709                 spin_unlock_irq(&cm_id_priv->lock);
1710                 if (!work)
1711                         return;
1712
1713                 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1714                                                 &work->cm_event);
1715                 cm_free_work(work);
1716         }
1717         cm_deref_id(cm_id_priv);
1718         if (ret)
1719                 cm_destroy_id(&cm_id_priv->id, ret);
1720 }
1721
1722 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1723                           struct cm_id_private *cm_id_priv,
1724                           enum cm_msg_response msg_mraed, u8 service_timeout,
1725                           const void *private_data, u8 private_data_len)
1726 {
1727         cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1728         cm_mra_set_msg_mraed(mra_msg, msg_mraed);
1729         mra_msg->local_comm_id = cm_id_priv->id.local_id;
1730         mra_msg->remote_comm_id = cm_id_priv->id.remote_id;
1731         cm_mra_set_service_timeout(mra_msg, service_timeout);
1732
1733         if (private_data && private_data_len)
1734                 memcpy(mra_msg->private_data, private_data, private_data_len);
1735 }
1736
1737 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1738                           struct cm_id_private *cm_id_priv,
1739                           enum ib_cm_rej_reason reason,
1740                           void *ari,
1741                           u8 ari_length,
1742                           const void *private_data,
1743                           u8 private_data_len)
1744 {
1745         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1746         rej_msg->remote_comm_id = cm_id_priv->id.remote_id;
1747
1748         switch(cm_id_priv->id.state) {
1749         case IB_CM_REQ_RCVD:
1750                 rej_msg->local_comm_id = 0;
1751                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1752                 break;
1753         case IB_CM_MRA_REQ_SENT:
1754                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1755                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1756                 break;
1757         case IB_CM_REP_RCVD:
1758         case IB_CM_MRA_REP_SENT:
1759                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1760                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP);
1761                 break;
1762         default:
1763                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1764                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER);
1765                 break;
1766         }
1767
1768         rej_msg->reason = cpu_to_be16(reason);
1769         if (ari && ari_length) {
1770                 cm_rej_set_reject_info_len(rej_msg, ari_length);
1771                 memcpy(rej_msg->ari, ari, ari_length);
1772         }
1773
1774         if (private_data && private_data_len)
1775                 memcpy(rej_msg->private_data, private_data, private_data_len);
1776 }
1777
1778 static void cm_dup_req_handler(struct cm_work *work,
1779                                struct cm_id_private *cm_id_priv)
1780 {
1781         struct ib_mad_send_buf *msg = NULL;
1782         int ret;
1783
1784         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1785                         counter[CM_REQ_COUNTER]);
1786
1787         /* Quick state check to discard duplicate REQs. */
1788         if (cm_id_priv->id.state == IB_CM_REQ_RCVD)
1789                 return;
1790
1791         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1792         if (ret)
1793                 return;
1794
1795         spin_lock_irq(&cm_id_priv->lock);
1796         switch (cm_id_priv->id.state) {
1797         case IB_CM_MRA_REQ_SENT:
1798                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1799                               CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1800                               cm_id_priv->private_data,
1801                               cm_id_priv->private_data_len);
1802                 break;
1803         case IB_CM_TIMEWAIT:
1804                 cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv,
1805                               IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);
1806                 break;
1807         default:
1808                 goto unlock;
1809         }
1810         spin_unlock_irq(&cm_id_priv->lock);
1811
1812         ret = ib_post_send_mad(msg, NULL);
1813         if (ret)
1814                 goto free;
1815         return;
1816
1817 unlock: spin_unlock_irq(&cm_id_priv->lock);
1818 free:   cm_free_msg(msg);
1819 }
1820
1821 static struct cm_id_private * cm_match_req(struct cm_work *work,
1822                                            struct cm_id_private *cm_id_priv)
1823 {
1824         struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1825         struct cm_timewait_info *timewait_info;
1826         struct cm_req_msg *req_msg;
1827         struct ib_cm_id *cm_id;
1828
1829         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1830
1831         /* Check for possible duplicate REQ. */
1832         spin_lock_irq(&cm.lock);
1833         timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1834         if (timewait_info) {
1835                 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
1836                                            timewait_info->work.remote_id);
1837                 spin_unlock_irq(&cm.lock);
1838                 if (cur_cm_id_priv) {
1839                         cm_dup_req_handler(work, cur_cm_id_priv);
1840                         cm_deref_id(cur_cm_id_priv);
1841                 }
1842                 return NULL;
1843         }
1844
1845         /* Check for stale connections. */
1846         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1847         if (timewait_info) {
1848                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1849                 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
1850                                            timewait_info->work.remote_id);
1851
1852                 spin_unlock_irq(&cm.lock);
1853                 cm_issue_rej(work->port, work->mad_recv_wc,
1854                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
1855                              NULL, 0);
1856                 if (cur_cm_id_priv) {
1857                         cm_id = &cur_cm_id_priv->id;
1858                         ib_send_cm_dreq(cm_id, NULL, 0);
1859                         cm_deref_id(cur_cm_id_priv);
1860                 }
1861                 return NULL;
1862         }
1863
1864         /* Find matching listen request. */
1865         listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
1866                                            req_msg->service_id);
1867         if (!listen_cm_id_priv) {
1868                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1869                 spin_unlock_irq(&cm.lock);
1870                 cm_issue_rej(work->port, work->mad_recv_wc,
1871                              IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
1872                              NULL, 0);
1873                 goto out;
1874         }
1875         atomic_inc(&listen_cm_id_priv->refcount);
1876         atomic_inc(&cm_id_priv->refcount);
1877         cm_id_priv->id.state = IB_CM_REQ_RCVD;
1878         atomic_inc(&cm_id_priv->work_count);
1879         spin_unlock_irq(&cm.lock);
1880 out:
1881         return listen_cm_id_priv;
1882 }
1883
1884 /*
1885  * Work-around for inter-subnet connections.  If the LIDs are permissive,
1886  * we need to override the LID/SL data in the REQ with the LID information
1887  * in the work completion.
1888  */
1889 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
1890 {
1891         if (!cm_req_get_primary_subnet_local(req_msg)) {
1892                 if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
1893                         req_msg->primary_local_lid = ib_lid_be16(wc->slid);
1894                         cm_req_set_primary_sl(req_msg, wc->sl);
1895                 }
1896
1897                 if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE)
1898                         req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1899         }
1900
1901         if (!cm_req_get_alt_subnet_local(req_msg)) {
1902                 if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
1903                         req_msg->alt_local_lid = ib_lid_be16(wc->slid);
1904                         cm_req_set_alt_sl(req_msg, wc->sl);
1905                 }
1906
1907                 if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE)
1908                         req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1909         }
1910 }
1911
1912 static int cm_req_handler(struct cm_work *work)
1913 {
1914         struct ib_cm_id *cm_id;
1915         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
1916         struct cm_req_msg *req_msg;
1917         union ib_gid gid;
1918         struct ib_gid_attr gid_attr;
1919         const struct ib_global_route *grh;
1920         int ret;
1921
1922         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1923
1924         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
1925         if (IS_ERR(cm_id))
1926                 return PTR_ERR(cm_id);
1927
1928         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1929         cm_id_priv->id.remote_id = req_msg->local_comm_id;
1930         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
1931                                       work->mad_recv_wc->recv_buf.grh,
1932                                       &cm_id_priv->av);
1933         if (ret)
1934                 goto destroy;
1935         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1936                                                             id.local_id);
1937         if (IS_ERR(cm_id_priv->timewait_info)) {
1938                 ret = PTR_ERR(cm_id_priv->timewait_info);
1939                 goto destroy;
1940         }
1941         cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id;
1942         cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid;
1943         cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg);
1944
1945         listen_cm_id_priv = cm_match_req(work, cm_id_priv);
1946         if (!listen_cm_id_priv) {
1947                 pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__,
1948                          be32_to_cpu(cm_id->local_id));
1949                 ret = -EINVAL;
1950                 goto free_timeinfo;
1951         }
1952
1953         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
1954         cm_id_priv->id.context = listen_cm_id_priv->id.context;
1955         cm_id_priv->id.service_id = req_msg->service_id;
1956         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
1957
1958         cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
1959
1960         memset(&work->path[0], 0, sizeof(work->path[0]));
1961         if (cm_req_has_alt_path(req_msg))
1962                 memset(&work->path[1], 0, sizeof(work->path[1]));
1963         grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
1964         ret = ib_get_cached_gid(work->port->cm_dev->ib_device,
1965                                 work->port->port_num,
1966                                 grh->sgid_index,
1967                                 &gid, &gid_attr);
1968         if (ret) {
1969                 ib_send_cm_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
1970                 goto rejected;
1971         }
1972
1973         if (gid_attr.ndev) {
1974                 work->path[0].rec_type =
1975                         sa_conv_gid_to_pathrec_type(gid_attr.gid_type);
1976                 sa_path_set_ifindex(&work->path[0],
1977                                     gid_attr.ndev->ifindex);
1978                 sa_path_set_ndev(&work->path[0],
1979                                  dev_net(gid_attr.ndev));
1980                 dev_put(gid_attr.ndev);
1981         } else {
1982                 cm_path_set_rec_type(work->port->cm_dev->ib_device,
1983                                      work->port->port_num,
1984                                      &work->path[0],
1985                                      &req_msg->primary_local_gid);
1986         }
1987         if (cm_req_has_alt_path(req_msg))
1988                 work->path[1].rec_type = work->path[0].rec_type;
1989         cm_format_paths_from_req(req_msg, &work->path[0],
1990                                  &work->path[1]);
1991         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
1992                 sa_path_set_dmac(&work->path[0],
1993                                  cm_id_priv->av.ah_attr.roce.dmac);
1994         work->path[0].hop_limit = grh->hop_limit;
1995         ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av,
1996                                  cm_id_priv);
1997         if (ret) {
1998                 int err;
1999
2000                 err = ib_get_cached_gid(work->port->cm_dev->ib_device,
2001                                         work->port->port_num, 0,
2002                                         &work->path[0].sgid,
2003                                         NULL);
2004                 if (err)
2005                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
2006                                        NULL, 0, NULL, 0);
2007                 else
2008                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
2009                                        &work->path[0].sgid,
2010                                        sizeof(work->path[0].sgid),
2011                                        NULL, 0);
2012                 goto rejected;
2013         }
2014         if (cm_req_has_alt_path(req_msg)) {
2015                 ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av,
2016                                          cm_id_priv);
2017                 if (ret) {
2018                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
2019                                        &work->path[0].sgid,
2020                                        sizeof(work->path[0].sgid), NULL, 0);
2021                         goto rejected;
2022                 }
2023         }
2024         cm_id_priv->tid = req_msg->hdr.tid;
2025         cm_id_priv->timeout_ms = cm_convert_to_ms(
2026                                         cm_req_get_local_resp_timeout(req_msg));
2027         cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg);
2028         cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg);
2029         cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg);
2030         cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg);
2031         cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg);
2032         cm_id_priv->pkey = req_msg->pkey;
2033         cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg);
2034         cm_id_priv->retry_count = cm_req_get_retry_count(req_msg);
2035         cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
2036         cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2037
2038         cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2039         cm_process_work(cm_id_priv, work);
2040         cm_deref_id(listen_cm_id_priv);
2041         return 0;
2042
2043 rejected:
2044         atomic_dec(&cm_id_priv->refcount);
2045         cm_deref_id(listen_cm_id_priv);
2046 free_timeinfo:
2047         kfree(cm_id_priv->timewait_info);
2048 destroy:
2049         ib_destroy_cm_id(cm_id);
2050         return ret;
2051 }
2052
2053 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2054                           struct cm_id_private *cm_id_priv,
2055                           struct ib_cm_rep_param *param)
2056 {
2057         cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
2058         rep_msg->local_comm_id = cm_id_priv->id.local_id;
2059         rep_msg->remote_comm_id = cm_id_priv->id.remote_id;
2060         cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn));
2061         rep_msg->resp_resources = param->responder_resources;
2062         cm_rep_set_target_ack_delay(rep_msg,
2063                                     cm_id_priv->av.port->cm_dev->ack_delay);
2064         cm_rep_set_failover(rep_msg, param->failover_accepted);
2065         cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count);
2066         rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
2067
2068         if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2069                 rep_msg->initiator_depth = param->initiator_depth;
2070                 cm_rep_set_flow_ctrl(rep_msg, param->flow_control);
2071                 cm_rep_set_srq(rep_msg, param->srq);
2072                 cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num));
2073         } else {
2074                 cm_rep_set_srq(rep_msg, 1);
2075                 cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param->qp_num));
2076         }
2077
2078         if (param->private_data && param->private_data_len)
2079                 memcpy(rep_msg->private_data, param->private_data,
2080                        param->private_data_len);
2081 }
2082
2083 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2084                    struct ib_cm_rep_param *param)
2085 {
2086         struct cm_id_private *cm_id_priv;
2087         struct ib_mad_send_buf *msg;
2088         struct cm_rep_msg *rep_msg;
2089         unsigned long flags;
2090         int ret;
2091
2092         if (param->private_data &&
2093             param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2094                 return -EINVAL;
2095
2096         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2097         spin_lock_irqsave(&cm_id_priv->lock, flags);
2098         if (cm_id->state != IB_CM_REQ_RCVD &&
2099             cm_id->state != IB_CM_MRA_REQ_SENT) {
2100                 pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__,
2101                          be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
2102                 ret = -EINVAL;
2103                 goto out;
2104         }
2105
2106         ret = cm_alloc_msg(cm_id_priv, &msg);
2107         if (ret)
2108                 goto out;
2109
2110         rep_msg = (struct cm_rep_msg *) msg->mad;
2111         cm_format_rep(rep_msg, cm_id_priv, param);
2112         msg->timeout_ms = cm_id_priv->timeout_ms;
2113         msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2114
2115         ret = ib_post_send_mad(msg, NULL);
2116         if (ret) {
2117                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2118                 cm_free_msg(msg);
2119                 return ret;
2120         }
2121
2122         cm_id->state = IB_CM_REP_SENT;
2123         cm_id_priv->msg = msg;
2124         cm_id_priv->initiator_depth = param->initiator_depth;
2125         cm_id_priv->responder_resources = param->responder_resources;
2126         cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg);
2127         cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2128
2129 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2130         return ret;
2131 }
2132 EXPORT_SYMBOL(ib_send_cm_rep);
2133
2134 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2135                           struct cm_id_private *cm_id_priv,
2136                           const void *private_data,
2137                           u8 private_data_len)
2138 {
2139         cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2140         rtu_msg->local_comm_id = cm_id_priv->id.local_id;
2141         rtu_msg->remote_comm_id = cm_id_priv->id.remote_id;
2142
2143         if (private_data && private_data_len)
2144                 memcpy(rtu_msg->private_data, private_data, private_data_len);
2145 }
2146
2147 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2148                    const void *private_data,
2149                    u8 private_data_len)
2150 {
2151         struct cm_id_private *cm_id_priv;
2152         struct ib_mad_send_buf *msg;
2153         unsigned long flags;
2154         void *data;
2155         int ret;
2156
2157         if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2158                 return -EINVAL;
2159
2160         data = cm_copy_private_data(private_data, private_data_len);
2161         if (IS_ERR(data))
2162                 return PTR_ERR(data);
2163
2164         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2165         spin_lock_irqsave(&cm_id_priv->lock, flags);
2166         if (cm_id->state != IB_CM_REP_RCVD &&
2167             cm_id->state != IB_CM_MRA_REP_SENT) {
2168                 pr_debug("%s: local_id %d, cm_id->state %d\n", __func__,
2169                          be32_to_cpu(cm_id->local_id), cm_id->state);
2170                 ret = -EINVAL;
2171                 goto error;
2172         }
2173
2174         ret = cm_alloc_msg(cm_id_priv, &msg);
2175         if (ret)
2176                 goto error;
2177
2178         cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2179                       private_data, private_data_len);
2180
2181         ret = ib_post_send_mad(msg, NULL);
2182         if (ret) {
2183                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2184                 cm_free_msg(msg);
2185                 kfree(data);
2186                 return ret;
2187         }
2188
2189         cm_id->state = IB_CM_ESTABLISHED;
2190         cm_set_private_data(cm_id_priv, data, private_data_len);
2191         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2192         return 0;
2193
2194 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2195         kfree(data);
2196         return ret;
2197 }
2198 EXPORT_SYMBOL(ib_send_cm_rtu);
2199
2200 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2201 {
2202         struct cm_rep_msg *rep_msg;
2203         struct ib_cm_rep_event_param *param;
2204
2205         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2206         param = &work->cm_event.param.rep_rcvd;
2207         param->remote_ca_guid = rep_msg->local_ca_guid;
2208         param->remote_qkey = be32_to_cpu(rep_msg->local_qkey);
2209         param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2210         param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg));
2211         param->responder_resources = rep_msg->initiator_depth;
2212         param->initiator_depth = rep_msg->resp_resources;
2213         param->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
2214         param->failover_accepted = cm_rep_get_failover(rep_msg);
2215         param->flow_control = cm_rep_get_flow_ctrl(rep_msg);
2216         param->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
2217         param->srq = cm_rep_get_srq(rep_msg);
2218         work->cm_event.private_data = &rep_msg->private_data;
2219 }
2220
2221 static void cm_dup_rep_handler(struct cm_work *work)
2222 {
2223         struct cm_id_private *cm_id_priv;
2224         struct cm_rep_msg *rep_msg;
2225         struct ib_mad_send_buf *msg = NULL;
2226         int ret;
2227
2228         rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2229         cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id,
2230                                    rep_msg->local_comm_id);
2231         if (!cm_id_priv)
2232                 return;
2233
2234         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2235                         counter[CM_REP_COUNTER]);
2236         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2237         if (ret)
2238                 goto deref;
2239
2240         spin_lock_irq(&cm_id_priv->lock);
2241         if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2242                 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2243                               cm_id_priv->private_data,
2244                               cm_id_priv->private_data_len);
2245         else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2246                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2247                               CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2248                               cm_id_priv->private_data,
2249                               cm_id_priv->private_data_len);
2250         else
2251                 goto unlock;
2252         spin_unlock_irq(&cm_id_priv->lock);
2253
2254         ret = ib_post_send_mad(msg, NULL);
2255         if (ret)
2256                 goto free;
2257         goto deref;
2258
2259 unlock: spin_unlock_irq(&cm_id_priv->lock);
2260 free:   cm_free_msg(msg);
2261 deref:  cm_deref_id(cm_id_priv);
2262 }
2263
2264 static int cm_rep_handler(struct cm_work *work)
2265 {
2266         struct cm_id_private *cm_id_priv;
2267         struct cm_rep_msg *rep_msg;
2268         int ret;
2269         struct cm_id_private *cur_cm_id_priv;
2270         struct ib_cm_id *cm_id;
2271         struct cm_timewait_info *timewait_info;
2272
2273         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2274         cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0);
2275         if (!cm_id_priv) {
2276                 cm_dup_rep_handler(work);
2277                 pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__,
2278                          be32_to_cpu(rep_msg->remote_comm_id));
2279                 return -EINVAL;
2280         }
2281
2282         cm_format_rep_event(work, cm_id_priv->qp_type);
2283
2284         spin_lock_irq(&cm_id_priv->lock);
2285         switch (cm_id_priv->id.state) {
2286         case IB_CM_REQ_SENT:
2287         case IB_CM_MRA_REQ_RCVD:
2288                 break;
2289         default:
2290                 spin_unlock_irq(&cm_id_priv->lock);
2291                 ret = -EINVAL;
2292                 pr_debug("%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n",
2293                          __func__, cm_id_priv->id.state,
2294                          be32_to_cpu(rep_msg->local_comm_id),
2295                          be32_to_cpu(rep_msg->remote_comm_id));
2296                 goto error;
2297         }
2298
2299         cm_id_priv->timewait_info->work.remote_id = rep_msg->local_comm_id;
2300         cm_id_priv->timewait_info->remote_ca_guid = rep_msg->local_ca_guid;
2301         cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2302
2303         spin_lock(&cm.lock);
2304         /* Check for duplicate REP. */
2305         if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2306                 spin_unlock(&cm.lock);
2307                 spin_unlock_irq(&cm_id_priv->lock);
2308                 ret = -EINVAL;
2309                 pr_debug("%s: Failed to insert remote id %d\n", __func__,
2310                          be32_to_cpu(rep_msg->remote_comm_id));
2311                 goto error;
2312         }
2313         /* Check for a stale connection. */
2314         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2315         if (timewait_info) {
2316                 rb_erase(&cm_id_priv->timewait_info->remote_id_node,
2317                          &cm.remote_id_table);
2318                 cm_id_priv->timewait_info->inserted_remote_id = 0;
2319                 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
2320                                            timewait_info->work.remote_id);
2321
2322                 spin_unlock(&cm.lock);
2323                 spin_unlock_irq(&cm_id_priv->lock);
2324                 cm_issue_rej(work->port, work->mad_recv_wc,
2325                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2326                              NULL, 0);
2327                 ret = -EINVAL;
2328                 pr_debug("%s: Stale connection. local_comm_id %d, remote_comm_id %d\n",
2329                          __func__, be32_to_cpu(rep_msg->local_comm_id),
2330                          be32_to_cpu(rep_msg->remote_comm_id));
2331
2332                 if (cur_cm_id_priv) {
2333                         cm_id = &cur_cm_id_priv->id;
2334                         ib_send_cm_dreq(cm_id, NULL, 0);
2335                         cm_deref_id(cur_cm_id_priv);
2336                 }
2337
2338                 goto error;
2339         }
2340         spin_unlock(&cm.lock);
2341
2342         cm_id_priv->id.state = IB_CM_REP_RCVD;
2343         cm_id_priv->id.remote_id = rep_msg->local_comm_id;
2344         cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2345         cm_id_priv->initiator_depth = rep_msg->resp_resources;
2346         cm_id_priv->responder_resources = rep_msg->initiator_depth;
2347         cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg);
2348         cm_id_priv->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
2349         cm_id_priv->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
2350         cm_id_priv->av.timeout =
2351                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2352                                        cm_id_priv->av.timeout - 1);
2353         cm_id_priv->alt_av.timeout =
2354                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2355                                        cm_id_priv->alt_av.timeout - 1);
2356
2357         /* todo: handle peer_to_peer */
2358
2359         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2360         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2361         if (!ret)
2362                 list_add_tail(&work->list, &cm_id_priv->work_list);
2363         spin_unlock_irq(&cm_id_priv->lock);
2364
2365         if (ret)
2366                 cm_process_work(cm_id_priv, work);
2367         else
2368                 cm_deref_id(cm_id_priv);
2369         return 0;
2370
2371 error:
2372         cm_deref_id(cm_id_priv);
2373         return ret;
2374 }
2375
2376 static int cm_establish_handler(struct cm_work *work)
2377 {
2378         struct cm_id_private *cm_id_priv;
2379         int ret;
2380
2381         /* See comment in cm_establish about lookup. */
2382         cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2383         if (!cm_id_priv)
2384                 return -EINVAL;
2385
2386         spin_lock_irq(&cm_id_priv->lock);
2387         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2388                 spin_unlock_irq(&cm_id_priv->lock);
2389                 goto out;
2390         }
2391
2392         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2393         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2394         if (!ret)
2395                 list_add_tail(&work->list, &cm_id_priv->work_list);
2396         spin_unlock_irq(&cm_id_priv->lock);
2397
2398         if (ret)
2399                 cm_process_work(cm_id_priv, work);
2400         else
2401                 cm_deref_id(cm_id_priv);
2402         return 0;
2403 out:
2404         cm_deref_id(cm_id_priv);
2405         return -EINVAL;
2406 }
2407
2408 static int cm_rtu_handler(struct cm_work *work)
2409 {
2410         struct cm_id_private *cm_id_priv;
2411         struct cm_rtu_msg *rtu_msg;
2412         int ret;
2413
2414         rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2415         cm_id_priv = cm_acquire_id(rtu_msg->remote_comm_id,
2416                                    rtu_msg->local_comm_id);
2417         if (!cm_id_priv)
2418                 return -EINVAL;
2419
2420         work->cm_event.private_data = &rtu_msg->private_data;
2421
2422         spin_lock_irq(&cm_id_priv->lock);
2423         if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2424             cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2425                 spin_unlock_irq(&cm_id_priv->lock);
2426                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2427                                 counter[CM_RTU_COUNTER]);
2428                 goto out;
2429         }
2430         cm_id_priv->id.state = IB_CM_ESTABLISHED;
2431
2432         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2433         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2434         if (!ret)
2435                 list_add_tail(&work->list, &cm_id_priv->work_list);
2436         spin_unlock_irq(&cm_id_priv->lock);
2437
2438         if (ret)
2439                 cm_process_work(cm_id_priv, work);
2440         else
2441                 cm_deref_id(cm_id_priv);
2442         return 0;
2443 out:
2444         cm_deref_id(cm_id_priv);
2445         return -EINVAL;
2446 }
2447
2448 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2449                           struct cm_id_private *cm_id_priv,
2450                           const void *private_data,
2451                           u8 private_data_len)
2452 {
2453         cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2454                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_DREQ));
2455         dreq_msg->local_comm_id = cm_id_priv->id.local_id;
2456         dreq_msg->remote_comm_id = cm_id_priv->id.remote_id;
2457         cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv->remote_qpn);
2458
2459         if (private_data && private_data_len)
2460                 memcpy(dreq_msg->private_data, private_data, private_data_len);
2461 }
2462
2463 int ib_send_cm_dreq(struct ib_cm_id *cm_id,
2464                     const void *private_data,
2465                     u8 private_data_len)
2466 {
2467         struct cm_id_private *cm_id_priv;
2468         struct ib_mad_send_buf *msg;
2469         unsigned long flags;
2470         int ret;
2471
2472         if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2473                 return -EINVAL;
2474
2475         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2476         spin_lock_irqsave(&cm_id_priv->lock, flags);
2477         if (cm_id->state != IB_CM_ESTABLISHED) {
2478                 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2479                          be32_to_cpu(cm_id->local_id), cm_id->state);
2480                 ret = -EINVAL;
2481                 goto out;
2482         }
2483
2484         if (cm_id->lap_state == IB_CM_LAP_SENT ||
2485             cm_id->lap_state == IB_CM_MRA_LAP_RCVD)
2486                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2487
2488         ret = cm_alloc_msg(cm_id_priv, &msg);
2489         if (ret) {
2490                 cm_enter_timewait(cm_id_priv);
2491                 goto out;
2492         }
2493
2494         cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2495                        private_data, private_data_len);
2496         msg->timeout_ms = cm_id_priv->timeout_ms;
2497         msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2498
2499         ret = ib_post_send_mad(msg, NULL);
2500         if (ret) {
2501                 cm_enter_timewait(cm_id_priv);
2502                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2503                 cm_free_msg(msg);
2504                 return ret;
2505         }
2506
2507         cm_id->state = IB_CM_DREQ_SENT;
2508         cm_id_priv->msg = msg;
2509 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2510         return ret;
2511 }
2512 EXPORT_SYMBOL(ib_send_cm_dreq);
2513
2514 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2515                           struct cm_id_private *cm_id_priv,
2516                           const void *private_data,
2517                           u8 private_data_len)
2518 {
2519         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2520         drep_msg->local_comm_id = cm_id_priv->id.local_id;
2521         drep_msg->remote_comm_id = cm_id_priv->id.remote_id;
2522
2523         if (private_data && private_data_len)
2524                 memcpy(drep_msg->private_data, private_data, private_data_len);
2525 }
2526
2527 int ib_send_cm_drep(struct ib_cm_id *cm_id,
2528                     const void *private_data,
2529                     u8 private_data_len)
2530 {
2531         struct cm_id_private *cm_id_priv;
2532         struct ib_mad_send_buf *msg;
2533         unsigned long flags;
2534         void *data;
2535         int ret;
2536
2537         if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2538                 return -EINVAL;
2539
2540         data = cm_copy_private_data(private_data, private_data_len);
2541         if (IS_ERR(data))
2542                 return PTR_ERR(data);
2543
2544         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2545         spin_lock_irqsave(&cm_id_priv->lock, flags);
2546         if (cm_id->state != IB_CM_DREQ_RCVD) {
2547                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2548                 kfree(data);
2549                 pr_debug("%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n",
2550                          __func__, be32_to_cpu(cm_id->local_id), cm_id->state);
2551                 return -EINVAL;
2552         }
2553
2554         cm_set_private_data(cm_id_priv, data, private_data_len);
2555         cm_enter_timewait(cm_id_priv);
2556
2557         ret = cm_alloc_msg(cm_id_priv, &msg);
2558         if (ret)
2559                 goto out;
2560
2561         cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2562                        private_data, private_data_len);
2563
2564         ret = ib_post_send_mad(msg, NULL);
2565         if (ret) {
2566                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2567                 cm_free_msg(msg);
2568                 return ret;
2569         }
2570
2571 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2572         return ret;
2573 }
2574 EXPORT_SYMBOL(ib_send_cm_drep);
2575
2576 static int cm_issue_drep(struct cm_port *port,
2577                          struct ib_mad_recv_wc *mad_recv_wc)
2578 {
2579         struct ib_mad_send_buf *msg = NULL;
2580         struct cm_dreq_msg *dreq_msg;
2581         struct cm_drep_msg *drep_msg;
2582         int ret;
2583
2584         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2585         if (ret)
2586                 return ret;
2587
2588         dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2589         drep_msg = (struct cm_drep_msg *) msg->mad;
2590
2591         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2592         drep_msg->remote_comm_id = dreq_msg->local_comm_id;
2593         drep_msg->local_comm_id = dreq_msg->remote_comm_id;
2594
2595         ret = ib_post_send_mad(msg, NULL);
2596         if (ret)
2597                 cm_free_msg(msg);
2598
2599         return ret;
2600 }
2601
2602 static int cm_dreq_handler(struct cm_work *work)
2603 {
2604         struct cm_id_private *cm_id_priv;
2605         struct cm_dreq_msg *dreq_msg;
2606         struct ib_mad_send_buf *msg = NULL;
2607         int ret;
2608
2609         dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2610         cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id,
2611                                    dreq_msg->local_comm_id);
2612         if (!cm_id_priv) {
2613                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2614                                 counter[CM_DREQ_COUNTER]);
2615                 cm_issue_drep(work->port, work->mad_recv_wc);
2616                 pr_debug("%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n",
2617                          __func__, be32_to_cpu(dreq_msg->local_comm_id),
2618                          be32_to_cpu(dreq_msg->remote_comm_id));
2619                 return -EINVAL;
2620         }
2621
2622         work->cm_event.private_data = &dreq_msg->private_data;
2623
2624         spin_lock_irq(&cm_id_priv->lock);
2625         if (cm_id_priv->local_qpn != cm_dreq_get_remote_qpn(dreq_msg))
2626                 goto unlock;
2627
2628         switch (cm_id_priv->id.state) {
2629         case IB_CM_REP_SENT:
2630         case IB_CM_DREQ_SENT:
2631                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2632                 break;
2633         case IB_CM_ESTABLISHED:
2634                 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2635                     cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2636                         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2637                 break;
2638         case IB_CM_MRA_REP_RCVD:
2639                 break;
2640         case IB_CM_TIMEWAIT:
2641                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2642                                 counter[CM_DREQ_COUNTER]);
2643                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2644                 if (IS_ERR(msg))
2645                         goto unlock;
2646
2647                 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2648                                cm_id_priv->private_data,
2649                                cm_id_priv->private_data_len);
2650                 spin_unlock_irq(&cm_id_priv->lock);
2651
2652                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2653                     ib_post_send_mad(msg, NULL))
2654                         cm_free_msg(msg);
2655                 goto deref;
2656         case IB_CM_DREQ_RCVD:
2657                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2658                                 counter[CM_DREQ_COUNTER]);
2659                 goto unlock;
2660         default:
2661                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
2662                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
2663                          cm_id_priv->id.state);
2664                 goto unlock;
2665         }
2666         cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2667         cm_id_priv->tid = dreq_msg->hdr.tid;
2668         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2669         if (!ret)
2670                 list_add_tail(&work->list, &cm_id_priv->work_list);
2671         spin_unlock_irq(&cm_id_priv->lock);
2672
2673         if (ret)
2674                 cm_process_work(cm_id_priv, work);
2675         else
2676                 cm_deref_id(cm_id_priv);
2677         return 0;
2678
2679 unlock: spin_unlock_irq(&cm_id_priv->lock);
2680 deref:  cm_deref_id(cm_id_priv);
2681         return -EINVAL;
2682 }
2683
2684 static int cm_drep_handler(struct cm_work *work)
2685 {
2686         struct cm_id_private *cm_id_priv;
2687         struct cm_drep_msg *drep_msg;
2688         int ret;
2689
2690         drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2691         cm_id_priv = cm_acquire_id(drep_msg->remote_comm_id,
2692                                    drep_msg->local_comm_id);
2693         if (!cm_id_priv)
2694                 return -EINVAL;
2695
2696         work->cm_event.private_data = &drep_msg->private_data;
2697
2698         spin_lock_irq(&cm_id_priv->lock);
2699         if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2700             cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2701                 spin_unlock_irq(&cm_id_priv->lock);
2702                 goto out;
2703         }
2704         cm_enter_timewait(cm_id_priv);
2705
2706         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2707         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2708         if (!ret)
2709                 list_add_tail(&work->list, &cm_id_priv->work_list);
2710         spin_unlock_irq(&cm_id_priv->lock);
2711
2712         if (ret)
2713                 cm_process_work(cm_id_priv, work);
2714         else
2715                 cm_deref_id(cm_id_priv);
2716         return 0;
2717 out:
2718         cm_deref_id(cm_id_priv);
2719         return -EINVAL;
2720 }
2721
2722 int ib_send_cm_rej(struct ib_cm_id *cm_id,
2723                    enum ib_cm_rej_reason reason,
2724                    void *ari,
2725                    u8 ari_length,
2726                    const void *private_data,
2727                    u8 private_data_len)
2728 {
2729         struct cm_id_private *cm_id_priv;
2730         struct ib_mad_send_buf *msg;
2731         unsigned long flags;
2732         int ret;
2733
2734         if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2735             (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2736                 return -EINVAL;
2737
2738         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2739
2740         spin_lock_irqsave(&cm_id_priv->lock, flags);
2741         switch (cm_id->state) {
2742         case IB_CM_REQ_SENT:
2743         case IB_CM_MRA_REQ_RCVD:
2744         case IB_CM_REQ_RCVD:
2745         case IB_CM_MRA_REQ_SENT:
2746         case IB_CM_REP_RCVD:
2747         case IB_CM_MRA_REP_SENT:
2748                 ret = cm_alloc_msg(cm_id_priv, &msg);
2749                 if (!ret)
2750                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2751                                       cm_id_priv, reason, ari, ari_length,
2752                                       private_data, private_data_len);
2753
2754                 cm_reset_to_idle(cm_id_priv);
2755                 break;
2756         case IB_CM_REP_SENT:
2757         case IB_CM_MRA_REP_RCVD:
2758                 ret = cm_alloc_msg(cm_id_priv, &msg);
2759                 if (!ret)
2760                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2761                                       cm_id_priv, reason, ari, ari_length,
2762                                       private_data, private_data_len);
2763
2764                 cm_enter_timewait(cm_id_priv);
2765                 break;
2766         default:
2767                 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2768                          be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
2769                 ret = -EINVAL;
2770                 goto out;
2771         }
2772
2773         if (ret)
2774                 goto out;
2775
2776         ret = ib_post_send_mad(msg, NULL);
2777         if (ret)
2778                 cm_free_msg(msg);
2779
2780 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2781         return ret;
2782 }
2783 EXPORT_SYMBOL(ib_send_cm_rej);
2784
2785 static void cm_format_rej_event(struct cm_work *work)
2786 {
2787         struct cm_rej_msg *rej_msg;
2788         struct ib_cm_rej_event_param *param;
2789
2790         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2791         param = &work->cm_event.param.rej_rcvd;
2792         param->ari = rej_msg->ari;
2793         param->ari_length = cm_rej_get_reject_info_len(rej_msg);
2794         param->reason = __be16_to_cpu(rej_msg->reason);
2795         work->cm_event.private_data = &rej_msg->private_data;
2796 }
2797
2798 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2799 {
2800         struct cm_timewait_info *timewait_info;
2801         struct cm_id_private *cm_id_priv;
2802         __be32 remote_id;
2803
2804         remote_id = rej_msg->local_comm_id;
2805
2806         if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_TIMEOUT) {
2807                 spin_lock_irq(&cm.lock);
2808                 timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari),
2809                                                   remote_id);
2810                 if (!timewait_info) {
2811                         spin_unlock_irq(&cm.lock);
2812                         return NULL;
2813                 }
2814                 cm_id_priv = idr_find(&cm.local_id_table, (__force int)
2815                                       (timewait_info->work.local_id ^
2816                                        cm.random_id_operand));
2817                 if (cm_id_priv) {
2818                         if (cm_id_priv->id.remote_id == remote_id)
2819                                 atomic_inc(&cm_id_priv->refcount);
2820                         else
2821                                 cm_id_priv = NULL;
2822                 }
2823                 spin_unlock_irq(&cm.lock);
2824         } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ)
2825                 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0);
2826         else
2827                 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id);
2828
2829         return cm_id_priv;
2830 }
2831
2832 static int cm_rej_handler(struct cm_work *work)
2833 {
2834         struct cm_id_private *cm_id_priv;
2835         struct cm_rej_msg *rej_msg;
2836         int ret;
2837
2838         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2839         cm_id_priv = cm_acquire_rejected_id(rej_msg);
2840         if (!cm_id_priv)
2841                 return -EINVAL;
2842
2843         cm_format_rej_event(work);
2844
2845         spin_lock_irq(&cm_id_priv->lock);
2846         switch (cm_id_priv->id.state) {
2847         case IB_CM_REQ_SENT:
2848         case IB_CM_MRA_REQ_RCVD:
2849         case IB_CM_REP_SENT:
2850         case IB_CM_MRA_REP_RCVD:
2851                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2852                 /* fall through */
2853         case IB_CM_REQ_RCVD:
2854         case IB_CM_MRA_REQ_SENT:
2855                 if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_STALE_CONN)
2856                         cm_enter_timewait(cm_id_priv);
2857                 else
2858                         cm_reset_to_idle(cm_id_priv);
2859                 break;
2860         case IB_CM_DREQ_SENT:
2861                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2862                 /* fall through */
2863         case IB_CM_REP_RCVD:
2864         case IB_CM_MRA_REP_SENT:
2865                 cm_enter_timewait(cm_id_priv);
2866                 break;
2867         case IB_CM_ESTABLISHED:
2868                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
2869                     cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
2870                         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
2871                                 ib_cancel_mad(cm_id_priv->av.port->mad_agent,
2872                                               cm_id_priv->msg);
2873                         cm_enter_timewait(cm_id_priv);
2874                         break;
2875                 }
2876                 /* fall through */
2877         default:
2878                 spin_unlock_irq(&cm_id_priv->lock);
2879                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
2880                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
2881                          cm_id_priv->id.state);
2882                 ret = -EINVAL;
2883                 goto out;
2884         }
2885
2886         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2887         if (!ret)
2888                 list_add_tail(&work->list, &cm_id_priv->work_list);
2889         spin_unlock_irq(&cm_id_priv->lock);
2890
2891         if (ret)
2892                 cm_process_work(cm_id_priv, work);
2893         else
2894                 cm_deref_id(cm_id_priv);
2895         return 0;
2896 out:
2897         cm_deref_id(cm_id_priv);
2898         return -EINVAL;
2899 }
2900
2901 int ib_send_cm_mra(struct ib_cm_id *cm_id,
2902                    u8 service_timeout,
2903                    const void *private_data,
2904                    u8 private_data_len)
2905 {
2906         struct cm_id_private *cm_id_priv;
2907         struct ib_mad_send_buf *msg;
2908         enum ib_cm_state cm_state;
2909         enum ib_cm_lap_state lap_state;
2910         enum cm_msg_response msg_response;
2911         void *data;
2912         unsigned long flags;
2913         int ret;
2914
2915         if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
2916                 return -EINVAL;
2917
2918         data = cm_copy_private_data(private_data, private_data_len);
2919         if (IS_ERR(data))
2920                 return PTR_ERR(data);
2921
2922         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2923
2924         spin_lock_irqsave(&cm_id_priv->lock, flags);
2925         switch(cm_id_priv->id.state) {
2926         case IB_CM_REQ_RCVD:
2927                 cm_state = IB_CM_MRA_REQ_SENT;
2928                 lap_state = cm_id->lap_state;
2929                 msg_response = CM_MSG_RESPONSE_REQ;
2930                 break;
2931         case IB_CM_REP_RCVD:
2932                 cm_state = IB_CM_MRA_REP_SENT;
2933                 lap_state = cm_id->lap_state;
2934                 msg_response = CM_MSG_RESPONSE_REP;
2935                 break;
2936         case IB_CM_ESTABLISHED:
2937                 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
2938                         cm_state = cm_id->state;
2939                         lap_state = IB_CM_MRA_LAP_SENT;
2940                         msg_response = CM_MSG_RESPONSE_OTHER;
2941                         break;
2942                 }
2943                 /* fall through */
2944         default:
2945                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
2946                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
2947                          cm_id_priv->id.state);
2948                 ret = -EINVAL;
2949                 goto error1;
2950         }
2951
2952         if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
2953                 ret = cm_alloc_msg(cm_id_priv, &msg);
2954                 if (ret)
2955                         goto error1;
2956
2957                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2958                               msg_response, service_timeout,
2959                               private_data, private_data_len);
2960                 ret = ib_post_send_mad(msg, NULL);
2961                 if (ret)
2962                         goto error2;
2963         }
2964
2965         cm_id->state = cm_state;
2966         cm_id->lap_state = lap_state;
2967         cm_id_priv->service_timeout = service_timeout;
2968         cm_set_private_data(cm_id_priv, data, private_data_len);
2969         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2970         return 0;
2971
2972 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2973         kfree(data);
2974         return ret;
2975
2976 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2977         kfree(data);
2978         cm_free_msg(msg);
2979         return ret;
2980 }
2981 EXPORT_SYMBOL(ib_send_cm_mra);
2982
2983 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
2984 {
2985         switch (cm_mra_get_msg_mraed(mra_msg)) {
2986         case CM_MSG_RESPONSE_REQ:
2987                 return cm_acquire_id(mra_msg->remote_comm_id, 0);
2988         case CM_MSG_RESPONSE_REP:
2989         case CM_MSG_RESPONSE_OTHER:
2990                 return cm_acquire_id(mra_msg->remote_comm_id,
2991                                      mra_msg->local_comm_id);
2992         default:
2993                 return NULL;
2994         }
2995 }
2996
2997 static int cm_mra_handler(struct cm_work *work)
2998 {
2999         struct cm_id_private *cm_id_priv;
3000         struct cm_mra_msg *mra_msg;
3001         int timeout, ret;
3002
3003         mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3004         cm_id_priv = cm_acquire_mraed_id(mra_msg);
3005         if (!cm_id_priv)
3006                 return -EINVAL;
3007
3008         work->cm_event.private_data = &mra_msg->private_data;
3009         work->cm_event.param.mra_rcvd.service_timeout =
3010                                         cm_mra_get_service_timeout(mra_msg);
3011         timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) +
3012                   cm_convert_to_ms(cm_id_priv->av.timeout);
3013
3014         spin_lock_irq(&cm_id_priv->lock);
3015         switch (cm_id_priv->id.state) {
3016         case IB_CM_REQ_SENT:
3017                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ ||
3018                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
3019                                   cm_id_priv->msg, timeout))
3020                         goto out;
3021                 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3022                 break;
3023         case IB_CM_REP_SENT:
3024                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP ||
3025                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
3026                                   cm_id_priv->msg, timeout))
3027                         goto out;
3028                 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3029                 break;
3030         case IB_CM_ESTABLISHED:
3031                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER ||
3032                     cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3033                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
3034                                   cm_id_priv->msg, timeout)) {
3035                         if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3036                                 atomic_long_inc(&work->port->
3037                                                 counter_group[CM_RECV_DUPLICATES].
3038                                                 counter[CM_MRA_COUNTER]);
3039                         goto out;
3040                 }
3041                 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3042                 break;
3043         case IB_CM_MRA_REQ_RCVD:
3044         case IB_CM_MRA_REP_RCVD:
3045                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3046                                 counter[CM_MRA_COUNTER]);
3047                 /* fall through */
3048         default:
3049                 pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n",
3050                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
3051                          cm_id_priv->id.state);
3052                 goto out;
3053         }
3054
3055         cm_id_priv->msg->context[1] = (void *) (unsigned long)
3056                                       cm_id_priv->id.state;
3057         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3058         if (!ret)
3059                 list_add_tail(&work->list, &cm_id_priv->work_list);
3060         spin_unlock_irq(&cm_id_priv->lock);
3061
3062         if (ret)
3063                 cm_process_work(cm_id_priv, work);
3064         else
3065                 cm_deref_id(cm_id_priv);
3066         return 0;
3067 out:
3068         spin_unlock_irq(&cm_id_priv->lock);
3069         cm_deref_id(cm_id_priv);
3070         return -EINVAL;
3071 }
3072
3073 static void cm_format_lap(struct cm_lap_msg *lap_msg,
3074                           struct cm_id_private *cm_id_priv,
3075                           struct sa_path_rec *alternate_path,
3076                           const void *private_data,
3077                           u8 private_data_len)
3078 {
3079         bool alt_ext = false;
3080
3081         if (alternate_path->rec_type == SA_PATH_REC_TYPE_OPA)
3082                 alt_ext = opa_is_extended_lid(alternate_path->opa.dlid,
3083                                               alternate_path->opa.slid);
3084         cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID,
3085                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_LAP));
3086         lap_msg->local_comm_id = cm_id_priv->id.local_id;
3087         lap_msg->remote_comm_id = cm_id_priv->id.remote_id;
3088         cm_lap_set_remote_qpn(lap_msg, cm_id_priv->remote_qpn);
3089         /* todo: need remote CM response timeout */
3090         cm_lap_set_remote_resp_timeout(lap_msg, 0x1F);
3091         lap_msg->alt_local_lid =
3092                 htons(ntohl(sa_path_get_slid(alternate_path)));
3093         lap_msg->alt_remote_lid =
3094                 htons(ntohl(sa_path_get_dlid(alternate_path)));
3095         lap_msg->alt_local_gid = alternate_path->sgid;
3096         lap_msg->alt_remote_gid = alternate_path->dgid;
3097         if (alt_ext) {
3098                 lap_msg->alt_local_gid.global.interface_id
3099                         = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.slid));
3100                 lap_msg->alt_remote_gid.global.interface_id
3101                         = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.dlid));
3102         }
3103         cm_lap_set_flow_label(lap_msg, alternate_path->flow_label);
3104         cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class);
3105         lap_msg->alt_hop_limit = alternate_path->hop_limit;
3106         cm_lap_set_packet_rate(lap_msg, alternate_path->rate);
3107         cm_lap_set_sl(lap_msg, alternate_path->sl);
3108         cm_lap_set_subnet_local(lap_msg, 1); /* local only... */
3109         cm_lap_set_local_ack_timeout(lap_msg,
3110                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
3111                                alternate_path->packet_life_time));
3112
3113         if (private_data && private_data_len)
3114                 memcpy(lap_msg->private_data, private_data, private_data_len);
3115 }
3116
3117 int ib_send_cm_lap(struct ib_cm_id *cm_id,
3118                    struct sa_path_rec *alternate_path,
3119                    const void *private_data,
3120                    u8 private_data_len)
3121 {
3122         struct cm_id_private *cm_id_priv;
3123         struct ib_mad_send_buf *msg;
3124         unsigned long flags;
3125         int ret;
3126
3127         if (private_data && private_data_len > IB_CM_LAP_PRIVATE_DATA_SIZE)
3128                 return -EINVAL;
3129
3130         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3131         spin_lock_irqsave(&cm_id_priv->lock, flags);
3132         if (cm_id->state != IB_CM_ESTABLISHED ||
3133             (cm_id->lap_state != IB_CM_LAP_UNINIT &&
3134              cm_id->lap_state != IB_CM_LAP_IDLE)) {
3135                 ret = -EINVAL;
3136                 goto out;
3137         }
3138
3139         ret = cm_init_av_by_path(alternate_path, &cm_id_priv->alt_av,
3140                                  cm_id_priv);
3141         if (ret)
3142                 goto out;
3143         cm_id_priv->alt_av.timeout =
3144                         cm_ack_timeout(cm_id_priv->target_ack_delay,
3145                                        cm_id_priv->alt_av.timeout - 1);
3146
3147         ret = cm_alloc_msg(cm_id_priv, &msg);
3148         if (ret)
3149                 goto out;
3150
3151         cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv,
3152                       alternate_path, private_data, private_data_len);
3153         msg->timeout_ms = cm_id_priv->timeout_ms;
3154         msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED;
3155
3156         ret = ib_post_send_mad(msg, NULL);
3157         if (ret) {
3158                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3159                 cm_free_msg(msg);
3160                 return ret;
3161         }
3162
3163         cm_id->lap_state = IB_CM_LAP_SENT;
3164         cm_id_priv->msg = msg;
3165
3166 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3167         return ret;
3168 }
3169 EXPORT_SYMBOL(ib_send_cm_lap);
3170
3171 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3172                                         struct sa_path_rec *path)
3173 {
3174         u32 lid;
3175
3176         if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3177                 sa_path_set_dlid(path, ntohs(lap_msg->alt_local_lid));
3178                 sa_path_set_slid(path, ntohs(lap_msg->alt_remote_lid));
3179         } else {
3180                 lid = opa_get_lid_from_gid(&lap_msg->alt_local_gid);
3181                 sa_path_set_dlid(path, lid);
3182
3183                 lid = opa_get_lid_from_gid(&lap_msg->alt_remote_gid);
3184                 sa_path_set_slid(path, lid);
3185         }
3186 }
3187
3188 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3189                                     struct sa_path_rec *path,
3190                                     struct cm_lap_msg *lap_msg)
3191 {
3192         path->dgid = lap_msg->alt_local_gid;
3193         path->sgid = lap_msg->alt_remote_gid;
3194         path->flow_label = cm_lap_get_flow_label(lap_msg);
3195         path->hop_limit = lap_msg->alt_hop_limit;
3196         path->traffic_class = cm_lap_get_traffic_class(lap_msg);
3197         path->reversible = 1;
3198         path->pkey = cm_id_priv->pkey;
3199         path->sl = cm_lap_get_sl(lap_msg);
3200         path->mtu_selector = IB_SA_EQ;
3201         path->mtu = cm_id_priv->path_mtu;
3202         path->rate_selector = IB_SA_EQ;
3203         path->rate = cm_lap_get_packet_rate(lap_msg);
3204         path->packet_life_time_selector = IB_SA_EQ;
3205         path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg);
3206         path->packet_life_time -= (path->packet_life_time > 0);
3207         cm_format_path_lid_from_lap(lap_msg, path);
3208 }
3209
3210 static int cm_lap_handler(struct cm_work *work)
3211 {
3212         struct cm_id_private *cm_id_priv;
3213         struct cm_lap_msg *lap_msg;
3214         struct ib_cm_lap_event_param *param;
3215         struct ib_mad_send_buf *msg = NULL;
3216         int ret;
3217
3218         /* Currently Alternate path messages are not supported for
3219          * RoCE link layer.
3220          */
3221         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3222                                work->port->port_num))
3223                 return -EINVAL;
3224
3225         /* todo: verify LAP request and send reject APR if invalid. */
3226         lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3227         cm_id_priv = cm_acquire_id(lap_msg->remote_comm_id,
3228                                    lap_msg->local_comm_id);
3229         if (!cm_id_priv)
3230                 return -EINVAL;
3231
3232         param = &work->cm_event.param.lap_rcvd;
3233         memset(&work->path[0], 0, sizeof(work->path[1]));
3234         cm_path_set_rec_type(work->port->cm_dev->ib_device,
3235                              work->port->port_num,
3236                              &work->path[0],
3237                              &lap_msg->alt_local_gid);
3238         param->alternate_path = &work->path[0];
3239         cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3240         work->cm_event.private_data = &lap_msg->private_data;
3241
3242         spin_lock_irq(&cm_id_priv->lock);
3243         if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3244                 goto unlock;
3245
3246         switch (cm_id_priv->id.lap_state) {
3247         case IB_CM_LAP_UNINIT:
3248         case IB_CM_LAP_IDLE:
3249                 break;
3250         case IB_CM_MRA_LAP_SENT:
3251                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3252                                 counter[CM_LAP_COUNTER]);
3253                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3254                 if (IS_ERR(msg))
3255                         goto unlock;
3256
3257                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3258                               CM_MSG_RESPONSE_OTHER,
3259                               cm_id_priv->service_timeout,
3260                               cm_id_priv->private_data,
3261                               cm_id_priv->private_data_len);
3262                 spin_unlock_irq(&cm_id_priv->lock);
3263
3264                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3265                     ib_post_send_mad(msg, NULL))
3266                         cm_free_msg(msg);
3267                 goto deref;
3268         case IB_CM_LAP_RCVD:
3269                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3270                                 counter[CM_LAP_COUNTER]);
3271                 goto unlock;
3272         default:
3273                 goto unlock;
3274         }
3275
3276         ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3277                                  work->mad_recv_wc->recv_buf.grh,
3278                                  &cm_id_priv->av);
3279         if (ret)
3280                 goto unlock;
3281
3282         cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av,
3283                            cm_id_priv);
3284         cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3285         cm_id_priv->tid = lap_msg->hdr.tid;
3286         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3287         if (!ret)
3288                 list_add_tail(&work->list, &cm_id_priv->work_list);
3289         spin_unlock_irq(&cm_id_priv->lock);
3290
3291         if (ret)
3292                 cm_process_work(cm_id_priv, work);
3293         else
3294                 cm_deref_id(cm_id_priv);
3295         return 0;
3296
3297 unlock: spin_unlock_irq(&cm_id_priv->lock);
3298 deref:  cm_deref_id(cm_id_priv);
3299         return -EINVAL;
3300 }
3301
3302 static void cm_format_apr(struct cm_apr_msg *apr_msg,
3303                           struct cm_id_private *cm_id_priv,
3304                           enum ib_cm_apr_status status,
3305                           void *info,
3306                           u8 info_length,
3307                           const void *private_data,
3308                           u8 private_data_len)
3309 {
3310         cm_format_mad_hdr(&apr_msg->hdr, CM_APR_ATTR_ID, cm_id_priv->tid);
3311         apr_msg->local_comm_id = cm_id_priv->id.local_id;
3312         apr_msg->remote_comm_id = cm_id_priv->id.remote_id;
3313         apr_msg->ap_status = (u8) status;
3314
3315         if (info && info_length) {
3316                 apr_msg->info_length = info_length;
3317                 memcpy(apr_msg->info, info, info_length);
3318         }
3319
3320         if (private_data && private_data_len)
3321                 memcpy(apr_msg->private_data, private_data, private_data_len);
3322 }
3323
3324 int ib_send_cm_apr(struct ib_cm_id *cm_id,
3325                    enum ib_cm_apr_status status,
3326                    void *info,
3327                    u8 info_length,
3328                    const void *private_data,
3329                    u8 private_data_len)
3330 {
3331         struct cm_id_private *cm_id_priv;
3332         struct ib_mad_send_buf *msg;
3333         unsigned long flags;
3334         int ret;
3335
3336         if ((private_data && private_data_len > IB_CM_APR_PRIVATE_DATA_SIZE) ||
3337             (info && info_length > IB_CM_APR_INFO_LENGTH))
3338                 return -EINVAL;
3339
3340         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3341         spin_lock_irqsave(&cm_id_priv->lock, flags);
3342         if (cm_id->state != IB_CM_ESTABLISHED ||
3343             (cm_id->lap_state != IB_CM_LAP_RCVD &&
3344              cm_id->lap_state != IB_CM_MRA_LAP_SENT)) {
3345                 ret = -EINVAL;
3346                 goto out;
3347         }
3348
3349         ret = cm_alloc_msg(cm_id_priv, &msg);
3350         if (ret)
3351                 goto out;
3352
3353         cm_format_apr((struct cm_apr_msg *) msg->mad, cm_id_priv, status,
3354                       info, info_length, private_data, private_data_len);
3355         ret = ib_post_send_mad(msg, NULL);
3356         if (ret) {
3357                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3358                 cm_free_msg(msg);
3359                 return ret;
3360         }
3361
3362         cm_id->lap_state = IB_CM_LAP_IDLE;
3363 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3364         return ret;
3365 }
3366 EXPORT_SYMBOL(ib_send_cm_apr);
3367
3368 static int cm_apr_handler(struct cm_work *work)
3369 {
3370         struct cm_id_private *cm_id_priv;
3371         struct cm_apr_msg *apr_msg;
3372         int ret;
3373
3374         /* Currently Alternate path messages are not supported for
3375          * RoCE link layer.
3376          */
3377         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3378                                work->port->port_num))
3379                 return -EINVAL;
3380
3381         apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3382         cm_id_priv = cm_acquire_id(apr_msg->remote_comm_id,
3383                                    apr_msg->local_comm_id);
3384         if (!cm_id_priv)
3385                 return -EINVAL; /* Unmatched reply. */
3386
3387         work->cm_event.param.apr_rcvd.ap_status = apr_msg->ap_status;
3388         work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info;
3389         work->cm_event.param.apr_rcvd.info_len = apr_msg->info_length;
3390         work->cm_event.private_data = &apr_msg->private_data;
3391
3392         spin_lock_irq(&cm_id_priv->lock);
3393         if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3394             (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3395              cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3396                 spin_unlock_irq(&cm_id_priv->lock);
3397                 goto out;
3398         }
3399         cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3400         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3401         cm_id_priv->msg = NULL;
3402
3403         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3404         if (!ret)
3405                 list_add_tail(&work->list, &cm_id_priv->work_list);
3406         spin_unlock_irq(&cm_id_priv->lock);
3407
3408         if (ret)
3409                 cm_process_work(cm_id_priv, work);
3410         else
3411                 cm_deref_id(cm_id_priv);
3412         return 0;
3413 out:
3414         cm_deref_id(cm_id_priv);
3415         return -EINVAL;
3416 }
3417
3418 static int cm_timewait_handler(struct cm_work *work)
3419 {
3420         struct cm_timewait_info *timewait_info;
3421         struct cm_id_private *cm_id_priv;
3422         int ret;
3423
3424         timewait_info = (struct cm_timewait_info *)work;
3425         spin_lock_irq(&cm.lock);
3426         list_del(&timewait_info->list);
3427         spin_unlock_irq(&cm.lock);
3428
3429         cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3430                                    timewait_info->work.remote_id);
3431         if (!cm_id_priv)
3432                 return -EINVAL;
3433
3434         spin_lock_irq(&cm_id_priv->lock);
3435         if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3436             cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3437                 spin_unlock_irq(&cm_id_priv->lock);
3438                 goto out;
3439         }
3440         cm_id_priv->id.state = IB_CM_IDLE;
3441         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3442         if (!ret)
3443                 list_add_tail(&work->list, &cm_id_priv->work_list);
3444         spin_unlock_irq(&cm_id_priv->lock);
3445
3446         if (ret)
3447                 cm_process_work(cm_id_priv, work);
3448         else
3449                 cm_deref_id(cm_id_priv);
3450         return 0;
3451 out:
3452         cm_deref_id(cm_id_priv);
3453         return -EINVAL;
3454 }
3455
3456 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3457                                struct cm_id_private *cm_id_priv,
3458                                struct ib_cm_sidr_req_param *param)
3459 {
3460         cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3461                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_SIDR));
3462         sidr_req_msg->request_id = cm_id_priv->id.local_id;
3463         sidr_req_msg->pkey = param->path->pkey;
3464         sidr_req_msg->service_id = param->service_id;
3465
3466         if (param->private_data && param->private_data_len)
3467                 memcpy(sidr_req_msg->private_data, param->private_data,
3468                        param->private_data_len);
3469 }
3470
3471 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3472                         struct ib_cm_sidr_req_param *param)
3473 {
3474         struct cm_id_private *cm_id_priv;
3475         struct ib_mad_send_buf *msg;
3476         unsigned long flags;
3477         int ret;
3478
3479         if (!param->path || (param->private_data &&
3480              param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3481                 return -EINVAL;
3482
3483         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3484         ret = cm_init_av_by_path(param->path, &cm_id_priv->av, cm_id_priv);
3485         if (ret)
3486                 goto out;
3487
3488         cm_id->service_id = param->service_id;
3489         cm_id->service_mask = ~cpu_to_be64(0);
3490         cm_id_priv->timeout_ms = param->timeout_ms;
3491         cm_id_priv->max_cm_retries = param->max_cm_retries;
3492         ret = cm_alloc_msg(cm_id_priv, &msg);
3493         if (ret)
3494                 goto out;
3495
3496         cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv,
3497                            param);
3498         msg->timeout_ms = cm_id_priv->timeout_ms;
3499         msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;
3500
3501         spin_lock_irqsave(&cm_id_priv->lock, flags);
3502         if (cm_id->state == IB_CM_IDLE)
3503                 ret = ib_post_send_mad(msg, NULL);
3504         else
3505                 ret = -EINVAL;
3506
3507         if (ret) {
3508                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3509                 cm_free_msg(msg);
3510                 goto out;
3511         }
3512         cm_id->state = IB_CM_SIDR_REQ_SENT;
3513         cm_id_priv->msg = msg;
3514         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3515 out:
3516         return ret;
3517 }
3518 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3519
3520 static void cm_format_sidr_req_event(struct cm_work *work,
3521                                      struct ib_cm_id *listen_id)
3522 {
3523         struct cm_sidr_req_msg *sidr_req_msg;
3524         struct ib_cm_sidr_req_event_param *param;
3525
3526         sidr_req_msg = (struct cm_sidr_req_msg *)
3527                                 work->mad_recv_wc->recv_buf.mad;
3528         param = &work->cm_event.param.sidr_req_rcvd;
3529         param->pkey = __be16_to_cpu(sidr_req_msg->pkey);
3530         param->listen_id = listen_id;
3531         param->service_id = sidr_req_msg->service_id;
3532         param->bth_pkey = cm_get_bth_pkey(work);
3533         param->port = work->port->port_num;
3534         work->cm_event.private_data = &sidr_req_msg->private_data;
3535 }
3536
3537 static int cm_sidr_req_handler(struct cm_work *work)
3538 {
3539         struct ib_cm_id *cm_id;
3540         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
3541         struct cm_sidr_req_msg *sidr_req_msg;
3542         struct ib_wc *wc;
3543         int ret;
3544
3545         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
3546         if (IS_ERR(cm_id))
3547                 return PTR_ERR(cm_id);
3548         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3549
3550         /* Record SGID/SLID and request ID for lookup. */
3551         sidr_req_msg = (struct cm_sidr_req_msg *)
3552                                 work->mad_recv_wc->recv_buf.mad;
3553         wc = work->mad_recv_wc->wc;
3554         cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
3555         cm_id_priv->av.dgid.global.interface_id = 0;
3556         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3557                                       work->mad_recv_wc->recv_buf.grh,
3558                                       &cm_id_priv->av);
3559         if (ret)
3560                 goto out;
3561
3562         cm_id_priv->id.remote_id = sidr_req_msg->request_id;
3563         cm_id_priv->tid = sidr_req_msg->hdr.tid;
3564         atomic_inc(&cm_id_priv->work_count);
3565
3566         spin_lock_irq(&cm.lock);
3567         cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3568         if (cur_cm_id_priv) {
3569                 spin_unlock_irq(&cm.lock);
3570                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3571                                 counter[CM_SIDR_REQ_COUNTER]);
3572                 goto out; /* Duplicate message. */
3573         }
3574         cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3575         cur_cm_id_priv = cm_find_listen(cm_id->device,
3576                                         sidr_req_msg->service_id);
3577         if (!cur_cm_id_priv) {
3578                 spin_unlock_irq(&cm.lock);
3579                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED);
3580                 goto out; /* No match. */
3581         }
3582         atomic_inc(&cur_cm_id_priv->refcount);
3583         atomic_inc(&cm_id_priv->refcount);
3584         spin_unlock_irq(&cm.lock);
3585
3586         cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler;
3587         cm_id_priv->id.context = cur_cm_id_priv->id.context;
3588         cm_id_priv->id.service_id = sidr_req_msg->service_id;
3589         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3590
3591         cm_format_sidr_req_event(work, &cur_cm_id_priv->id);
3592         cm_process_work(cm_id_priv, work);
3593         cm_deref_id(cur_cm_id_priv);
3594         return 0;
3595 out:
3596         ib_destroy_cm_id(&cm_id_priv->id);
3597         return -EINVAL;
3598 }
3599
3600 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3601                                struct cm_id_private *cm_id_priv,
3602                                struct ib_cm_sidr_rep_param *param)
3603 {
3604         cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3605                           cm_id_priv->tid);
3606         sidr_rep_msg->request_id = cm_id_priv->id.remote_id;
3607         sidr_rep_msg->status = param->status;
3608         cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param->qp_num));
3609         sidr_rep_msg->service_id = cm_id_priv->id.service_id;
3610         sidr_rep_msg->qkey = cpu_to_be32(param->qkey);
3611
3612         if (param->info && param->info_length)
3613                 memcpy(sidr_rep_msg->info, param->info, param->info_length);
3614
3615         if (param->private_data && param->private_data_len)
3616                 memcpy(sidr_rep_msg->private_data, param->private_data,
3617                        param->private_data_len);
3618 }
3619
3620 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3621                         struct ib_cm_sidr_rep_param *param)
3622 {
3623         struct cm_id_private *cm_id_priv;
3624         struct ib_mad_send_buf *msg;
3625         unsigned long flags;
3626         int ret;
3627
3628         if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3629             (param->private_data &&
3630              param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3631                 return -EINVAL;
3632
3633         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3634         spin_lock_irqsave(&cm_id_priv->lock, flags);
3635         if (cm_id->state != IB_CM_SIDR_REQ_RCVD) {
3636                 ret = -EINVAL;
3637                 goto error;
3638         }
3639
3640         ret = cm_alloc_msg(cm_id_priv, &msg);
3641         if (ret)
3642                 goto error;
3643
3644         cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3645                            param);
3646         ret = ib_post_send_mad(msg, NULL);
3647         if (ret) {
3648                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3649                 cm_free_msg(msg);
3650                 return ret;
3651         }
3652         cm_id->state = IB_CM_IDLE;
3653         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3654
3655         spin_lock_irqsave(&cm.lock, flags);
3656         if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3657                 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3658                 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3659         }
3660         spin_unlock_irqrestore(&cm.lock, flags);
3661         return 0;
3662
3663 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3664         return ret;
3665 }
3666 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3667
3668 static void cm_format_sidr_rep_event(struct cm_work *work)
3669 {
3670         struct cm_sidr_rep_msg *sidr_rep_msg;
3671         struct ib_cm_sidr_rep_event_param *param;
3672
3673         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3674                                 work->mad_recv_wc->recv_buf.mad;
3675         param = &work->cm_event.param.sidr_rep_rcvd;
3676         param->status = sidr_rep_msg->status;
3677         param->qkey = be32_to_cpu(sidr_rep_msg->qkey);
3678         param->qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg));
3679         param->info = &sidr_rep_msg->info;
3680         param->info_len = sidr_rep_msg->info_length;
3681         work->cm_event.private_data = &sidr_rep_msg->private_data;
3682 }
3683
3684 static int cm_sidr_rep_handler(struct cm_work *work)
3685 {
3686         struct cm_sidr_rep_msg *sidr_rep_msg;
3687         struct cm_id_private *cm_id_priv;
3688
3689         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3690                                 work->mad_recv_wc->recv_buf.mad;
3691         cm_id_priv = cm_acquire_id(sidr_rep_msg->request_id, 0);
3692         if (!cm_id_priv)
3693                 return -EINVAL; /* Unmatched reply. */
3694
3695         spin_lock_irq(&cm_id_priv->lock);
3696         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3697                 spin_unlock_irq(&cm_id_priv->lock);
3698                 goto out;
3699         }
3700         cm_id_priv->id.state = IB_CM_IDLE;
3701         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3702         spin_unlock_irq(&cm_id_priv->lock);
3703
3704         cm_format_sidr_rep_event(work);
3705         cm_process_work(cm_id_priv, work);
3706         return 0;
3707 out:
3708         cm_deref_id(cm_id_priv);
3709         return -EINVAL;
3710 }
3711
3712 static void cm_process_send_error(struct ib_mad_send_buf *msg,
3713                                   enum ib_wc_status wc_status)
3714 {
3715         struct cm_id_private *cm_id_priv;
3716         struct ib_cm_event cm_event;
3717         enum ib_cm_state state;
3718         int ret;
3719
3720         memset(&cm_event, 0, sizeof cm_event);
3721         cm_id_priv = msg->context[0];
3722
3723         /* Discard old sends or ones without a response. */
3724         spin_lock_irq(&cm_id_priv->lock);
3725         state = (enum ib_cm_state) (unsigned long) msg->context[1];
3726         if (msg != cm_id_priv->msg || state != cm_id_priv->id.state)
3727                 goto discard;
3728
3729         pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n",
3730                              state, ib_wc_status_msg(wc_status));
3731         switch (state) {
3732         case IB_CM_REQ_SENT:
3733         case IB_CM_MRA_REQ_RCVD:
3734                 cm_reset_to_idle(cm_id_priv);
3735                 cm_event.event = IB_CM_REQ_ERROR;
3736                 break;
3737         case IB_CM_REP_SENT:
3738         case IB_CM_MRA_REP_RCVD:
3739                 cm_reset_to_idle(cm_id_priv);
3740                 cm_event.event = IB_CM_REP_ERROR;
3741                 break;
3742         case IB_CM_DREQ_SENT:
3743                 cm_enter_timewait(cm_id_priv);
3744                 cm_event.event = IB_CM_DREQ_ERROR;
3745                 break;
3746         case IB_CM_SIDR_REQ_SENT:
3747                 cm_id_priv->id.state = IB_CM_IDLE;
3748                 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3749                 break;
3750         default:
3751                 goto discard;
3752         }
3753         spin_unlock_irq(&cm_id_priv->lock);
3754         cm_event.param.send_status = wc_status;
3755
3756         /* No other events can occur on the cm_id at this point. */
3757         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3758         cm_free_msg(msg);
3759         if (ret)
3760                 ib_destroy_cm_id(&cm_id_priv->id);
3761         return;
3762 discard:
3763         spin_unlock_irq(&cm_id_priv->lock);
3764         cm_free_msg(msg);
3765 }
3766
3767 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3768                             struct ib_mad_send_wc *mad_send_wc)
3769 {
3770         struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3771         struct cm_port *port;
3772         u16 attr_index;
3773
3774         port = mad_agent->context;
3775         attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3776                                   msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3777
3778         /*
3779          * If the send was in response to a received message (context[0] is not
3780          * set to a cm_id), and is not a REJ, then it is a send that was
3781          * manually retried.
3782          */
3783         if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
3784                 msg->retries = 1;
3785
3786         atomic_long_add(1 + msg->retries,
3787                         &port->counter_group[CM_XMIT].counter[attr_index]);
3788         if (msg->retries)
3789                 atomic_long_add(msg->retries,
3790                                 &port->counter_group[CM_XMIT_RETRIES].
3791                                 counter[attr_index]);
3792
3793         switch (mad_send_wc->status) {
3794         case IB_WC_SUCCESS:
3795         case IB_WC_WR_FLUSH_ERR:
3796                 cm_free_msg(msg);
3797                 break;
3798         default:
3799                 if (msg->context[0] && msg->context[1])
3800                         cm_process_send_error(msg, mad_send_wc->status);
3801                 else
3802                         cm_free_msg(msg);
3803                 break;
3804         }
3805 }
3806
3807 static void cm_work_handler(struct work_struct *_work)
3808 {
3809         struct cm_work *work = container_of(_work, struct cm_work, work.work);
3810         int ret;
3811
3812         switch (work->cm_event.event) {
3813         case IB_CM_REQ_RECEIVED:
3814                 ret = cm_req_handler(work);
3815                 break;
3816         case IB_CM_MRA_RECEIVED:
3817                 ret = cm_mra_handler(work);
3818                 break;
3819         case IB_CM_REJ_RECEIVED:
3820                 ret = cm_rej_handler(work);
3821                 break;
3822         case IB_CM_REP_RECEIVED:
3823                 ret = cm_rep_handler(work);
3824                 break;
3825         case IB_CM_RTU_RECEIVED:
3826                 ret = cm_rtu_handler(work);
3827                 break;
3828         case IB_CM_USER_ESTABLISHED:
3829                 ret = cm_establish_handler(work);
3830                 break;
3831         case IB_CM_DREQ_RECEIVED:
3832                 ret = cm_dreq_handler(work);
3833                 break;
3834         case IB_CM_DREP_RECEIVED:
3835                 ret = cm_drep_handler(work);
3836                 break;
3837         case IB_CM_SIDR_REQ_RECEIVED:
3838                 ret = cm_sidr_req_handler(work);
3839                 break;
3840         case IB_CM_SIDR_REP_RECEIVED:
3841                 ret = cm_sidr_rep_handler(work);
3842                 break;
3843         case IB_CM_LAP_RECEIVED:
3844                 ret = cm_lap_handler(work);
3845                 break;
3846         case IB_CM_APR_RECEIVED:
3847                 ret = cm_apr_handler(work);
3848                 break;
3849         case IB_CM_TIMEWAIT_EXIT:
3850                 ret = cm_timewait_handler(work);
3851                 break;
3852         default:
3853                 pr_debug("cm_event.event: 0x%x\n", work->cm_event.event);
3854                 ret = -EINVAL;
3855                 break;
3856         }
3857         if (ret)
3858                 cm_free_work(work);
3859 }
3860
3861 static int cm_establish(struct ib_cm_id *cm_id)
3862 {
3863         struct cm_id_private *cm_id_priv;
3864         struct cm_work *work;
3865         unsigned long flags;
3866         int ret = 0;
3867         struct cm_device *cm_dev;
3868
3869         cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3870         if (!cm_dev)
3871                 return -ENODEV;
3872
3873         work = kmalloc(sizeof *work, GFP_ATOMIC);
3874         if (!work)
3875                 return -ENOMEM;
3876
3877         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3878         spin_lock_irqsave(&cm_id_priv->lock, flags);
3879         switch (cm_id->state)
3880         {
3881         case IB_CM_REP_SENT:
3882         case IB_CM_MRA_REP_RCVD:
3883                 cm_id->state = IB_CM_ESTABLISHED;
3884                 break;
3885         case IB_CM_ESTABLISHED:
3886                 ret = -EISCONN;
3887                 break;
3888         default:
3889                 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
3890                          be32_to_cpu(cm_id->local_id), cm_id->state);
3891                 ret = -EINVAL;
3892                 break;
3893         }
3894         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3895
3896         if (ret) {
3897                 kfree(work);
3898                 goto out;
3899         }
3900
3901         /*
3902          * The CM worker thread may try to destroy the cm_id before it
3903          * can execute this work item.  To prevent potential deadlock,
3904          * we need to find the cm_id once we're in the context of the
3905          * worker thread, rather than holding a reference on it.
3906          */
3907         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3908         work->local_id = cm_id->local_id;
3909         work->remote_id = cm_id->remote_id;
3910         work->mad_recv_wc = NULL;
3911         work->cm_event.event = IB_CM_USER_ESTABLISHED;
3912
3913         /* Check if the device started its remove_one */
3914         spin_lock_irqsave(&cm.lock, flags);
3915         if (!cm_dev->going_down) {
3916                 queue_delayed_work(cm.wq, &work->work, 0);
3917         } else {
3918                 kfree(work);
3919                 ret = -ENODEV;
3920         }
3921         spin_unlock_irqrestore(&cm.lock, flags);
3922
3923 out:
3924         return ret;
3925 }
3926
3927 static int cm_migrate(struct ib_cm_id *cm_id)
3928 {
3929         struct cm_id_private *cm_id_priv;
3930         struct cm_av tmp_av;
3931         unsigned long flags;
3932         int tmp_send_port_not_ready;
3933         int ret = 0;
3934
3935         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3936         spin_lock_irqsave(&cm_id_priv->lock, flags);
3937         if (cm_id->state == IB_CM_ESTABLISHED &&
3938             (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3939              cm_id->lap_state == IB_CM_LAP_IDLE)) {
3940                 cm_id->lap_state = IB_CM_LAP_IDLE;
3941                 /* Swap address vector */
3942                 tmp_av = cm_id_priv->av;
3943                 cm_id_priv->av = cm_id_priv->alt_av;
3944                 cm_id_priv->alt_av = tmp_av;
3945                 /* Swap port send ready state */
3946                 tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready;
3947                 cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready;
3948                 cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready;
3949         } else
3950                 ret = -EINVAL;
3951         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3952
3953         return ret;
3954 }
3955
3956 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3957 {
3958         int ret;
3959
3960         switch (event) {
3961         case IB_EVENT_COMM_EST:
3962                 ret = cm_establish(cm_id);
3963                 break;
3964         case IB_EVENT_PATH_MIG:
3965                 ret = cm_migrate(cm_id);
3966                 break;
3967         default:
3968                 ret = -EINVAL;
3969         }
3970         return ret;
3971 }
3972 EXPORT_SYMBOL(ib_cm_notify);
3973
3974 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3975                             struct ib_mad_send_buf *send_buf,
3976                             struct ib_mad_recv_wc *mad_recv_wc)
3977 {
3978         struct cm_port *port = mad_agent->context;
3979         struct cm_work *work;
3980         enum ib_cm_event_type event;
3981         bool alt_path = false;
3982         u16 attr_id;
3983         int paths = 0;
3984         int going_down = 0;
3985
3986         switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
3987         case CM_REQ_ATTR_ID:
3988                 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
3989                                                 mad_recv_wc->recv_buf.mad);
3990                 paths = 1 + (alt_path != 0);
3991                 event = IB_CM_REQ_RECEIVED;
3992                 break;
3993         case CM_MRA_ATTR_ID:
3994                 event = IB_CM_MRA_RECEIVED;
3995                 break;
3996         case CM_REJ_ATTR_ID:
3997                 event = IB_CM_REJ_RECEIVED;
3998                 break;
3999         case CM_REP_ATTR_ID:
4000                 event = IB_CM_REP_RECEIVED;
4001                 break;
4002         case CM_RTU_ATTR_ID:
4003                 event = IB_CM_RTU_RECEIVED;
4004                 break;
4005         case CM_DREQ_ATTR_ID:
4006                 event = IB_CM_DREQ_RECEIVED;
4007                 break;
4008         case CM_DREP_ATTR_ID:
4009                 event = IB_CM_DREP_RECEIVED;
4010                 break;
4011         case CM_SIDR_REQ_ATTR_ID:
4012                 event = IB_CM_SIDR_REQ_RECEIVED;
4013                 break;
4014         case CM_SIDR_REP_ATTR_ID:
4015                 event = IB_CM_SIDR_REP_RECEIVED;
4016                 break;
4017         case CM_LAP_ATTR_ID:
4018                 paths = 1;
4019                 event = IB_CM_LAP_RECEIVED;
4020                 break;
4021         case CM_APR_ATTR_ID:
4022                 event = IB_CM_APR_RECEIVED;
4023                 break;
4024         default:
4025                 ib_free_recv_mad(mad_recv_wc);
4026                 return;
4027         }
4028
4029         attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4030         atomic_long_inc(&port->counter_group[CM_RECV].
4031                         counter[attr_id - CM_ATTR_ID_OFFSET]);
4032
4033         work = kmalloc(sizeof(*work) + sizeof(struct sa_path_rec) * paths,
4034                        GFP_KERNEL);
4035         if (!work) {
4036                 ib_free_recv_mad(mad_recv_wc);
4037                 return;
4038         }
4039
4040         INIT_DELAYED_WORK(&work->work, cm_work_handler);
4041         work->cm_event.event = event;
4042         work->mad_recv_wc = mad_recv_wc;
4043         work->port = port;
4044
4045         /* Check if the device started its remove_one */
4046         spin_lock_irq(&cm.lock);
4047         if (!port->cm_dev->going_down)
4048                 queue_delayed_work(cm.wq, &work->work, 0);
4049         else
4050                 going_down = 1;
4051         spin_unlock_irq(&cm.lock);
4052
4053         if (going_down) {
4054                 kfree(work);
4055                 ib_free_recv_mad(mad_recv_wc);
4056         }
4057 }
4058
4059 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4060                                 struct ib_qp_attr *qp_attr,
4061                                 int *qp_attr_mask)
4062 {
4063         unsigned long flags;
4064         int ret;
4065
4066         spin_lock_irqsave(&cm_id_priv->lock, flags);
4067         switch (cm_id_priv->id.state) {
4068         case IB_CM_REQ_SENT:
4069         case IB_CM_MRA_REQ_RCVD:
4070         case IB_CM_REQ_RCVD:
4071         case IB_CM_MRA_REQ_SENT:
4072         case IB_CM_REP_RCVD:
4073         case IB_CM_MRA_REP_SENT:
4074         case IB_CM_REP_SENT:
4075         case IB_CM_MRA_REP_RCVD:
4076         case IB_CM_ESTABLISHED:
4077                 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4078                                 IB_QP_PKEY_INDEX | IB_QP_PORT;
4079                 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4080                 if (cm_id_priv->responder_resources)
4081                         qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4082                                                     IB_ACCESS_REMOTE_ATOMIC;
4083                 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4084                 qp_attr->port_num = cm_id_priv->av.port->port_num;
4085                 ret = 0;
4086                 break;
4087         default:
4088                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4089                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
4090                          cm_id_priv->id.state);
4091                 ret = -EINVAL;
4092                 break;
4093         }
4094         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4095         return ret;
4096 }
4097
4098 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4099                                struct ib_qp_attr *qp_attr,
4100                                int *qp_attr_mask)
4101 {
4102         unsigned long flags;
4103         int ret;
4104
4105         spin_lock_irqsave(&cm_id_priv->lock, flags);
4106         switch (cm_id_priv->id.state) {
4107         case IB_CM_REQ_RCVD:
4108         case IB_CM_MRA_REQ_SENT:
4109         case IB_CM_REP_RCVD:
4110         case IB_CM_MRA_REP_SENT:
4111         case IB_CM_REP_SENT:
4112         case IB_CM_MRA_REP_RCVD:
4113         case IB_CM_ESTABLISHED:
4114                 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4115                                 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4116                 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4117                 qp_attr->path_mtu = cm_id_priv->path_mtu;
4118                 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4119                 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4120                 if (cm_id_priv->qp_type == IB_QPT_RC ||
4121                     cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4122                         *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4123                                          IB_QP_MIN_RNR_TIMER;
4124                         qp_attr->max_dest_rd_atomic =
4125                                         cm_id_priv->responder_resources;
4126                         qp_attr->min_rnr_timer = 0;
4127                 }
4128                 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4129                         *qp_attr_mask |= IB_QP_ALT_PATH;
4130                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4131                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4132                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4133                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4134                 }
4135                 ret = 0;
4136                 break;
4137         default:
4138                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4139                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
4140                          cm_id_priv->id.state);
4141                 ret = -EINVAL;
4142                 break;
4143         }
4144         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4145         return ret;
4146 }
4147
4148 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4149                                struct ib_qp_attr *qp_attr,
4150                                int *qp_attr_mask)
4151 {
4152         unsigned long flags;
4153         int ret;
4154
4155         spin_lock_irqsave(&cm_id_priv->lock, flags);
4156         switch (cm_id_priv->id.state) {
4157         /* Allow transition to RTS before sending REP */
4158         case IB_CM_REQ_RCVD:
4159         case IB_CM_MRA_REQ_SENT:
4160
4161         case IB_CM_REP_RCVD:
4162         case IB_CM_MRA_REP_SENT:
4163         case IB_CM_REP_SENT:
4164         case IB_CM_MRA_REP_RCVD:
4165         case IB_CM_ESTABLISHED:
4166                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4167                         *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4168                         qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4169                         switch (cm_id_priv->qp_type) {
4170                         case IB_QPT_RC:
4171                         case IB_QPT_XRC_INI:
4172                                 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4173                                                  IB_QP_MAX_QP_RD_ATOMIC;
4174                                 qp_attr->retry_cnt = cm_id_priv->retry_count;
4175                                 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4176                                 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4177                                 /* fall through */
4178                         case IB_QPT_XRC_TGT:
4179                                 *qp_attr_mask |= IB_QP_TIMEOUT;
4180                                 qp_attr->timeout = cm_id_priv->av.timeout;
4181                                 break;
4182                         default:
4183                                 break;
4184                         }
4185                         if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4186                                 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4187                                 qp_attr->path_mig_state = IB_MIG_REARM;
4188                         }
4189                 } else {
4190                         *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4191                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4192                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4193                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4194                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4195                         qp_attr->path_mig_state = IB_MIG_REARM;
4196                 }
4197                 ret = 0;
4198                 break;
4199         default:
4200                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4201                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
4202                          cm_id_priv->id.state);
4203                 ret = -EINVAL;
4204                 break;
4205         }
4206         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4207         return ret;
4208 }
4209
4210 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4211                        struct ib_qp_attr *qp_attr,
4212                        int *qp_attr_mask)
4213 {
4214         struct cm_id_private *cm_id_priv;
4215         int ret;
4216
4217         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4218         switch (qp_attr->qp_state) {
4219         case IB_QPS_INIT:
4220                 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4221                 break;
4222         case IB_QPS_RTR:
4223                 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4224                 break;
4225         case IB_QPS_RTS:
4226                 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4227                 break;
4228         default:
4229                 ret = -EINVAL;
4230                 break;
4231         }
4232         return ret;
4233 }
4234 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4235
4236 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
4237                                char *buf)
4238 {
4239         struct cm_counter_group *group;
4240         struct cm_counter_attribute *cm_attr;
4241
4242         group = container_of(obj, struct cm_counter_group, obj);
4243         cm_attr = container_of(attr, struct cm_counter_attribute, attr);
4244
4245         return sprintf(buf, "%ld\n",
4246                        atomic_long_read(&group->counter[cm_attr->index]));
4247 }
4248
4249 static const struct sysfs_ops cm_counter_ops = {
4250         .show = cm_show_counter
4251 };
4252
4253 static struct kobj_type cm_counter_obj_type = {
4254         .sysfs_ops = &cm_counter_ops,
4255         .default_attrs = cm_counter_default_attrs
4256 };
4257
4258 static void cm_release_port_obj(struct kobject *obj)
4259 {
4260         struct cm_port *cm_port;
4261
4262         cm_port = container_of(obj, struct cm_port, port_obj);
4263         kfree(cm_port);
4264 }
4265
4266 static struct kobj_type cm_port_obj_type = {
4267         .release = cm_release_port_obj
4268 };
4269
4270 static char *cm_devnode(struct device *dev, umode_t *mode)
4271 {
4272         if (mode)
4273                 *mode = 0666;
4274         return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
4275 }
4276
4277 struct class cm_class = {
4278         .owner   = THIS_MODULE,
4279         .name    = "infiniband_cm",
4280         .devnode = cm_devnode,
4281 };
4282 EXPORT_SYMBOL(cm_class);
4283
4284 static int cm_create_port_fs(struct cm_port *port)
4285 {
4286         int i, ret;
4287
4288         ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type,
4289                                    &port->cm_dev->device->kobj,
4290                                    "%d", port->port_num);
4291         if (ret) {
4292                 kfree(port);
4293                 return ret;
4294         }
4295
4296         for (i = 0; i < CM_COUNTER_GROUPS; i++) {
4297                 ret = kobject_init_and_add(&port->counter_group[i].obj,
4298                                            &cm_counter_obj_type,
4299                                            &port->port_obj,
4300                                            "%s", counter_group_names[i]);
4301                 if (ret)
4302                         goto error;
4303         }
4304
4305         return 0;
4306
4307 error:
4308         while (i--)
4309                 kobject_put(&port->counter_group[i].obj);
4310         kobject_put(&port->port_obj);
4311         return ret;
4312
4313 }
4314
4315 static void cm_remove_port_fs(struct cm_port *port)
4316 {
4317         int i;
4318
4319         for (i = 0; i < CM_COUNTER_GROUPS; i++)
4320                 kobject_put(&port->counter_group[i].obj);
4321
4322         kobject_put(&port->port_obj);
4323 }
4324
4325 static void cm_add_one(struct ib_device *ib_device)
4326 {
4327         struct cm_device *cm_dev;
4328         struct cm_port *port;
4329         struct ib_mad_reg_req reg_req = {
4330                 .mgmt_class = IB_MGMT_CLASS_CM,
4331                 .mgmt_class_version = IB_CM_CLASS_VERSION,
4332         };
4333         struct ib_port_modify port_modify = {
4334                 .set_port_cap_mask = IB_PORT_CM_SUP
4335         };
4336         unsigned long flags;
4337         int ret;
4338         int count = 0;
4339         u8 i;
4340
4341         cm_dev = kzalloc(sizeof(*cm_dev) + sizeof(*port) *
4342                          ib_device->phys_port_cnt, GFP_KERNEL);
4343         if (!cm_dev)
4344                 return;
4345
4346         cm_dev->ib_device = ib_device;
4347         cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4348         cm_dev->going_down = 0;
4349         cm_dev->device = device_create(&cm_class, &ib_device->dev,
4350                                        MKDEV(0, 0), NULL,
4351                                        "%s", ib_device->name);
4352         if (IS_ERR(cm_dev->device)) {
4353                 kfree(cm_dev);
4354                 return;
4355         }
4356
4357         set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4358         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4359                 if (!rdma_cap_ib_cm(ib_device, i))
4360                         continue;
4361
4362                 port = kzalloc(sizeof *port, GFP_KERNEL);
4363                 if (!port)
4364                         goto error1;
4365
4366                 cm_dev->port[i-1] = port;
4367                 port->cm_dev = cm_dev;
4368                 port->port_num = i;
4369
4370                 INIT_LIST_HEAD(&port->cm_priv_prim_list);
4371                 INIT_LIST_HEAD(&port->cm_priv_altr_list);
4372
4373                 ret = cm_create_port_fs(port);
4374                 if (ret)
4375                         goto error1;
4376
4377                 port->mad_agent = ib_register_mad_agent(ib_device, i,
4378                                                         IB_QPT_GSI,
4379                                                         &reg_req,
4380                                                         0,
4381                                                         cm_send_handler,
4382                                                         cm_recv_handler,
4383                                                         port,
4384                                                         0);
4385                 if (IS_ERR(port->mad_agent))
4386                         goto error2;
4387
4388                 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4389                 if (ret)
4390                         goto error3;
4391
4392                 count++;
4393         }
4394
4395         if (!count)
4396                 goto free;
4397
4398         ib_set_client_data(ib_device, &cm_client, cm_dev);
4399
4400         write_lock_irqsave(&cm.device_lock, flags);
4401         list_add_tail(&cm_dev->list, &cm.device_list);
4402         write_unlock_irqrestore(&cm.device_lock, flags);
4403         return;
4404
4405 error3:
4406         ib_unregister_mad_agent(port->mad_agent);
4407 error2:
4408         cm_remove_port_fs(port);
4409 error1:
4410         port_modify.set_port_cap_mask = 0;
4411         port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4412         while (--i) {
4413                 if (!rdma_cap_ib_cm(ib_device, i))
4414                         continue;
4415
4416                 port = cm_dev->port[i-1];
4417                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4418                 ib_unregister_mad_agent(port->mad_agent);
4419                 cm_remove_port_fs(port);
4420         }
4421 free:
4422         device_unregister(cm_dev->device);
4423         kfree(cm_dev);
4424 }
4425
4426 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4427 {
4428         struct cm_device *cm_dev = client_data;
4429         struct cm_port *port;
4430         struct cm_id_private *cm_id_priv;
4431         struct ib_mad_agent *cur_mad_agent;
4432         struct ib_port_modify port_modify = {
4433                 .clr_port_cap_mask = IB_PORT_CM_SUP
4434         };
4435         unsigned long flags;
4436         int i;
4437
4438         if (!cm_dev)
4439                 return;
4440
4441         write_lock_irqsave(&cm.device_lock, flags);
4442         list_del(&cm_dev->list);
4443         write_unlock_irqrestore(&cm.device_lock, flags);
4444
4445         spin_lock_irq(&cm.lock);
4446         cm_dev->going_down = 1;
4447         spin_unlock_irq(&cm.lock);
4448
4449         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4450                 if (!rdma_cap_ib_cm(ib_device, i))
4451                         continue;
4452
4453                 port = cm_dev->port[i-1];
4454                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4455                 /* Mark all the cm_id's as not valid */
4456                 spin_lock_irq(&cm.lock);
4457                 list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list)
4458                         cm_id_priv->altr_send_port_not_ready = 1;
4459                 list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list)
4460                         cm_id_priv->prim_send_port_not_ready = 1;
4461                 spin_unlock_irq(&cm.lock);
4462                 /*
4463                  * We flush the queue here after the going_down set, this
4464                  * verify that no new works will be queued in the recv handler,
4465                  * after that we can call the unregister_mad_agent
4466                  */
4467                 flush_workqueue(cm.wq);
4468                 spin_lock_irq(&cm.state_lock);
4469                 cur_mad_agent = port->mad_agent;
4470                 port->mad_agent = NULL;
4471                 spin_unlock_irq(&cm.state_lock);
4472                 ib_unregister_mad_agent(cur_mad_agent);
4473                 cm_remove_port_fs(port);
4474         }
4475
4476         device_unregister(cm_dev->device);
4477         kfree(cm_dev);
4478 }
4479
4480 static int __init ib_cm_init(void)
4481 {
4482         int ret;
4483
4484         memset(&cm, 0, sizeof cm);
4485         INIT_LIST_HEAD(&cm.device_list);
4486         rwlock_init(&cm.device_lock);
4487         spin_lock_init(&cm.lock);
4488         spin_lock_init(&cm.state_lock);
4489         cm.listen_service_table = RB_ROOT;
4490         cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4491         cm.remote_id_table = RB_ROOT;
4492         cm.remote_qp_table = RB_ROOT;
4493         cm.remote_sidr_table = RB_ROOT;
4494         idr_init(&cm.local_id_table);
4495         get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4496         INIT_LIST_HEAD(&cm.timewait_list);
4497
4498         ret = class_register(&cm_class);
4499         if (ret) {
4500                 ret = -ENOMEM;
4501                 goto error1;
4502         }
4503
4504         cm.wq = alloc_workqueue("ib_cm", 0, 1);
4505         if (!cm.wq) {
4506                 ret = -ENOMEM;
4507                 goto error2;
4508         }
4509
4510         ret = ib_register_client(&cm_client);
4511         if (ret)
4512                 goto error3;
4513
4514         return 0;
4515 error3:
4516         destroy_workqueue(cm.wq);
4517 error2:
4518         class_unregister(&cm_class);
4519 error1:
4520         idr_destroy(&cm.local_id_table);
4521         return ret;
4522 }
4523
4524 static void __exit ib_cm_cleanup(void)
4525 {
4526         struct cm_timewait_info *timewait_info, *tmp;
4527
4528         spin_lock_irq(&cm.lock);
4529         list_for_each_entry(timewait_info, &cm.timewait_list, list)
4530                 cancel_delayed_work(&timewait_info->work.work);
4531         spin_unlock_irq(&cm.lock);
4532
4533         ib_unregister_client(&cm_client);
4534         destroy_workqueue(cm.wq);
4535
4536         list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4537                 list_del(&timewait_info->list);
4538                 kfree(timewait_info);
4539         }
4540
4541         class_unregister(&cm_class);
4542         idr_destroy(&cm.local_id_table);
4543 }
4544
4545 module_init(ib_cm_init);
4546 module_exit(ib_cm_cleanup);
4547