net/smc/smc_llc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Link Layer Control (LLC)
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Klaus Wacker <[email protected]>
  10  *              Ursula Braun <[email protected]>
  11  */
  12
  13 #include <net/tcp.h>
  14 #include <rdma/ib_verbs.h>
  15
  16 #include "smc.h"
  17 #include "smc_core.h"
  18 #include "smc_clc.h"
  19 #include "smc_llc.h"
  20 #include "smc_pnet.h"
  21
  22 #define SMC_LLC_DATA_LEN                40
  23
  24 struct smc_llc_hdr {
  25         struct smc_wr_rx_hdr common;
  26         union {
  27                 struct {
  28                         u8 length;      /* 44 */
  29         #if defined(__BIG_ENDIAN_BITFIELD)
  30                         u8 reserved:4,
  31                            add_link_rej_rsn:4;
  32 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  33                         u8 add_link_rej_rsn:4,
  34                            reserved:4;
  35 #endif
  36                 };
  37                 u16 length_v2;  /* 44 - 8192*/
  38         };
  39         u8 flags;
  40 } __packed;             /* format defined in
  41                          * IBM Shared Memory Communications Version 2
  42                          * (https://www.ibm.com/support/pages/node/6326337)
  43                          */
  44
  45 #define SMC_LLC_FLAG_NO_RMBE_EYEC       0x03
  46
  47 struct smc_llc_msg_confirm_link {       /* type 0x01 */
  48         struct smc_llc_hdr hd;
  49         u8 sender_mac[ETH_ALEN];
  50         u8 sender_gid[SMC_GID_SIZE];
  51         u8 sender_qp_num[3];
  52         u8 link_num;
  53         u8 link_uid[SMC_LGR_ID_SIZE];
  54         u8 max_links;
  55         u8 reserved[9];
  56 };
  57
  58 #define SMC_LLC_FLAG_ADD_LNK_REJ        0x40
  59 #define SMC_LLC_REJ_RSN_NO_ALT_PATH     1
  60
  61 #define SMC_LLC_ADD_LNK_MAX_LINKS       2
  62
  63 struct smc_llc_msg_add_link {           /* type 0x02 */
  64         struct smc_llc_hdr hd;
  65         u8 sender_mac[ETH_ALEN];
  66         u8 reserved2[2];
  67         u8 sender_gid[SMC_GID_SIZE];
  68         u8 sender_qp_num[3];
  69         u8 link_num;
  70 #if defined(__BIG_ENDIAN_BITFIELD)
  71         u8 reserved3 : 4,
  72            qp_mtu   : 4;
  73 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  74         u8 qp_mtu   : 4,
  75            reserved3 : 4;
  76 #endif
  77         u8 initial_psn[3];
  78         u8 reserved[8];
  79 };
  80
  81 struct smc_llc_msg_add_link_cont_rt {
  82         __be32 rmb_key;
  83         __be32 rmb_key_new;
  84         __be64 rmb_vaddr_new;
  85 };
  86
  87 struct smc_llc_msg_add_link_v2_ext {
  88 #if defined(__BIG_ENDIAN_BITFIELD)
  89         u8 v2_direct : 1,
  90            reserved  : 7;
  91 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  92         u8 reserved  : 7,
  93            v2_direct : 1;
  94 #endif
  95         u8 reserved2;
  96         u8 client_target_gid[SMC_GID_SIZE];
  97         u8 reserved3[8];
  98         u16 num_rkeys;
  99         struct smc_llc_msg_add_link_cont_rt rt[];
 100 } __packed;             /* format defined in
 101                          * IBM Shared Memory Communications Version 2
 102                          * (https://www.ibm.com/support/pages/node/6326337)
 103                          */
 104
 105 struct smc_llc_msg_req_add_link_v2 {
 106         struct smc_llc_hdr hd;
 107         u8 reserved[20];
 108         u8 gid_cnt;
 109         u8 reserved2[3];
 110         u8 gid[][SMC_GID_SIZE];
 111 };
 112
 113 #define SMC_LLC_RKEYS_PER_CONT_MSG      2
 114
 115 struct smc_llc_msg_add_link_cont {      /* type 0x03 */
 116         struct smc_llc_hdr hd;
 117         u8 link_num;
 118         u8 num_rkeys;
 119         u8 reserved2[2];
 120         struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
 121         u8 reserved[4];
 122 } __packed;                     /* format defined in RFC7609 */
 123
 124 #define SMC_LLC_FLAG_DEL_LINK_ALL       0x40
 125 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
 126
 127 struct smc_llc_msg_del_link {           /* type 0x04 */
 128         struct smc_llc_hdr hd;
 129         u8 link_num;
 130         __be32 reason;
 131         u8 reserved[35];
 132 } __packed;                     /* format defined in RFC7609 */
 133
 134 struct smc_llc_msg_test_link {          /* type 0x07 */
 135         struct smc_llc_hdr hd;
 136         u8 user_data[16];
 137         u8 reserved[24];
 138 };
 139
 140 struct smc_rmb_rtoken {
 141         union {
 142                 u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
 143                                 /* is actually the num of rtokens, first */
 144                                 /* rtoken is always for the current link */
 145                 u8 link_id;     /* link id of the rtoken */
 146         };
 147         __be32 rmb_key;
 148         __be64 rmb_vaddr;
 149 } __packed;                     /* format defined in RFC7609 */
 150
 151 #define SMC_LLC_RKEYS_PER_MSG           3
 152 #define SMC_LLC_RKEYS_PER_MSG_V2        255
 153
 154 struct smc_llc_msg_confirm_rkey {       /* type 0x06 */
 155         struct smc_llc_hdr hd;
 156         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
 157         u8 reserved;
 158 };
 159
 160 #define SMC_LLC_DEL_RKEY_MAX    8
 161 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
 162 #define SMC_LLC_FLAG_RKEY_NEG   0x20
 163
 164 struct smc_llc_msg_delete_rkey {        /* type 0x09 */
 165         struct smc_llc_hdr hd;
 166         u8 num_rkeys;
 167         u8 err_mask;
 168         u8 reserved[2];
 169         __be32 rkey[8];
 170         u8 reserved2[4];
 171 };
 172
 173 struct smc_llc_msg_delete_rkey_v2 {     /* type 0x29 */
 174         struct smc_llc_hdr hd;
 175         u8 num_rkeys;
 176         u8 num_inval_rkeys;
 177         u8 reserved[2];
 178         __be32 rkey[];
 179 };
 180
 181 union smc_llc_msg {
 182         struct smc_llc_msg_confirm_link confirm_link;
 183         struct smc_llc_msg_add_link add_link;
 184         struct smc_llc_msg_req_add_link_v2 req_add_link;
 185         struct smc_llc_msg_add_link_cont add_link_cont;
 186         struct smc_llc_msg_del_link delete_link;
 187
 188         struct smc_llc_msg_confirm_rkey confirm_rkey;
 189         struct smc_llc_msg_delete_rkey delete_rkey;
 190
 191         struct smc_llc_msg_test_link test_link;
 192         struct {
 193                 struct smc_llc_hdr hdr;
 194                 u8 data[SMC_LLC_DATA_LEN];
 195         } raw;
 196 };
 197
 198 #define SMC_LLC_FLAG_RESP               0x80
 199
 200 struct smc_llc_qentry {
 201         struct list_head list;
 202         struct smc_link *link;
 203         union smc_llc_msg msg;
 204 };
 205
 206 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
 207
 208 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
 209 {
 210         struct smc_llc_qentry *qentry = flow->qentry;
 211
 212         flow->qentry = NULL;
 213         return qentry;
 214 }
 215
 216 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
 217 {
 218         struct smc_llc_qentry *qentry;
 219
 220         if (flow->qentry) {
 221                 qentry = flow->qentry;
 222                 flow->qentry = NULL;
 223                 kfree(qentry);
 224         }
 225 }
 226
 227 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
 228                                            struct smc_llc_qentry *qentry)
 229 {
 230         flow->qentry = qentry;
 231 }
 232
 233 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
 234                                   struct smc_llc_qentry *qentry)
 235 {
 236         u8 msg_type = qentry->msg.raw.hdr.common.llc_type;
 237
 238         if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
 239             flow_type != msg_type && !lgr->delayed_event) {
 240                 lgr->delayed_event = qentry;
 241                 return;
 242         }
 243         /* drop parallel or already-in-progress llc requests */
 244         if (flow_type != msg_type)
 245                 pr_warn_once("smc: SMC-R lg %*phN dropped parallel "
 246                              "LLC msg: msg %d flow %d role %d\n",
 247                              SMC_LGR_ID_SIZE, &lgr->id,
 248                              qentry->msg.raw.hdr.common.type,
 249                              flow_type, lgr->role);
 250         kfree(qentry);
 251 }
 252
 253 /* try to start a new llc flow, initiated by an incoming llc msg */
 254 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
 255                                struct smc_llc_qentry *qentry)
 256 {
 257         struct smc_link_group *lgr = qentry->link->lgr;
 258
 259         spin_lock_bh(&lgr->llc_flow_lock);
 260         if (flow->type) {
 261                 /* a flow is already active */
 262                 smc_llc_flow_parallel(lgr, flow->type, qentry);
 263                 spin_unlock_bh(&lgr->llc_flow_lock);
 264                 return false;
 265         }
 266         switch (qentry->msg.raw.hdr.common.llc_type) {
 267         case SMC_LLC_ADD_LINK:
 268                 flow->type = SMC_LLC_FLOW_ADD_LINK;
 269                 break;
 270         case SMC_LLC_DELETE_LINK:
 271                 flow->type = SMC_LLC_FLOW_DEL_LINK;
 272                 break;
 273         case SMC_LLC_CONFIRM_RKEY:
 274         case SMC_LLC_DELETE_RKEY:
 275                 flow->type = SMC_LLC_FLOW_RKEY;
 276                 break;
 277         default:
 278                 flow->type = SMC_LLC_FLOW_NONE;
 279         }
 280         smc_llc_flow_qentry_set(flow, qentry);
 281         spin_unlock_bh(&lgr->llc_flow_lock);
 282         return true;
 283 }
 284
 285 /* start a new local llc flow, wait till current flow finished */
 286 int smc_llc_flow_initiate(struct smc_link_group *lgr,
 287                           enum smc_llc_flowtype type)
 288 {
 289         enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
 290         int rc;
 291
 292         /* all flows except confirm_rkey and delete_rkey are exclusive,
 293          * confirm/delete rkey flows can run concurrently (local and remote)
 294          */
 295         if (type == SMC_LLC_FLOW_RKEY)
 296                 allowed_remote = SMC_LLC_FLOW_RKEY;
 297 again:
 298         if (list_empty(&lgr->list))
 299                 return -ENODEV;
 300         spin_lock_bh(&lgr->llc_flow_lock);
 301         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 302             (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 303              lgr->llc_flow_rmt.type == allowed_remote)) {
 304                 lgr->llc_flow_lcl.type = type;
 305                 spin_unlock_bh(&lgr->llc_flow_lock);
 306                 return 0;
 307         }
 308         spin_unlock_bh(&lgr->llc_flow_lock);
 309         rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
 310                                 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 311                                  (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 312                                   lgr->llc_flow_rmt.type == allowed_remote))),
 313                                 SMC_LLC_WAIT_TIME * 10);
 314         if (!rc)
 315                 return -ETIMEDOUT;
 316         goto again;
 317 }
 318
 319 /* finish the current llc flow */
 320 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
 321 {
 322         spin_lock_bh(&lgr->llc_flow_lock);
 323         memset(flow, 0, sizeof(*flow));
 324         flow->type = SMC_LLC_FLOW_NONE;
 325         spin_unlock_bh(&lgr->llc_flow_lock);
 326         if (!list_empty(&lgr->list) && lgr->delayed_event &&
 327             flow == &lgr->llc_flow_lcl)
 328                 schedule_work(&lgr->llc_event_work);
 329         else
 330                 wake_up(&lgr->llc_flow_waiter);
 331 }
 332
 333 /* lnk is optional and used for early wakeup when link goes down, useful in
 334  * cases where we wait for a response on the link after we sent a request
 335  */
 336 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
 337                                     struct smc_link *lnk,
 338                                     int time_out, u8 exp_msg)
 339 {
 340         struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
 341         u8 rcv_msg;
 342
 343         wait_event_timeout(lgr->llc_msg_waiter,
 344                            (flow->qentry ||
 345                             (lnk && !smc_link_usable(lnk)) ||
 346                             list_empty(&lgr->list)),
 347                            time_out);
 348         if (!flow->qentry ||
 349             (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
 350                 smc_llc_flow_qentry_del(flow);
 351                 goto out;
 352         }
 353         rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type;
 354         if (exp_msg && rcv_msg != exp_msg) {
 355                 if (exp_msg == SMC_LLC_ADD_LINK &&
 356                     rcv_msg == SMC_LLC_DELETE_LINK) {
 357                         /* flow_start will delay the unexpected msg */
 358                         smc_llc_flow_start(&lgr->llc_flow_lcl,
 359                                            smc_llc_flow_qentry_clr(flow));
 360                         return NULL;
 361                 }
 362                 pr_warn_once("smc: SMC-R lg %*phN dropped unexpected LLC msg: "
 363                              "msg %d exp %d flow %d role %d flags %x\n",
 364                              SMC_LGR_ID_SIZE, &lgr->id, rcv_msg, exp_msg,
 365                              flow->type, lgr->role,
 366                              flow->qentry->msg.raw.hdr.flags);
 367                 smc_llc_flow_qentry_del(flow);
 368         }
 369 out:
 370         return flow->qentry;
 371 }
 372
 373 /********************************** send *************************************/
 374
 375 struct smc_llc_tx_pend {
 376 };
 377
 378 /* handler for send/transmission completion of an LLC msg */
 379 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
 380                                struct smc_link *link,
 381                                enum ib_wc_status wc_status)
 382 {
 383         /* future work: handle wc_status error for recovery and failover */
 384 }
 385
 386 /**
 387  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
 388  * @link: Pointer to SMC link used for sending LLC control message.
 389  * @wr_buf: Out variable returning pointer to work request payload buffer.
 390  * @pend: Out variable returning pointer to private pending WR tracking.
 391  *        It's the context the transmit complete handler will get.
 392  *
 393  * Reserves and pre-fills an entry for a pending work request send/tx.
 394  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
 395  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
 396  *
 397  * Return: 0 on success, otherwise an error value.
 398  */
 399 static int smc_llc_add_pending_send(struct smc_link *link,
 400                                     struct smc_wr_buf **wr_buf,
 401                                     struct smc_wr_tx_pend_priv **pend)
 402 {
 403         int rc;
 404
 405         rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
 406                                      pend);
 407         if (rc < 0)
 408                 return rc;
 409         BUILD_BUG_ON_MSG(
 410                 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
 411                 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
 412         BUILD_BUG_ON_MSG(
 413                 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
 414                 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
 415         BUILD_BUG_ON_MSG(
 416                 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
 417                 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
 418         return 0;
 419 }
 420
 421 static int smc_llc_add_pending_send_v2(struct smc_link *link,
 422                                        struct smc_wr_v2_buf **wr_buf,
 423                                        struct smc_wr_tx_pend_priv **pend)
 424 {
 425         int rc;
 426
 427         rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend);
 428         if (rc < 0)
 429                 return rc;
 430         return 0;
 431 }
 432
 433 static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr,
 434                                  struct smc_link_group *lgr, size_t len)
 435 {
 436         if (lgr->smc_version == SMC_V2) {
 437                 hdr->common.llc_version = SMC_V2;
 438                 hdr->length_v2 = len;
 439         } else {
 440                 hdr->common.llc_version = 0;
 441                 hdr->length = len;
 442         }
 443 }
 444
 445 /* high-level API to send LLC confirm link */
 446 int smc_llc_send_confirm_link(struct smc_link *link,
 447                               enum smc_llc_reqresp reqresp)
 448 {
 449         struct smc_llc_msg_confirm_link *confllc;
 450         struct smc_wr_tx_pend_priv *pend;
 451         struct smc_wr_buf *wr_buf;
 452         int rc;
 453
 454         if (!smc_wr_tx_link_hold(link))
 455                 return -ENOLINK;
 456         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 457         if (rc)
 458                 goto put_out;
 459         confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
 460         memset(confllc, 0, sizeof(*confllc));
 461         confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK;
 462         smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc));
 463         confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
 464         if (reqresp == SMC_LLC_RESP)
 465                 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
 466         memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
 467                ETH_ALEN);
 468         memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
 469         hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
 470         confllc->link_num = link->link_id;
 471         memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
 472         confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
 473         /* send llc message */
 474         rc = smc_wr_tx_send(link, pend);
 475 put_out:
 476         smc_wr_tx_link_put(link);
 477         return rc;
 478 }
 479
 480 /* send LLC confirm rkey request */
 481 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
 482                                      struct smc_buf_desc *rmb_desc)
 483 {
 484         struct smc_llc_msg_confirm_rkey *rkeyllc;
 485         struct smc_wr_tx_pend_priv *pend;
 486         struct smc_wr_buf *wr_buf;
 487         struct smc_link *link;
 488         int i, rc, rtok_ix;
 489
 490         if (!smc_wr_tx_link_hold(send_link))
 491                 return -ENOLINK;
 492         rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
 493         if (rc)
 494                 goto put_out;
 495         rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
 496         memset(rkeyllc, 0, sizeof(*rkeyllc));
 497         rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY;
 498         smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc));
 499
 500         rtok_ix = 1;
 501         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 502                 link = &send_link->lgr->lnk[i];
 503                 if (smc_link_active(link) && link != send_link) {
 504                         rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
 505                         rkeyllc->rtoken[rtok_ix].rmb_key =
 506                                 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 507                         rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64(
 508                                 (u64)sg_dma_address(
 509                                         rmb_desc->sgt[link->link_idx].sgl));
 510                         rtok_ix++;
 511                 }
 512         }
 513         /* rkey of send_link is in rtoken[0] */
 514         rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
 515         rkeyllc->rtoken[0].rmb_key =
 516                 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey);
 517         rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
 518                 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
 519         /* send llc message */
 520         rc = smc_wr_tx_send(send_link, pend);
 521 put_out:
 522         smc_wr_tx_link_put(send_link);
 523         return rc;
 524 }
 525
 526 /* send LLC delete rkey request */
 527 static int smc_llc_send_delete_rkey(struct smc_link *link,
 528                                     struct smc_buf_desc *rmb_desc)
 529 {
 530         struct smc_llc_msg_delete_rkey *rkeyllc;
 531         struct smc_wr_tx_pend_priv *pend;
 532         struct smc_wr_buf *wr_buf;
 533         int rc;
 534
 535         if (!smc_wr_tx_link_hold(link))
 536                 return -ENOLINK;
 537         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 538         if (rc)
 539                 goto put_out;
 540         rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
 541         memset(rkeyllc, 0, sizeof(*rkeyllc));
 542         rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY;
 543         smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc));
 544         rkeyllc->num_rkeys = 1;
 545         rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 546         /* send llc message */
 547         rc = smc_wr_tx_send(link, pend);
 548 put_out:
 549         smc_wr_tx_link_put(link);
 550         return rc;
 551 }
 552
 553 /* return first buffer from any of the next buf lists */
 554 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
 555                                                   int *buf_lst)
 556 {
 557         struct smc_buf_desc *buf_pos;
 558
 559         while (*buf_lst < SMC_RMBE_SIZES) {
 560                 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
 561                                                    struct smc_buf_desc, list);
 562                 if (buf_pos)
 563                         return buf_pos;
 564                 (*buf_lst)++;
 565         }
 566         return NULL;
 567 }
 568
 569 /* return next rmb from buffer lists */
 570 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
 571                                                  int *buf_lst,
 572                                                  struct smc_buf_desc *buf_pos)
 573 {
 574         struct smc_buf_desc *buf_next;
 575
 576         if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
 577                 (*buf_lst)++;
 578                 return _smc_llc_get_next_rmb(lgr, buf_lst);
 579         }
 580         buf_next = list_next_entry(buf_pos, list);
 581         return buf_next;
 582 }
 583
 584 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
 585                                                   int *buf_lst)
 586 {
 587         *buf_lst = 0;
 588         return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
 589 }
 590
 591 static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext,
 592                                struct smc_link *link, struct smc_link *link_new)
 593 {
 594         struct smc_link_group *lgr = link->lgr;
 595         struct smc_buf_desc *buf_pos;
 596         int prim_lnk_idx, lnk_idx, i;
 597         struct smc_buf_desc *rmb;
 598         int len = sizeof(*ext);
 599         int buf_lst;
 600
 601         ext->v2_direct = !lgr->uses_gateway;
 602         memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE);
 603
 604         prim_lnk_idx = link->link_idx;
 605         lnk_idx = link_new->link_idx;
 606         mutex_lock(&lgr->rmbs_lock);
 607         ext->num_rkeys = lgr->conns_num;
 608         if (!ext->num_rkeys)
 609                 goto out;
 610         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
 611         for (i = 0; i < ext->num_rkeys; i++) {
 612                 if (!buf_pos)
 613                         break;
 614                 rmb = buf_pos;
 615                 ext->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey);
 616                 ext->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey);
 617                 ext->rt[i].rmb_vaddr_new =
 618                         cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
 619                 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
 620                 while (buf_pos && !(buf_pos)->used)
 621                         buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
 622         }
 623         len += i * sizeof(ext->rt[0]);
 624 out:
 625         mutex_unlock(&lgr->rmbs_lock);
 626         return len;
 627 }
 628
 629 /* send ADD LINK request or response */
 630 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 631                           struct smc_link *link_new,
 632                           enum smc_llc_reqresp reqresp)
 633 {
 634         struct smc_llc_msg_add_link_v2_ext *ext = NULL;
 635         struct smc_llc_msg_add_link *addllc;
 636         struct smc_wr_tx_pend_priv *pend;
 637         int len = sizeof(*addllc);
 638         int rc;
 639
 640         if (!smc_wr_tx_link_hold(link))
 641                 return -ENOLINK;
 642         if (link->lgr->smc_version == SMC_V2) {
 643                 struct smc_wr_v2_buf *wr_buf;
 644
 645                 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
 646                 if (rc)
 647                         goto put_out;
 648                 addllc = (struct smc_llc_msg_add_link *)wr_buf;
 649                 ext = (struct smc_llc_msg_add_link_v2_ext *)
 650                                                 &wr_buf->raw[sizeof(*addllc)];
 651                 memset(ext, 0, SMC_WR_TX_SIZE);
 652         } else {
 653                 struct smc_wr_buf *wr_buf;
 654
 655                 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 656                 if (rc)
 657                         goto put_out;
 658                 addllc = (struct smc_llc_msg_add_link *)wr_buf;
 659         }
 660
 661         memset(addllc, 0, sizeof(*addllc));
 662         addllc->hd.common.llc_type = SMC_LLC_ADD_LINK;
 663         if (reqresp == SMC_LLC_RESP)
 664                 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
 665         memcpy(addllc->sender_mac, mac, ETH_ALEN);
 666         memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
 667         if (link_new) {
 668                 addllc->link_num = link_new->link_id;
 669                 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
 670                 hton24(addllc->initial_psn, link_new->psn_initial);
 671                 if (reqresp == SMC_LLC_REQ)
 672                         addllc->qp_mtu = link_new->path_mtu;
 673                 else
 674                         addllc->qp_mtu = min(link_new->path_mtu,
 675                                              link_new->peer_mtu);
 676         }
 677         if (ext && link_new)
 678                 len += smc_llc_fill_ext_v2(ext, link, link_new);
 679         smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len);
 680         /* send llc message */
 681         if (link->lgr->smc_version == SMC_V2)
 682                 rc = smc_wr_tx_v2_send(link, pend, len);
 683         else
 684                 rc = smc_wr_tx_send(link, pend);
 685 put_out:
 686         smc_wr_tx_link_put(link);
 687         return rc;
 688 }
 689
 690 /* send DELETE LINK request or response */
 691 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
 692                              enum smc_llc_reqresp reqresp, bool orderly,
 693                              u32 reason)
 694 {
 695         struct smc_llc_msg_del_link *delllc;
 696         struct smc_wr_tx_pend_priv *pend;
 697         struct smc_wr_buf *wr_buf;
 698         int rc;
 699
 700         if (!smc_wr_tx_link_hold(link))
 701                 return -ENOLINK;
 702         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 703         if (rc)
 704                 goto put_out;
 705         delllc = (struct smc_llc_msg_del_link *)wr_buf;
 706
 707         memset(delllc, 0, sizeof(*delllc));
 708         delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK;
 709         smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc));
 710         if (reqresp == SMC_LLC_RESP)
 711                 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
 712         if (orderly)
 713                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
 714         if (link_del_id)
 715                 delllc->link_num = link_del_id;
 716         else
 717                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
 718         delllc->reason = htonl(reason);
 719         /* send llc message */
 720         rc = smc_wr_tx_send(link, pend);
 721 put_out:
 722         smc_wr_tx_link_put(link);
 723         return rc;
 724 }
 725
 726 /* send LLC test link request */
 727 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
 728 {
 729         struct smc_llc_msg_test_link *testllc;
 730         struct smc_wr_tx_pend_priv *pend;
 731         struct smc_wr_buf *wr_buf;
 732         int rc;
 733
 734         if (!smc_wr_tx_link_hold(link))
 735                 return -ENOLINK;
 736         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 737         if (rc)
 738                 goto put_out;
 739         testllc = (struct smc_llc_msg_test_link *)wr_buf;
 740         memset(testllc, 0, sizeof(*testllc));
 741         testllc->hd.common.llc_type = SMC_LLC_TEST_LINK;
 742         smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc));
 743         memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
 744         /* send llc message */
 745         rc = smc_wr_tx_send(link, pend);
 746 put_out:
 747         smc_wr_tx_link_put(link);
 748         return rc;
 749 }
 750
 751 /* schedule an llc send on link, may wait for buffers */
 752 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
 753 {
 754         struct smc_wr_tx_pend_priv *pend;
 755         struct smc_wr_buf *wr_buf;
 756         int rc;
 757
 758         if (!smc_wr_tx_link_hold(link))
 759                 return -ENOLINK;
 760         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 761         if (rc)
 762                 goto put_out;
 763         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 764         rc = smc_wr_tx_send(link, pend);
 765 put_out:
 766         smc_wr_tx_link_put(link);
 767         return rc;
 768 }
 769
 770 /* schedule an llc send on link, may wait for buffers,
 771  * and wait for send completion notification.
 772  * @return 0 on success
 773  */
 774 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
 775 {
 776         struct smc_wr_tx_pend_priv *pend;
 777         struct smc_wr_buf *wr_buf;
 778         int rc;
 779
 780         if (!smc_wr_tx_link_hold(link))
 781                 return -ENOLINK;
 782         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 783         if (rc)
 784                 goto put_out;
 785         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 786         rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
 787 put_out:
 788         smc_wr_tx_link_put(link);
 789         return rc;
 790 }
 791
 792 /********************************* receive ***********************************/
 793
 794 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
 795                                   enum smc_lgr_type lgr_new_t)
 796 {
 797         int i;
 798
 799         if (lgr->type == SMC_LGR_SYMMETRIC ||
 800             (lgr->type != SMC_LGR_SINGLE &&
 801              (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 802               lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
 803                 return -EMLINK;
 804
 805         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 806             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
 807                 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
 808                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 809                                 return i;
 810         } else {
 811                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
 812                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 813                                 return i;
 814         }
 815         return -EMLINK;
 816 }
 817
 818 /* send one add_link_continue msg */
 819 static int smc_llc_add_link_cont(struct smc_link *link,
 820                                  struct smc_link *link_new, u8 *num_rkeys_todo,
 821                                  int *buf_lst, struct smc_buf_desc **buf_pos)
 822 {
 823         struct smc_llc_msg_add_link_cont *addc_llc;
 824         struct smc_link_group *lgr = link->lgr;
 825         int prim_lnk_idx, lnk_idx, i, rc;
 826         struct smc_wr_tx_pend_priv *pend;
 827         struct smc_wr_buf *wr_buf;
 828         struct smc_buf_desc *rmb;
 829         u8 n;
 830
 831         if (!smc_wr_tx_link_hold(link))
 832                 return -ENOLINK;
 833         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 834         if (rc)
 835                 goto put_out;
 836         addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
 837         memset(addc_llc, 0, sizeof(*addc_llc));
 838
 839         prim_lnk_idx = link->link_idx;
 840         lnk_idx = link_new->link_idx;
 841         addc_llc->link_num = link_new->link_id;
 842         addc_llc->num_rkeys = *num_rkeys_todo;
 843         n = *num_rkeys_todo;
 844         for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
 845                 if (!*buf_pos) {
 846                         addc_llc->num_rkeys = addc_llc->num_rkeys -
 847                                               *num_rkeys_todo;
 848                         *num_rkeys_todo = 0;
 849                         break;
 850                 }
 851                 rmb = *buf_pos;
 852
 853                 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey);
 854                 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey);
 855                 addc_llc->rt[i].rmb_vaddr_new =
 856                         cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
 857
 858                 (*num_rkeys_todo)--;
 859                 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 860                 while (*buf_pos && !(*buf_pos)->used)
 861                         *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 862         }
 863         addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT;
 864         addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
 865         if (lgr->role == SMC_CLNT)
 866                 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
 867         rc = smc_wr_tx_send(link, pend);
 868 put_out:
 869         smc_wr_tx_link_put(link);
 870         return rc;
 871 }
 872
 873 static int smc_llc_cli_rkey_exchange(struct smc_link *link,
 874                                      struct smc_link *link_new)
 875 {
 876         struct smc_llc_msg_add_link_cont *addc_llc;
 877         struct smc_link_group *lgr = link->lgr;
 878         u8 max, num_rkeys_send, num_rkeys_recv;
 879         struct smc_llc_qentry *qentry;
 880         struct smc_buf_desc *buf_pos;
 881         int buf_lst;
 882         int rc = 0;
 883         int i;
 884
 885         mutex_lock(&lgr->rmbs_lock);
 886         num_rkeys_send = lgr->conns_num;
 887         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
 888         do {
 889                 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
 890                                       SMC_LLC_ADD_LINK_CONT);
 891                 if (!qentry) {
 892                         rc = -ETIMEDOUT;
 893                         break;
 894                 }
 895                 addc_llc = &qentry->msg.add_link_cont;
 896                 num_rkeys_recv = addc_llc->num_rkeys;
 897                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
 898                 for (i = 0; i < max; i++) {
 899                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
 900                                        addc_llc->rt[i].rmb_key,
 901                                        addc_llc->rt[i].rmb_vaddr_new,
 902                                        addc_llc->rt[i].rmb_key_new);
 903                         num_rkeys_recv--;
 904                 }
 905                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 906                 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
 907                                            &buf_lst, &buf_pos);
 908                 if (rc)
 909                         break;
 910         } while (num_rkeys_send || num_rkeys_recv);
 911
 912         mutex_unlock(&lgr->rmbs_lock);
 913         return rc;
 914 }
 915
 916 /* prepare and send an add link reject response */
 917 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
 918 {
 919         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 920         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
 921         qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
 922         smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
 923                              sizeof(qentry->msg));
 924         return smc_llc_send_message(qentry->link, &qentry->msg);
 925 }
 926
 927 static int smc_llc_cli_conf_link(struct smc_link *link,
 928                                  struct smc_init_info *ini,
 929                                  struct smc_link *link_new,
 930                                  enum smc_lgr_type lgr_new_t)
 931 {
 932         struct smc_link_group *lgr = link->lgr;
 933         struct smc_llc_qentry *qentry = NULL;
 934         int rc = 0;
 935
 936         /* receive CONFIRM LINK request over RoCE fabric */
 937         qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
 938         if (!qentry) {
 939                 rc = smc_llc_send_delete_link(link, link_new->link_id,
 940                                               SMC_LLC_REQ, false,
 941                                               SMC_LLC_DEL_LOST_PATH);
 942                 return -ENOLINK;
 943         }
 944         if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
 945                 /* received DELETE_LINK instead */
 946                 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 947                 smc_llc_send_message(link, &qentry->msg);
 948                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 949                 return -ENOLINK;
 950         }
 951         smc_llc_save_peer_uid(qentry);
 952         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 953
 954         rc = smc_ib_modify_qp_rts(link_new);
 955         if (rc) {
 956                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 957                                          false, SMC_LLC_DEL_LOST_PATH);
 958                 return -ENOLINK;
 959         }
 960         smc_wr_remember_qp_attr(link_new);
 961
 962         rc = smcr_buf_reg_lgr(link_new);
 963         if (rc) {
 964                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 965                                          false, SMC_LLC_DEL_LOST_PATH);
 966                 return -ENOLINK;
 967         }
 968
 969         /* send CONFIRM LINK response over RoCE fabric */
 970         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
 971         if (rc) {
 972                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 973                                          false, SMC_LLC_DEL_LOST_PATH);
 974                 return -ENOLINK;
 975         }
 976         smc_llc_link_active(link_new);
 977         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 978             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
 979                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
 980         else
 981                 smcr_lgr_set_type(lgr, lgr_new_t);
 982         return 0;
 983 }
 984
 985 static void smc_llc_save_add_link_rkeys(struct smc_link *link,
 986                                         struct smc_link *link_new)
 987 {
 988         struct smc_llc_msg_add_link_v2_ext *ext;
 989         struct smc_link_group *lgr = link->lgr;
 990         int max, i;
 991
 992         ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 +
 993                                                      SMC_WR_TX_SIZE);
 994         max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
 995         mutex_lock(&lgr->rmbs_lock);
 996         for (i = 0; i < max; i++) {
 997                 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
 998                                ext->rt[i].rmb_key,
 999                                ext->rt[i].rmb_vaddr_new,
1000                                ext->rt[i].rmb_key_new);
1001         }
1002         mutex_unlock(&lgr->rmbs_lock);
1003 }
1004
1005 static void smc_llc_save_add_link_info(struct smc_link *link,
1006                                        struct smc_llc_msg_add_link *add_llc)
1007 {
1008         link->peer_qpn = ntoh24(add_llc->sender_qp_num);
1009         memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
1010         memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
1011         link->peer_psn = ntoh24(add_llc->initial_psn);
1012         link->peer_mtu = add_llc->qp_mtu;
1013 }
1014
1015 /* as an SMC client, process an add link request */
1016 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
1017 {
1018         struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
1019         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1020         struct smc_link_group *lgr = smc_get_lgr(link);
1021         struct smc_init_info *ini = NULL;
1022         struct smc_link *lnk_new = NULL;
1023         int lnk_idx, rc = 0;
1024
1025         if (!llc->qp_mtu)
1026                 goto out_reject;
1027
1028         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1029         if (!ini) {
1030                 rc = -ENOMEM;
1031                 goto out_reject;
1032         }
1033
1034         ini->vlan_id = lgr->vlan_id;
1035         if (lgr->smc_version == SMC_V2) {
1036                 ini->check_smcrv2 = true;
1037                 ini->smcrv2.saddr = lgr->saddr;
1038                 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid);
1039         }
1040         smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1041         if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1042             (lgr->smc_version == SMC_V2 ||
1043              !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1044                 if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2)
1045                         goto out_reject;
1046                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1047         }
1048         if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1049                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1050                 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1051                 ini->smcrv2.ib_port_v2 = link->ibport;
1052         } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1053                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1054                 ini->ib_dev = link->smcibdev;
1055                 ini->ib_port = link->ibport;
1056         }
1057         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1058         if (lnk_idx < 0)
1059                 goto out_reject;
1060         lnk_new = &lgr->lnk[lnk_idx];
1061         rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini);
1062         if (rc)
1063                 goto out_reject;
1064         smc_llc_save_add_link_info(lnk_new, llc);
1065         lnk_new->link_id = llc->link_num;       /* SMC server assigns link id */
1066         smc_llc_link_set_uid(lnk_new);
1067
1068         rc = smc_ib_ready_link(lnk_new);
1069         if (rc)
1070                 goto out_clear_lnk;
1071
1072         rc = smcr_buf_map_lgr(lnk_new);
1073         if (rc)
1074                 goto out_clear_lnk;
1075
1076         rc = smc_llc_send_add_link(link,
1077                                    lnk_new->smcibdev->mac[lnk_new->ibport - 1],
1078                                    lnk_new->gid, lnk_new, SMC_LLC_RESP);
1079         if (rc)
1080                 goto out_clear_lnk;
1081         if (lgr->smc_version == SMC_V2) {
1082                 smc_llc_save_add_link_rkeys(link, lnk_new);
1083         } else {
1084                 rc = smc_llc_cli_rkey_exchange(link, lnk_new);
1085                 if (rc) {
1086                         rc = 0;
1087                         goto out_clear_lnk;
1088                 }
1089         }
1090         rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t);
1091         if (!rc)
1092                 goto out;
1093 out_clear_lnk:
1094         lnk_new->state = SMC_LNK_INACTIVE;
1095         smcr_link_clear(lnk_new, false);
1096 out_reject:
1097         smc_llc_cli_add_link_reject(qentry);
1098 out:
1099         kfree(ini);
1100         kfree(qentry);
1101         return rc;
1102 }
1103
1104 static void smc_llc_send_request_add_link(struct smc_link *link)
1105 {
1106         struct smc_llc_msg_req_add_link_v2 *llc;
1107         struct smc_wr_tx_pend_priv *pend;
1108         struct smc_wr_v2_buf *wr_buf;
1109         struct smc_gidlist gidlist;
1110         int rc, len, i;
1111
1112         if (!smc_wr_tx_link_hold(link))
1113                 return;
1114         if (link->lgr->type == SMC_LGR_SYMMETRIC ||
1115             link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1116                 goto put_out;
1117
1118         smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid);
1119         if (gidlist.len <= 1)
1120                 goto put_out;
1121
1122         rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
1123         if (rc)
1124                 goto put_out;
1125         llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf;
1126         memset(llc, 0, SMC_WR_TX_SIZE);
1127
1128         llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK;
1129         for (i = 0; i < gidlist.len; i++)
1130                 memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0]));
1131         llc->gid_cnt = gidlist.len;
1132         len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0]));
1133         smc_llc_init_msg_hdr(&llc->hd, link->lgr, len);
1134         rc = smc_wr_tx_v2_send(link, pend, len);
1135         if (!rc)
1136                 /* set REQ_ADD_LINK flow and wait for response from peer */
1137                 link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;
1138 put_out:
1139         smc_wr_tx_link_put(link);
1140 }
1141
1142 /* as an SMC client, invite server to start the add_link processing */
1143 static void smc_llc_cli_add_link_invite(struct smc_link *link,
1144                                         struct smc_llc_qentry *qentry)
1145 {
1146         struct smc_link_group *lgr = smc_get_lgr(link);
1147         struct smc_init_info *ini = NULL;
1148
1149         if (lgr->smc_version == SMC_V2) {
1150                 smc_llc_send_request_add_link(link);
1151                 goto out;
1152         }
1153
1154         if (lgr->type == SMC_LGR_SYMMETRIC ||
1155             lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1156                 goto out;
1157
1158         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1159         if (!ini)
1160                 goto out;
1161
1162         ini->vlan_id = lgr->vlan_id;
1163         smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1164         if (!ini->ib_dev)
1165                 goto out;
1166
1167         smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1],
1168                               ini->ib_gid, NULL, SMC_LLC_REQ);
1169 out:
1170         kfree(ini);
1171         kfree(qentry);
1172 }
1173
1174 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
1175 {
1176         int i;
1177
1178         for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
1179                 if (llc->raw.data[i])
1180                         return false;
1181         return true;
1182 }
1183
1184 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
1185 {
1186         if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK &&
1187             smc_llc_is_empty_llc_message(llc))
1188                 return true;
1189         return false;
1190 }
1191
1192 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
1193 {
1194         struct smc_llc_qentry *qentry;
1195
1196         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1197
1198         mutex_lock(&lgr->llc_conf_mutex);
1199         if (smc_llc_is_local_add_link(&qentry->msg))
1200                 smc_llc_cli_add_link_invite(qentry->link, qentry);
1201         else
1202                 smc_llc_cli_add_link(qentry->link, qentry);
1203         mutex_unlock(&lgr->llc_conf_mutex);
1204 }
1205
1206 static int smc_llc_active_link_count(struct smc_link_group *lgr)
1207 {
1208         int i, link_count = 0;
1209
1210         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1211                 if (!smc_link_active(&lgr->lnk[i]))
1212                         continue;
1213                 link_count++;
1214         }
1215         return link_count;
1216 }
1217
1218 /* find the asymmetric link when 3 links are established  */
1219 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
1220 {
1221         int asym_idx = -ENOENT;
1222         int i, j, k;
1223         bool found;
1224
1225         /* determine asymmetric link */
1226         found = false;
1227         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1228                 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
1229                         if (!smc_link_usable(&lgr->lnk[i]) ||
1230                             !smc_link_usable(&lgr->lnk[j]))
1231                                 continue;
1232                         if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
1233                                     SMC_GID_SIZE)) {
1234                                 found = true;   /* asym_lnk is i or j */
1235                                 break;
1236                         }
1237                 }
1238                 if (found)
1239                         break;
1240         }
1241         if (!found)
1242                 goto out; /* no asymmetric link */
1243         for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
1244                 if (!smc_link_usable(&lgr->lnk[k]))
1245                         continue;
1246                 if (k != i &&
1247                     !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
1248                             SMC_GID_SIZE)) {
1249                         asym_idx = i;
1250                         break;
1251                 }
1252                 if (k != j &&
1253                     !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
1254                             SMC_GID_SIZE)) {
1255                         asym_idx = j;
1256                         break;
1257                 }
1258         }
1259 out:
1260         return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1261 }
1262
1263 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1264 {
1265         struct smc_link *lnk_new = NULL, *lnk_asym;
1266         struct smc_llc_qentry *qentry;
1267         int rc;
1268
1269         lnk_asym = smc_llc_find_asym_link(lgr);
1270         if (!lnk_asym)
1271                 return; /* no asymmetric link */
1272         if (!smc_link_downing(&lnk_asym->state))
1273                 return;
1274         lnk_new = smc_switch_conns(lgr, lnk_asym, false);
1275         smc_wr_tx_wait_no_pending_sends(lnk_asym);
1276         if (!lnk_new)
1277                 goto out_free;
1278         /* change flow type from ADD_LINK into DEL_LINK */
1279         lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1280         rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
1281                                       true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1282         if (rc) {
1283                 smcr_link_down_cond(lnk_new);
1284                 goto out_free;
1285         }
1286         qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
1287                               SMC_LLC_DELETE_LINK);
1288         if (!qentry) {
1289                 smcr_link_down_cond(lnk_new);
1290                 goto out_free;
1291         }
1292         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1293 out_free:
1294         smcr_link_clear(lnk_asym, true);
1295 }
1296
1297 static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1298                                      struct smc_link *link_new)
1299 {
1300         struct smc_llc_msg_add_link_cont *addc_llc;
1301         struct smc_link_group *lgr = link->lgr;
1302         u8 max, num_rkeys_send, num_rkeys_recv;
1303         struct smc_llc_qentry *qentry = NULL;
1304         struct smc_buf_desc *buf_pos;
1305         int buf_lst;
1306         int rc = 0;
1307         int i;
1308
1309         mutex_lock(&lgr->rmbs_lock);
1310         num_rkeys_send = lgr->conns_num;
1311         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
1312         do {
1313                 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
1314                                       &buf_lst, &buf_pos);
1315                 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1316                                       SMC_LLC_ADD_LINK_CONT);
1317                 if (!qentry) {
1318                         rc = -ETIMEDOUT;
1319                         goto out;
1320                 }
1321                 addc_llc = &qentry->msg.add_link_cont;
1322                 num_rkeys_recv = addc_llc->num_rkeys;
1323                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1324                 for (i = 0; i < max; i++) {
1325                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1326                                        addc_llc->rt[i].rmb_key,
1327                                        addc_llc->rt[i].rmb_vaddr_new,
1328                                        addc_llc->rt[i].rmb_key_new);
1329                         num_rkeys_recv--;
1330                 }
1331                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1332         } while (num_rkeys_send || num_rkeys_recv);
1333 out:
1334         mutex_unlock(&lgr->rmbs_lock);
1335         return rc;
1336 }
1337
1338 static int smc_llc_srv_conf_link(struct smc_link *link,
1339                                  struct smc_link *link_new,
1340                                  enum smc_lgr_type lgr_new_t)
1341 {
1342         struct smc_link_group *lgr = link->lgr;
1343         struct smc_llc_qentry *qentry = NULL;
1344         int rc;
1345
1346         /* send CONFIRM LINK request over the RoCE fabric */
1347         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1348         if (rc)
1349                 return -ENOLINK;
1350         /* receive CONFIRM LINK response over the RoCE fabric */
1351         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);
1352         if (!qentry ||
1353             qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
1354                 /* send DELETE LINK */
1355                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1356                                          false, SMC_LLC_DEL_LOST_PATH);
1357                 if (qentry)
1358                         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1359                 return -ENOLINK;
1360         }
1361         smc_llc_save_peer_uid(qentry);
1362         smc_llc_link_active(link_new);
1363         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1364             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1365                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1366         else
1367                 smcr_lgr_set_type(lgr, lgr_new_t);
1368         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1369         return 0;
1370 }
1371
1372 static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry)
1373 {
1374         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
1375         smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
1376                              sizeof(qentry->msg));
1377         memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data));
1378         smc_llc_send_message(qentry->link, &qentry->msg);
1379 }
1380
1381 int smc_llc_srv_add_link(struct smc_link *link,
1382                          struct smc_llc_qentry *req_qentry)
1383 {
1384         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1385         struct smc_link_group *lgr = link->lgr;
1386         struct smc_llc_msg_add_link *add_llc;
1387         struct smc_llc_qentry *qentry = NULL;
1388         bool send_req_add_link_resp = false;
1389         struct smc_link *link_new = NULL;
1390         struct smc_init_info *ini = NULL;
1391         int lnk_idx, rc = 0;
1392
1393         if (req_qentry &&
1394             req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK)
1395                 send_req_add_link_resp = true;
1396
1397         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1398         if (!ini) {
1399                 rc = -ENOMEM;
1400                 goto out;
1401         }
1402
1403         /* ignore client add link recommendation, start new flow */
1404         ini->vlan_id = lgr->vlan_id;
1405         if (lgr->smc_version == SMC_V2) {
1406                 ini->check_smcrv2 = true;
1407                 ini->smcrv2.saddr = lgr->saddr;
1408                 if (send_req_add_link_resp) {
1409                         struct smc_llc_msg_req_add_link_v2 *req_add =
1410                                 &req_qentry->msg.req_add_link;
1411
1412                         ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]);
1413                 }
1414         }
1415         smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1416         if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1417                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1418                 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1419                 ini->smcrv2.ib_port_v2 = link->ibport;
1420         } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1421                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1422                 ini->ib_dev = link->smcibdev;
1423                 ini->ib_port = link->ibport;
1424         }
1425         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1426         if (lnk_idx < 0) {
1427                 rc = 0;
1428                 goto out;
1429         }
1430
1431         rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini);
1432         if (rc)
1433                 goto out;
1434         link_new = &lgr->lnk[lnk_idx];
1435
1436         rc = smcr_buf_map_lgr(link_new);
1437         if (rc)
1438                 goto out_err;
1439
1440         rc = smc_llc_send_add_link(link,
1441                                    link_new->smcibdev->mac[link_new->ibport-1],
1442                                    link_new->gid, link_new, SMC_LLC_REQ);
1443         if (rc)
1444                 goto out_err;
1445         send_req_add_link_resp = false;
1446         /* receive ADD LINK response over the RoCE fabric */
1447         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1448         if (!qentry) {
1449                 rc = -ETIMEDOUT;
1450                 goto out_err;
1451         }
1452         add_llc = &qentry->msg.add_link;
1453         if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1454                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1455                 rc = -ENOLINK;
1456                 goto out_err;
1457         }
1458         if (lgr->type == SMC_LGR_SINGLE &&
1459             (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1460              (lgr->smc_version == SMC_V2 ||
1461               !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) {
1462                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1463         }
1464         smc_llc_save_add_link_info(link_new, add_llc);
1465         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1466
1467         rc = smc_ib_ready_link(link_new);
1468         if (rc)
1469                 goto out_err;
1470         rc = smcr_buf_reg_lgr(link_new);
1471         if (rc)
1472                 goto out_err;
1473         if (lgr->smc_version == SMC_V2) {
1474                 smc_llc_save_add_link_rkeys(link, link_new);
1475         } else {
1476                 rc = smc_llc_srv_rkey_exchange(link, link_new);
1477                 if (rc)
1478                         goto out_err;
1479         }
1480         rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1481         if (rc)
1482                 goto out_err;
1483         kfree(ini);
1484         return 0;
1485 out_err:
1486         if (link_new) {
1487                 link_new->state = SMC_LNK_INACTIVE;
1488                 smcr_link_clear(link_new, false);
1489         }
1490 out:
1491         kfree(ini);
1492         if (send_req_add_link_resp)
1493                 smc_llc_send_req_add_link_response(req_qentry);
1494         return rc;
1495 }
1496
1497 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1498 {
1499         struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1500         struct smc_llc_qentry *qentry;
1501         int rc;
1502
1503         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1504
1505         mutex_lock(&lgr->llc_conf_mutex);
1506         rc = smc_llc_srv_add_link(link, qentry);
1507         if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1508                 /* delete any asymmetric link */
1509                 smc_llc_delete_asym_link(lgr);
1510         }
1511         mutex_unlock(&lgr->llc_conf_mutex);
1512         kfree(qentry);
1513 }
1514
1515 /* enqueue a local add_link req to trigger a new add_link flow */
1516 void smc_llc_add_link_local(struct smc_link *link)
1517 {
1518         struct smc_llc_msg_add_link add_llc = {};
1519
1520         add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK;
1521         smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc));
1522         /* no dev and port needed */
1523         smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1524 }
1525
1526 /* worker to process an add link message */
1527 static void smc_llc_add_link_work(struct work_struct *work)
1528 {
1529         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1530                                                   llc_add_link_work);
1531
1532         if (list_empty(&lgr->list)) {
1533                 /* link group is terminating */
1534                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1535                 goto out;
1536         }
1537
1538         if (lgr->role == SMC_CLNT)
1539                 smc_llc_process_cli_add_link(lgr);
1540         else
1541                 smc_llc_process_srv_add_link(lgr);
1542 out:
1543         if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)
1544                 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1545 }
1546
1547 /* enqueue a local del_link msg to trigger a new del_link flow,
1548  * called only for role SMC_SERV
1549  */
1550 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1551 {
1552         struct smc_llc_msg_del_link del_llc = {};
1553
1554         del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1555         smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc));
1556         del_llc.link_num = del_link_id;
1557         del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1558         del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1559         smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1560 }
1561
1562 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1563 {
1564         struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1565         struct smc_llc_msg_del_link *del_llc;
1566         struct smc_llc_qentry *qentry;
1567         int active_links;
1568         int lnk_idx;
1569
1570         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1571         lnk = qentry->link;
1572         del_llc = &qentry->msg.delete_link;
1573
1574         if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1575                 smc_lgr_terminate_sched(lgr);
1576                 goto out;
1577         }
1578         mutex_lock(&lgr->llc_conf_mutex);
1579         /* delete single link */
1580         for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1581                 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1582                         continue;
1583                 lnk_del = &lgr->lnk[lnk_idx];
1584                 break;
1585         }
1586         del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1587         if (!lnk_del) {
1588                 /* link was not found */
1589                 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1590                 smc_llc_send_message(lnk, &qentry->msg);
1591                 goto out_unlock;
1592         }
1593         lnk_asym = smc_llc_find_asym_link(lgr);
1594
1595         del_llc->reason = 0;
1596         smc_llc_send_message(lnk, &qentry->msg); /* response */
1597
1598         if (smc_link_downing(&lnk_del->state))
1599                 smc_switch_conns(lgr, lnk_del, false);
1600         smcr_link_clear(lnk_del, true);
1601
1602         active_links = smc_llc_active_link_count(lgr);
1603         if (lnk_del == lnk_asym) {
1604                 /* expected deletion of asym link, don't change lgr state */
1605         } else if (active_links == 1) {
1606                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1607         } else if (!active_links) {
1608                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1609                 smc_lgr_terminate_sched(lgr);
1610         }
1611 out_unlock:
1612         mutex_unlock(&lgr->llc_conf_mutex);
1613 out:
1614         kfree(qentry);
1615 }
1616
1617 /* try to send a DELETE LINK ALL request on any active link,
1618  * waiting for send completion
1619  */
1620 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1621 {
1622         struct smc_llc_msg_del_link delllc = {};
1623         int i;
1624
1625         delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1626         smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc));
1627         if (ord)
1628                 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1629         delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1630         delllc.reason = htonl(rsn);
1631
1632         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1633                 if (!smc_link_usable(&lgr->lnk[i]))
1634                         continue;
1635                 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1636                         break;
1637         }
1638 }
1639
1640 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1641 {
1642         struct smc_llc_msg_del_link *del_llc;
1643         struct smc_link *lnk, *lnk_del;
1644         struct smc_llc_qentry *qentry;
1645         int active_links;
1646         int i;
1647
1648         mutex_lock(&lgr->llc_conf_mutex);
1649         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1650         lnk = qentry->link;
1651         del_llc = &qentry->msg.delete_link;
1652
1653         if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1654                 /* delete entire lgr */
1655                 smc_llc_send_link_delete_all(lgr, true, ntohl(
1656                                               qentry->msg.delete_link.reason));
1657                 smc_lgr_terminate_sched(lgr);
1658                 goto out;
1659         }
1660         /* delete single link */
1661         lnk_del = NULL;
1662         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1663                 if (lgr->lnk[i].link_id == del_llc->link_num) {
1664                         lnk_del = &lgr->lnk[i];
1665                         break;
1666                 }
1667         }
1668         if (!lnk_del)
1669                 goto out; /* asymmetric link already deleted */
1670
1671         if (smc_link_downing(&lnk_del->state)) {
1672                 if (smc_switch_conns(lgr, lnk_del, false))
1673                         smc_wr_tx_wait_no_pending_sends(lnk_del);
1674         }
1675         if (!list_empty(&lgr->list)) {
1676                 /* qentry is either a request from peer (send it back to
1677                  * initiate the DELETE_LINK processing), or a locally
1678                  * enqueued DELETE_LINK request (forward it)
1679                  */
1680                 if (!smc_llc_send_message(lnk, &qentry->msg)) {
1681                         struct smc_llc_qentry *qentry2;
1682
1683                         qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1684                                                SMC_LLC_DELETE_LINK);
1685                         if (qentry2)
1686                                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1687                 }
1688         }
1689         smcr_link_clear(lnk_del, true);
1690
1691         active_links = smc_llc_active_link_count(lgr);
1692         if (active_links == 1) {
1693                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1694         } else if (!active_links) {
1695                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1696                 smc_lgr_terminate_sched(lgr);
1697         }
1698
1699         if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1700                 /* trigger setup of asymm alt link */
1701                 smc_llc_add_link_local(lnk);
1702         }
1703 out:
1704         mutex_unlock(&lgr->llc_conf_mutex);
1705         kfree(qentry);
1706 }
1707
1708 static void smc_llc_delete_link_work(struct work_struct *work)
1709 {
1710         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1711                                                   llc_del_link_work);
1712
1713         if (list_empty(&lgr->list)) {
1714                 /* link group is terminating */
1715                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1716                 goto out;
1717         }
1718
1719         if (lgr->role == SMC_CLNT)
1720                 smc_llc_process_cli_delete_link(lgr);
1721         else
1722                 smc_llc_process_srv_delete_link(lgr);
1723 out:
1724         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1725 }
1726
1727 /* process a confirm_rkey request from peer, remote flow */
1728 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1729 {
1730         struct smc_llc_msg_confirm_rkey *llc;
1731         struct smc_llc_qentry *qentry;
1732         struct smc_link *link;
1733         int num_entries;
1734         int rk_idx;
1735         int i;
1736
1737         qentry = lgr->llc_flow_rmt.qentry;
1738         llc = &qentry->msg.confirm_rkey;
1739         link = qentry->link;
1740
1741         num_entries = llc->rtoken[0].num_rkeys;
1742         if (num_entries > SMC_LLC_RKEYS_PER_MSG)
1743                 goto out_err;
1744         /* first rkey entry is for receiving link */
1745         rk_idx = smc_rtoken_add(link,
1746                                 llc->rtoken[0].rmb_vaddr,
1747                                 llc->rtoken[0].rmb_key);
1748         if (rk_idx < 0)
1749                 goto out_err;
1750
1751         for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1752                 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1753                                 llc->rtoken[i].rmb_vaddr,
1754                                 llc->rtoken[i].rmb_key);
1755         /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1756         goto out;
1757 out_err:
1758         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1759         llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1760 out:
1761         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1762         smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1763         smc_llc_send_message(link, &qentry->msg);
1764         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1765 }
1766
1767 /* process a delete_rkey request from peer, remote flow */
1768 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1769 {
1770         struct smc_llc_msg_delete_rkey *llc;
1771         struct smc_llc_qentry *qentry;
1772         struct smc_link *link;
1773         u8 err_mask = 0;
1774         int i, max;
1775
1776         qentry = lgr->llc_flow_rmt.qentry;
1777         llc = &qentry->msg.delete_rkey;
1778         link = qentry->link;
1779
1780         if (lgr->smc_version == SMC_V2) {
1781                 struct smc_llc_msg_delete_rkey_v2 *llcv2;
1782
1783                 memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc));
1784                 llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2;
1785                 llcv2->num_inval_rkeys = 0;
1786
1787                 max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1788                 for (i = 0; i < max; i++) {
1789                         if (smc_rtoken_delete(link, llcv2->rkey[i]))
1790                                 llcv2->num_inval_rkeys++;
1791                 }
1792                 memset(&llc->rkey[0], 0, sizeof(llc->rkey));
1793                 memset(&llc->reserved2, 0, sizeof(llc->reserved2));
1794                 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1795                 if (llcv2->num_inval_rkeys) {
1796                         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1797                         llc->err_mask = llcv2->num_inval_rkeys;
1798                 }
1799                 goto finish;
1800         }
1801
1802         max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1803         for (i = 0; i < max; i++) {
1804                 if (smc_rtoken_delete(link, llc->rkey[i]))
1805                         err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1806         }
1807         if (err_mask) {
1808                 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1809                 llc->err_mask = err_mask;
1810         }
1811 finish:
1812         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1813         smc_llc_send_message(link, &qentry->msg);
1814         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1815 }
1816
1817 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1818 {
1819         pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: "
1820                             "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type);
1821         smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1822         smc_lgr_terminate_sched(lgr);
1823 }
1824
1825 /* flush the llc event queue */
1826 static void smc_llc_event_flush(struct smc_link_group *lgr)
1827 {
1828         struct smc_llc_qentry *qentry, *q;
1829
1830         spin_lock_bh(&lgr->llc_event_q_lock);
1831         list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1832                 list_del_init(&qentry->list);
1833                 kfree(qentry);
1834         }
1835         spin_unlock_bh(&lgr->llc_event_q_lock);
1836 }
1837
1838 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1839 {
1840         union smc_llc_msg *llc = &qentry->msg;
1841         struct smc_link *link = qentry->link;
1842         struct smc_link_group *lgr = link->lgr;
1843
1844         if (!smc_link_usable(link))
1845                 goto out;
1846
1847         switch (llc->raw.hdr.common.llc_type) {
1848         case SMC_LLC_TEST_LINK:
1849                 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1850                 smc_llc_send_message(link, llc);
1851                 break;
1852         case SMC_LLC_ADD_LINK:
1853                 if (list_empty(&lgr->list))
1854                         goto out;       /* lgr is terminating */
1855                 if (lgr->role == SMC_CLNT) {
1856                         if (smc_llc_is_local_add_link(llc)) {
1857                                 if (lgr->llc_flow_lcl.type ==
1858                                     SMC_LLC_FLOW_ADD_LINK)
1859                                         break;  /* add_link in progress */
1860                                 if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1861                                                        qentry)) {
1862                                         schedule_work(&lgr->llc_add_link_work);
1863                                 }
1864                                 return;
1865                         }
1866                         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1867                             !lgr->llc_flow_lcl.qentry) {
1868                                 /* a flow is waiting for this message */
1869                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1870                                                         qentry);
1871                                 wake_up(&lgr->llc_msg_waiter);
1872                                 return;
1873                         }
1874                         if (lgr->llc_flow_lcl.type ==
1875                                         SMC_LLC_FLOW_REQ_ADD_LINK) {
1876                                 /* server started add_link processing */
1877                                 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1878                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1879                                                         qentry);
1880                                 schedule_work(&lgr->llc_add_link_work);
1881                                 return;
1882                         }
1883                         if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1884                                 schedule_work(&lgr->llc_add_link_work);
1885                         }
1886                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1887                         /* as smc server, handle client suggestion */
1888                         schedule_work(&lgr->llc_add_link_work);
1889                 }
1890                 return;
1891         case SMC_LLC_CONFIRM_LINK:
1892         case SMC_LLC_ADD_LINK_CONT:
1893                 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1894                         /* a flow is waiting for this message */
1895                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1896                         wake_up(&lgr->llc_msg_waiter);
1897                         return;
1898                 }
1899                 break;
1900         case SMC_LLC_DELETE_LINK:
1901                 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1902                     !lgr->llc_flow_lcl.qentry) {
1903                         /* DEL LINK REQ during ADD LINK SEQ */
1904                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1905                         wake_up(&lgr->llc_msg_waiter);
1906                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1907                         schedule_work(&lgr->llc_del_link_work);
1908                 }
1909                 return;
1910         case SMC_LLC_CONFIRM_RKEY:
1911                 /* new request from remote, assign to remote flow */
1912                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1913                         /* process here, does not wait for more llc msgs */
1914                         smc_llc_rmt_conf_rkey(lgr);
1915                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1916                 }
1917                 return;
1918         case SMC_LLC_CONFIRM_RKEY_CONT:
1919                 /* not used because max links is 3, and 3 rkeys fit into
1920                  * one CONFIRM_RKEY message
1921                  */
1922                 break;
1923         case SMC_LLC_DELETE_RKEY:
1924                 /* new request from remote, assign to remote flow */
1925                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1926                         /* process here, does not wait for more llc msgs */
1927                         smc_llc_rmt_delete_rkey(lgr);
1928                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1929                 }
1930                 return;
1931         case SMC_LLC_REQ_ADD_LINK:
1932                 /* handle response here, smc_llc_flow_stop() cannot be called
1933                  * in tasklet context
1934                  */
1935                 if (lgr->role == SMC_CLNT &&
1936                     lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&
1937                     (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) {
1938                         smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl);
1939                 } else if (lgr->role == SMC_SERV) {
1940                         if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1941                                 /* as smc server, handle client suggestion */
1942                                 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1943                                 schedule_work(&lgr->llc_add_link_work);
1944                         }
1945                         return;
1946                 }
1947                 break;
1948         default:
1949                 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1950                 break;
1951         }
1952 out:
1953         kfree(qentry);
1954 }
1955
1956 /* worker to process llc messages on the event queue */
1957 static void smc_llc_event_work(struct work_struct *work)
1958 {
1959         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1960                                                   llc_event_work);
1961         struct smc_llc_qentry *qentry;
1962
1963         if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1964                 qentry = lgr->delayed_event;
1965                 lgr->delayed_event = NULL;
1966                 if (smc_link_usable(qentry->link))
1967                         smc_llc_event_handler(qentry);
1968                 else
1969                         kfree(qentry);
1970         }
1971
1972 again:
1973         spin_lock_bh(&lgr->llc_event_q_lock);
1974         if (!list_empty(&lgr->llc_event_q)) {
1975                 qentry = list_first_entry(&lgr->llc_event_q,
1976                                           struct smc_llc_qentry, list);
1977                 list_del_init(&qentry->list);
1978                 spin_unlock_bh(&lgr->llc_event_q_lock);
1979                 smc_llc_event_handler(qentry);
1980                 goto again;
1981         }
1982         spin_unlock_bh(&lgr->llc_event_q_lock);
1983 }
1984
1985 /* process llc responses in tasklet context */
1986 static void smc_llc_rx_response(struct smc_link *link,
1987                                 struct smc_llc_qentry *qentry)
1988 {
1989         enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
1990         struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
1991         u8 llc_type = qentry->msg.raw.hdr.common.llc_type;
1992
1993         switch (llc_type) {
1994         case SMC_LLC_TEST_LINK:
1995                 if (smc_link_active(link))
1996                         complete(&link->llc_testlink_resp);
1997                 break;
1998         case SMC_LLC_ADD_LINK:
1999         case SMC_LLC_ADD_LINK_CONT:
2000         case SMC_LLC_CONFIRM_LINK:
2001                 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
2002                         break;  /* drop out-of-flow response */
2003                 goto assign;
2004         case SMC_LLC_DELETE_LINK:
2005                 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
2006                         break;  /* drop out-of-flow response */
2007                 goto assign;
2008         case SMC_LLC_CONFIRM_RKEY:
2009         case SMC_LLC_DELETE_RKEY:
2010                 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
2011                         break;  /* drop out-of-flow response */
2012                 goto assign;
2013         case SMC_LLC_CONFIRM_RKEY_CONT:
2014                 /* not used because max links is 3 */
2015                 break;
2016         default:
2017                 smc_llc_protocol_violation(link->lgr,
2018                                            qentry->msg.raw.hdr.common.type);
2019                 break;
2020         }
2021         kfree(qentry);
2022         return;
2023 assign:
2024         /* assign responses to the local flow, we requested them */
2025         smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
2026         wake_up(&link->lgr->llc_msg_waiter);
2027 }
2028
2029 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
2030 {
2031         struct smc_link_group *lgr = link->lgr;
2032         struct smc_llc_qentry *qentry;
2033         unsigned long flags;
2034
2035         qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
2036         if (!qentry)
2037                 return;
2038         qentry->link = link;
2039         INIT_LIST_HEAD(&qentry->list);
2040         memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
2041
2042         /* process responses immediately */
2043         if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) &&
2044             llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) {
2045                 smc_llc_rx_response(link, qentry);
2046                 return;
2047         }
2048
2049         /* add requests to event queue */
2050         spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
2051         list_add_tail(&qentry->list, &lgr->llc_event_q);
2052         spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
2053         queue_work(system_highpri_wq, &lgr->llc_event_work);
2054 }
2055
2056 /* copy received msg and add it to the event queue */
2057 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
2058 {
2059         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
2060         union smc_llc_msg *llc = buf;
2061
2062         if (wc->byte_len < sizeof(*llc))
2063                 return; /* short message */
2064         if (!llc->raw.hdr.common.llc_version) {
2065                 if (llc->raw.hdr.length != sizeof(*llc))
2066                         return; /* invalid message */
2067         } else {
2068                 if (llc->raw.hdr.length_v2 < sizeof(*llc))
2069                         return; /* invalid message */
2070         }
2071
2072         smc_llc_enqueue(link, llc);
2073 }
2074
2075 /***************************** worker, utils *********************************/
2076
2077 static void smc_llc_testlink_work(struct work_struct *work)
2078 {
2079         struct smc_link *link = container_of(to_delayed_work(work),
2080                                              struct smc_link, llc_testlink_wrk);
2081         unsigned long next_interval;
2082         unsigned long expire_time;
2083         u8 user_data[16] = { 0 };
2084         int rc;
2085
2086         if (!smc_link_active(link))
2087                 return;         /* don't reschedule worker */
2088         expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
2089         if (time_is_after_jiffies(expire_time)) {
2090                 next_interval = expire_time - jiffies;
2091                 goto out;
2092         }
2093         reinit_completion(&link->llc_testlink_resp);
2094         smc_llc_send_test_link(link, user_data);
2095         /* receive TEST LINK response over RoCE fabric */
2096         rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
2097                                                        SMC_LLC_WAIT_TIME);
2098         if (!smc_link_active(link))
2099                 return;         /* link state changed */
2100         if (rc <= 0) {
2101                 smcr_link_down_cond_sched(link);
2102                 return;
2103         }
2104         next_interval = link->llc_testlink_time;
2105 out:
2106         schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
2107 }
2108
2109 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
2110 {
2111         struct net *net = sock_net(smc->clcsock->sk);
2112
2113         INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
2114         INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
2115         INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
2116         INIT_LIST_HEAD(&lgr->llc_event_q);
2117         spin_lock_init(&lgr->llc_event_q_lock);
2118         spin_lock_init(&lgr->llc_flow_lock);
2119         init_waitqueue_head(&lgr->llc_flow_waiter);
2120         init_waitqueue_head(&lgr->llc_msg_waiter);
2121         mutex_init(&lgr->llc_conf_mutex);
2122         lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
2123 }
2124
2125 /* called after lgr was removed from lgr_list */
2126 void smc_llc_lgr_clear(struct smc_link_group *lgr)
2127 {
2128         smc_llc_event_flush(lgr);
2129         wake_up_all(&lgr->llc_flow_waiter);
2130         wake_up_all(&lgr->llc_msg_waiter);
2131         cancel_work_sync(&lgr->llc_event_work);
2132         cancel_work_sync(&lgr->llc_add_link_work);
2133         cancel_work_sync(&lgr->llc_del_link_work);
2134         if (lgr->delayed_event) {
2135                 kfree(lgr->delayed_event);
2136                 lgr->delayed_event = NULL;
2137         }
2138 }
2139
2140 int smc_llc_link_init(struct smc_link *link)
2141 {
2142         init_completion(&link->llc_testlink_resp);
2143         INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2144         return 0;
2145 }
2146
2147 void smc_llc_link_active(struct smc_link *link)
2148 {
2149         pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, "
2150                             "peerid %*phN, ibdev %s, ibport %d\n",
2151                             SMC_LGR_ID_SIZE, &link->lgr->id,
2152                             SMC_LGR_ID_SIZE, &link->link_uid,
2153                             SMC_LGR_ID_SIZE, &link->peer_link_uid,
2154                             link->smcibdev->ibdev->name, link->ibport);
2155         link->state = SMC_LNK_ACTIVE;
2156         if (link->lgr->llc_testlink_time) {
2157                 link->llc_testlink_time = link->lgr->llc_testlink_time;
2158                 schedule_delayed_work(&link->llc_testlink_wrk,
2159                                       link->llc_testlink_time);
2160         }
2161 }
2162
2163 /* called in worker context */
2164 void smc_llc_link_clear(struct smc_link *link, bool log)
2165 {
2166         if (log)
2167                 pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN"
2168                                     ", peerid %*phN, ibdev %s, ibport %d\n",
2169                                     SMC_LGR_ID_SIZE, &link->lgr->id,
2170                                     SMC_LGR_ID_SIZE, &link->link_uid,
2171                                     SMC_LGR_ID_SIZE, &link->peer_link_uid,
2172                                     link->smcibdev->ibdev->name, link->ibport);
2173         complete(&link->llc_testlink_resp);
2174         cancel_delayed_work_sync(&link->llc_testlink_wrk);
2175 }
2176
2177 /* register a new rtoken at the remote peer (for all links) */
2178 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
2179                             struct smc_buf_desc *rmb_desc)
2180 {
2181         struct smc_link_group *lgr = send_link->lgr;
2182         struct smc_llc_qentry *qentry = NULL;
2183         int rc = 0;
2184
2185         rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
2186         if (rc)
2187                 goto out;
2188         /* receive CONFIRM RKEY response from server over RoCE fabric */
2189         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2190                               SMC_LLC_CONFIRM_RKEY);
2191         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2192                 rc = -EFAULT;
2193 out:
2194         if (qentry)
2195                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2196         return rc;
2197 }
2198
2199 /* unregister an rtoken at the remote peer */
2200 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
2201                            struct smc_buf_desc *rmb_desc)
2202 {
2203         struct smc_llc_qentry *qentry = NULL;
2204         struct smc_link *send_link;
2205         int rc = 0;
2206
2207         send_link = smc_llc_usable_link(lgr);
2208         if (!send_link)
2209                 return -ENOLINK;
2210
2211         /* protected by llc_flow control */
2212         rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
2213         if (rc)
2214                 goto out;
2215         /* receive DELETE RKEY response from server over RoCE fabric */
2216         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2217                               SMC_LLC_DELETE_RKEY);
2218         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2219                 rc = -EFAULT;
2220 out:
2221         if (qentry)
2222                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2223         return rc;
2224 }
2225
2226 void smc_llc_link_set_uid(struct smc_link *link)
2227 {
2228         __be32 link_uid;
2229
2230         link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
2231         memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
2232 }
2233
2234 /* save peers link user id, used for debug purposes */
2235 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
2236 {
2237         memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
2238                SMC_LGR_ID_SIZE);
2239 }
2240
2241 /* evaluate confirm link request or response */
2242 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
2243                            enum smc_llc_reqresp type)
2244 {
2245         if (type == SMC_LLC_REQ) {      /* SMC server assigns link_id */
2246                 qentry->link->link_id = qentry->msg.confirm_link.link_num;
2247                 smc_llc_link_set_uid(qentry->link);
2248         }
2249         if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
2250                 return -ENOTSUPP;
2251         return 0;
2252 }
2253
2254 /***************************** init, exit, misc ******************************/
2255
2256 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
2257         {
2258                 .handler        = smc_llc_rx_handler,
2259                 .type           = SMC_LLC_CONFIRM_LINK
2260         },
2261         {
2262                 .handler        = smc_llc_rx_handler,
2263                 .type           = SMC_LLC_TEST_LINK
2264         },
2265         {
2266                 .handler        = smc_llc_rx_handler,
2267                 .type           = SMC_LLC_ADD_LINK
2268         },
2269         {
2270                 .handler        = smc_llc_rx_handler,
2271                 .type           = SMC_LLC_ADD_LINK_CONT
2272         },
2273         {
2274                 .handler        = smc_llc_rx_handler,
2275                 .type           = SMC_LLC_DELETE_LINK
2276         },
2277         {
2278                 .handler        = smc_llc_rx_handler,
2279                 .type           = SMC_LLC_CONFIRM_RKEY
2280         },
2281         {
2282                 .handler        = smc_llc_rx_handler,
2283                 .type           = SMC_LLC_CONFIRM_RKEY_CONT
2284         },
2285         {
2286                 .handler        = smc_llc_rx_handler,
2287                 .type           = SMC_LLC_DELETE_RKEY
2288         },
2289         /* V2 types */
2290         {
2291                 .handler        = smc_llc_rx_handler,
2292                 .type           = SMC_LLC_CONFIRM_LINK_V2
2293         },
2294         {
2295                 .handler        = smc_llc_rx_handler,
2296                 .type           = SMC_LLC_TEST_LINK_V2
2297         },
2298         {
2299                 .handler        = smc_llc_rx_handler,
2300                 .type           = SMC_LLC_ADD_LINK_V2
2301         },
2302         {
2303                 .handler        = smc_llc_rx_handler,
2304                 .type           = SMC_LLC_DELETE_LINK_V2
2305         },
2306         {
2307                 .handler        = smc_llc_rx_handler,
2308                 .type           = SMC_LLC_REQ_ADD_LINK_V2
2309         },
2310         {
2311                 .handler        = smc_llc_rx_handler,
2312                 .type           = SMC_LLC_CONFIRM_RKEY_V2
2313         },
2314         {
2315                 .handler        = smc_llc_rx_handler,
2316                 .type           = SMC_LLC_DELETE_RKEY_V2
2317         },
2318         {
2319                 .handler        = NULL,
2320         }
2321 };
2322
2323 int __init smc_llc_init(void)
2324 {
2325         struct smc_wr_rx_handler *handler;
2326         int rc = 0;
2327
2328         for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
2329                 INIT_HLIST_NODE(&handler->list);
2330                 rc = smc_wr_rx_register_handler(handler);
2331                 if (rc)
2332                         break;
2333         }
2334         return rc;
2335 }