net/smc/smc_wr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  * Work Requests exploiting Infiniband API
   6  *
   7  * Work requests (WR) of type ib_post_send or ib_post_recv respectively
   8  * are submitted to either RC SQ or RC RQ respectively
   9  * (reliably connected send/receive queue)
  10  * and become work queue entries (WQEs).
  11  * While an SQ WR/WQE is pending, we track it until transmission completion.
  12  * Through a send or receive completion queue (CQ) respectively,
  13  * we get completion queue entries (CQEs) [aka work completions (WCs)].
  14  * Since the CQ callback is called from IRQ context, we split work by using
  15  * bottom halves implemented by tasklets.
  16  *
  17  * SMC uses this to exchange LLC (link layer control)
  18  * and CDC (connection data control) messages.
  19  *
  20  * Copyright IBM Corp. 2016
  21  *
  22  * Author(s):  Steffen Maier <[email protected]>
  23  */
  24
  25 #include <linux/atomic.h>
  26 #include <linux/hashtable.h>
  27 #include <linux/wait.h>
  28 #include <rdma/ib_verbs.h>
  29 #include <asm/div64.h>
  30
  31 #include "smc.h"
  32 #include "smc_wr.h"
  33
  34 #define SMC_WR_MAX_POLL_CQE 10  /* max. # of compl. queue elements in 1 poll */
  35
  36 #define SMC_WR_RX_HASH_BITS 4
  37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
  38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
  39
  40 struct smc_wr_tx_pend { /* control data for a pending send request */
  41         u64                     wr_id;          /* work request id sent */
  42         smc_wr_tx_handler       handler;
  43         enum ib_wc_status       wc_status;      /* CQE status */
  44         struct smc_link         *link;
  45         u32                     idx;
  46         struct smc_wr_tx_pend_priv priv;
  47 };
  48
  49 /******************************** send queue *********************************/
  50
  51 /*------------------------------- completion --------------------------------*/
  52
  53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
  54 {
  55         u32 i;
  56
  57         for (i = 0; i < link->wr_tx_cnt; i++) {
  58                 if (link->wr_tx_pends[i].wr_id == wr_id)
  59                         return i;
  60         }
  61         return link->wr_tx_cnt;
  62 }
  63
  64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
  65 {
  66         struct smc_wr_tx_pend pnd_snd;
  67         struct smc_link *link;
  68         u32 pnd_snd_idx;
  69         int i;
  70
  71         link = wc->qp->qp_context;
  72
  73         if (wc->opcode == IB_WC_REG_MR) {
  74                 if (wc->status)
  75                         link->wr_reg_state = FAILED;
  76                 else
  77                         link->wr_reg_state = CONFIRMED;
  78                 wake_up(&link->wr_reg_wait);
  79                 return;
  80         }
  81
  82         pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
  83         if (pnd_snd_idx == link->wr_tx_cnt)
  84                 return;
  85         link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
  86         memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
  87         /* clear the full struct smc_wr_tx_pend including .priv */
  88         memset(&link->wr_tx_pends[pnd_snd_idx], 0,
  89                sizeof(link->wr_tx_pends[pnd_snd_idx]));
  90         memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
  91                sizeof(link->wr_tx_bufs[pnd_snd_idx]));
  92         if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
  93                 return;
  94         if (wc->status) {
  95                 struct smc_link_group *lgr;
  96
  97                 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
  98                         /* clear full struct smc_wr_tx_pend including .priv */
  99                         memset(&link->wr_tx_pends[i], 0,
 100                                sizeof(link->wr_tx_pends[i]));
 101                         memset(&link->wr_tx_bufs[i], 0,
 102                                sizeof(link->wr_tx_bufs[i]));
 103                         clear_bit(i, link->wr_tx_mask);
 104                 }
 105                 /* terminate connections of this link group abnormally */
 106                 lgr = container_of(link, struct smc_link_group,
 107                                    lnk[SMC_SINGLE_LINK]);
 108                 smc_lgr_terminate(lgr);
 109         }
 110         if (pnd_snd.handler)
 111                 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
 112         wake_up(&link->wr_tx_wait);
 113 }
 114
 115 static void smc_wr_tx_tasklet_fn(unsigned long data)
 116 {
 117         struct smc_ib_device *dev = (struct smc_ib_device *)data;
 118         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
 119         int i = 0, rc;
 120         int polled = 0;
 121
 122 again:
 123         polled++;
 124         do {
 125                 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
 126                 if (polled == 1) {
 127                         ib_req_notify_cq(dev->roce_cq_send,
 128                                          IB_CQ_NEXT_COMP |
 129                                          IB_CQ_REPORT_MISSED_EVENTS);
 130                 }
 131                 if (!rc)
 132                         break;
 133                 for (i = 0; i < rc; i++)
 134                         smc_wr_tx_process_cqe(&wc[i]);
 135         } while (rc > 0);
 136         if (polled == 1)
 137                 goto again;
 138 }
 139
 140 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
 141 {
 142         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
 143
 144         tasklet_schedule(&dev->send_tasklet);
 145 }
 146
 147 /*---------------------------- request submission ---------------------------*/
 148
 149 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
 150 {
 151         *idx = link->wr_tx_cnt;
 152         for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
 153                 if (!test_and_set_bit(*idx, link->wr_tx_mask))
 154                         return 0;
 155         }
 156         *idx = link->wr_tx_cnt;
 157         return -EBUSY;
 158 }
 159
 160 /**
 161  * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
 162  *                      and sets info for pending transmit tracking
 163  * @link:               Pointer to smc_link used to later send the message.
 164  * @handler:            Send completion handler function pointer.
 165  * @wr_buf:             Out value returns pointer to message buffer.
 166  * @wr_pend_priv:       Out value returns pointer serving as handler context.
 167  *
 168  * Return: 0 on success, or -errno on error.
 169  */
 170 int smc_wr_tx_get_free_slot(struct smc_link *link,
 171                             smc_wr_tx_handler handler,
 172                             struct smc_wr_buf **wr_buf,
 173                             struct smc_wr_tx_pend_priv **wr_pend_priv)
 174 {
 175         struct smc_wr_tx_pend *wr_pend;
 176         struct ib_send_wr *wr_ib;
 177         u64 wr_id;
 178         u32 idx;
 179         int rc;
 180
 181         *wr_buf = NULL;
 182         *wr_pend_priv = NULL;
 183         if (in_softirq()) {
 184                 rc = smc_wr_tx_get_free_slot_index(link, &idx);
 185                 if (rc)
 186                         return rc;
 187         } else {
 188                 rc = wait_event_interruptible_timeout(
 189                         link->wr_tx_wait,
 190                         (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
 191                         SMC_WR_TX_WAIT_FREE_SLOT_TIME);
 192                 if (!rc) {
 193                         /* timeout - terminate connections */
 194                         struct smc_link_group *lgr;
 195
 196                         lgr = container_of(link, struct smc_link_group,
 197                                            lnk[SMC_SINGLE_LINK]);
 198                         smc_lgr_terminate(lgr);
 199                         return -EPIPE;
 200                 }
 201                 if (rc == -ERESTARTSYS)
 202                         return -EINTR;
 203                 if (idx == link->wr_tx_cnt)
 204                         return -EPIPE;
 205         }
 206         wr_id = smc_wr_tx_get_next_wr_id(link);
 207         wr_pend = &link->wr_tx_pends[idx];
 208         wr_pend->wr_id = wr_id;
 209         wr_pend->handler = handler;
 210         wr_pend->link = link;
 211         wr_pend->idx = idx;
 212         wr_ib = &link->wr_tx_ibs[idx];
 213         wr_ib->wr_id = wr_id;
 214         *wr_buf = &link->wr_tx_bufs[idx];
 215         *wr_pend_priv = &wr_pend->priv;
 216         return 0;
 217 }
 218
 219 int smc_wr_tx_put_slot(struct smc_link *link,
 220                        struct smc_wr_tx_pend_priv *wr_pend_priv)
 221 {
 222         struct smc_wr_tx_pend *pend;
 223
 224         pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
 225         if (pend->idx < link->wr_tx_cnt) {
 226                 /* clear the full struct smc_wr_tx_pend including .priv */
 227                 memset(&link->wr_tx_pends[pend->idx], 0,
 228                        sizeof(link->wr_tx_pends[pend->idx]));
 229                 memset(&link->wr_tx_bufs[pend->idx], 0,
 230                        sizeof(link->wr_tx_bufs[pend->idx]));
 231                 test_and_clear_bit(pend->idx, link->wr_tx_mask);
 232                 return 1;
 233         }
 234
 235         return 0;
 236 }
 237
 238 /* Send prepared WR slot via ib_post_send.
 239  * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
 240  */
 241 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
 242 {
 243         struct ib_send_wr *failed_wr = NULL;
 244         struct smc_wr_tx_pend *pend;
 245         int rc;
 246
 247         ib_req_notify_cq(link->smcibdev->roce_cq_send,
 248                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
 249         pend = container_of(priv, struct smc_wr_tx_pend, priv);
 250         rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx],
 251                           &failed_wr);
 252         if (rc)
 253                 smc_wr_tx_put_slot(link, priv);
 254         return rc;
 255 }
 256
 257 /* Register a memory region and wait for result. */
 258 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
 259 {
 260         struct ib_send_wr *failed_wr = NULL;
 261         int rc;
 262
 263         ib_req_notify_cq(link->smcibdev->roce_cq_send,
 264                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
 265         link->wr_reg_state = POSTED;
 266         link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
 267         link->wr_reg.mr = mr;
 268         link->wr_reg.key = mr->rkey;
 269         failed_wr = &link->wr_reg.wr;
 270         rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr);
 271         WARN_ON(failed_wr != &link->wr_reg.wr);
 272         if (rc)
 273                 return rc;
 274
 275         rc = wait_event_interruptible_timeout(link->wr_reg_wait,
 276                                               (link->wr_reg_state != POSTED),
 277                                               SMC_WR_REG_MR_WAIT_TIME);
 278         if (!rc) {
 279                 /* timeout - terminate connections */
 280                 struct smc_link_group *lgr;
 281
 282                 lgr = container_of(link, struct smc_link_group,
 283                                    lnk[SMC_SINGLE_LINK]);
 284                 smc_lgr_terminate(lgr);
 285                 return -EPIPE;
 286         }
 287         if (rc == -ERESTARTSYS)
 288                 return -EINTR;
 289         switch (link->wr_reg_state) {
 290         case CONFIRMED:
 291                 rc = 0;
 292                 break;
 293         case FAILED:
 294                 rc = -EIO;
 295                 break;
 296         case POSTED:
 297                 rc = -EPIPE;
 298                 break;
 299         }
 300         return rc;
 301 }
 302
 303 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_rx_hdr_type,
 304                              smc_wr_tx_filter filter,
 305                              smc_wr_tx_dismisser dismisser,
 306                              unsigned long data)
 307 {
 308         struct smc_wr_tx_pend_priv *tx_pend;
 309         struct smc_wr_rx_hdr *wr_rx;
 310         int i;
 311
 312         for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
 313                 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i];
 314                 if (wr_rx->type != wr_rx_hdr_type)
 315                         continue;
 316                 tx_pend = &link->wr_tx_pends[i].priv;
 317                 if (filter(tx_pend, data))
 318                         dismisser(tx_pend);
 319         }
 320 }
 321
 322 bool smc_wr_tx_has_pending(struct smc_link *link, u8 wr_rx_hdr_type,
 323                            smc_wr_tx_filter filter, unsigned long data)
 324 {
 325         struct smc_wr_tx_pend_priv *tx_pend;
 326         struct smc_wr_rx_hdr *wr_rx;
 327         int i;
 328
 329         for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
 330                 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i];
 331                 if (wr_rx->type != wr_rx_hdr_type)
 332                         continue;
 333                 tx_pend = &link->wr_tx_pends[i].priv;
 334                 if (filter(tx_pend, data))
 335                         return true;
 336         }
 337         return false;
 338 }
 339
 340 /****************************** receive queue ********************************/
 341
 342 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
 343 {
 344         struct smc_wr_rx_handler *h_iter;
 345         int rc = 0;
 346
 347         spin_lock(&smc_wr_rx_hash_lock);
 348         hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
 349                 if (h_iter->type == handler->type) {
 350                         rc = -EEXIST;
 351                         goto out_unlock;
 352                 }
 353         }
 354         hash_add(smc_wr_rx_hash, &handler->list, handler->type);
 355 out_unlock:
 356         spin_unlock(&smc_wr_rx_hash_lock);
 357         return rc;
 358 }
 359
 360 /* Demultiplex a received work request based on the message type to its handler.
 361  * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
 362  * and not being modified any more afterwards so we don't need to lock it.
 363  */
 364 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
 365 {
 366         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
 367         struct smc_wr_rx_handler *handler;
 368         struct smc_wr_rx_hdr *wr_rx;
 369         u64 temp_wr_id;
 370         u32 index;
 371
 372         if (wc->byte_len < sizeof(*wr_rx))
 373                 return; /* short message */
 374         temp_wr_id = wc->wr_id;
 375         index = do_div(temp_wr_id, link->wr_rx_cnt);
 376         wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
 377         hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
 378                 if (handler->type == wr_rx->type)
 379                         handler->handler(wc, wr_rx);
 380         }
 381 }
 382
 383 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
 384 {
 385         struct smc_link *link;
 386         int i;
 387
 388         for (i = 0; i < num; i++) {
 389                 link = wc[i].qp->qp_context;
 390                 if (wc[i].status == IB_WC_SUCCESS) {
 391                         smc_wr_rx_demultiplex(&wc[i]);
 392                         smc_wr_rx_post(link); /* refill WR RX */
 393                 } else {
 394                         struct smc_link_group *lgr;
 395
 396                         /* handle status errors */
 397                         switch (wc[i].status) {
 398                         case IB_WC_RETRY_EXC_ERR:
 399                         case IB_WC_RNR_RETRY_EXC_ERR:
 400                         case IB_WC_WR_FLUSH_ERR:
 401                                 /* terminate connections of this link group
 402                                  * abnormally
 403                                  */
 404                                 lgr = container_of(link, struct smc_link_group,
 405                                                    lnk[SMC_SINGLE_LINK]);
 406                                 smc_lgr_terminate(lgr);
 407                                 break;
 408                         default:
 409                                 smc_wr_rx_post(link); /* refill WR RX */
 410                                 break;
 411                         }
 412                 }
 413         }
 414 }
 415
 416 static void smc_wr_rx_tasklet_fn(unsigned long data)
 417 {
 418         struct smc_ib_device *dev = (struct smc_ib_device *)data;
 419         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
 420         int polled = 0;
 421         int rc;
 422
 423 again:
 424         polled++;
 425         do {
 426                 memset(&wc, 0, sizeof(wc));
 427                 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
 428                 if (polled == 1) {
 429                         ib_req_notify_cq(dev->roce_cq_recv,
 430                                          IB_CQ_SOLICITED_MASK
 431                                          | IB_CQ_REPORT_MISSED_EVENTS);
 432                 }
 433                 if (!rc)
 434                         break;
 435                 smc_wr_rx_process_cqes(&wc[0], rc);
 436         } while (rc > 0);
 437         if (polled == 1)
 438                 goto again;
 439 }
 440
 441 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
 442 {
 443         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
 444
 445         tasklet_schedule(&dev->recv_tasklet);
 446 }
 447
 448 int smc_wr_rx_post_init(struct smc_link *link)
 449 {
 450         u32 i;
 451         int rc = 0;
 452
 453         for (i = 0; i < link->wr_rx_cnt; i++)
 454                 rc = smc_wr_rx_post(link);
 455         return rc;
 456 }
 457
 458 /***************************** init, exit, misc ******************************/
 459
 460 void smc_wr_remember_qp_attr(struct smc_link *lnk)
 461 {
 462         struct ib_qp_attr *attr = &lnk->qp_attr;
 463         struct ib_qp_init_attr init_attr;
 464
 465         memset(attr, 0, sizeof(*attr));
 466         memset(&init_attr, 0, sizeof(init_attr));
 467         ib_query_qp(lnk->roce_qp, attr,
 468                     IB_QP_STATE |
 469                     IB_QP_CUR_STATE |
 470                     IB_QP_PKEY_INDEX |
 471                     IB_QP_PORT |
 472                     IB_QP_QKEY |
 473                     IB_QP_AV |
 474                     IB_QP_PATH_MTU |
 475                     IB_QP_TIMEOUT |
 476                     IB_QP_RETRY_CNT |
 477                     IB_QP_RNR_RETRY |
 478                     IB_QP_RQ_PSN |
 479                     IB_QP_ALT_PATH |
 480                     IB_QP_MIN_RNR_TIMER |
 481                     IB_QP_SQ_PSN |
 482                     IB_QP_PATH_MIG_STATE |
 483                     IB_QP_CAP |
 484                     IB_QP_DEST_QPN,
 485                     &init_attr);
 486
 487         lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
 488                                lnk->qp_attr.cap.max_send_wr);
 489         lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
 490                                lnk->qp_attr.cap.max_recv_wr);
 491 }
 492
 493 static void smc_wr_init_sge(struct smc_link *lnk)
 494 {
 495         u32 i;
 496
 497         for (i = 0; i < lnk->wr_tx_cnt; i++) {
 498                 lnk->wr_tx_sges[i].addr =
 499                         lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
 500                 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
 501                 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
 502                 lnk->wr_tx_ibs[i].next = NULL;
 503                 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
 504                 lnk->wr_tx_ibs[i].num_sge = 1;
 505                 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
 506                 lnk->wr_tx_ibs[i].send_flags =
 507                         IB_SEND_SIGNALED | IB_SEND_SOLICITED;
 508         }
 509         for (i = 0; i < lnk->wr_rx_cnt; i++) {
 510                 lnk->wr_rx_sges[i].addr =
 511                         lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
 512                 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
 513                 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
 514                 lnk->wr_rx_ibs[i].next = NULL;
 515                 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
 516                 lnk->wr_rx_ibs[i].num_sge = 1;
 517         }
 518         lnk->wr_reg.wr.next = NULL;
 519         lnk->wr_reg.wr.num_sge = 0;
 520         lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
 521         lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
 522         lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
 523 }
 524
 525 void smc_wr_free_link(struct smc_link *lnk)
 526 {
 527         struct ib_device *ibdev;
 528
 529         memset(lnk->wr_tx_mask, 0,
 530                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
 531
 532         if (!lnk->smcibdev)
 533                 return;
 534         ibdev = lnk->smcibdev->ibdev;
 535
 536         if (lnk->wr_rx_dma_addr) {
 537                 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
 538                                     SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
 539                                     DMA_FROM_DEVICE);
 540                 lnk->wr_rx_dma_addr = 0;
 541         }
 542         if (lnk->wr_tx_dma_addr) {
 543                 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
 544                                     SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
 545                                     DMA_TO_DEVICE);
 546                 lnk->wr_tx_dma_addr = 0;
 547         }
 548 }
 549
 550 void smc_wr_free_link_mem(struct smc_link *lnk)
 551 {
 552         kfree(lnk->wr_tx_pends);
 553         lnk->wr_tx_pends = NULL;
 554         kfree(lnk->wr_tx_mask);
 555         lnk->wr_tx_mask = NULL;
 556         kfree(lnk->wr_tx_sges);
 557         lnk->wr_tx_sges = NULL;
 558         kfree(lnk->wr_rx_sges);
 559         lnk->wr_rx_sges = NULL;
 560         kfree(lnk->wr_rx_ibs);
 561         lnk->wr_rx_ibs = NULL;
 562         kfree(lnk->wr_tx_ibs);
 563         lnk->wr_tx_ibs = NULL;
 564         kfree(lnk->wr_tx_bufs);
 565         lnk->wr_tx_bufs = NULL;
 566         kfree(lnk->wr_rx_bufs);
 567         lnk->wr_rx_bufs = NULL;
 568 }
 569
 570 int smc_wr_alloc_link_mem(struct smc_link *link)
 571 {
 572         /* allocate link related memory */
 573         link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
 574         if (!link->wr_tx_bufs)
 575                 goto no_mem;
 576         link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
 577                                    GFP_KERNEL);
 578         if (!link->wr_rx_bufs)
 579                 goto no_mem_wr_tx_bufs;
 580         link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
 581                                   GFP_KERNEL);
 582         if (!link->wr_tx_ibs)
 583                 goto no_mem_wr_rx_bufs;
 584         link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
 585                                   sizeof(link->wr_rx_ibs[0]),
 586                                   GFP_KERNEL);
 587         if (!link->wr_rx_ibs)
 588                 goto no_mem_wr_tx_ibs;
 589         link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
 590                                    GFP_KERNEL);
 591         if (!link->wr_tx_sges)
 592                 goto no_mem_wr_rx_ibs;
 593         link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
 594                                    sizeof(link->wr_rx_sges[0]),
 595                                    GFP_KERNEL);
 596         if (!link->wr_rx_sges)
 597                 goto no_mem_wr_tx_sges;
 598         link->wr_tx_mask = kzalloc(
 599                 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*link->wr_tx_mask),
 600                 GFP_KERNEL);
 601         if (!link->wr_tx_mask)
 602                 goto no_mem_wr_rx_sges;
 603         link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
 604                                     sizeof(link->wr_tx_pends[0]),
 605                                     GFP_KERNEL);
 606         if (!link->wr_tx_pends)
 607                 goto no_mem_wr_tx_mask;
 608         return 0;
 609
 610 no_mem_wr_tx_mask:
 611         kfree(link->wr_tx_mask);
 612 no_mem_wr_rx_sges:
 613         kfree(link->wr_rx_sges);
 614 no_mem_wr_tx_sges:
 615         kfree(link->wr_tx_sges);
 616 no_mem_wr_rx_ibs:
 617         kfree(link->wr_rx_ibs);
 618 no_mem_wr_tx_ibs:
 619         kfree(link->wr_tx_ibs);
 620 no_mem_wr_rx_bufs:
 621         kfree(link->wr_rx_bufs);
 622 no_mem_wr_tx_bufs:
 623         kfree(link->wr_tx_bufs);
 624 no_mem:
 625         return -ENOMEM;
 626 }
 627
 628 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
 629 {
 630         tasklet_kill(&smcibdev->recv_tasklet);
 631         tasklet_kill(&smcibdev->send_tasklet);
 632 }
 633
 634 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
 635 {
 636         tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
 637                      (unsigned long)smcibdev);
 638         tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
 639                      (unsigned long)smcibdev);
 640 }
 641
 642 int smc_wr_create_link(struct smc_link *lnk)
 643 {
 644         struct ib_device *ibdev = lnk->smcibdev->ibdev;
 645         int rc = 0;
 646
 647         smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
 648         lnk->wr_rx_id = 0;
 649         lnk->wr_rx_dma_addr = ib_dma_map_single(
 650                 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
 651                 DMA_FROM_DEVICE);
 652         if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
 653                 lnk->wr_rx_dma_addr = 0;
 654                 rc = -EIO;
 655                 goto out;
 656         }
 657         lnk->wr_tx_dma_addr = ib_dma_map_single(
 658                 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
 659                 DMA_TO_DEVICE);
 660         if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
 661                 rc = -EIO;
 662                 goto dma_unmap;
 663         }
 664         smc_wr_init_sge(lnk);
 665         memset(lnk->wr_tx_mask, 0,
 666                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
 667         init_waitqueue_head(&lnk->wr_tx_wait);
 668         init_waitqueue_head(&lnk->wr_reg_wait);
 669         return rc;
 670
 671 dma_unmap:
 672         ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
 673                             SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
 674                             DMA_FROM_DEVICE);
 675         lnk->wr_rx_dma_addr = 0;
 676 out:
 677         return rc;
 678 }