]> Git Repo - linux.git/blame - net/smc/smc_wr.c
projects / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'sched_ext-for-6.14-rc2-fixes' of git://git.kernel.org/pub/scm/linux/kernel...
[linux.git] / net / smc / smc_wr.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
f38ba179
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Work Requests exploiting Infiniband API
6 *
7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
8 * are submitted to either RC SQ or RC RQ respectively
9 * (reliably connected send/receive queue)
10 * and become work queue entries (WQEs).
11 * While an SQ WR/WQE is pending, we track it until transmission completion.
12 * Through a send or receive completion queue (CQ) respectively,
13 * we get completion queue entries (CQEs) [aka work completions (WCs)].
14 * Since the CQ callback is called from IRQ context, we split work by using
15 * bottom halves implemented by tasklets.
16 *
17 * SMC uses this to exchange LLC (link layer control)
18 * and CDC (connection data control) messages.
19 *
20 * Copyright IBM Corp. 2016
21 *
22 * Author(s): Steffen Maier <[email protected]>
23 */
24
25#include <linux/atomic.h>
26#include <linux/hashtable.h>
27#include <linux/wait.h>
28#include <rdma/ib_verbs.h>
29#include <asm/div64.h>
30
31#include "smc.h"
32#include "smc_wr.h"
33
34#define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */
35
36#define SMC_WR_RX_HASH_BITS 4
37static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
38static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
39
40struct smc_wr_tx_pend { /* control data for a pending send request */
41 u64 wr_id; /* work request id sent */
42 smc_wr_tx_handler handler;
43 enum ib_wc_status wc_status; /* CQE status */
44 struct smc_link *link;
45 u32 idx;
46 struct smc_wr_tx_pend_priv priv;
09c61d24 47 u8 compl_requested;
f38ba179
UB		 48};
49
50/******************************** send queue *********************************/
51
52/*------------------------------- completion --------------------------------*/
53
6a37ad3d
UB		 54/* returns true if at least one tx work request is pending on the given link */
55static inline bool smc_wr_is_tx_pend(struct smc_link *link)
56{
49dc9013 57 return !bitmap_empty(link->wr_tx_mask, link->wr_tx_cnt);
6a37ad3d
UB		 58}
59
60/* wait till all pending tx work requests on the given link are completed */
349d4312 61void smc_wr_tx_wait_no_pending_sends(struct smc_link *link)
6a37ad3d 62{
349d4312 63 wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link));
6a37ad3d
UB		 64}
65
f38ba179
UB		 66static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
67{
68 u32 i;
69
70 for (i = 0; i < link->wr_tx_cnt; i++) {
71 if (link->wr_tx_pends[i].wr_id == wr_id)
72 return i;
73 }
74 return link->wr_tx_cnt;
75}
76
77static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
78{
79 struct smc_wr_tx_pend pnd_snd;
80 struct smc_link *link;
81 u32 pnd_snd_idx;
f38ba179
UB		 82
83 link = wc->qp->qp_context;
652a1e41
UB		 84
85 if (wc->opcode == IB_WC_REG_MR) {
86 if (wc->status)
87 link->wr_reg_state = FAILED;
88 else
89 link->wr_reg_state = CONFIRMED;
15e1b99a 90 smc_wr_wakeup_reg_wait(link);
652a1e41
UB		 91 return;
92 }
93
f38ba179 94 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
8799e310
KG		 95 if (pnd_snd_idx == link->wr_tx_cnt) {
96 if (link->lgr->smc_version != SMC_V2 ||
97 link->wr_tx_v2_pend->wr_id != wc->wr_id)
98 return;
99 link->wr_tx_v2_pend->wc_status = wc->status;
100 memcpy(&pnd_snd, link->wr_tx_v2_pend, sizeof(pnd_snd));
101 /* clear the full struct smc_wr_tx_pend including .priv */
102 memset(link->wr_tx_v2_pend, 0,
103 sizeof(*link->wr_tx_v2_pend));
104 memset(link->lgr->wr_tx_buf_v2, 0,
105 sizeof(*link->lgr->wr_tx_buf_v2));
106 } else {
107 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
108 if (link->wr_tx_pends[pnd_snd_idx].compl_requested)
109 complete(&link->wr_tx_compl[pnd_snd_idx]);
110 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx],
111 sizeof(pnd_snd));
112 /* clear the full struct smc_wr_tx_pend including .priv */
113 memset(&link->wr_tx_pends[pnd_snd_idx], 0,
114 sizeof(link->wr_tx_pends[pnd_snd_idx]));
115 memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
116 sizeof(link->wr_tx_bufs[pnd_snd_idx]));
117 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
118 return;
119 }
120
f38ba179 121 if (wc->status) {
8799e310
KG		 122 if (link->lgr->smc_version == SMC_V2) {
123 memset(link->wr_tx_v2_pend, 0,
124 sizeof(*link->wr_tx_v2_pend));
125 memset(link->lgr->wr_tx_buf_v2, 0,
126 sizeof(*link->lgr->wr_tx_buf_v2));
127 }
87523930
KG		 128 /* terminate link */
129 smcr_link_down_cond_sched(link);
f38ba179
UB		 130 }
131 if (pnd_snd.handler)
132 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
133 wake_up(&link->wr_tx_wait);
134}
135
fcb8e3a3 136static void smc_wr_tx_tasklet_fn(struct tasklet_struct *t)
f38ba179 137{
fcb8e3a3 138 struct smc_ib_device *dev = from_tasklet(dev, t, send_tasklet);
f38ba179
UB		 139 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
140 int i = 0, rc;
141 int polled = 0;
142
143again:
144 polled++;
145 do {
86e780d3 146 memset(&wc, 0, sizeof(wc));
f38ba179
UB		 147 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
148 if (polled == 1) {
149 ib_req_notify_cq(dev->roce_cq_send,
150 IB_CQ_NEXT_COMP |
151 IB_CQ_REPORT_MISSED_EVENTS);
152 }
153 if (!rc)
154 break;
155 for (i = 0; i < rc; i++)
156 smc_wr_tx_process_cqe(&wc[i]);
157 } while (rc > 0);
158 if (polled == 1)
159 goto again;
160}
161
162void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
163{
164 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
165
166 tasklet_schedule(&dev->send_tasklet);
167}
168
169/*---------------------------- request submission ---------------------------*/
170
171static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
172{
173 *idx = link->wr_tx_cnt;
90cee52f 174 if (!smc_link_sendable(link))
b7eede75 175 return -ENOLINK;
f38ba179
UB		 176 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
177 if (!test_and_set_bit(*idx, link->wr_tx_mask))
178 return 0;
179 }
180 *idx = link->wr_tx_cnt;
181 return -EBUSY;
182}
183
184/**
185 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
186 * and sets info for pending transmit tracking
187 * @link: Pointer to smc_link used to later send the message.
188 * @handler: Send completion handler function pointer.
189 * @wr_buf: Out value returns pointer to message buffer.
ad6f317f 190 * @wr_rdma_buf: Out value returns pointer to rdma work request.
f38ba179
UB		 191 * @wr_pend_priv: Out value returns pointer serving as handler context.
192 *
193 * Return: 0 on success, or -errno on error.
194 */
195int smc_wr_tx_get_free_slot(struct smc_link *link,
196 smc_wr_tx_handler handler,
197 struct smc_wr_buf **wr_buf,
ad6f317f 198 struct smc_rdma_wr **wr_rdma_buf,
f38ba179
UB		 199 struct smc_wr_tx_pend_priv **wr_pend_priv)
200{
15e1b99a 201 struct smc_link_group *lgr = smc_get_lgr(link);
f38ba179 202 struct smc_wr_tx_pend *wr_pend;
1a0a04c7 203 u32 idx = link->wr_tx_cnt;
f38ba179
UB		 204 struct ib_send_wr *wr_ib;
205 u64 wr_id;
f38ba179
UB		 206 int rc;
207
208 *wr_buf = NULL;
209 *wr_pend_priv = NULL;
15e1b99a 210 if (in_softirq() || lgr->terminating) {
f38ba179
UB		 211 rc = smc_wr_tx_get_free_slot_index(link, &idx);
212 if (rc)
213 return rc;
214 } else {
15e1b99a 215 rc = wait_event_interruptible_timeout(
f38ba179 216 link->wr_tx_wait,
90cee52f 217 !smc_link_sendable(link) ||
15e1b99a 218 lgr->terminating ||
f38ba179
UB		 219 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
220 SMC_WR_TX_WAIT_FREE_SLOT_TIME);
221 if (!rc) {
87523930
KG		 222 /* timeout - terminate link */
223 smcr_link_down_cond_sched(link);
f38ba179
UB		 224 return -EPIPE;
225 }
f38ba179
UB		 226 if (idx == link->wr_tx_cnt)
227 return -EPIPE;
228 }
229 wr_id = smc_wr_tx_get_next_wr_id(link);
230 wr_pend = &link->wr_tx_pends[idx];
231 wr_pend->wr_id = wr_id;
232 wr_pend->handler = handler;
233 wr_pend->link = link;
234 wr_pend->idx = idx;
235 wr_ib = &link->wr_tx_ibs[idx];
236 wr_ib->wr_id = wr_id;
237 *wr_buf = &link->wr_tx_bufs[idx];
ad6f317f
UB		 238 if (wr_rdma_buf)
239 *wr_rdma_buf = &link->wr_tx_rdmas[idx];
f38ba179
UB		 240 *wr_pend_priv = &wr_pend->priv;
241 return 0;
242}
243
b4ba4652
KG		 244int smc_wr_tx_get_v2_slot(struct smc_link *link,
245 smc_wr_tx_handler handler,
246 struct smc_wr_v2_buf **wr_buf,
247 struct smc_wr_tx_pend_priv **wr_pend_priv)
248{
249 struct smc_wr_tx_pend *wr_pend;
250 struct ib_send_wr *wr_ib;
251 u64 wr_id;
252
253 if (link->wr_tx_v2_pend->idx == link->wr_tx_cnt)
254 return -EBUSY;
255
256 *wr_buf = NULL;
257 *wr_pend_priv = NULL;
258 wr_id = smc_wr_tx_get_next_wr_id(link);
259 wr_pend = link->wr_tx_v2_pend;
260 wr_pend->wr_id = wr_id;
261 wr_pend->handler = handler;
262 wr_pend->link = link;
263 wr_pend->idx = link->wr_tx_cnt;
264 wr_ib = link->wr_tx_v2_ib;
265 wr_ib->wr_id = wr_id;
266 *wr_buf = link->lgr->wr_tx_buf_v2;
267 *wr_pend_priv = &wr_pend->priv;
268 return 0;
269}
270
f38ba179
UB		 271int smc_wr_tx_put_slot(struct smc_link *link,
272 struct smc_wr_tx_pend_priv *wr_pend_priv)
273{
274 struct smc_wr_tx_pend *pend;
275
276 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
277 if (pend->idx < link->wr_tx_cnt) {
e438bae4
UB		 278 u32 idx = pend->idx;
279
f38ba179 280 /* clear the full struct smc_wr_tx_pend including .priv */
46ad0222
KG		 281 memset(&link->wr_tx_pends[idx], 0,
282 sizeof(link->wr_tx_pends[idx]));
283 memset(&link->wr_tx_bufs[idx], 0,
284 sizeof(link->wr_tx_bufs[idx]));
e438bae4 285 test_and_clear_bit(idx, link->wr_tx_mask);
6a37ad3d 286 wake_up(&link->wr_tx_wait);
f38ba179 287 return 1;
8799e310
KG		 288 } else if (link->lgr->smc_version == SMC_V2 &&
289 pend->idx == link->wr_tx_cnt) {
290 /* Large v2 buffer */
291 memset(&link->wr_tx_v2_pend, 0,
292 sizeof(link->wr_tx_v2_pend));
293 memset(&link->lgr->wr_tx_buf_v2, 0,
294 sizeof(link->lgr->wr_tx_buf_v2));
295 return 1;
f38ba179
UB		 296 }
297
298 return 0;
299}
300
301/* Send prepared WR slot via ib_post_send.
302 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
303 */
304int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
305{
f38ba179
UB		 306 struct smc_wr_tx_pend *pend;
307 int rc;
308
309 ib_req_notify_cq(link->smcibdev->roce_cq_send,
8301fa44 310 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
f38ba179 311 pend = container_of(priv, struct smc_wr_tx_pend, priv);
2e3bbe46 312 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
b4772b3a 313 if (rc) {
f38ba179 314 smc_wr_tx_put_slot(link, priv);
87523930 315 smcr_link_down_cond_sched(link);
b4772b3a 316 }
f38ba179
UB		 317 return rc;
318}
319
b4ba4652
KG		 320int smc_wr_tx_v2_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
321 int len)
322{
323 int rc;
324
325 link->wr_tx_v2_ib->sg_list[0].length = len;
326 ib_req_notify_cq(link->smcibdev->roce_cq_send,
327 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
328 rc = ib_post_send(link->roce_qp, link->wr_tx_v2_ib, NULL);
329 if (rc) {
330 smc_wr_tx_put_slot(link, priv);
331 smcr_link_down_cond_sched(link);
332 }
333 return rc;
334}
335
09c61d24
KG		 336/* Send prepared WR slot via ib_post_send and wait for send completion
337 * notification.
338 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
339 */
340int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
341 unsigned long timeout)
342{
343 struct smc_wr_tx_pend *pend;
6d7373da 344 u32 pnd_idx;
09c61d24
KG		 345 int rc;
346
347 pend = container_of(priv, struct smc_wr_tx_pend, priv);
348 pend->compl_requested = 1;
6d7373da
KG		 349 pnd_idx = pend->idx;
350 init_completion(&link->wr_tx_compl[pnd_idx]);
09c61d24
KG		 351
352 rc = smc_wr_tx_send(link, priv);
353 if (rc)
354 return rc;
355 /* wait for completion by smc_wr_tx_process_cqe() */
356 rc = wait_for_completion_interruptible_timeout(
6d7373da 357 &link->wr_tx_compl[pnd_idx], timeout);
09c61d24
KG		 358 if (rc <= 0)
359 rc = -ENODATA;
360 if (rc > 0)
361 rc = 0;
362 return rc;
363}
364
652a1e41
UB		 365/* Register a memory region and wait for result. */
366int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
367{
652a1e41
UB		 368 int rc;
369
370 ib_req_notify_cq(link->smcibdev->roce_cq_send,
371 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
372 link->wr_reg_state = POSTED;
373 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
374 link->wr_reg.mr = mr;
375 link->wr_reg.key = mr->rkey;
2e3bbe46 376 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
652a1e41
UB		 377 if (rc)
378 return rc;
379
79a22238 380 percpu_ref_get(&link->wr_reg_refs);
652a1e41
UB		 381 rc = wait_event_interruptible_timeout(link->wr_reg_wait,
382 (link->wr_reg_state != POSTED),
383 SMC_WR_REG_MR_WAIT_TIME);
79a22238 384 percpu_ref_put(&link->wr_reg_refs);
652a1e41 385 if (!rc) {
87523930
KG		 386 /* timeout - terminate link */
387 smcr_link_down_cond_sched(link);
652a1e41
UB		 388 return -EPIPE;
389 }
390 if (rc == -ERESTARTSYS)
391 return -EINTR;
392 switch (link->wr_reg_state) {
393 case CONFIRMED:
394 rc = 0;
395 break;
396 case FAILED:
397 rc = -EIO;
398 break;
399 case POSTED:
400 rc = -EPIPE;
401 break;
402 }
403 return rc;
404}
405
f38ba179
UB		 406/****************************** receive queue ********************************/
407
408int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
409{
410 struct smc_wr_rx_handler *h_iter;
411 int rc = 0;
412
413 spin_lock(&smc_wr_rx_hash_lock);
414 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
415 if (h_iter->type == handler->type) {
416 rc = -EEXIST;
417 goto out_unlock;
418 }
419 }
420 hash_add(smc_wr_rx_hash, &handler->list, handler->type);
421out_unlock:
422 spin_unlock(&smc_wr_rx_hash_lock);
423 return rc;
424}
425
426/* Demultiplex a received work request based on the message type to its handler.
427 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
428 * and not being modified any more afterwards so we don't need to lock it.
429 */
430static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
431{
432 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
433 struct smc_wr_rx_handler *handler;
434 struct smc_wr_rx_hdr *wr_rx;
435 u64 temp_wr_id;
436 u32 index;
437
438 if (wc->byte_len < sizeof(*wr_rx))
439 return; /* short message */
440 temp_wr_id = wc->wr_id;
441 index = do_div(temp_wr_id, link->wr_rx_cnt);
27ef6a99 442 wr_rx = (struct smc_wr_rx_hdr *)(link->wr_rx_bufs + index * link->wr_rx_buflen);
f38ba179
UB		 443 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
444 if (handler->type == wr_rx->type)
445 handler->handler(wc, wr_rx);
446 }
447}
448
449static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
450{
451 struct smc_link *link;
452 int i;
453
454 for (i = 0; i < num; i++) {
455 link = wc[i].qp->qp_context;
e9b1a4f8 456 link->wr_rx_id_compl = wc[i].wr_id;
f38ba179 457 if (wc[i].status == IB_WC_SUCCESS) {
877ae5be 458 link->wr_rx_tstamp = jiffies;
f38ba179
UB		 459 smc_wr_rx_demultiplex(&wc[i]);
460 smc_wr_rx_post(link); /* refill WR RX */
461 } else {
462 /* handle status errors */
463 switch (wc[i].status) {
464 case IB_WC_RETRY_EXC_ERR:
465 case IB_WC_RNR_RETRY_EXC_ERR:
466 case IB_WC_WR_FLUSH_ERR:
87523930 467 smcr_link_down_cond_sched(link);
e9b1a4f8
YL		 468 if (link->wr_rx_id_compl == link->wr_rx_id)
469 wake_up(&link->wr_rx_empty_wait);
f38ba179
UB		 470 break;
471 default:
472 smc_wr_rx_post(link); /* refill WR RX */
473 break;
474 }
475 }
476 }
477}
478
fcb8e3a3 479static void smc_wr_rx_tasklet_fn(struct tasklet_struct *t)
f38ba179 480{
fcb8e3a3 481 struct smc_ib_device *dev = from_tasklet(dev, t, recv_tasklet);
f38ba179
UB		 482 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
483 int polled = 0;
484 int rc;
485
486again:
487 polled++;
488 do {
489 memset(&wc, 0, sizeof(wc));
490 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
491 if (polled == 1) {
492 ib_req_notify_cq(dev->roce_cq_recv,
493 IB_CQ_SOLICITED_MASK
494 | IB_CQ_REPORT_MISSED_EVENTS);
495 }
496 if (!rc)
497 break;
498 smc_wr_rx_process_cqes(&wc[0], rc);
499 } while (rc > 0);
500 if (polled == 1)
501 goto again;
502}
503
504void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
505{
506 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
507
508 tasklet_schedule(&dev->recv_tasklet);
509}
510
511int smc_wr_rx_post_init(struct smc_link *link)
512{
513 u32 i;
514 int rc = 0;
515
516 for (i = 0; i < link->wr_rx_cnt; i++)
517 rc = smc_wr_rx_post(link);
518 return rc;
519}
520
521/***************************** init, exit, misc ******************************/
522
523void smc_wr_remember_qp_attr(struct smc_link *lnk)
524{
525 struct ib_qp_attr *attr = &lnk->qp_attr;
526 struct ib_qp_init_attr init_attr;
527
528 memset(attr, 0, sizeof(*attr));
529 memset(&init_attr, 0, sizeof(init_attr));
530 ib_query_qp(lnk->roce_qp, attr,
531 IB_QP_STATE |
532 IB_QP_CUR_STATE |
533 IB_QP_PKEY_INDEX |
534 IB_QP_PORT |
535 IB_QP_QKEY |
536 IB_QP_AV |
537 IB_QP_PATH_MTU |
538 IB_QP_TIMEOUT |
539 IB_QP_RETRY_CNT |
540 IB_QP_RNR_RETRY |
541 IB_QP_RQ_PSN |
542 IB_QP_ALT_PATH |
543 IB_QP_MIN_RNR_TIMER |
544 IB_QP_SQ_PSN |
545 IB_QP_PATH_MIG_STATE |
546 IB_QP_CAP |
547 IB_QP_DEST_QPN,
548 &init_attr);
549
550 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
551 lnk->qp_attr.cap.max_send_wr);
552 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
553 lnk->qp_attr.cap.max_recv_wr);
554}
555
556static void smc_wr_init_sge(struct smc_link *lnk)
557{
b632eb06 558 bool send_inline = (lnk->qp_attr.cap.max_inline_data > SMC_WR_TX_SIZE);
f38ba179
UB		 559 u32 i;
560
561 for (i = 0; i < lnk->wr_tx_cnt; i++) {
b632eb06 562 lnk->wr_tx_sges[i].addr = send_inline ? (uintptr_t)(&lnk->wr_tx_bufs[i]) :
f38ba179
UB		 563 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
564 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
bd4ad577 565 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
ad6f317f
UB		 566 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey =
567 lnk->roce_pd->local_dma_lkey;
568 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey =
569 lnk->roce_pd->local_dma_lkey;
570 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey =
571 lnk->roce_pd->local_dma_lkey;
572 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey =
573 lnk->roce_pd->local_dma_lkey;
f38ba179
UB		 574 lnk->wr_tx_ibs[i].next = NULL;
575 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
576 lnk->wr_tx_ibs[i].num_sge = 1;
577 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
578 lnk->wr_tx_ibs[i].send_flags =
2c9c1682 579 IB_SEND_SIGNALED | IB_SEND_SOLICITED;
b632eb06
GW		 580 if (send_inline)
581 lnk->wr_tx_ibs[i].send_flags |= IB_SEND_INLINE;
ad6f317f
UB		 582 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE;
583 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE;
584 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list =
585 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge;
586 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list =
587 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge;
f38ba179 588 }
8799e310
KG		 589
590 if (lnk->lgr->smc_version == SMC_V2) {
591 lnk->wr_tx_v2_sge->addr = lnk->wr_tx_v2_dma_addr;
592 lnk->wr_tx_v2_sge->length = SMC_WR_BUF_V2_SIZE;
593 lnk->wr_tx_v2_sge->lkey = lnk->roce_pd->local_dma_lkey;
594
595 lnk->wr_tx_v2_ib->next = NULL;
596 lnk->wr_tx_v2_ib->sg_list = lnk->wr_tx_v2_sge;
597 lnk->wr_tx_v2_ib->num_sge = 1;
598 lnk->wr_tx_v2_ib->opcode = IB_WR_SEND;
599 lnk->wr_tx_v2_ib->send_flags =
600 IB_SEND_SIGNALED | IB_SEND_SOLICITED;
601 }
602
603 /* With SMC-Rv2 there can be messages larger than SMC_WR_TX_SIZE.
604 * Each ib_recv_wr gets 2 sges, the second one is a spillover buffer
605 * and the same buffer for all sges. When a larger message arrived then
606 * the content of the first small sge is copied to the beginning of
607 * the larger spillover buffer, allowing easy data mapping.
608 */
f38ba179 609 for (i = 0; i < lnk->wr_rx_cnt; i++) {
27ef6a99 610 int x = i * lnk->wr_rx_sge_cnt;
8799e310
KG		 611
612 lnk->wr_rx_sges[x].addr =
27ef6a99
GW		 613 lnk->wr_rx_dma_addr + i * lnk->wr_rx_buflen;
614 lnk->wr_rx_sges[x].length = smc_link_shared_v2_rxbuf(lnk) ?
615 SMC_WR_TX_SIZE : lnk->wr_rx_buflen;
8799e310 616 lnk->wr_rx_sges[x].lkey = lnk->roce_pd->local_dma_lkey;
27ef6a99 617 if (lnk->lgr->smc_version == SMC_V2 && smc_link_shared_v2_rxbuf(lnk)) {
8799e310
KG		 618 lnk->wr_rx_sges[x + 1].addr =
619 lnk->wr_rx_v2_dma_addr + SMC_WR_TX_SIZE;
620 lnk->wr_rx_sges[x + 1].length =
621 SMC_WR_BUF_V2_SIZE - SMC_WR_TX_SIZE;
622 lnk->wr_rx_sges[x + 1].lkey =
623 lnk->roce_pd->local_dma_lkey;
624 }
f38ba179 625 lnk->wr_rx_ibs[i].next = NULL;
8799e310 626 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[x];
27ef6a99 627 lnk->wr_rx_ibs[i].num_sge = lnk->wr_rx_sge_cnt;
f38ba179 628 }
652a1e41
UB		 629 lnk->wr_reg.wr.next = NULL;
630 lnk->wr_reg.wr.num_sge = 0;
631 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
632 lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
633 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
f38ba179
UB		 634}
635
636void smc_wr_free_link(struct smc_link *lnk)
637{
638 struct ib_device *ibdev;
639
b7eede75
KG		 640 if (!lnk->smcibdev)
641 return;
642 ibdev = lnk->smcibdev->ibdev;
643
e9b1a4f8 644 smc_wr_drain_cq(lnk);
8f3d65c1
KG		 645 smc_wr_wakeup_reg_wait(lnk);
646 smc_wr_wakeup_tx_wait(lnk);
647
349d4312 648 smc_wr_tx_wait_no_pending_sends(lnk);
79a22238
KS		 649 percpu_ref_kill(&lnk->wr_reg_refs);
650 wait_for_completion(&lnk->reg_ref_comp);
25c12b45 651 percpu_ref_exit(&lnk->wr_reg_refs);
79a22238
KS		 652 percpu_ref_kill(&lnk->wr_tx_refs);
653 wait_for_completion(&lnk->tx_ref_comp);
25c12b45 654 percpu_ref_exit(&lnk->wr_tx_refs);
f38ba179 655
f38ba179
UB		 656 if (lnk->wr_rx_dma_addr) {
657 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
27ef6a99 658 lnk->wr_rx_buflen * lnk->wr_rx_cnt,
f38ba179
UB		 659 DMA_FROM_DEVICE);
660 lnk->wr_rx_dma_addr = 0;
661 }
8799e310
KG		 662 if (lnk->wr_rx_v2_dma_addr) {
663 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr,
664 SMC_WR_BUF_V2_SIZE,
665 DMA_FROM_DEVICE);
666 lnk->wr_rx_v2_dma_addr = 0;
667 }
f38ba179
UB		 668 if (lnk->wr_tx_dma_addr) {
669 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
670 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
671 DMA_TO_DEVICE);
672 lnk->wr_tx_dma_addr = 0;
673 }
8799e310
KG		 674 if (lnk->wr_tx_v2_dma_addr) {
675 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr,
676 SMC_WR_BUF_V2_SIZE,
677 DMA_TO_DEVICE);
678 lnk->wr_tx_v2_dma_addr = 0;
679 }
680}
681
682void smc_wr_free_lgr_mem(struct smc_link_group *lgr)
683{
684 if (lgr->smc_version < SMC_V2)
685 return;
686
687 kfree(lgr->wr_rx_buf_v2);
688 lgr->wr_rx_buf_v2 = NULL;
689 kfree(lgr->wr_tx_buf_v2);
690 lgr->wr_tx_buf_v2 = NULL;
f38ba179
UB		 691}
692
693void smc_wr_free_link_mem(struct smc_link *lnk)
694{
8799e310
KG		 695 kfree(lnk->wr_tx_v2_ib);
696 lnk->wr_tx_v2_ib = NULL;
697 kfree(lnk->wr_tx_v2_sge);
698 lnk->wr_tx_v2_sge = NULL;
699 kfree(lnk->wr_tx_v2_pend);
700 lnk->wr_tx_v2_pend = NULL;
09c61d24
KG		 701 kfree(lnk->wr_tx_compl);
702 lnk->wr_tx_compl = NULL;
f38ba179
UB		 703 kfree(lnk->wr_tx_pends);
704 lnk->wr_tx_pends = NULL;
49dc9013 705 bitmap_free(lnk->wr_tx_mask);
f38ba179
UB		 706 lnk->wr_tx_mask = NULL;
707 kfree(lnk->wr_tx_sges);
708 lnk->wr_tx_sges = NULL;
ad6f317f
UB		 709 kfree(lnk->wr_tx_rdma_sges);
710 lnk->wr_tx_rdma_sges = NULL;
f38ba179
UB		 711 kfree(lnk->wr_rx_sges);
712 lnk->wr_rx_sges = NULL;
ad6f317f
UB		 713 kfree(lnk->wr_tx_rdmas);
714 lnk->wr_tx_rdmas = NULL;
f38ba179
UB		 715 kfree(lnk->wr_rx_ibs);
716 lnk->wr_rx_ibs = NULL;
717 kfree(lnk->wr_tx_ibs);
718 lnk->wr_tx_ibs = NULL;
719 kfree(lnk->wr_tx_bufs);
720 lnk->wr_tx_bufs = NULL;
721 kfree(lnk->wr_rx_bufs);
722 lnk->wr_rx_bufs = NULL;
723}
724
8799e310
KG		 725int smc_wr_alloc_lgr_mem(struct smc_link_group *lgr)
726{
727 if (lgr->smc_version < SMC_V2)
728 return 0;
729
730 lgr->wr_rx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL);
731 if (!lgr->wr_rx_buf_v2)
732 return -ENOMEM;
733 lgr->wr_tx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL);
734 if (!lgr->wr_tx_buf_v2) {
735 kfree(lgr->wr_rx_buf_v2);
736 return -ENOMEM;
737 }
738 return 0;
739}
740
f38ba179
UB		 741int smc_wr_alloc_link_mem(struct smc_link *link)
742{
743 /* allocate link related memory */
744 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
745 if (!link->wr_tx_bufs)
746 goto no_mem;
27ef6a99 747 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, link->wr_rx_buflen,
f38ba179
UB		 748 GFP_KERNEL);
749 if (!link->wr_rx_bufs)
750 goto no_mem_wr_tx_bufs;
751 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
752 GFP_KERNEL);
753 if (!link->wr_tx_ibs)
754 goto no_mem_wr_rx_bufs;
755 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
756 sizeof(link->wr_rx_ibs[0]),
757 GFP_KERNEL);
758 if (!link->wr_rx_ibs)
759 goto no_mem_wr_tx_ibs;
ad6f317f
UB		 760 link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,
761 sizeof(link->wr_tx_rdmas[0]),
762 GFP_KERNEL);
763 if (!link->wr_tx_rdmas)
764 goto no_mem_wr_rx_ibs;
765 link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,
766 sizeof(link->wr_tx_rdma_sges[0]),
767 GFP_KERNEL);
768 if (!link->wr_tx_rdma_sges)
769 goto no_mem_wr_tx_rdmas;
f38ba179
UB		 770 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
771 GFP_KERNEL);
772 if (!link->wr_tx_sges)
ad6f317f 773 goto no_mem_wr_tx_rdma_sges;
f38ba179 774 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
27ef6a99 775 sizeof(link->wr_rx_sges[0]) * link->wr_rx_sge_cnt,
f38ba179
UB		 776 GFP_KERNEL);
777 if (!link->wr_rx_sges)
778 goto no_mem_wr_tx_sges;
49dc9013 779 link->wr_tx_mask = bitmap_zalloc(SMC_WR_BUF_CNT, GFP_KERNEL);
f38ba179
UB		 780 if (!link->wr_tx_mask)
781 goto no_mem_wr_rx_sges;
782 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
783 sizeof(link->wr_tx_pends[0]),
784 GFP_KERNEL);
785 if (!link->wr_tx_pends)
786 goto no_mem_wr_tx_mask;
09c61d24
KG		 787 link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT,
788 sizeof(link->wr_tx_compl[0]),
789 GFP_KERNEL);
790 if (!link->wr_tx_compl)
791 goto no_mem_wr_tx_pends;
8799e310
KG		 792
793 if (link->lgr->smc_version == SMC_V2) {
794 link->wr_tx_v2_ib = kzalloc(sizeof(*link->wr_tx_v2_ib),
795 GFP_KERNEL);
796 if (!link->wr_tx_v2_ib)
797 goto no_mem_tx_compl;
798 link->wr_tx_v2_sge = kzalloc(sizeof(*link->wr_tx_v2_sge),
799 GFP_KERNEL);
800 if (!link->wr_tx_v2_sge)
801 goto no_mem_v2_ib;
802 link->wr_tx_v2_pend = kzalloc(sizeof(*link->wr_tx_v2_pend),
803 GFP_KERNEL);
804 if (!link->wr_tx_v2_pend)
805 goto no_mem_v2_sge;
806 }
f38ba179
UB		 807 return 0;
808
8799e310
KG		 809no_mem_v2_sge:
810 kfree(link->wr_tx_v2_sge);
811no_mem_v2_ib:
812 kfree(link->wr_tx_v2_ib);
813no_mem_tx_compl:
814 kfree(link->wr_tx_compl);
09c61d24
KG		 815no_mem_wr_tx_pends:
816 kfree(link->wr_tx_pends);
f38ba179
UB		 817no_mem_wr_tx_mask:
818 kfree(link->wr_tx_mask);
819no_mem_wr_rx_sges:
820 kfree(link->wr_rx_sges);
821no_mem_wr_tx_sges:
822 kfree(link->wr_tx_sges);
ad6f317f
UB		 823no_mem_wr_tx_rdma_sges:
824 kfree(link->wr_tx_rdma_sges);
825no_mem_wr_tx_rdmas:
826 kfree(link->wr_tx_rdmas);
f38ba179
UB		 827no_mem_wr_rx_ibs:
828 kfree(link->wr_rx_ibs);
829no_mem_wr_tx_ibs:
830 kfree(link->wr_tx_ibs);
831no_mem_wr_rx_bufs:
832 kfree(link->wr_rx_bufs);
833no_mem_wr_tx_bufs:
834 kfree(link->wr_tx_bufs);
835no_mem:
836 return -ENOMEM;
837}
838
839void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
840{
841 tasklet_kill(&smcibdev->recv_tasklet);
842 tasklet_kill(&smcibdev->send_tasklet);
843}
844
845void smc_wr_add_dev(struct smc_ib_device *smcibdev)
846{
fcb8e3a3
AP		 847 tasklet_setup(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn);
848 tasklet_setup(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn);
f38ba179
UB		 849}
850
79a22238
KS		 851static void smcr_wr_tx_refs_free(struct percpu_ref *ref)
852{
853 struct smc_link *lnk = container_of(ref, struct smc_link, wr_tx_refs);
854
855 complete(&lnk->tx_ref_comp);
856}
857
858static void smcr_wr_reg_refs_free(struct percpu_ref *ref)
859{
860 struct smc_link *lnk = container_of(ref, struct smc_link, wr_reg_refs);
861
862 complete(&lnk->reg_ref_comp);
863}
864
f38ba179
UB		 865int smc_wr_create_link(struct smc_link *lnk)
866{
867 struct ib_device *ibdev = lnk->smcibdev->ibdev;
868 int rc = 0;
869
870 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
871 lnk->wr_rx_id = 0;
872 lnk->wr_rx_dma_addr = ib_dma_map_single(
27ef6a99 873 ibdev, lnk->wr_rx_bufs, lnk->wr_rx_buflen * lnk->wr_rx_cnt,
f38ba179
UB		 874 DMA_FROM_DEVICE);
875 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
876 lnk->wr_rx_dma_addr = 0;
877 rc = -EIO;
878 goto out;
879 }
8799e310 880 if (lnk->lgr->smc_version == SMC_V2) {
27ef6a99
GW		 881 if (smc_link_shared_v2_rxbuf(lnk)) {
882 lnk->wr_rx_v2_dma_addr =
883 ib_dma_map_single(ibdev, lnk->lgr->wr_rx_buf_v2,
884 SMC_WR_BUF_V2_SIZE, DMA_FROM_DEVICE);
885 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_v2_dma_addr)) {
886 lnk->wr_rx_v2_dma_addr = 0;
887 rc = -EIO;
888 goto dma_unmap;
889 }
8799e310
KG		 890 }
891 lnk->wr_tx_v2_dma_addr = ib_dma_map_single(ibdev,
892 lnk->lgr->wr_tx_buf_v2, SMC_WR_BUF_V2_SIZE,
893 DMA_TO_DEVICE);
894 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_v2_dma_addr)) {
895 lnk->wr_tx_v2_dma_addr = 0;
896 rc = -EIO;
897 goto dma_unmap;
898 }
899 }
f38ba179
UB		 900 lnk->wr_tx_dma_addr = ib_dma_map_single(
901 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
902 DMA_TO_DEVICE);
903 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
904 rc = -EIO;
905 goto dma_unmap;
906 }
907 smc_wr_init_sge(lnk);
49dc9013 908 bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT);
652a1e41 909 init_waitqueue_head(&lnk->wr_tx_wait);
79a22238
KS		 910 rc = percpu_ref_init(&lnk->wr_tx_refs, smcr_wr_tx_refs_free, 0, GFP_KERNEL);
911 if (rc)
912 goto dma_unmap;
913 init_completion(&lnk->tx_ref_comp);
652a1e41 914 init_waitqueue_head(&lnk->wr_reg_wait);
79a22238
KS		 915 rc = percpu_ref_init(&lnk->wr_reg_refs, smcr_wr_reg_refs_free, 0, GFP_KERNEL);
916 if (rc)
25c12b45 917 goto cancel_ref;
79a22238 918 init_completion(&lnk->reg_ref_comp);
e9b1a4f8 919 init_waitqueue_head(&lnk->wr_rx_empty_wait);
f38ba179
UB		 920 return rc;
921
25c12b45
KS		 922cancel_ref:
923 percpu_ref_exit(&lnk->wr_tx_refs);
f38ba179 924dma_unmap:
8799e310
KG		 925 if (lnk->wr_rx_v2_dma_addr) {
926 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr,
927 SMC_WR_BUF_V2_SIZE,
928 DMA_FROM_DEVICE);
929 lnk->wr_rx_v2_dma_addr = 0;
930 }
931 if (lnk->wr_tx_v2_dma_addr) {
932 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr,
933 SMC_WR_BUF_V2_SIZE,
934 DMA_TO_DEVICE);
935 lnk->wr_tx_v2_dma_addr = 0;
936 }
f38ba179 937 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
27ef6a99 938 lnk->wr_rx_buflen * lnk->wr_rx_cnt,
f38ba179
UB		 939 DMA_FROM_DEVICE);
940 lnk->wr_rx_dma_addr = 0;
941out:
942 return rc;
943}
Empty description
This page took 0.931406 seconds and 4 git commands to generate.