]> Git Repo - linux.git/blame - net/smc/smc_llc.c
projects / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
[linux.git] / net / smc / smc_llc.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
9bf9abea
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
9bf9abea
UB		 7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <[email protected]>
10 * Ursula Braun <[email protected]>
11 */
12
13#include <net/tcp.h>
14#include <rdma/ib_verbs.h>
15
16#include "smc.h"
17#include "smc_core.h"
18#include "smc_clc.h"
19#include "smc_llc.h"
336ba09f 20#include "smc_pnet.h"
9bf9abea 21
0f627126
SR		 22#define SMC_LLC_DATA_LEN 40
23
24struct smc_llc_hdr {
25 struct smc_wr_rx_hdr common;
26 u8 length; /* 44 */
52bedf37
KG		 27#if defined(__BIG_ENDIAN_BITFIELD)
28 u8 reserved:4,
29 add_link_rej_rsn:4;
30#elif defined(__LITTLE_ENDIAN_BITFIELD)
31 u8 add_link_rej_rsn:4,
32 reserved:4;
33#endif
0f627126
SR		 34 u8 flags;
35};
36
75d320d6
KG		 37#define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
38
0f627126
SR		 39struct smc_llc_msg_confirm_link { /* type 0x01 */
40 struct smc_llc_hdr hd;
41 u8 sender_mac[ETH_ALEN];
42 u8 sender_gid[SMC_GID_SIZE];
43 u8 sender_qp_num[3];
44 u8 link_num;
45 u8 link_uid[SMC_LGR_ID_SIZE];
46 u8 max_links;
47 u8 reserved[9];
48};
49
52bedf37
KG		 50#define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
51#define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
52
53#define SMC_LLC_ADD_LNK_MAX_LINKS 2
54
55struct smc_llc_msg_add_link { /* type 0x02 */
56 struct smc_llc_hdr hd;
57 u8 sender_mac[ETH_ALEN];
58 u8 reserved2[2];
59 u8 sender_gid[SMC_GID_SIZE];
60 u8 sender_qp_num[3];
61 u8 link_num;
fbed3b37
KG		 62#if defined(__BIG_ENDIAN_BITFIELD)
63 u8 reserved3 : 4,
64 qp_mtu : 4;
65#elif defined(__LITTLE_ENDIAN_BITFIELD)
66 u8 qp_mtu : 4,
67 reserved3 : 4;
68#endif
52bedf37
KG		 69 u8 initial_psn[3];
70 u8 reserved[8];
71};
72
87f88cda
KG		 73struct smc_llc_msg_add_link_cont_rt {
74 __be32 rmb_key;
75 __be32 rmb_key_new;
76 __be64 rmb_vaddr_new;
77};
78
79#define SMC_LLC_RKEYS_PER_CONT_MSG 2
80
81struct smc_llc_msg_add_link_cont { /* type 0x03 */
82 struct smc_llc_hdr hd;
83 u8 link_num;
84 u8 num_rkeys;
85 u8 reserved2[2];
86 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
87 u8 reserved[4];
88} __packed; /* format defined in RFC7609 */
89
52bedf37
KG		 90#define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
91#define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
92
93struct smc_llc_msg_del_link { /* type 0x04 */
94 struct smc_llc_hdr hd;
95 u8 link_num;
96 __be32 reason;
97 u8 reserved[35];
98} __packed; /* format defined in RFC7609 */
99
313164da
KG		 100struct smc_llc_msg_test_link { /* type 0x07 */
101 struct smc_llc_hdr hd;
102 u8 user_data[16];
103 u8 reserved[24];
104};
105
4ed75de5
KG		 106struct smc_rmb_rtoken {
107 union {
108 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
109 /* is actually the num of rtokens, first */
110 /* rtoken is always for the current link */
111 u8 link_id; /* link id of the rtoken */
112 };
113 __be32 rmb_key;
114 __be64 rmb_vaddr;
115} __packed; /* format defined in RFC7609 */
116
117#define SMC_LLC_RKEYS_PER_MSG 3
118
119struct smc_llc_msg_confirm_rkey { /* type 0x06 */
120 struct smc_llc_hdr hd;
121 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
122 u8 reserved;
123};
124
4ed75de5 125#define SMC_LLC_DEL_RKEY_MAX 8
3bc67e09 126#define SMC_LLC_FLAG_RKEY_RETRY 0x10
4ed75de5
KG		 127#define SMC_LLC_FLAG_RKEY_NEG 0x20
128
129struct smc_llc_msg_delete_rkey { /* type 0x09 */
130 struct smc_llc_hdr hd;
131 u8 num_rkeys;
132 u8 err_mask;
133 u8 reserved[2];
134 __be32 rkey[8];
135 u8 reserved2[4];
136};
137
0f627126
SR		 138union smc_llc_msg {
139 struct smc_llc_msg_confirm_link confirm_link;
52bedf37 140 struct smc_llc_msg_add_link add_link;
87f88cda 141 struct smc_llc_msg_add_link_cont add_link_cont;
52bedf37 142 struct smc_llc_msg_del_link delete_link;
4ed75de5
KG		 143
144 struct smc_llc_msg_confirm_rkey confirm_rkey;
4ed75de5
KG		 145 struct smc_llc_msg_delete_rkey delete_rkey;
146
313164da 147 struct smc_llc_msg_test_link test_link;
0f627126
SR		 148 struct {
149 struct smc_llc_hdr hdr;
150 u8 data[SMC_LLC_DATA_LEN];
151 } raw;
152};
153
154#define SMC_LLC_FLAG_RESP 0x80
155
6c8968c4
KG		 156struct smc_llc_qentry {
157 struct list_head list;
158 struct smc_link *link;
159 union smc_llc_msg msg;
160};
161
4dadd151
KG		 162static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
163
555da9af
KG		 164struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
165{
166 struct smc_llc_qentry *qentry = flow->qentry;
167
168 flow->qentry = NULL;
169 return qentry;
170}
171
172void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
173{
174 struct smc_llc_qentry *qentry;
175
176 if (flow->qentry) {
177 qentry = flow->qentry;
178 flow->qentry = NULL;
179 kfree(qentry);
180 }
181}
182
183static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
184 struct smc_llc_qentry *qentry)
185{
186 flow->qentry = qentry;
187}
188
189/* try to start a new llc flow, initiated by an incoming llc msg */
190static bool smc_llc_flow_start(struct smc_llc_flow *flow,
191 struct smc_llc_qentry *qentry)
192{
193 struct smc_link_group *lgr = qentry->link->lgr;
194
195 spin_lock_bh(&lgr->llc_flow_lock);
196 if (flow->type) {
197 /* a flow is already active */
198 if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK ||
199 qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) &&
200 !lgr->delayed_event) {
201 lgr->delayed_event = qentry;
202 } else {
203 /* forget this llc request */
204 kfree(qentry);
205 }
206 spin_unlock_bh(&lgr->llc_flow_lock);
207 return false;
208 }
209 switch (qentry->msg.raw.hdr.common.type) {
210 case SMC_LLC_ADD_LINK:
211 flow->type = SMC_LLC_FLOW_ADD_LINK;
212 break;
213 case SMC_LLC_DELETE_LINK:
214 flow->type = SMC_LLC_FLOW_DEL_LINK;
215 break;
216 case SMC_LLC_CONFIRM_RKEY:
217 case SMC_LLC_DELETE_RKEY:
218 flow->type = SMC_LLC_FLOW_RKEY;
219 break;
220 default:
221 flow->type = SMC_LLC_FLOW_NONE;
222 }
223 if (qentry == lgr->delayed_event)
224 lgr->delayed_event = NULL;
225 spin_unlock_bh(&lgr->llc_flow_lock);
226 smc_llc_flow_qentry_set(flow, qentry);
227 return true;
228}
229
230/* start a new local llc flow, wait till current flow finished */
231int smc_llc_flow_initiate(struct smc_link_group *lgr,
232 enum smc_llc_flowtype type)
233{
234 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
235 int rc;
236
237 /* all flows except confirm_rkey and delete_rkey are exclusive,
238 * confirm/delete rkey flows can run concurrently (local and remote)
239 */
240 if (type == SMC_LLC_FLOW_RKEY)
241 allowed_remote = SMC_LLC_FLOW_RKEY;
242again:
243 if (list_empty(&lgr->list))
244 return -ENODEV;
245 spin_lock_bh(&lgr->llc_flow_lock);
246 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
247 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
248 lgr->llc_flow_rmt.type == allowed_remote)) {
249 lgr->llc_flow_lcl.type = type;
250 spin_unlock_bh(&lgr->llc_flow_lock);
251 return 0;
252 }
253 spin_unlock_bh(&lgr->llc_flow_lock);
254 rc = wait_event_interruptible_timeout(lgr->llc_waiter,
255 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
256 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
257 lgr->llc_flow_rmt.type == allowed_remote)),
258 SMC_LLC_WAIT_TIME);
259 if (!rc)
260 return -ETIMEDOUT;
261 goto again;
262}
263
264/* finish the current llc flow */
265void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
266{
267 spin_lock_bh(&lgr->llc_flow_lock);
268 memset(flow, 0, sizeof(*flow));
269 flow->type = SMC_LLC_FLOW_NONE;
270 spin_unlock_bh(&lgr->llc_flow_lock);
271 if (!list_empty(&lgr->list) && lgr->delayed_event &&
272 flow == &lgr->llc_flow_lcl)
273 schedule_work(&lgr->llc_event_work);
274 else
275 wake_up_interruptible(&lgr->llc_waiter);
276}
277
278/* lnk is optional and used for early wakeup when link goes down, useful in
279 * cases where we wait for a response on the link after we sent a request
280 */
281struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
282 struct smc_link *lnk,
283 int time_out, u8 exp_msg)
284{
285 struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
286
287 wait_event_interruptible_timeout(lgr->llc_waiter,
288 (flow->qentry ||
289 (lnk && !smc_link_usable(lnk)) ||
290 list_empty(&lgr->list)),
291 time_out);
292 if (!flow->qentry ||
293 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
294 smc_llc_flow_qentry_del(flow);
295 goto out;
296 }
297 if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) {
298 if (exp_msg == SMC_LLC_ADD_LINK &&
299 flow->qentry->msg.raw.hdr.common.type ==
300 SMC_LLC_DELETE_LINK) {
301 /* flow_start will delay the unexpected msg */
302 smc_llc_flow_start(&lgr->llc_flow_lcl,
303 smc_llc_flow_qentry_clr(flow));
304 return NULL;
305 }
306 smc_llc_flow_qentry_del(flow);
307 }
308out:
309 return flow->qentry;
310}
311
9bf9abea
UB		 312/********************************** send *************************************/
313
314struct smc_llc_tx_pend {
315};
316
317/* handler for send/transmission completion of an LLC msg */
318static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
319 struct smc_link *link,
320 enum ib_wc_status wc_status)
321{
322 /* future work: handle wc_status error for recovery and failover */
323}
324
325/**
326 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
327 * @link: Pointer to SMC link used for sending LLC control message.
328 * @wr_buf: Out variable returning pointer to work request payload buffer.
329 * @pend: Out variable returning pointer to private pending WR tracking.
330 * It's the context the transmit complete handler will get.
331 *
332 * Reserves and pre-fills an entry for a pending work request send/tx.
333 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
334 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
335 *
336 * Return: 0 on success, otherwise an error value.
337 */
338static int smc_llc_add_pending_send(struct smc_link *link,
339 struct smc_wr_buf **wr_buf,
340 struct smc_wr_tx_pend_priv **pend)
341{
342 int rc;
343
ad6f317f
UB		 344 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
345 pend);
9bf9abea
UB		 346 if (rc < 0)
347 return rc;
348 BUILD_BUG_ON_MSG(
349 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
350 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
351 BUILD_BUG_ON_MSG(
352 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
353 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
354 BUILD_BUG_ON_MSG(
355 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
356 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
357 return 0;
358}
359
360/* high-level API to send LLC confirm link */
947541f3 361int smc_llc_send_confirm_link(struct smc_link *link,
9bf9abea
UB		 362 enum smc_llc_reqresp reqresp)
363{
9bf9abea
UB		 364 struct smc_llc_msg_confirm_link *confllc;
365 struct smc_wr_tx_pend_priv *pend;
366 struct smc_wr_buf *wr_buf;
367 int rc;
368
369 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
370 if (rc)
371 return rc;
372 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
373 memset(confllc, 0, sizeof(*confllc));
374 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
375 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
75d320d6 376 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
9bf9abea
UB		 377 if (reqresp == SMC_LLC_RESP)
378 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
947541f3
UB		 379 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
380 ETH_ALEN);
7005ada6 381 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
9bf9abea 382 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
2be922f3 383 confllc->link_num = link->link_id;
45fa8da0 384 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
b1570a87 385 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
52bedf37
KG		 386 /* send llc message */
387 rc = smc_wr_tx_send(link, pend);
388 return rc;
389}
390
44aa81ce 391/* send LLC confirm rkey request */
3d88a21b 392static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
44aa81ce
KG		 393 struct smc_buf_desc *rmb_desc)
394{
395 struct smc_llc_msg_confirm_rkey *rkeyllc;
396 struct smc_wr_tx_pend_priv *pend;
397 struct smc_wr_buf *wr_buf;
3d88a21b
KG		 398 struct smc_link *link;
399 int i, rc, rtok_ix;
44aa81ce 400
3d88a21b 401 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
44aa81ce
KG		 402 if (rc)
403 return rc;
404 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
405 memset(rkeyllc, 0, sizeof(*rkeyllc));
406 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
407 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
3d88a21b
KG		 408
409 rtok_ix = 1;
410 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
411 link = &send_link->lgr->lnk[i];
412 if (link->state == SMC_LNK_ACTIVE && link != send_link) {
413 rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
414 rkeyllc->rtoken[rtok_ix].rmb_key =
415 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
416 rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64(
417 (u64)sg_dma_address(
418 rmb_desc->sgt[link->link_idx].sgl));
419 rtok_ix++;
420 }
421 }
422 /* rkey of send_link is in rtoken[0] */
423 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
44aa81ce 424 rkeyllc->rtoken[0].rmb_key =
3d88a21b 425 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey);
44aa81ce 426 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
3d88a21b 427 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
44aa81ce 428 /* send llc message */
3d88a21b 429 rc = smc_wr_tx_send(send_link, pend);
44aa81ce
KG		 430 return rc;
431}
432
60e03c62
KG		 433/* send LLC delete rkey request */
434static int smc_llc_send_delete_rkey(struct smc_link *link,
435 struct smc_buf_desc *rmb_desc)
436{
437 struct smc_llc_msg_delete_rkey *rkeyllc;
438 struct smc_wr_tx_pend_priv *pend;
439 struct smc_wr_buf *wr_buf;
440 int rc;
441
442 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
443 if (rc)
444 return rc;
445 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
446 memset(rkeyllc, 0, sizeof(*rkeyllc));
447 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
448 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
449 rkeyllc->num_rkeys = 1;
387707fd 450 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
60e03c62
KG		 451 /* send llc message */
452 rc = smc_wr_tx_send(link, pend);
453 return rc;
454}
455
52bedf37 456/* send ADD LINK request or response */
7005ada6 457int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
fbed3b37 458 struct smc_link *link_new,
52bedf37
KG		 459 enum smc_llc_reqresp reqresp)
460{
461 struct smc_llc_msg_add_link *addllc;
462 struct smc_wr_tx_pend_priv *pend;
463 struct smc_wr_buf *wr_buf;
464 int rc;
465
466 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
467 if (rc)
468 return rc;
469 addllc = (struct smc_llc_msg_add_link *)wr_buf;
fbed3b37
KG		 470
471 memset(addllc, 0, sizeof(*addllc));
472 addllc->hd.common.type = SMC_LLC_ADD_LINK;
473 addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
474 if (reqresp == SMC_LLC_RESP)
475 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
476 memcpy(addllc->sender_mac, mac, ETH_ALEN);
477 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
478 if (link_new) {
479 addllc->link_num = link_new->link_id;
480 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
481 hton24(addllc->initial_psn, link_new->psn_initial);
482 if (reqresp == SMC_LLC_REQ)
483 addllc->qp_mtu = link_new->path_mtu;
484 else
485 addllc->qp_mtu = min(link_new->path_mtu,
486 link_new->peer_mtu);
487 }
52bedf37
KG		 488 /* send llc message */
489 rc = smc_wr_tx_send(link, pend);
490 return rc;
491}
492
493/* send DELETE LINK request or response */
fbed3b37
KG		 494int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
495 enum smc_llc_reqresp reqresp, bool orderly,
496 u32 reason)
52bedf37
KG		 497{
498 struct smc_llc_msg_del_link *delllc;
499 struct smc_wr_tx_pend_priv *pend;
500 struct smc_wr_buf *wr_buf;
501 int rc;
502
503 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
504 if (rc)
505 return rc;
506 delllc = (struct smc_llc_msg_del_link *)wr_buf;
fbed3b37
KG		 507
508 memset(delllc, 0, sizeof(*delllc));
509 delllc->hd.common.type = SMC_LLC_DELETE_LINK;
510 delllc->hd.length = sizeof(struct smc_llc_msg_del_link);
511 if (reqresp == SMC_LLC_RESP)
512 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
513 if (orderly)
514 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
515 if (link_del_id)
516 delllc->link_num = link_del_id;
517 else
518 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
519 delllc->reason = htonl(reason);
9bf9abea
UB		 520 /* send llc message */
521 rc = smc_wr_tx_send(link, pend);
522 return rc;
523}
524
d97935fa
KG		 525/* send LLC test link request */
526static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
313164da
KG		 527{
528 struct smc_llc_msg_test_link *testllc;
529 struct smc_wr_tx_pend_priv *pend;
530 struct smc_wr_buf *wr_buf;
531 int rc;
532
533 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
534 if (rc)
535 return rc;
536 testllc = (struct smc_llc_msg_test_link *)wr_buf;
537 memset(testllc, 0, sizeof(*testllc));
538 testllc->hd.common.type = SMC_LLC_TEST_LINK;
539 testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
313164da
KG		 540 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
541 /* send llc message */
542 rc = smc_wr_tx_send(link, pend);
543 return rc;
544}
545
6c8968c4
KG		 546/* schedule an llc send on link, may wait for buffers */
547static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
4ed75de5
KG		 548{
549 struct smc_wr_tx_pend_priv *pend;
550 struct smc_wr_buf *wr_buf;
551 int rc;
552
6c8968c4
KG		 553 if (!smc_link_usable(link))
554 return -ENOLINK;
555 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
4ed75de5 556 if (rc)
6c8968c4
KG		 557 return rc;
558 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
559 return smc_wr_tx_send(link, pend);
4ed75de5
KG		 560}
561
f3811fd7
KG		 562/* schedule an llc send on link, may wait for buffers,
563 * and wait for send completion notification.
564 * @return 0 on success
565 */
566static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
567{
568 struct smc_wr_tx_pend_priv *pend;
569 struct smc_wr_buf *wr_buf;
570 int rc;
571
572 if (!smc_link_usable(link))
573 return -ENOLINK;
574 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
575 if (rc)
576 return rc;
577 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
578 return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
579}
580
9bf9abea
UB		 581/********************************* receive ***********************************/
582
336ba09f
KG		 583static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
584 enum smc_lgr_type lgr_new_t)
585{
586 int i;
587
588 if (lgr->type == SMC_LGR_SYMMETRIC ||
589 (lgr->type != SMC_LGR_SINGLE &&
590 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
591 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
592 return -EMLINK;
593
594 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
595 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
596 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
597 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
598 return i;
599 } else {
600 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
601 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
602 return i;
603 }
604 return -EMLINK;
605}
606
87f88cda
KG		 607/* return first buffer from any of the next buf lists */
608static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
609 int *buf_lst)
610{
611 struct smc_buf_desc *buf_pos;
612
613 while (*buf_lst < SMC_RMBE_SIZES) {
614 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
615 struct smc_buf_desc, list);
616 if (buf_pos)
617 return buf_pos;
618 (*buf_lst)++;
619 }
620 return NULL;
621}
622
623/* return next rmb from buffer lists */
624static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
625 int *buf_lst,
626 struct smc_buf_desc *buf_pos)
627{
628 struct smc_buf_desc *buf_next;
629
630 if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
631 (*buf_lst)++;
632 return _smc_llc_get_next_rmb(lgr, buf_lst);
633 }
634 buf_next = list_next_entry(buf_pos, list);
635 return buf_next;
636}
637
638static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
639 int *buf_lst)
640{
641 *buf_lst = 0;
642 return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
643}
644
645/* send one add_link_continue msg */
646static int smc_llc_add_link_cont(struct smc_link *link,
647 struct smc_link *link_new, u8 *num_rkeys_todo,
648 int *buf_lst, struct smc_buf_desc **buf_pos)
649{
650 struct smc_llc_msg_add_link_cont *addc_llc;
651 struct smc_link_group *lgr = link->lgr;
652 int prim_lnk_idx, lnk_idx, i, rc;
653 struct smc_wr_tx_pend_priv *pend;
654 struct smc_wr_buf *wr_buf;
655 struct smc_buf_desc *rmb;
656 u8 n;
657
658 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
659 if (rc)
660 return rc;
661 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
662 memset(addc_llc, 0, sizeof(*addc_llc));
663
664 prim_lnk_idx = link->link_idx;
665 lnk_idx = link_new->link_idx;
666 addc_llc->link_num = link_new->link_id;
667 addc_llc->num_rkeys = *num_rkeys_todo;
668 n = *num_rkeys_todo;
669 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
670 if (!*buf_pos) {
671 addc_llc->num_rkeys = addc_llc->num_rkeys -
672 *num_rkeys_todo;
673 *num_rkeys_todo = 0;
674 break;
675 }
676 rmb = *buf_pos;
677
678 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey);
679 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey);
680 addc_llc->rt[i].rmb_vaddr_new =
681 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
682
683 (*num_rkeys_todo)--;
684 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
685 while (*buf_pos && !(*buf_pos)->used)
686 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
687 }
688 addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT;
689 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
690 if (lgr->role == SMC_CLNT)
691 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
692 return smc_wr_tx_send(link, pend);
693}
694
695static int smc_llc_cli_rkey_exchange(struct smc_link *link,
696 struct smc_link *link_new)
697{
698 struct smc_llc_msg_add_link_cont *addc_llc;
699 struct smc_link_group *lgr = link->lgr;
700 u8 max, num_rkeys_send, num_rkeys_recv;
701 struct smc_llc_qentry *qentry;
702 struct smc_buf_desc *buf_pos;
703 int buf_lst;
704 int rc = 0;
705 int i;
706
707 mutex_lock(&lgr->rmbs_lock);
708 num_rkeys_send = lgr->conns_num;
709 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
710 do {
711 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
712 SMC_LLC_ADD_LINK_CONT);
713 if (!qentry) {
714 rc = -ETIMEDOUT;
715 break;
716 }
717 addc_llc = &qentry->msg.add_link_cont;
718 num_rkeys_recv = addc_llc->num_rkeys;
719 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
720 for (i = 0; i < max; i++) {
721 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
722 addc_llc->rt[i].rmb_key,
723 addc_llc->rt[i].rmb_vaddr_new,
724 addc_llc->rt[i].rmb_key_new);
725 num_rkeys_recv--;
726 }
727 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
728 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
729 &buf_lst, &buf_pos);
730 if (rc)
731 break;
732 } while (num_rkeys_send || num_rkeys_recv);
733
734 mutex_unlock(&lgr->rmbs_lock);
735 return rc;
736}
737
336ba09f
KG		 738/* prepare and send an add link reject response */
739static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
740{
741 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
742 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
743 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
744 return smc_llc_send_message(qentry->link, &qentry->msg);
745}
746
b1570a87
KG		 747static int smc_llc_cli_conf_link(struct smc_link *link,
748 struct smc_init_info *ini,
749 struct smc_link *link_new,
750 enum smc_lgr_type lgr_new_t)
751{
752 struct smc_link_group *lgr = link->lgr;
b1570a87
KG		 753 struct smc_llc_qentry *qentry = NULL;
754 int rc = 0;
755
756 /* receive CONFIRM LINK request over RoCE fabric */
757 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
758 if (!qentry) {
759 rc = smc_llc_send_delete_link(link, link_new->link_id,
760 SMC_LLC_REQ, false,
761 SMC_LLC_DEL_LOST_PATH);
762 return -ENOLINK;
763 }
764 if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) {
765 /* received DELETE_LINK instead */
b1570a87
KG		 766 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
767 smc_llc_send_message(link, &qentry->msg);
768 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
769 return -ENOLINK;
770 }
649758ff 771 smc_llc_save_peer_uid(qentry);
b1570a87
KG		 772 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
773
774 rc = smc_ib_modify_qp_rts(link_new);
775 if (rc) {
776 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
777 false, SMC_LLC_DEL_LOST_PATH);
778 return -ENOLINK;
779 }
780 smc_wr_remember_qp_attr(link_new);
781
782 rc = smcr_buf_reg_lgr(link_new);
783 if (rc) {
784 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
785 false, SMC_LLC_DEL_LOST_PATH);
786 return -ENOLINK;
787 }
788
789 /* send CONFIRM LINK response over RoCE fabric */
790 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
791 if (rc) {
792 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
793 false, SMC_LLC_DEL_LOST_PATH);
794 return -ENOLINK;
795 }
796 smc_llc_link_active(link_new);
ad6c111b
KG		 797 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
798 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
799 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
800 else
801 smcr_lgr_set_type(lgr, lgr_new_t);
b1570a87
KG		 802 return 0;
803}
804
336ba09f
KG		 805static void smc_llc_save_add_link_info(struct smc_link *link,
806 struct smc_llc_msg_add_link *add_llc)
807{
808 link->peer_qpn = ntoh24(add_llc->sender_qp_num);
809 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
810 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
811 link->peer_psn = ntoh24(add_llc->initial_psn);
812 link->peer_mtu = add_llc->qp_mtu;
813}
814
815/* as an SMC client, process an add link request */
816int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
817{
818 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
819 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
820 struct smc_link_group *lgr = smc_get_lgr(link);
821 struct smc_link *lnk_new = NULL;
822 struct smc_init_info ini;
823 int lnk_idx, rc = 0;
824
825 ini.vlan_id = lgr->vlan_id;
826 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev);
827 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
828 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) {
829 if (!ini.ib_dev)
830 goto out_reject;
831 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
832 }
833 if (!ini.ib_dev) {
834 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
835 ini.ib_dev = link->smcibdev;
836 ini.ib_port = link->ibport;
837 }
838 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
839 if (lnk_idx < 0)
840 goto out_reject;
841 lnk_new = &lgr->lnk[lnk_idx];
842 rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini);
843 if (rc)
844 goto out_reject;
845 smc_llc_save_add_link_info(lnk_new, llc);
45fa8da0
KG		 846 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */
847 smc_llc_link_set_uid(lnk_new);
336ba09f
KG		 848
849 rc = smc_ib_ready_link(lnk_new);
850 if (rc)
851 goto out_clear_lnk;
852
853 rc = smcr_buf_map_lgr(lnk_new);
854 if (rc)
855 goto out_clear_lnk;
856
857 rc = smc_llc_send_add_link(link,
858 lnk_new->smcibdev->mac[ini.ib_port - 1],
859 lnk_new->gid, lnk_new, SMC_LLC_RESP);
860 if (rc)
861 goto out_clear_lnk;
87f88cda 862 rc = smc_llc_cli_rkey_exchange(link, lnk_new);
336ba09f
KG		 863 if (rc) {
864 rc = 0;
865 goto out_clear_lnk;
866 }
b1570a87 867 rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t);
336ba09f
KG		 868 if (!rc)
869 goto out;
870out_clear_lnk:
0a99be43 871 smcr_link_clear(lnk_new, false);
336ba09f
KG		 872out_reject:
873 smc_llc_cli_add_link_reject(qentry);
874out:
875 kfree(qentry);
876 return rc;
877}
878
b1570a87
KG		 879static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
880{
881 struct smc_llc_qentry *qentry;
882
883 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
884
885 mutex_lock(&lgr->llc_conf_mutex);
886 smc_llc_cli_add_link(qentry->link, qentry);
887 mutex_unlock(&lgr->llc_conf_mutex);
888}
889
9c416878
KG		 890static int smc_llc_active_link_count(struct smc_link_group *lgr)
891{
892 int i, link_count = 0;
893
894 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
895 if (!smc_link_usable(&lgr->lnk[i]))
896 continue;
897 link_count++;
898 }
899 return link_count;
900}
901
c9a5d243
KG		 902/* find the asymmetric link when 3 links are established */
903static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
904{
905 int asym_idx = -ENOENT;
906 int i, j, k;
907 bool found;
908
909 /* determine asymmetric link */
910 found = false;
911 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
912 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
913 if (!smc_link_usable(&lgr->lnk[i]) ||
914 !smc_link_usable(&lgr->lnk[j]))
915 continue;
916 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
917 SMC_GID_SIZE)) {
918 found = true; /* asym_lnk is i or j */
919 break;
920 }
921 }
922 if (found)
923 break;
924 }
925 if (!found)
926 goto out; /* no asymmetric link */
927 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
928 if (!smc_link_usable(&lgr->lnk[k]))
929 continue;
930 if (k != i &&
931 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
932 SMC_GID_SIZE)) {
933 asym_idx = i;
934 break;
935 }
936 if (k != j &&
937 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
938 SMC_GID_SIZE)) {
939 asym_idx = j;
940 break;
941 }
942 }
943out:
944 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
945}
946
947static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
948{
949 struct smc_link *lnk_new = NULL, *lnk_asym;
950 struct smc_llc_qentry *qentry;
951 int rc;
952
953 lnk_asym = smc_llc_find_asym_link(lgr);
954 if (!lnk_asym)
955 return; /* no asymmetric link */
956 if (!smc_link_downing(&lnk_asym->state))
957 return;
c6f02ebe 958 lnk_new = smc_switch_conns(lgr, lnk_asym, false);
c9a5d243
KG		 959 smc_wr_tx_wait_no_pending_sends(lnk_asym);
960 if (!lnk_new)
961 goto out_free;
962 /* change flow type from ADD_LINK into DEL_LINK */
963 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
964 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
965 true, SMC_LLC_DEL_NO_ASYM_NEEDED);
966 if (rc) {
967 smcr_link_down_cond(lnk_new);
968 goto out_free;
969 }
970 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
971 SMC_LLC_DELETE_LINK);
972 if (!qentry) {
973 smcr_link_down_cond(lnk_new);
974 goto out_free;
975 }
976 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
977out_free:
0a99be43 978 smcr_link_clear(lnk_asym, true);
c9a5d243
KG		 979}
980
57b49924
KG		 981static int smc_llc_srv_rkey_exchange(struct smc_link *link,
982 struct smc_link *link_new)
983{
984 struct smc_llc_msg_add_link_cont *addc_llc;
985 struct smc_link_group *lgr = link->lgr;
986 u8 max, num_rkeys_send, num_rkeys_recv;
987 struct smc_llc_qentry *qentry = NULL;
988 struct smc_buf_desc *buf_pos;
989 int buf_lst;
990 int rc = 0;
991 int i;
992
993 mutex_lock(&lgr->rmbs_lock);
994 num_rkeys_send = lgr->conns_num;
995 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
996 do {
997 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
998 &buf_lst, &buf_pos);
999 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1000 SMC_LLC_ADD_LINK_CONT);
1001 if (!qentry) {
1002 rc = -ETIMEDOUT;
1003 goto out;
1004 }
1005 addc_llc = &qentry->msg.add_link_cont;
1006 num_rkeys_recv = addc_llc->num_rkeys;
1007 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1008 for (i = 0; i < max; i++) {
1009 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1010 addc_llc->rt[i].rmb_key,
1011 addc_llc->rt[i].rmb_vaddr_new,
1012 addc_llc->rt[i].rmb_key_new);
1013 num_rkeys_recv--;
1014 }
1015 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1016 } while (num_rkeys_send || num_rkeys_recv);
1017out:
1018 mutex_unlock(&lgr->rmbs_lock);
1019 return rc;
1020}
1021
1551c95b
KG		 1022static int smc_llc_srv_conf_link(struct smc_link *link,
1023 struct smc_link *link_new,
1024 enum smc_lgr_type lgr_new_t)
1025{
1026 struct smc_link_group *lgr = link->lgr;
1027 struct smc_llc_qentry *qentry = NULL;
1028 int rc;
1029
1030 /* send CONFIRM LINK request over the RoCE fabric */
1031 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1032 if (rc)
1033 return -ENOLINK;
1034 /* receive CONFIRM LINK response over the RoCE fabric */
1035 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME,
1036 SMC_LLC_CONFIRM_LINK);
1037 if (!qentry) {
1038 /* send DELETE LINK */
1039 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1040 false, SMC_LLC_DEL_LOST_PATH);
1041 return -ENOLINK;
1042 }
649758ff 1043 smc_llc_save_peer_uid(qentry);
1551c95b 1044 smc_llc_link_active(link_new);
ad6c111b
KG		 1045 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1046 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1047 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1048 else
1049 smcr_lgr_set_type(lgr, lgr_new_t);
1551c95b
KG		 1050 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1051 return 0;
1052}
1053
2d2209f2
KG		 1054int smc_llc_srv_add_link(struct smc_link *link)
1055{
1056 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1057 struct smc_link_group *lgr = link->lgr;
1058 struct smc_llc_msg_add_link *add_llc;
1059 struct smc_llc_qentry *qentry = NULL;
1060 struct smc_link *link_new;
1061 struct smc_init_info ini;
1062 int lnk_idx, rc = 0;
1063
1064 /* ignore client add link recommendation, start new flow */
1065 ini.vlan_id = lgr->vlan_id;
1066 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev);
1067 if (!ini.ib_dev) {
1068 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1069 ini.ib_dev = link->smcibdev;
1070 ini.ib_port = link->ibport;
1071 }
1072 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1073 if (lnk_idx < 0)
1074 return 0;
1075
1076 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini);
1077 if (rc)
1078 return rc;
1079 link_new = &lgr->lnk[lnk_idx];
1080 rc = smc_llc_send_add_link(link,
1081 link_new->smcibdev->mac[ini.ib_port - 1],
1082 link_new->gid, link_new, SMC_LLC_REQ);
1083 if (rc)
1084 goto out_err;
1085 /* receive ADD LINK response over the RoCE fabric */
1086 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1087 if (!qentry) {
1088 rc = -ETIMEDOUT;
1089 goto out_err;
1090 }
1091 add_llc = &qentry->msg.add_link;
1092 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1093 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1094 rc = -ENOLINK;
1095 goto out_err;
1096 }
1097 if (lgr->type == SMC_LGR_SINGLE &&
1098 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1099 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1100 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1101 }
1102 smc_llc_save_add_link_info(link_new, add_llc);
1103 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1104
1105 rc = smc_ib_ready_link(link_new);
1106 if (rc)
1107 goto out_err;
1108 rc = smcr_buf_map_lgr(link_new);
1109 if (rc)
1110 goto out_err;
1111 rc = smcr_buf_reg_lgr(link_new);
1112 if (rc)
1113 goto out_err;
57b49924 1114 rc = smc_llc_srv_rkey_exchange(link, link_new);
2d2209f2
KG		 1115 if (rc)
1116 goto out_err;
1551c95b 1117 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
2d2209f2
KG		 1118 if (rc)
1119 goto out_err;
1120 return 0;
1121out_err:
0a99be43 1122 smcr_link_clear(link_new, false);
2d2209f2
KG		 1123 return rc;
1124}
1125
1126static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1127{
1128 struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1129 int rc;
1130
1131 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1132
1133 mutex_lock(&lgr->llc_conf_mutex);
1134 rc = smc_llc_srv_add_link(link);
1135 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1136 /* delete any asymmetric link */
c9a5d243 1137 smc_llc_delete_asym_link(lgr);
2d2209f2
KG		 1138 }
1139 mutex_unlock(&lgr->llc_conf_mutex);
1140}
1141
4dadd151
KG		 1142/* enqueue a local add_link req to trigger a new add_link flow, only as SERV */
1143void smc_llc_srv_add_link_local(struct smc_link *link)
1144{
1145 struct smc_llc_msg_add_link add_llc = {0};
1146
1147 add_llc.hd.length = sizeof(add_llc);
1148 add_llc.hd.common.type = SMC_LLC_ADD_LINK;
1149 /* no dev and port needed, we as server ignore client data anyway */
1150 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1151}
1152
b45e7f98
KG		 1153/* worker to process an add link message */
1154static void smc_llc_add_link_work(struct work_struct *work)
1155{
1156 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1157 llc_add_link_work);
1158
1159 if (list_empty(&lgr->list)) {
1160 /* link group is terminating */
1161 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1162 goto out;
1163 }
1164
b1570a87
KG		 1165 if (lgr->role == SMC_CLNT)
1166 smc_llc_process_cli_add_link(lgr);
2d2209f2
KG		 1167 else
1168 smc_llc_process_srv_add_link(lgr);
b45e7f98
KG		 1169out:
1170 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1171}
1172
4dadd151
KG		 1173/* enqueue a local del_link msg to trigger a new del_link flow,
1174 * called only for role SMC_SERV
1175 */
1176void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1177{
1178 struct smc_llc_msg_del_link del_llc = {0};
1179
1180 del_llc.hd.length = sizeof(del_llc);
1181 del_llc.hd.common.type = SMC_LLC_DELETE_LINK;
1182 del_llc.link_num = del_link_id;
1183 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1184 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1185 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1186}
1187
9c416878
KG		 1188static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1189{
1190 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1191 struct smc_llc_msg_del_link *del_llc;
1192 struct smc_llc_qentry *qentry;
1193 int active_links;
1194 int lnk_idx;
1195
1196 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1197 lnk = qentry->link;
1198 del_llc = &qentry->msg.delete_link;
1199
1200 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1201 smc_lgr_terminate_sched(lgr);
1202 goto out;
1203 }
1204 mutex_lock(&lgr->llc_conf_mutex);
1205 /* delete single link */
1206 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1207 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1208 continue;
1209 lnk_del = &lgr->lnk[lnk_idx];
1210 break;
1211 }
1212 del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1213 if (!lnk_del) {
1214 /* link was not found */
1215 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1216 smc_llc_send_message(lnk, &qentry->msg);
1217 goto out_unlock;
1218 }
1219 lnk_asym = smc_llc_find_asym_link(lgr);
1220
1221 del_llc->reason = 0;
1222 smc_llc_send_message(lnk, &qentry->msg); /* response */
1223
1224 if (smc_link_downing(&lnk_del->state)) {
c6f02ebe 1225 smc_switch_conns(lgr, lnk_del, false);
9c416878
KG		 1226 smc_wr_tx_wait_no_pending_sends(lnk_del);
1227 }
0a99be43 1228 smcr_link_clear(lnk_del, true);
9c416878
KG		 1229
1230 active_links = smc_llc_active_link_count(lgr);
1231 if (lnk_del == lnk_asym) {
1232 /* expected deletion of asym link, don't change lgr state */
1233 } else if (active_links == 1) {
ad6c111b 1234 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
9c416878 1235 } else if (!active_links) {
ad6c111b 1236 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
9c416878
KG		 1237 smc_lgr_terminate_sched(lgr);
1238 }
1239out_unlock:
1240 mutex_unlock(&lgr->llc_conf_mutex);
1241out:
1242 kfree(qentry);
1243}
1244
f3811fd7
KG		 1245/* try to send a DELETE LINK ALL request on any active link,
1246 * waiting for send completion
1247 */
1248void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1249{
1250 struct smc_llc_msg_del_link delllc = {0};
1251 int i;
1252
1253 delllc.hd.common.type = SMC_LLC_DELETE_LINK;
1254 delllc.hd.length = sizeof(delllc);
1255 if (ord)
1256 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1257 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1258 delllc.reason = htonl(rsn);
1259
1260 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1261 if (!smc_link_usable(&lgr->lnk[i]))
1262 continue;
1263 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1264 break;
1265 }
1266}
1267
08ae27dd
KG		 1268static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1269{
1270 struct smc_llc_msg_del_link *del_llc;
1271 struct smc_link *lnk, *lnk_del;
1272 struct smc_llc_qentry *qentry;
1273 int active_links;
1274 int i;
1275
1276 mutex_lock(&lgr->llc_conf_mutex);
1277 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1278 lnk = qentry->link;
1279 del_llc = &qentry->msg.delete_link;
1280
1281 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1282 /* delete entire lgr */
f3811fd7
KG		 1283 smc_llc_send_link_delete_all(lgr, true, ntohl(
1284 qentry->msg.delete_link.reason));
08ae27dd
KG		 1285 smc_lgr_terminate_sched(lgr);
1286 goto out;
1287 }
1288 /* delete single link */
1289 lnk_del = NULL;
1290 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1291 if (lgr->lnk[i].link_id == del_llc->link_num) {
1292 lnk_del = &lgr->lnk[i];
1293 break;
1294 }
1295 }
1296 if (!lnk_del)
1297 goto out; /* asymmetric link already deleted */
1298
1299 if (smc_link_downing(&lnk_del->state)) {
c6f02ebe 1300 smc_switch_conns(lgr, lnk_del, false);
08ae27dd
KG		 1301 smc_wr_tx_wait_no_pending_sends(lnk_del);
1302 }
1303 if (!list_empty(&lgr->list)) {
1304 /* qentry is either a request from peer (send it back to
1305 * initiate the DELETE_LINK processing), or a locally
1306 * enqueued DELETE_LINK request (forward it)
1307 */
1308 if (!smc_llc_send_message(lnk, &qentry->msg)) {
08ae27dd
KG		 1309 struct smc_llc_qentry *qentry2;
1310
1311 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1312 SMC_LLC_DELETE_LINK);
ca7e3edc 1313 if (qentry2)
08ae27dd 1314 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
08ae27dd
KG		 1315 }
1316 }
0a99be43 1317 smcr_link_clear(lnk_del, true);
08ae27dd
KG		 1318
1319 active_links = smc_llc_active_link_count(lgr);
1320 if (active_links == 1) {
ad6c111b 1321 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
08ae27dd 1322 } else if (!active_links) {
ad6c111b 1323 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
08ae27dd
KG		 1324 smc_lgr_terminate_sched(lgr);
1325 }
1326
1327 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1328 /* trigger setup of asymm alt link */
4dadd151 1329 smc_llc_srv_add_link_local(lnk);
08ae27dd
KG		 1330 }
1331out:
1332 mutex_unlock(&lgr->llc_conf_mutex);
1333 kfree(qentry);
1334}
1335
9ec6bf19 1336static void smc_llc_delete_link_work(struct work_struct *work)
52bedf37 1337{
9ec6bf19
KG		 1338 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1339 llc_del_link_work);
52bedf37 1340
9ec6bf19
KG		 1341 if (list_empty(&lgr->list)) {
1342 /* link group is terminating */
1343 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1344 goto out;
52bedf37 1345 }
9c416878
KG		 1346
1347 if (lgr->role == SMC_CLNT)
1348 smc_llc_process_cli_delete_link(lgr);
08ae27dd
KG		 1349 else
1350 smc_llc_process_srv_delete_link(lgr);
9ec6bf19
KG		 1351out:
1352 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
52bedf37
KG		 1353}
1354
3bc67e09
KG		 1355/* process a confirm_rkey request from peer, remote flow */
1356static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
4ed75de5 1357{
3bc67e09
KG		 1358 struct smc_llc_msg_confirm_rkey *llc;
1359 struct smc_llc_qentry *qentry;
1360 struct smc_link *link;
1361 int num_entries;
1362 int rk_idx;
1363 int i;
1364
1365 qentry = lgr->llc_flow_rmt.qentry;
1366 llc = &qentry->msg.confirm_rkey;
1367 link = qentry->link;
1368
1369 num_entries = llc->rtoken[0].num_rkeys;
1370 /* first rkey entry is for receiving link */
1371 rk_idx = smc_rtoken_add(link,
1372 llc->rtoken[0].rmb_vaddr,
1373 llc->rtoken[0].rmb_key);
1374 if (rk_idx < 0)
1375 goto out_err;
1376
1377 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1378 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1379 llc->rtoken[i].rmb_vaddr,
1380 llc->rtoken[i].rmb_key);
1381 /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1382 goto out;
1383out_err:
1384 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1385 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1386out:
ef79d439 1387 llc->hd.flags |= SMC_LLC_FLAG_RESP;
3bc67e09
KG		 1388 smc_llc_send_message(link, &qentry->msg);
1389 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
4ed75de5
KG		 1390}
1391
218b24fe
KG		 1392/* process a delete_rkey request from peer, remote flow */
1393static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
4ed75de5 1394{
218b24fe
KG		 1395 struct smc_llc_msg_delete_rkey *llc;
1396 struct smc_llc_qentry *qentry;
1397 struct smc_link *link;
4ed75de5
KG		 1398 u8 err_mask = 0;
1399 int i, max;
1400
218b24fe
KG		 1401 qentry = lgr->llc_flow_rmt.qentry;
1402 llc = &qentry->msg.delete_rkey;
1403 link = qentry->link;
1404
ef79d439
KG		 1405 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1406 for (i = 0; i < max; i++) {
1407 if (smc_rtoken_delete(link, llc->rkey[i]))
1408 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1409 }
ef79d439
KG		 1410 if (err_mask) {
1411 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1412 llc->err_mask = err_mask;
4ed75de5 1413 }
218b24fe
KG		 1414 llc->hd.flags |= SMC_LLC_FLAG_RESP;
1415 smc_llc_send_message(link, &qentry->msg);
1416 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1417}
ef79d439 1418
3e0c40af
KG		 1419static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1420{
1421 pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: "
1422 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type);
1423 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1424 smc_lgr_terminate_sched(lgr);
1425}
1426
6c8968c4 1427/* flush the llc event queue */
00a049cf 1428static void smc_llc_event_flush(struct smc_link_group *lgr)
9bf9abea 1429{
6c8968c4
KG		 1430 struct smc_llc_qentry *qentry, *q;
1431
1432 spin_lock_bh(&lgr->llc_event_q_lock);
1433 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1434 list_del_init(&qentry->list);
1435 kfree(qentry);
1436 }
1437 spin_unlock_bh(&lgr->llc_event_q_lock);
1438}
1439
1440static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1441{
1442 union smc_llc_msg *llc = &qentry->msg;
1443 struct smc_link *link = qentry->link;
0fb0b02b 1444 struct smc_link_group *lgr = link->lgr;
9bf9abea 1445
d854fcbf 1446 if (!smc_link_usable(link))
6c8968c4 1447 goto out;
313164da
KG		 1448
1449 switch (llc->raw.hdr.common.type) {
1450 case SMC_LLC_TEST_LINK:
56e8091c
KG		 1451 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1452 smc_llc_send_message(link, llc);
313164da 1453 break;
52bedf37 1454 case SMC_LLC_ADD_LINK:
0fb0b02b
KG		 1455 if (list_empty(&lgr->list))
1456 goto out; /* lgr is terminating */
1457 if (lgr->role == SMC_CLNT) {
1458 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) {
1459 /* a flow is waiting for this message */
1460 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1461 qentry);
1462 wake_up_interruptible(&lgr->llc_waiter);
1463 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1464 qentry)) {
b45e7f98 1465 schedule_work(&lgr->llc_add_link_work);
0fb0b02b
KG		 1466 }
1467 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1468 /* as smc server, handle client suggestion */
b45e7f98 1469 schedule_work(&lgr->llc_add_link_work);
0fb0b02b
KG		 1470 }
1471 return;
1472 case SMC_LLC_CONFIRM_LINK:
87f88cda 1473 case SMC_LLC_ADD_LINK_CONT:
0fb0b02b
KG		 1474 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1475 /* a flow is waiting for this message */
1476 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1477 wake_up_interruptible(&lgr->llc_waiter);
1478 return;
1479 }
52bedf37
KG		 1480 break;
1481 case SMC_LLC_DELETE_LINK:
9ec6bf19
KG		 1482 if (lgr->role == SMC_CLNT) {
1483 /* server requests to delete this link, send response */
1484 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1485 /* DEL LINK REQ during ADD LINK SEQ */
1486 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1487 qentry);
1488 wake_up_interruptible(&lgr->llc_waiter);
1489 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1490 qentry)) {
1491 schedule_work(&lgr->llc_del_link_work);
1492 }
1493 } else {
1494 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1495 !lgr->llc_flow_lcl.qentry) {
1496 /* DEL LINK REQ during ADD LINK SEQ */
1497 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1498 qentry);
1499 wake_up_interruptible(&lgr->llc_waiter);
1500 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1501 qentry)) {
1502 schedule_work(&lgr->llc_del_link_work);
1503 }
1504 }
1505 return;
4ed75de5 1506 case SMC_LLC_CONFIRM_RKEY:
3bc67e09
KG		 1507 /* new request from remote, assign to remote flow */
1508 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1509 /* process here, does not wait for more llc msgs */
1510 smc_llc_rmt_conf_rkey(lgr);
1511 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1512 }
1513 return;
4ed75de5 1514 case SMC_LLC_CONFIRM_RKEY_CONT:
42d18acc
KG		 1515 /* not used because max links is 3, and 3 rkeys fit into
1516 * one CONFIRM_RKEY message
1517 */
4ed75de5
KG		 1518 break;
1519 case SMC_LLC_DELETE_RKEY:
218b24fe
KG		 1520 /* new request from remote, assign to remote flow */
1521 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1522 /* process here, does not wait for more llc msgs */
1523 smc_llc_rmt_delete_rkey(lgr);
1524 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1525 }
1526 return;
3e0c40af
KG		 1527 default:
1528 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1529 break;
313164da 1530 }
6c8968c4
KG		 1531out:
1532 kfree(qentry);
1533}
1534
1535/* worker to process llc messages on the event queue */
1536static void smc_llc_event_work(struct work_struct *work)
1537{
1538 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1539 llc_event_work);
1540 struct smc_llc_qentry *qentry;
1541
555da9af
KG		 1542 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1543 if (smc_link_usable(lgr->delayed_event->link)) {
1544 smc_llc_event_handler(lgr->delayed_event);
1545 } else {
1546 qentry = lgr->delayed_event;
1547 lgr->delayed_event = NULL;
1548 kfree(qentry);
1549 }
1550 }
1551
6c8968c4
KG		 1552again:
1553 spin_lock_bh(&lgr->llc_event_q_lock);
1554 if (!list_empty(&lgr->llc_event_q)) {
1555 qentry = list_first_entry(&lgr->llc_event_q,
1556 struct smc_llc_qentry, list);
1557 list_del_init(&qentry->list);
1558 spin_unlock_bh(&lgr->llc_event_q_lock);
1559 smc_llc_event_handler(qentry);
1560 goto again;
1561 }
1562 spin_unlock_bh(&lgr->llc_event_q_lock);
1563}
1564
ef79d439 1565/* process llc responses in tasklet context */
a6688d91
KG		 1566static void smc_llc_rx_response(struct smc_link *link,
1567 struct smc_llc_qentry *qentry)
ef79d439 1568{
a6688d91 1569 u8 llc_type = qentry->msg.raw.hdr.common.type;
ef79d439 1570
a6688d91 1571 switch (llc_type) {
ef79d439
KG		 1572 case SMC_LLC_TEST_LINK:
1573 if (link->state == SMC_LNK_ACTIVE)
1574 complete(&link->llc_testlink_resp);
1575 break;
ef79d439 1576 case SMC_LLC_ADD_LINK:
9ec6bf19 1577 case SMC_LLC_DELETE_LINK:
4667bb4a 1578 case SMC_LLC_CONFIRM_LINK:
87f88cda 1579 case SMC_LLC_ADD_LINK_CONT:
3d88a21b 1580 case SMC_LLC_CONFIRM_RKEY:
6d74c3a8 1581 case SMC_LLC_DELETE_RKEY:
4667bb4a
KG		 1582 /* assign responses to the local flow, we requested them */
1583 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
1584 wake_up_interruptible(&link->lgr->llc_waiter);
1585 return;
ef79d439 1586 case SMC_LLC_CONFIRM_RKEY_CONT:
42d18acc 1587 /* not used because max links is 3 */
ef79d439 1588 break;
3e0c40af
KG		 1589 default:
1590 smc_llc_protocol_violation(link->lgr, llc_type);
1591 break;
ef79d439 1592 }
a6688d91 1593 kfree(qentry);
ef79d439
KG		 1594}
1595
a6688d91 1596static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
6c8968c4 1597{
6c8968c4
KG		 1598 struct smc_link_group *lgr = link->lgr;
1599 struct smc_llc_qentry *qentry;
6c8968c4
KG		 1600 unsigned long flags;
1601
6c8968c4
KG		 1602 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
1603 if (!qentry)
1604 return;
1605 qentry->link = link;
1606 INIT_LIST_HEAD(&qentry->list);
1607 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
a6688d91
KG		 1608
1609 /* process responses immediately */
1610 if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
1611 smc_llc_rx_response(link, qentry);
1612 return;
1613 }
1614
1615 /* add requests to event queue */
6c8968c4
KG		 1616 spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
1617 list_add_tail(&qentry->list, &lgr->llc_event_q);
1618 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
1619 schedule_work(&link->lgr->llc_event_work);
9bf9abea
UB		 1620}
1621
a6688d91
KG		 1622/* copy received msg and add it to the event queue */
1623static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
1624{
1625 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
1626 union smc_llc_msg *llc = buf;
1627
1628 if (wc->byte_len < sizeof(*llc))
1629 return; /* short message */
1630 if (llc->raw.hdr.length != sizeof(*llc))
1631 return; /* invalid message */
1632
1633 smc_llc_enqueue(link, llc);
1634}
1635
44aa81ce 1636/***************************** worker, utils *********************************/
877ae5be
KG		 1637
1638static void smc_llc_testlink_work(struct work_struct *work)
1639{
1640 struct smc_link *link = container_of(to_delayed_work(work),
1641 struct smc_link, llc_testlink_wrk);
1642 unsigned long next_interval;
877ae5be
KG		 1643 unsigned long expire_time;
1644 u8 user_data[16] = { 0 };
1645 int rc;
1646
877ae5be
KG		 1647 if (link->state != SMC_LNK_ACTIVE)
1648 return; /* don't reschedule worker */
1649 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
1650 if (time_is_after_jiffies(expire_time)) {
1651 next_interval = expire_time - jiffies;
1652 goto out;
1653 }
1654 reinit_completion(&link->llc_testlink_resp);
d97935fa 1655 smc_llc_send_test_link(link, user_data);
877ae5be
KG		 1656 /* receive TEST LINK response over RoCE fabric */
1657 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
1658 SMC_LLC_WAIT_TIME);
1020e1ef
KG		 1659 if (link->state != SMC_LNK_ACTIVE)
1660 return; /* link state changed */
877ae5be 1661 if (rc <= 0) {
87523930 1662 smcr_link_down_cond_sched(link);
877ae5be
KG		 1663 return;
1664 }
1665 next_interval = link->llc_testlink_time;
1666out:
1020e1ef 1667 schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
877ae5be
KG		 1668}
1669
00a049cf
KG		 1670void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
1671{
1672 struct net *net = sock_net(smc->clcsock->sk);
1673
1674 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
b45e7f98 1675 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
9ec6bf19 1676 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
00a049cf
KG		 1677 INIT_LIST_HEAD(&lgr->llc_event_q);
1678 spin_lock_init(&lgr->llc_event_q_lock);
555da9af
KG		 1679 spin_lock_init(&lgr->llc_flow_lock);
1680 init_waitqueue_head(&lgr->llc_waiter);
d5500667 1681 mutex_init(&lgr->llc_conf_mutex);
00a049cf
KG		 1682 lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
1683}
1684
1685/* called after lgr was removed from lgr_list */
1686void smc_llc_lgr_clear(struct smc_link_group *lgr)
1687{
1688 smc_llc_event_flush(lgr);
555da9af 1689 wake_up_interruptible_all(&lgr->llc_waiter);
00a049cf 1690 cancel_work_sync(&lgr->llc_event_work);
b45e7f98 1691 cancel_work_sync(&lgr->llc_add_link_work);
9ec6bf19 1692 cancel_work_sync(&lgr->llc_del_link_work);
555da9af
KG		 1693 if (lgr->delayed_event) {
1694 kfree(lgr->delayed_event);
1695 lgr->delayed_event = NULL;
1696 }
00a049cf
KG		 1697}
1698
2a4c57a9 1699int smc_llc_link_init(struct smc_link *link)
877ae5be
KG		 1700{
1701 init_completion(&link->llc_testlink_resp);
1702 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2a4c57a9 1703 return 0;
b32cf4ab
KG		 1704}
1705
00a049cf 1706void smc_llc_link_active(struct smc_link *link)
b32cf4ab 1707{
0a99be43
KG		 1708 pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, "
1709 "peerid %*phN, ibdev %s, ibport %d\n",
1710 SMC_LGR_ID_SIZE, &link->lgr->id,
1711 SMC_LGR_ID_SIZE, &link->link_uid,
1712 SMC_LGR_ID_SIZE, &link->peer_link_uid,
1713 link->smcibdev->ibdev->name, link->ibport);
877ae5be 1714 link->state = SMC_LNK_ACTIVE;
00a049cf
KG		 1715 if (link->lgr->llc_testlink_time) {
1716 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ;
1020e1ef
KG		 1717 schedule_delayed_work(&link->llc_testlink_wrk,
1718 link->llc_testlink_time);
877ae5be
KG		 1719 }
1720}
1721
877ae5be 1722/* called in worker context */
0a99be43 1723void smc_llc_link_clear(struct smc_link *link, bool log)
877ae5be 1724{
0a99be43
KG		 1725 if (log)
1726 pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN"
1727 ", peerid %*phN, ibdev %s, ibport %d\n",
1728 SMC_LGR_ID_SIZE, &link->lgr->id,
1729 SMC_LGR_ID_SIZE, &link->link_uid,
1730 SMC_LGR_ID_SIZE, &link->peer_link_uid,
1731 link->smcibdev->ibdev->name, link->ibport);
2140ac26
KG		 1732 complete(&link->llc_testlink_resp);
1733 cancel_delayed_work_sync(&link->llc_testlink_wrk);
1734 smc_wr_wakeup_reg_wait(link);
1735 smc_wr_wakeup_tx_wait(link);
877ae5be
KG		 1736}
1737
3d88a21b
KG		 1738/* register a new rtoken at the remote peer (for all links) */
1739int smc_llc_do_confirm_rkey(struct smc_link *send_link,
44aa81ce
KG		 1740 struct smc_buf_desc *rmb_desc)
1741{
3d88a21b
KG		 1742 struct smc_link_group *lgr = send_link->lgr;
1743 struct smc_llc_qentry *qentry = NULL;
1744 int rc = 0;
44aa81ce 1745
3d88a21b
KG		 1746 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
1747 if (rc)
1748 goto out;
44aa81ce 1749 /* receive CONFIRM RKEY response from server over RoCE fabric */
3d88a21b
KG		 1750 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
1751 SMC_LLC_CONFIRM_RKEY);
1752 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
1753 rc = -EFAULT;
1754out:
1755 if (qentry)
1756 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
3d88a21b 1757 return rc;
44aa81ce
KG		 1758}
1759
60e03c62 1760/* unregister an rtoken at the remote peer */
6d74c3a8 1761int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
60e03c62
KG		 1762 struct smc_buf_desc *rmb_desc)
1763{
6d74c3a8
KG		 1764 struct smc_llc_qentry *qentry = NULL;
1765 struct smc_link *send_link;
0b29ec64 1766 int rc = 0;
60e03c62 1767
6d74c3a8
KG		 1768 send_link = smc_llc_usable_link(lgr);
1769 if (!send_link)
1770 return -ENOLINK;
1771
6d74c3a8
KG		 1772 /* protected by llc_flow control */
1773 rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
60e03c62
KG		 1774 if (rc)
1775 goto out;
1776 /* receive DELETE RKEY response from server over RoCE fabric */
6d74c3a8
KG		 1777 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
1778 SMC_LLC_DELETE_RKEY);
1779 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
60e03c62 1780 rc = -EFAULT;
60e03c62 1781out:
6d74c3a8
KG		 1782 if (qentry)
1783 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
60e03c62
KG		 1784 return rc;
1785}
1786
45fa8da0
KG		 1787void smc_llc_link_set_uid(struct smc_link *link)
1788{
1789 __be32 link_uid;
1790
1791 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
1792 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
1793}
1794
649758ff
KG		 1795/* save peers link user id, used for debug purposes */
1796void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
1797{
1798 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
1799 SMC_LGR_ID_SIZE);
1800}
1801
92334cfc
KG		 1802/* evaluate confirm link request or response */
1803int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
1804 enum smc_llc_reqresp type)
1805{
45fa8da0 1806 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */
92334cfc 1807 qentry->link->link_id = qentry->msg.confirm_link.link_num;
45fa8da0
KG		 1808 smc_llc_link_set_uid(qentry->link);
1809 }
92334cfc
KG		 1810 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
1811 return -ENOTSUPP;
1812 return 0;
1813}
1814
9bf9abea
UB		 1815/***************************** init, exit, misc ******************************/
1816
1817static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
1818 {
1819 .handler = smc_llc_rx_handler,
1820 .type = SMC_LLC_CONFIRM_LINK
1821 },
313164da
KG		 1822 {
1823 .handler = smc_llc_rx_handler,
1824 .type = SMC_LLC_TEST_LINK
1825 },
52bedf37
KG		 1826 {
1827 .handler = smc_llc_rx_handler,
1828 .type = SMC_LLC_ADD_LINK
1829 },
87f88cda
KG		 1830 {
1831 .handler = smc_llc_rx_handler,
1832 .type = SMC_LLC_ADD_LINK_CONT
1833 },
52bedf37
KG		 1834 {
1835 .handler = smc_llc_rx_handler,
1836 .type = SMC_LLC_DELETE_LINK
1837 },
4ed75de5
KG		 1838 {
1839 .handler = smc_llc_rx_handler,
1840 .type = SMC_LLC_CONFIRM_RKEY
1841 },
1842 {
1843 .handler = smc_llc_rx_handler,
1844 .type = SMC_LLC_CONFIRM_RKEY_CONT
1845 },
1846 {
1847 .handler = smc_llc_rx_handler,
1848 .type = SMC_LLC_DELETE_RKEY
1849 },
9bf9abea
UB		 1850 {
1851 .handler = NULL,
1852 }
1853};
1854
1855int __init smc_llc_init(void)
1856{
1857 struct smc_wr_rx_handler *handler;
1858 int rc = 0;
1859
1860 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
1861 INIT_HLIST_NODE(&handler->list);
1862 rc = smc_wr_rx_register_handler(handler);
1863 if (rc)
1864 break;
1865 }
1866 return rc;
1867}
Empty description
This page took 0.557211 seconds and 4 git commands to generate.