1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10 * Ursula Braun <ubraun@linux.vnet.ibm.com>
11 */
12
13 #include <net/tcp.h>
14 #include <rdma/ib_verbs.h>
15
16 #include "smc.h"
17 #include "smc_core.h"
18 #include "smc_clc.h"
19 #include "smc_llc.h"
20
21 #define SMC_LLC_DATA_LEN 40
22
23 struct smc_llc_hdr {
24 struct smc_wr_rx_hdr common;
25 u8 length; /* 44 */
26 #if defined(__BIG_ENDIAN_BITFIELD)
27 u8 reserved:4,
28 add_link_rej_rsn:4;
29 #elif defined(__LITTLE_ENDIAN_BITFIELD)
30 u8 add_link_rej_rsn:4,
31 reserved:4;
32 #endif
33 u8 flags;
34 };
35
36 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
37
38 struct smc_llc_msg_confirm_link { /* type 0x01 */
39 struct smc_llc_hdr hd;
40 u8 sender_mac[ETH_ALEN];
41 u8 sender_gid[SMC_GID_SIZE];
42 u8 sender_qp_num[3];
43 u8 link_num;
44 u8 link_uid[SMC_LGR_ID_SIZE];
45 u8 max_links;
46 u8 reserved[9];
47 };
48
49 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
50 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
51
52 #define SMC_LLC_ADD_LNK_MAX_LINKS 2
53
54 struct smc_llc_msg_add_link { /* type 0x02 */
55 struct smc_llc_hdr hd;
56 u8 sender_mac[ETH_ALEN];
57 u8 reserved2[2];
58 u8 sender_gid[SMC_GID_SIZE];
59 u8 sender_qp_num[3];
60 u8 link_num;
61 u8 flags2; /* QP mtu */
62 u8 initial_psn[3];
63 u8 reserved[8];
64 };
65
66 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
67 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
68
69 struct smc_llc_msg_del_link { /* type 0x04 */
70 struct smc_llc_hdr hd;
71 u8 link_num;
72 __be32 reason;
73 u8 reserved[35];
74 } __packed; /* format defined in RFC7609 */
75
76 struct smc_llc_msg_test_link { /* type 0x07 */
77 struct smc_llc_hdr hd;
78 u8 user_data[16];
79 u8 reserved[24];
80 };
81
82 struct smc_rmb_rtoken {
83 union {
84 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
85 /* is actually the num of rtokens, first */
86 /* rtoken is always for the current link */
87 u8 link_id; /* link id of the rtoken */
88 };
89 __be32 rmb_key;
90 __be64 rmb_vaddr;
91 } __packed; /* format defined in RFC7609 */
92
93 #define SMC_LLC_RKEYS_PER_MSG 3
94
95 struct smc_llc_msg_confirm_rkey { /* type 0x06 */
96 struct smc_llc_hdr hd;
97 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
98 u8 reserved;
99 };
100
101 struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */
102 struct smc_llc_hdr hd;
103 u8 num_rkeys;
104 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
105 };
106
107 #define SMC_LLC_DEL_RKEY_MAX 8
108 #define SMC_LLC_FLAG_RKEY_NEG 0x20
109
110 struct smc_llc_msg_delete_rkey { /* type 0x09 */
111 struct smc_llc_hdr hd;
112 u8 num_rkeys;
113 u8 err_mask;
114 u8 reserved[2];
115 __be32 rkey[8];
116 u8 reserved2[4];
117 };
118
119 union smc_llc_msg {
120 struct smc_llc_msg_confirm_link confirm_link;
121 struct smc_llc_msg_add_link add_link;
122 struct smc_llc_msg_del_link delete_link;
123
124 struct smc_llc_msg_confirm_rkey confirm_rkey;
125 struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont;
126 struct smc_llc_msg_delete_rkey delete_rkey;
127
128 struct smc_llc_msg_test_link test_link;
129 struct {
130 struct smc_llc_hdr hdr;
131 u8 data[SMC_LLC_DATA_LEN];
132 } raw;
133 };
134
135 #define SMC_LLC_FLAG_RESP 0x80
136
137 /********************************** send *************************************/
138
139 struct smc_llc_tx_pend {
140 };
141
142 /* handler for send/transmission completion of an LLC msg */
smc_llc_tx_handler(struct smc_wr_tx_pend_priv * pend,struct smc_link * link,enum ib_wc_status wc_status)143 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
144 struct smc_link *link,
145 enum ib_wc_status wc_status)
146 {
147 /* future work: handle wc_status error for recovery and failover */
148 }
149
150 /**
151 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
152 * @link: Pointer to SMC link used for sending LLC control message.
153 * @wr_buf: Out variable returning pointer to work request payload buffer.
154 * @pend: Out variable returning pointer to private pending WR tracking.
155 * It's the context the transmit complete handler will get.
156 *
157 * Reserves and pre-fills an entry for a pending work request send/tx.
158 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
159 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
160 *
161 * Return: 0 on success, otherwise an error value.
162 */
smc_llc_add_pending_send(struct smc_link * link,struct smc_wr_buf ** wr_buf,struct smc_wr_tx_pend_priv ** pend)163 static int smc_llc_add_pending_send(struct smc_link *link,
164 struct smc_wr_buf **wr_buf,
165 struct smc_wr_tx_pend_priv **pend)
166 {
167 int rc;
168
169 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
170 pend);
171 if (rc < 0)
172 return rc;
173 BUILD_BUG_ON_MSG(
174 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
175 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
176 BUILD_BUG_ON_MSG(
177 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
178 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
179 BUILD_BUG_ON_MSG(
180 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
181 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
182 return 0;
183 }
184
185 /* high-level API to send LLC confirm link */
smc_llc_send_confirm_link(struct smc_link * link,enum smc_llc_reqresp reqresp)186 int smc_llc_send_confirm_link(struct smc_link *link,
187 enum smc_llc_reqresp reqresp)
188 {
189 struct smc_link_group *lgr = smc_get_lgr(link);
190 struct smc_llc_msg_confirm_link *confllc;
191 struct smc_wr_tx_pend_priv *pend;
192 struct smc_wr_buf *wr_buf;
193 int rc;
194
195 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
196 if (rc)
197 return rc;
198 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
199 memset(confllc, 0, sizeof(*confllc));
200 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
201 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
202 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
203 if (reqresp == SMC_LLC_RESP)
204 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
205 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
206 ETH_ALEN);
207 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
208 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
209 confllc->link_num = link->link_id;
210 memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
211 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
212 /* send llc message */
213 rc = smc_wr_tx_send(link, pend);
214 return rc;
215 }
216
217 /* send LLC confirm rkey request */
smc_llc_send_confirm_rkey(struct smc_link * link,struct smc_buf_desc * rmb_desc)218 static int smc_llc_send_confirm_rkey(struct smc_link *link,
219 struct smc_buf_desc *rmb_desc)
220 {
221 struct smc_llc_msg_confirm_rkey *rkeyllc;
222 struct smc_wr_tx_pend_priv *pend;
223 struct smc_wr_buf *wr_buf;
224 int rc;
225
226 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
227 if (rc)
228 return rc;
229 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
230 memset(rkeyllc, 0, sizeof(*rkeyllc));
231 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
232 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
233 rkeyllc->rtoken[0].rmb_key =
234 htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey);
235 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
236 (u64)sg_dma_address(rmb_desc->sgt[SMC_SINGLE_LINK].sgl));
237 /* send llc message */
238 rc = smc_wr_tx_send(link, pend);
239 return rc;
240 }
241
242 /* send LLC delete rkey request */
smc_llc_send_delete_rkey(struct smc_link * link,struct smc_buf_desc * rmb_desc)243 static int smc_llc_send_delete_rkey(struct smc_link *link,
244 struct smc_buf_desc *rmb_desc)
245 {
246 struct smc_llc_msg_delete_rkey *rkeyllc;
247 struct smc_wr_tx_pend_priv *pend;
248 struct smc_wr_buf *wr_buf;
249 int rc;
250
251 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
252 if (rc)
253 return rc;
254 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
255 memset(rkeyllc, 0, sizeof(*rkeyllc));
256 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
257 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
258 rkeyllc->num_rkeys = 1;
259 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey);
260 /* send llc message */
261 rc = smc_wr_tx_send(link, pend);
262 return rc;
263 }
264
265 /* prepare an add link message */
smc_llc_prep_add_link(struct smc_llc_msg_add_link * addllc,struct smc_link * link,u8 mac[],u8 gid[],enum smc_llc_reqresp reqresp)266 static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
267 struct smc_link *link, u8 mac[], u8 gid[],
268 enum smc_llc_reqresp reqresp)
269 {
270 memset(addllc, 0, sizeof(*addllc));
271 addllc->hd.common.type = SMC_LLC_ADD_LINK;
272 addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
273 if (reqresp == SMC_LLC_RESP) {
274 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
275 /* always reject more links for now */
276 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
277 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
278 }
279 memcpy(addllc->sender_mac, mac, ETH_ALEN);
280 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
281 }
282
283 /* send ADD LINK request or response */
smc_llc_send_add_link(struct smc_link * link,u8 mac[],u8 gid[],enum smc_llc_reqresp reqresp)284 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
285 enum smc_llc_reqresp reqresp)
286 {
287 struct smc_llc_msg_add_link *addllc;
288 struct smc_wr_tx_pend_priv *pend;
289 struct smc_wr_buf *wr_buf;
290 int rc;
291
292 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
293 if (rc)
294 return rc;
295 addllc = (struct smc_llc_msg_add_link *)wr_buf;
296 smc_llc_prep_add_link(addllc, link, mac, gid, reqresp);
297 /* send llc message */
298 rc = smc_wr_tx_send(link, pend);
299 return rc;
300 }
301
302 /* prepare a delete link message */
smc_llc_prep_delete_link(struct smc_llc_msg_del_link * delllc,struct smc_link * link,enum smc_llc_reqresp reqresp,bool orderly)303 static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
304 struct smc_link *link,
305 enum smc_llc_reqresp reqresp, bool orderly)
306 {
307 memset(delllc, 0, sizeof(*delllc));
308 delllc->hd.common.type = SMC_LLC_DELETE_LINK;
309 delllc->hd.length = sizeof(struct smc_llc_msg_add_link);
310 if (reqresp == SMC_LLC_RESP)
311 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
312 /* DEL_LINK_ALL because only 1 link supported */
313 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
314 if (orderly)
315 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
316 delllc->link_num = link->link_id;
317 }
318
319 /* send DELETE LINK request or response */
smc_llc_send_delete_link(struct smc_link * link,enum smc_llc_reqresp reqresp,bool orderly)320 int smc_llc_send_delete_link(struct smc_link *link,
321 enum smc_llc_reqresp reqresp, bool orderly)
322 {
323 struct smc_llc_msg_del_link *delllc;
324 struct smc_wr_tx_pend_priv *pend;
325 struct smc_wr_buf *wr_buf;
326 int rc;
327
328 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
329 if (rc)
330 return rc;
331 delllc = (struct smc_llc_msg_del_link *)wr_buf;
332 smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
333 /* send llc message */
334 rc = smc_wr_tx_send(link, pend);
335 return rc;
336 }
337
338 /* send LLC test link request */
smc_llc_send_test_link(struct smc_link * link,u8 user_data[16])339 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
340 {
341 struct smc_llc_msg_test_link *testllc;
342 struct smc_wr_tx_pend_priv *pend;
343 struct smc_wr_buf *wr_buf;
344 int rc;
345
346 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
347 if (rc)
348 return rc;
349 testllc = (struct smc_llc_msg_test_link *)wr_buf;
350 memset(testllc, 0, sizeof(*testllc));
351 testllc->hd.common.type = SMC_LLC_TEST_LINK;
352 testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
353 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
354 /* send llc message */
355 rc = smc_wr_tx_send(link, pend);
356 return rc;
357 }
358
359 struct smc_llc_send_work {
360 struct work_struct work;
361 struct smc_link *link;
362 int llclen;
363 union smc_llc_msg llcbuf;
364 };
365
366 /* worker that sends a prepared message */
smc_llc_send_message_work(struct work_struct * work)367 static void smc_llc_send_message_work(struct work_struct *work)
368 {
369 struct smc_llc_send_work *llcwrk = container_of(work,
370 struct smc_llc_send_work, work);
371 struct smc_wr_tx_pend_priv *pend;
372 struct smc_wr_buf *wr_buf;
373 int rc;
374
375 if (llcwrk->link->state == SMC_LNK_INACTIVE)
376 goto out;
377 rc = smc_llc_add_pending_send(llcwrk->link, &wr_buf, &pend);
378 if (rc)
379 goto out;
380 memcpy(wr_buf, &llcwrk->llcbuf, llcwrk->llclen);
381 smc_wr_tx_send(llcwrk->link, pend);
382 out:
383 kfree(llcwrk);
384 }
385
386 /* copy llcbuf and schedule an llc send on link */
smc_llc_send_message(struct smc_link * link,void * llcbuf,int llclen)387 static int smc_llc_send_message(struct smc_link *link, void *llcbuf, int llclen)
388 {
389 struct smc_llc_send_work *wrk = kmalloc(sizeof(*wrk), GFP_ATOMIC);
390
391 if (!wrk)
392 return -ENOMEM;
393 INIT_WORK(&wrk->work, smc_llc_send_message_work);
394 wrk->link = link;
395 wrk->llclen = llclen;
396 memcpy(&wrk->llcbuf, llcbuf, llclen);
397 queue_work(link->llc_wq, &wrk->work);
398 return 0;
399 }
400
401 /********************************* receive ***********************************/
402
smc_llc_rx_confirm_link(struct smc_link * link,struct smc_llc_msg_confirm_link * llc)403 static void smc_llc_rx_confirm_link(struct smc_link *link,
404 struct smc_llc_msg_confirm_link *llc)
405 {
406 struct smc_link_group *lgr = smc_get_lgr(link);
407 int conf_rc;
408
409 /* RMBE eyecatchers are not supported */
410 if (llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)
411 conf_rc = 0;
412 else
413 conf_rc = ENOTSUPP;
414
415 if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
416 if (lgr->role == SMC_SERV &&
417 link->state == SMC_LNK_ACTIVATING) {
418 link->llc_confirm_resp_rc = conf_rc;
419 complete(&link->llc_confirm_resp);
420 }
421 } else {
422 if (lgr->role == SMC_CLNT &&
423 link->state == SMC_LNK_ACTIVATING) {
424 link->llc_confirm_rc = conf_rc;
425 link->link_id = llc->link_num;
426 complete(&link->llc_confirm);
427 }
428 }
429 }
430
smc_llc_rx_add_link(struct smc_link * link,struct smc_llc_msg_add_link * llc)431 static void smc_llc_rx_add_link(struct smc_link *link,
432 struct smc_llc_msg_add_link *llc)
433 {
434 struct smc_link_group *lgr = smc_get_lgr(link);
435
436 if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
437 if (link->state == SMC_LNK_ACTIVATING)
438 complete(&link->llc_add_resp);
439 } else {
440 if (link->state == SMC_LNK_ACTIVATING) {
441 complete(&link->llc_add);
442 return;
443 }
444
445 if (lgr->role == SMC_SERV) {
446 smc_llc_prep_add_link(llc, link,
447 link->smcibdev->mac[link->ibport - 1],
448 link->gid, SMC_LLC_REQ);
449
450 } else {
451 smc_llc_prep_add_link(llc, link,
452 link->smcibdev->mac[link->ibport - 1],
453 link->gid, SMC_LLC_RESP);
454 }
455 smc_llc_send_message(link, llc, sizeof(*llc));
456 }
457 }
458
smc_llc_rx_delete_link(struct smc_link * link,struct smc_llc_msg_del_link * llc)459 static void smc_llc_rx_delete_link(struct smc_link *link,
460 struct smc_llc_msg_del_link *llc)
461 {
462 struct smc_link_group *lgr = smc_get_lgr(link);
463
464 if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
465 if (lgr->role == SMC_SERV)
466 smc_lgr_schedule_free_work_fast(lgr);
467 } else {
468 smc_lgr_forget(lgr);
469 smc_llc_link_deleting(link);
470 if (lgr->role == SMC_SERV) {
471 /* client asks to delete this link, send request */
472 smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
473 } else {
474 /* server requests to delete this link, send response */
475 smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
476 }
477 smc_llc_send_message(link, llc, sizeof(*llc));
478 smc_lgr_schedule_free_work_fast(lgr);
479 }
480 }
481
smc_llc_rx_test_link(struct smc_link * link,struct smc_llc_msg_test_link * llc)482 static void smc_llc_rx_test_link(struct smc_link *link,
483 struct smc_llc_msg_test_link *llc)
484 {
485 if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
486 if (link->state == SMC_LNK_ACTIVE)
487 complete(&link->llc_testlink_resp);
488 } else {
489 llc->hd.flags |= SMC_LLC_FLAG_RESP;
490 smc_llc_send_message(link, llc, sizeof(*llc));
491 }
492 }
493
smc_llc_rx_confirm_rkey(struct smc_link * link,struct smc_llc_msg_confirm_rkey * llc)494 static void smc_llc_rx_confirm_rkey(struct smc_link *link,
495 struct smc_llc_msg_confirm_rkey *llc)
496 {
497 int rc;
498
499 if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
500 link->llc_confirm_rkey_rc = llc->hd.flags &
501 SMC_LLC_FLAG_RKEY_NEG;
502 complete(&link->llc_confirm_rkey);
503 } else {
504 rc = smc_rtoken_add(smc_get_lgr(link),
505 llc->rtoken[0].rmb_vaddr,
506 llc->rtoken[0].rmb_key);
507
508 /* ignore rtokens for other links, we have only one link */
509
510 llc->hd.flags |= SMC_LLC_FLAG_RESP;
511 if (rc < 0)
512 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
513 smc_llc_send_message(link, llc, sizeof(*llc));
514 }
515 }
516
smc_llc_rx_confirm_rkey_cont(struct smc_link * link,struct smc_llc_msg_confirm_rkey_cont * llc)517 static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link,
518 struct smc_llc_msg_confirm_rkey_cont *llc)
519 {
520 if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
521 /* unused as long as we don't send this type of msg */
522 } else {
523 /* ignore rtokens for other links, we have only one link */
524 llc->hd.flags |= SMC_LLC_FLAG_RESP;
525 smc_llc_send_message(link, llc, sizeof(*llc));
526 }
527 }
528
smc_llc_rx_delete_rkey(struct smc_link * link,struct smc_llc_msg_delete_rkey * llc)529 static void smc_llc_rx_delete_rkey(struct smc_link *link,
530 struct smc_llc_msg_delete_rkey *llc)
531 {
532 u8 err_mask = 0;
533 int i, max;
534
535 if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
536 link->llc_delete_rkey_rc = llc->hd.flags &
537 SMC_LLC_FLAG_RKEY_NEG;
538 complete(&link->llc_delete_rkey);
539 } else {
540 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
541 for (i = 0; i < max; i++) {
542 if (smc_rtoken_delete(smc_get_lgr(link), llc->rkey[i]))
543 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
544 }
545
546 if (err_mask) {
547 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
548 llc->err_mask = err_mask;
549 }
550
551 llc->hd.flags |= SMC_LLC_FLAG_RESP;
552 smc_llc_send_message(link, llc, sizeof(*llc));
553 }
554 }
555
smc_llc_rx_handler(struct ib_wc * wc,void * buf)556 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
557 {
558 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
559 union smc_llc_msg *llc = buf;
560
561 if (wc->byte_len < sizeof(*llc))
562 return; /* short message */
563 if (llc->raw.hdr.length != sizeof(*llc))
564 return; /* invalid message */
565 if (link->state == SMC_LNK_INACTIVE)
566 return; /* link not active, drop msg */
567
568 switch (llc->raw.hdr.common.type) {
569 case SMC_LLC_TEST_LINK:
570 smc_llc_rx_test_link(link, &llc->test_link);
571 break;
572 case SMC_LLC_CONFIRM_LINK:
573 smc_llc_rx_confirm_link(link, &llc->confirm_link);
574 break;
575 case SMC_LLC_ADD_LINK:
576 smc_llc_rx_add_link(link, &llc->add_link);
577 break;
578 case SMC_LLC_DELETE_LINK:
579 smc_llc_rx_delete_link(link, &llc->delete_link);
580 break;
581 case SMC_LLC_CONFIRM_RKEY:
582 smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey);
583 break;
584 case SMC_LLC_CONFIRM_RKEY_CONT:
585 smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont);
586 break;
587 case SMC_LLC_DELETE_RKEY:
588 smc_llc_rx_delete_rkey(link, &llc->delete_rkey);
589 break;
590 }
591 }
592
593 /***************************** worker, utils *********************************/
594
smc_llc_testlink_work(struct work_struct * work)595 static void smc_llc_testlink_work(struct work_struct *work)
596 {
597 struct smc_link *link = container_of(to_delayed_work(work),
598 struct smc_link, llc_testlink_wrk);
599 unsigned long next_interval;
600 unsigned long expire_time;
601 u8 user_data[16] = { 0 };
602 int rc;
603
604 if (link->state != SMC_LNK_ACTIVE)
605 return; /* don't reschedule worker */
606 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
607 if (time_is_after_jiffies(expire_time)) {
608 next_interval = expire_time - jiffies;
609 goto out;
610 }
611 reinit_completion(&link->llc_testlink_resp);
612 smc_llc_send_test_link(link, user_data);
613 /* receive TEST LINK response over RoCE fabric */
614 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
615 SMC_LLC_WAIT_TIME);
616 if (rc <= 0) {
617 smc_lgr_terminate(smc_get_lgr(link));
618 return;
619 }
620 next_interval = link->llc_testlink_time;
621 out:
622 queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk,
623 next_interval);
624 }
625
smc_llc_link_init(struct smc_link * link)626 int smc_llc_link_init(struct smc_link *link)
627 {
628 struct smc_link_group *lgr = smc_get_lgr(link);
629 link->llc_wq = alloc_ordered_workqueue("llc_wq-%x:%x)", WQ_MEM_RECLAIM,
630 *((u32 *)lgr->id),
631 link->link_id);
632 if (!link->llc_wq)
633 return -ENOMEM;
634 init_completion(&link->llc_confirm);
635 init_completion(&link->llc_confirm_resp);
636 init_completion(&link->llc_add);
637 init_completion(&link->llc_add_resp);
638 init_completion(&link->llc_confirm_rkey);
639 init_completion(&link->llc_delete_rkey);
640 mutex_init(&link->llc_delete_rkey_mutex);
641 init_completion(&link->llc_testlink_resp);
642 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
643 return 0;
644 }
645
smc_llc_link_active(struct smc_link * link,int testlink_time)646 void smc_llc_link_active(struct smc_link *link, int testlink_time)
647 {
648 link->state = SMC_LNK_ACTIVE;
649 if (testlink_time) {
650 link->llc_testlink_time = testlink_time * HZ;
651 queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk,
652 link->llc_testlink_time);
653 }
654 }
655
smc_llc_link_deleting(struct smc_link * link)656 void smc_llc_link_deleting(struct smc_link *link)
657 {
658 link->state = SMC_LNK_DELETING;
659 }
660
661 /* called in tasklet context */
smc_llc_link_inactive(struct smc_link * link)662 void smc_llc_link_inactive(struct smc_link *link)
663 {
664 link->state = SMC_LNK_INACTIVE;
665 cancel_delayed_work(&link->llc_testlink_wrk);
666 }
667
668 /* called in worker context */
smc_llc_link_clear(struct smc_link * link)669 void smc_llc_link_clear(struct smc_link *link)
670 {
671 flush_workqueue(link->llc_wq);
672 destroy_workqueue(link->llc_wq);
673 }
674
675 /* register a new rtoken at the remote peer */
smc_llc_do_confirm_rkey(struct smc_link * link,struct smc_buf_desc * rmb_desc)676 int smc_llc_do_confirm_rkey(struct smc_link *link,
677 struct smc_buf_desc *rmb_desc)
678 {
679 int rc;
680
681 /* protected by mutex smc_create_lgr_pending */
682 reinit_completion(&link->llc_confirm_rkey);
683 rc = smc_llc_send_confirm_rkey(link, rmb_desc);
684 if (rc)
685 return rc;
686 /* receive CONFIRM RKEY response from server over RoCE fabric */
687 rc = wait_for_completion_interruptible_timeout(&link->llc_confirm_rkey,
688 SMC_LLC_WAIT_TIME);
689 if (rc <= 0 || link->llc_confirm_rkey_rc)
690 return -EFAULT;
691 return 0;
692 }
693
694 /* unregister an rtoken at the remote peer */
smc_llc_do_delete_rkey(struct smc_link * link,struct smc_buf_desc * rmb_desc)695 int smc_llc_do_delete_rkey(struct smc_link *link,
696 struct smc_buf_desc *rmb_desc)
697 {
698 int rc;
699
700 mutex_lock(&link->llc_delete_rkey_mutex);
701 reinit_completion(&link->llc_delete_rkey);
702 rc = smc_llc_send_delete_rkey(link, rmb_desc);
703 if (rc)
704 goto out;
705 /* receive DELETE RKEY response from server over RoCE fabric */
706 rc = wait_for_completion_interruptible_timeout(&link->llc_delete_rkey,
707 SMC_LLC_WAIT_TIME);
708 if (rc <= 0 || link->llc_delete_rkey_rc)
709 rc = -EFAULT;
710 else
711 rc = 0;
712 out:
713 mutex_unlock(&link->llc_delete_rkey_mutex);
714 return rc;
715 }
716
717 /***************************** init, exit, misc ******************************/
718
719 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
720 {
721 .handler = smc_llc_rx_handler,
722 .type = SMC_LLC_CONFIRM_LINK
723 },
724 {
725 .handler = smc_llc_rx_handler,
726 .type = SMC_LLC_TEST_LINK
727 },
728 {
729 .handler = smc_llc_rx_handler,
730 .type = SMC_LLC_ADD_LINK
731 },
732 {
733 .handler = smc_llc_rx_handler,
734 .type = SMC_LLC_DELETE_LINK
735 },
736 {
737 .handler = smc_llc_rx_handler,
738 .type = SMC_LLC_CONFIRM_RKEY
739 },
740 {
741 .handler = smc_llc_rx_handler,
742 .type = SMC_LLC_CONFIRM_RKEY_CONT
743 },
744 {
745 .handler = smc_llc_rx_handler,
746 .type = SMC_LLC_DELETE_RKEY
747 },
748 {
749 .handler = NULL,
750 }
751 };
752
smc_llc_init(void)753 int __init smc_llc_init(void)
754 {
755 struct smc_wr_rx_handler *handler;
756 int rc = 0;
757
758 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
759 INIT_HLIST_NODE(&handler->list);
760 rc = smc_wr_rx_register_handler(handler);
761 if (rc)
762 break;
763 }
764 return rc;
765 }
766