1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Work Requests exploiting Infiniband API
6 *
7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
8 * are submitted to either RC SQ or RC RQ respectively
9 * (reliably connected send/receive queue)
10 * and become work queue entries (WQEs).
11 * While an SQ WR/WQE is pending, we track it until transmission completion.
12 * Through a send or receive completion queue (CQ) respectively,
13 * we get completion queue entries (CQEs) [aka work completions (WCs)].
14 * Since the CQ callback is called from IRQ context, we split work by using
15 * bottom halves implemented by tasklets.
16 *
17 * SMC uses this to exchange LLC (link layer control)
18 * and CDC (connection data control) messages.
19 *
20 * Copyright IBM Corp. 2016
21 *
22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com>
23 */
24
25 #include <linux/atomic.h>
26 #include <linux/hashtable.h>
27 #include <linux/wait.h>
28 #include <rdma/ib_verbs.h>
29 #include <asm/div64.h>
30
31 #include "smc.h"
32 #include "smc_wr.h"
33
34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */
35
36 #define SMC_WR_RX_HASH_BITS 4
37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
39
40 struct smc_wr_tx_pend { /* control data for a pending send request */
41 u64 wr_id; /* work request id sent */
42 smc_wr_tx_handler handler;
43 enum ib_wc_status wc_status; /* CQE status */
44 struct smc_link *link;
45 u32 idx;
46 struct smc_wr_tx_pend_priv priv;
47 };
48
49 /******************************** send queue *********************************/
50
51 /*------------------------------- completion --------------------------------*/
52
smc_wr_tx_find_pending_index(struct smc_link * link,u64 wr_id)53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
54 {
55 u32 i;
56
57 for (i = 0; i < link->wr_tx_cnt; i++) {
58 if (link->wr_tx_pends[i].wr_id == wr_id)
59 return i;
60 }
61 return link->wr_tx_cnt;
62 }
63
smc_wr_tx_process_cqe(struct ib_wc * wc)64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
65 {
66 struct smc_wr_tx_pend pnd_snd;
67 struct smc_link *link;
68 u32 pnd_snd_idx;
69 int i;
70
71 link = wc->qp->qp_context;
72
73 if (wc->opcode == IB_WC_REG_MR) {
74 if (wc->status)
75 link->wr_reg_state = FAILED;
76 else
77 link->wr_reg_state = CONFIRMED;
78 wake_up(&link->wr_reg_wait);
79 return;
80 }
81
82 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
83 if (pnd_snd_idx == link->wr_tx_cnt)
84 return;
85 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
86 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
87 /* clear the full struct smc_wr_tx_pend including .priv */
88 memset(&link->wr_tx_pends[pnd_snd_idx], 0,
89 sizeof(link->wr_tx_pends[pnd_snd_idx]));
90 memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
91 sizeof(link->wr_tx_bufs[pnd_snd_idx]));
92 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
93 return;
94 if (wc->status) {
95 struct smc_link_group *lgr;
96
97 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
98 /* clear full struct smc_wr_tx_pend including .priv */
99 memset(&link->wr_tx_pends[i], 0,
100 sizeof(link->wr_tx_pends[i]));
101 memset(&link->wr_tx_bufs[i], 0,
102 sizeof(link->wr_tx_bufs[i]));
103 clear_bit(i, link->wr_tx_mask);
104 }
105 /* terminate connections of this link group abnormally */
106 lgr = container_of(link, struct smc_link_group,
107 lnk[SMC_SINGLE_LINK]);
108 smc_lgr_terminate(lgr);
109 }
110 if (pnd_snd.handler)
111 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
112 wake_up(&link->wr_tx_wait);
113 }
114
smc_wr_tx_tasklet_fn(unsigned long data)115 static void smc_wr_tx_tasklet_fn(unsigned long data)
116 {
117 struct smc_ib_device *dev = (struct smc_ib_device *)data;
118 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
119 int i = 0, rc;
120 int polled = 0;
121
122 again:
123 polled++;
124 do {
125 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
126 if (polled == 1) {
127 ib_req_notify_cq(dev->roce_cq_send,
128 IB_CQ_NEXT_COMP |
129 IB_CQ_REPORT_MISSED_EVENTS);
130 }
131 if (!rc)
132 break;
133 for (i = 0; i < rc; i++)
134 smc_wr_tx_process_cqe(&wc[i]);
135 } while (rc > 0);
136 if (polled == 1)
137 goto again;
138 }
139
smc_wr_tx_cq_handler(struct ib_cq * ib_cq,void * cq_context)140 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
141 {
142 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
143
144 tasklet_schedule(&dev->send_tasklet);
145 }
146
147 /*---------------------------- request submission ---------------------------*/
148
smc_wr_tx_get_free_slot_index(struct smc_link * link,u32 * idx)149 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
150 {
151 *idx = link->wr_tx_cnt;
152 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
153 if (!test_and_set_bit(*idx, link->wr_tx_mask))
154 return 0;
155 }
156 *idx = link->wr_tx_cnt;
157 return -EBUSY;
158 }
159
160 /**
161 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
162 * and sets info for pending transmit tracking
163 * @link: Pointer to smc_link used to later send the message.
164 * @handler: Send completion handler function pointer.
165 * @wr_buf: Out value returns pointer to message buffer.
166 * @wr_pend_priv: Out value returns pointer serving as handler context.
167 *
168 * Return: 0 on success, or -errno on error.
169 */
smc_wr_tx_get_free_slot(struct smc_link * link,smc_wr_tx_handler handler,struct smc_wr_buf ** wr_buf,struct smc_wr_tx_pend_priv ** wr_pend_priv)170 int smc_wr_tx_get_free_slot(struct smc_link *link,
171 smc_wr_tx_handler handler,
172 struct smc_wr_buf **wr_buf,
173 struct smc_wr_tx_pend_priv **wr_pend_priv)
174 {
175 struct smc_wr_tx_pend *wr_pend;
176 struct ib_send_wr *wr_ib;
177 u64 wr_id;
178 u32 idx;
179 int rc;
180
181 *wr_buf = NULL;
182 *wr_pend_priv = NULL;
183 if (in_softirq()) {
184 rc = smc_wr_tx_get_free_slot_index(link, &idx);
185 if (rc)
186 return rc;
187 } else {
188 rc = wait_event_interruptible_timeout(
189 link->wr_tx_wait,
190 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
191 SMC_WR_TX_WAIT_FREE_SLOT_TIME);
192 if (!rc) {
193 /* timeout - terminate connections */
194 struct smc_link_group *lgr;
195
196 lgr = container_of(link, struct smc_link_group,
197 lnk[SMC_SINGLE_LINK]);
198 smc_lgr_terminate(lgr);
199 return -EPIPE;
200 }
201 if (rc == -ERESTARTSYS)
202 return -EINTR;
203 if (idx == link->wr_tx_cnt)
204 return -EPIPE;
205 }
206 wr_id = smc_wr_tx_get_next_wr_id(link);
207 wr_pend = &link->wr_tx_pends[idx];
208 wr_pend->wr_id = wr_id;
209 wr_pend->handler = handler;
210 wr_pend->link = link;
211 wr_pend->idx = idx;
212 wr_ib = &link->wr_tx_ibs[idx];
213 wr_ib->wr_id = wr_id;
214 *wr_buf = &link->wr_tx_bufs[idx];
215 *wr_pend_priv = &wr_pend->priv;
216 return 0;
217 }
218
smc_wr_tx_put_slot(struct smc_link * link,struct smc_wr_tx_pend_priv * wr_pend_priv)219 int smc_wr_tx_put_slot(struct smc_link *link,
220 struct smc_wr_tx_pend_priv *wr_pend_priv)
221 {
222 struct smc_wr_tx_pend *pend;
223
224 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
225 if (pend->idx < link->wr_tx_cnt) {
226 u32 idx = pend->idx;
227
228 /* clear the full struct smc_wr_tx_pend including .priv */
229 memset(&link->wr_tx_pends[pend->idx], 0,
230 sizeof(link->wr_tx_pends[pend->idx]));
231 memset(&link->wr_tx_bufs[pend->idx], 0,
232 sizeof(link->wr_tx_bufs[pend->idx]));
233 test_and_clear_bit(idx, link->wr_tx_mask);
234 return 1;
235 }
236
237 return 0;
238 }
239
240 /* Send prepared WR slot via ib_post_send.
241 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
242 */
smc_wr_tx_send(struct smc_link * link,struct smc_wr_tx_pend_priv * priv)243 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
244 {
245 struct ib_send_wr *failed_wr = NULL;
246 struct smc_wr_tx_pend *pend;
247 int rc;
248
249 ib_req_notify_cq(link->smcibdev->roce_cq_send,
250 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
251 pend = container_of(priv, struct smc_wr_tx_pend, priv);
252 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx],
253 &failed_wr);
254 if (rc)
255 smc_wr_tx_put_slot(link, priv);
256 return rc;
257 }
258
259 /* Register a memory region and wait for result. */
smc_wr_reg_send(struct smc_link * link,struct ib_mr * mr)260 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
261 {
262 struct ib_send_wr *failed_wr = NULL;
263 int rc;
264
265 ib_req_notify_cq(link->smcibdev->roce_cq_send,
266 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
267 link->wr_reg_state = POSTED;
268 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
269 link->wr_reg.mr = mr;
270 link->wr_reg.key = mr->rkey;
271 failed_wr = &link->wr_reg.wr;
272 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr);
273 WARN_ON(failed_wr != &link->wr_reg.wr);
274 if (rc)
275 return rc;
276
277 rc = wait_event_interruptible_timeout(link->wr_reg_wait,
278 (link->wr_reg_state != POSTED),
279 SMC_WR_REG_MR_WAIT_TIME);
280 if (!rc) {
281 /* timeout - terminate connections */
282 struct smc_link_group *lgr;
283
284 lgr = container_of(link, struct smc_link_group,
285 lnk[SMC_SINGLE_LINK]);
286 smc_lgr_terminate(lgr);
287 return -EPIPE;
288 }
289 if (rc == -ERESTARTSYS)
290 return -EINTR;
291 switch (link->wr_reg_state) {
292 case CONFIRMED:
293 rc = 0;
294 break;
295 case FAILED:
296 rc = -EIO;
297 break;
298 case POSTED:
299 rc = -EPIPE;
300 break;
301 }
302 return rc;
303 }
304
smc_wr_tx_dismiss_slots(struct smc_link * link,u8 wr_rx_hdr_type,smc_wr_tx_filter filter,smc_wr_tx_dismisser dismisser,unsigned long data)305 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_rx_hdr_type,
306 smc_wr_tx_filter filter,
307 smc_wr_tx_dismisser dismisser,
308 unsigned long data)
309 {
310 struct smc_wr_tx_pend_priv *tx_pend;
311 struct smc_wr_rx_hdr *wr_rx;
312 int i;
313
314 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
315 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i];
316 if (wr_rx->type != wr_rx_hdr_type)
317 continue;
318 tx_pend = &link->wr_tx_pends[i].priv;
319 if (filter(tx_pend, data))
320 dismisser(tx_pend);
321 }
322 }
323
smc_wr_tx_has_pending(struct smc_link * link,u8 wr_rx_hdr_type,smc_wr_tx_filter filter,unsigned long data)324 bool smc_wr_tx_has_pending(struct smc_link *link, u8 wr_rx_hdr_type,
325 smc_wr_tx_filter filter, unsigned long data)
326 {
327 struct smc_wr_tx_pend_priv *tx_pend;
328 struct smc_wr_rx_hdr *wr_rx;
329 int i;
330
331 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
332 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i];
333 if (wr_rx->type != wr_rx_hdr_type)
334 continue;
335 tx_pend = &link->wr_tx_pends[i].priv;
336 if (filter(tx_pend, data))
337 return true;
338 }
339 return false;
340 }
341
342 /****************************** receive queue ********************************/
343
smc_wr_rx_register_handler(struct smc_wr_rx_handler * handler)344 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
345 {
346 struct smc_wr_rx_handler *h_iter;
347 int rc = 0;
348
349 spin_lock(&smc_wr_rx_hash_lock);
350 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
351 if (h_iter->type == handler->type) {
352 rc = -EEXIST;
353 goto out_unlock;
354 }
355 }
356 hash_add(smc_wr_rx_hash, &handler->list, handler->type);
357 out_unlock:
358 spin_unlock(&smc_wr_rx_hash_lock);
359 return rc;
360 }
361
362 /* Demultiplex a received work request based on the message type to its handler.
363 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
364 * and not being modified any more afterwards so we don't need to lock it.
365 */
smc_wr_rx_demultiplex(struct ib_wc * wc)366 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
367 {
368 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
369 struct smc_wr_rx_handler *handler;
370 struct smc_wr_rx_hdr *wr_rx;
371 u64 temp_wr_id;
372 u32 index;
373
374 if (wc->byte_len < sizeof(*wr_rx))
375 return; /* short message */
376 temp_wr_id = wc->wr_id;
377 index = do_div(temp_wr_id, link->wr_rx_cnt);
378 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
379 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
380 if (handler->type == wr_rx->type)
381 handler->handler(wc, wr_rx);
382 }
383 }
384
smc_wr_rx_process_cqes(struct ib_wc wc[],int num)385 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
386 {
387 struct smc_link *link;
388 int i;
389
390 for (i = 0; i < num; i++) {
391 link = wc[i].qp->qp_context;
392 if (wc[i].status == IB_WC_SUCCESS) {
393 smc_wr_rx_demultiplex(&wc[i]);
394 smc_wr_rx_post(link); /* refill WR RX */
395 } else {
396 struct smc_link_group *lgr;
397
398 /* handle status errors */
399 switch (wc[i].status) {
400 case IB_WC_RETRY_EXC_ERR:
401 case IB_WC_RNR_RETRY_EXC_ERR:
402 case IB_WC_WR_FLUSH_ERR:
403 /* terminate connections of this link group
404 * abnormally
405 */
406 lgr = container_of(link, struct smc_link_group,
407 lnk[SMC_SINGLE_LINK]);
408 smc_lgr_terminate(lgr);
409 break;
410 default:
411 smc_wr_rx_post(link); /* refill WR RX */
412 break;
413 }
414 }
415 }
416 }
417
smc_wr_rx_tasklet_fn(unsigned long data)418 static void smc_wr_rx_tasklet_fn(unsigned long data)
419 {
420 struct smc_ib_device *dev = (struct smc_ib_device *)data;
421 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
422 int polled = 0;
423 int rc;
424
425 again:
426 polled++;
427 do {
428 memset(&wc, 0, sizeof(wc));
429 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
430 if (polled == 1) {
431 ib_req_notify_cq(dev->roce_cq_recv,
432 IB_CQ_SOLICITED_MASK
433 | IB_CQ_REPORT_MISSED_EVENTS);
434 }
435 if (!rc)
436 break;
437 smc_wr_rx_process_cqes(&wc[0], rc);
438 } while (rc > 0);
439 if (polled == 1)
440 goto again;
441 }
442
smc_wr_rx_cq_handler(struct ib_cq * ib_cq,void * cq_context)443 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
444 {
445 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
446
447 tasklet_schedule(&dev->recv_tasklet);
448 }
449
smc_wr_rx_post_init(struct smc_link * link)450 int smc_wr_rx_post_init(struct smc_link *link)
451 {
452 u32 i;
453 int rc = 0;
454
455 for (i = 0; i < link->wr_rx_cnt; i++)
456 rc = smc_wr_rx_post(link);
457 return rc;
458 }
459
460 /***************************** init, exit, misc ******************************/
461
smc_wr_remember_qp_attr(struct smc_link * lnk)462 void smc_wr_remember_qp_attr(struct smc_link *lnk)
463 {
464 struct ib_qp_attr *attr = &lnk->qp_attr;
465 struct ib_qp_init_attr init_attr;
466
467 memset(attr, 0, sizeof(*attr));
468 memset(&init_attr, 0, sizeof(init_attr));
469 ib_query_qp(lnk->roce_qp, attr,
470 IB_QP_STATE |
471 IB_QP_CUR_STATE |
472 IB_QP_PKEY_INDEX |
473 IB_QP_PORT |
474 IB_QP_QKEY |
475 IB_QP_AV |
476 IB_QP_PATH_MTU |
477 IB_QP_TIMEOUT |
478 IB_QP_RETRY_CNT |
479 IB_QP_RNR_RETRY |
480 IB_QP_RQ_PSN |
481 IB_QP_ALT_PATH |
482 IB_QP_MIN_RNR_TIMER |
483 IB_QP_SQ_PSN |
484 IB_QP_PATH_MIG_STATE |
485 IB_QP_CAP |
486 IB_QP_DEST_QPN,
487 &init_attr);
488
489 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
490 lnk->qp_attr.cap.max_send_wr);
491 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
492 lnk->qp_attr.cap.max_recv_wr);
493 }
494
smc_wr_init_sge(struct smc_link * lnk)495 static void smc_wr_init_sge(struct smc_link *lnk)
496 {
497 u32 i;
498
499 for (i = 0; i < lnk->wr_tx_cnt; i++) {
500 lnk->wr_tx_sges[i].addr =
501 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
502 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
503 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
504 lnk->wr_tx_ibs[i].next = NULL;
505 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
506 lnk->wr_tx_ibs[i].num_sge = 1;
507 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
508 lnk->wr_tx_ibs[i].send_flags =
509 IB_SEND_SIGNALED | IB_SEND_SOLICITED;
510 }
511 for (i = 0; i < lnk->wr_rx_cnt; i++) {
512 lnk->wr_rx_sges[i].addr =
513 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
514 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
515 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
516 lnk->wr_rx_ibs[i].next = NULL;
517 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
518 lnk->wr_rx_ibs[i].num_sge = 1;
519 }
520 lnk->wr_reg.wr.next = NULL;
521 lnk->wr_reg.wr.num_sge = 0;
522 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
523 lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
524 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
525 }
526
smc_wr_free_link(struct smc_link * lnk)527 void smc_wr_free_link(struct smc_link *lnk)
528 {
529 struct ib_device *ibdev;
530
531 memset(lnk->wr_tx_mask, 0,
532 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
533
534 if (!lnk->smcibdev)
535 return;
536 ibdev = lnk->smcibdev->ibdev;
537
538 if (lnk->wr_rx_dma_addr) {
539 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
540 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
541 DMA_FROM_DEVICE);
542 lnk->wr_rx_dma_addr = 0;
543 }
544 if (lnk->wr_tx_dma_addr) {
545 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
546 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
547 DMA_TO_DEVICE);
548 lnk->wr_tx_dma_addr = 0;
549 }
550 }
551
smc_wr_free_link_mem(struct smc_link * lnk)552 void smc_wr_free_link_mem(struct smc_link *lnk)
553 {
554 kfree(lnk->wr_tx_pends);
555 lnk->wr_tx_pends = NULL;
556 kfree(lnk->wr_tx_mask);
557 lnk->wr_tx_mask = NULL;
558 kfree(lnk->wr_tx_sges);
559 lnk->wr_tx_sges = NULL;
560 kfree(lnk->wr_rx_sges);
561 lnk->wr_rx_sges = NULL;
562 kfree(lnk->wr_rx_ibs);
563 lnk->wr_rx_ibs = NULL;
564 kfree(lnk->wr_tx_ibs);
565 lnk->wr_tx_ibs = NULL;
566 kfree(lnk->wr_tx_bufs);
567 lnk->wr_tx_bufs = NULL;
568 kfree(lnk->wr_rx_bufs);
569 lnk->wr_rx_bufs = NULL;
570 }
571
smc_wr_alloc_link_mem(struct smc_link * link)572 int smc_wr_alloc_link_mem(struct smc_link *link)
573 {
574 /* allocate link related memory */
575 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
576 if (!link->wr_tx_bufs)
577 goto no_mem;
578 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
579 GFP_KERNEL);
580 if (!link->wr_rx_bufs)
581 goto no_mem_wr_tx_bufs;
582 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
583 GFP_KERNEL);
584 if (!link->wr_tx_ibs)
585 goto no_mem_wr_rx_bufs;
586 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
587 sizeof(link->wr_rx_ibs[0]),
588 GFP_KERNEL);
589 if (!link->wr_rx_ibs)
590 goto no_mem_wr_tx_ibs;
591 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
592 GFP_KERNEL);
593 if (!link->wr_tx_sges)
594 goto no_mem_wr_rx_ibs;
595 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
596 sizeof(link->wr_rx_sges[0]),
597 GFP_KERNEL);
598 if (!link->wr_rx_sges)
599 goto no_mem_wr_tx_sges;
600 link->wr_tx_mask = kzalloc(
601 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*link->wr_tx_mask),
602 GFP_KERNEL);
603 if (!link->wr_tx_mask)
604 goto no_mem_wr_rx_sges;
605 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
606 sizeof(link->wr_tx_pends[0]),
607 GFP_KERNEL);
608 if (!link->wr_tx_pends)
609 goto no_mem_wr_tx_mask;
610 return 0;
611
612 no_mem_wr_tx_mask:
613 kfree(link->wr_tx_mask);
614 no_mem_wr_rx_sges:
615 kfree(link->wr_rx_sges);
616 no_mem_wr_tx_sges:
617 kfree(link->wr_tx_sges);
618 no_mem_wr_rx_ibs:
619 kfree(link->wr_rx_ibs);
620 no_mem_wr_tx_ibs:
621 kfree(link->wr_tx_ibs);
622 no_mem_wr_rx_bufs:
623 kfree(link->wr_rx_bufs);
624 no_mem_wr_tx_bufs:
625 kfree(link->wr_tx_bufs);
626 no_mem:
627 return -ENOMEM;
628 }
629
smc_wr_remove_dev(struct smc_ib_device * smcibdev)630 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
631 {
632 tasklet_kill(&smcibdev->recv_tasklet);
633 tasklet_kill(&smcibdev->send_tasklet);
634 }
635
smc_wr_add_dev(struct smc_ib_device * smcibdev)636 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
637 {
638 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
639 (unsigned long)smcibdev);
640 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
641 (unsigned long)smcibdev);
642 }
643
smc_wr_create_link(struct smc_link * lnk)644 int smc_wr_create_link(struct smc_link *lnk)
645 {
646 struct ib_device *ibdev = lnk->smcibdev->ibdev;
647 int rc = 0;
648
649 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
650 lnk->wr_rx_id = 0;
651 lnk->wr_rx_dma_addr = ib_dma_map_single(
652 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
653 DMA_FROM_DEVICE);
654 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
655 lnk->wr_rx_dma_addr = 0;
656 rc = -EIO;
657 goto out;
658 }
659 lnk->wr_tx_dma_addr = ib_dma_map_single(
660 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
661 DMA_TO_DEVICE);
662 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
663 rc = -EIO;
664 goto dma_unmap;
665 }
666 smc_wr_init_sge(lnk);
667 memset(lnk->wr_tx_mask, 0,
668 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
669 init_waitqueue_head(&lnk->wr_tx_wait);
670 init_waitqueue_head(&lnk->wr_reg_wait);
671 return rc;
672
673 dma_unmap:
674 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
675 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
676 DMA_FROM_DEVICE);
677 lnk->wr_rx_dma_addr = 0;
678 out:
679 return rc;
680 }
681