1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Basic Transport Functions exploiting Infiniband API
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11
12 #include <linux/socket.h>
13 #include <linux/if_vlan.h>
14 #include <linux/random.h>
15 #include <linux/workqueue.h>
16 #include <net/tcp.h>
17 #include <net/sock.h>
18 #include <rdma/ib_verbs.h>
19 #include <rdma/ib_cache.h>
20
21 #include "smc.h"
22 #include "smc_clc.h"
23 #include "smc_core.h"
24 #include "smc_ib.h"
25 #include "smc_wr.h"
26 #include "smc_llc.h"
27 #include "smc_cdc.h"
28 #include "smc_close.h"
29 #include "smc_ism.h"
30
31 #define SMC_LGR_NUM_INCR 256
32 #define SMC_LGR_FREE_DELAY_SERV (600 * HZ)
33 #define SMC_LGR_FREE_DELAY_CLNT (SMC_LGR_FREE_DELAY_SERV + 10 * HZ)
34 #define SMC_LGR_FREE_DELAY_FAST (8 * HZ)
35
36 static struct smc_lgr_list smc_lgr_list = { /* established link groups */
37 .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock),
38 .list = LIST_HEAD_INIT(smc_lgr_list.list),
39 .num = 0,
40 };
41
42 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
43 struct smc_buf_desc *buf_desc);
44
smc_lgr_schedule_free_work(struct smc_link_group * lgr)45 static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)
46 {
47 /* client link group creation always follows the server link group
48 * creation. For client use a somewhat higher removal delay time,
49 * otherwise there is a risk of out-of-sync link groups.
50 */
51 mod_delayed_work(system_wq, &lgr->free_work,
52 (!lgr->is_smcd && lgr->role == SMC_CLNT) ?
53 SMC_LGR_FREE_DELAY_CLNT : SMC_LGR_FREE_DELAY_SERV);
54 }
55
smc_lgr_schedule_free_work_fast(struct smc_link_group * lgr)56 void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr)
57 {
58 mod_delayed_work(system_wq, &lgr->free_work, SMC_LGR_FREE_DELAY_FAST);
59 }
60
61 /* Register connection's alert token in our lookup structure.
62 * To use rbtrees we have to implement our own insert core.
63 * Requires @conns_lock
64 * @smc connection to register
65 * Returns 0 on success, != otherwise.
66 */
smc_lgr_add_alert_token(struct smc_connection * conn)67 static void smc_lgr_add_alert_token(struct smc_connection *conn)
68 {
69 struct rb_node **link, *parent = NULL;
70 u32 token = conn->alert_token_local;
71
72 link = &conn->lgr->conns_all.rb_node;
73 while (*link) {
74 struct smc_connection *cur = rb_entry(*link,
75 struct smc_connection, alert_node);
76
77 parent = *link;
78 if (cur->alert_token_local > token)
79 link = &parent->rb_left;
80 else
81 link = &parent->rb_right;
82 }
83 /* Put the new node there */
84 rb_link_node(&conn->alert_node, parent, link);
85 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);
86 }
87
88 /* Register connection in link group by assigning an alert token
89 * registered in a search tree.
90 * Requires @conns_lock
91 * Note that '0' is a reserved value and not assigned.
92 */
smc_lgr_register_conn(struct smc_connection * conn)93 static void smc_lgr_register_conn(struct smc_connection *conn)
94 {
95 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
96 static atomic_t nexttoken = ATOMIC_INIT(0);
97
98 /* find a new alert_token_local value not yet used by some connection
99 * in this link group
100 */
101 sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */
102 while (!conn->alert_token_local) {
103 conn->alert_token_local = atomic_inc_return(&nexttoken);
104 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))
105 conn->alert_token_local = 0;
106 }
107 smc_lgr_add_alert_token(conn);
108 conn->lgr->conns_num++;
109 }
110
111 /* Unregister connection and reset the alert token of the given connection<
112 */
__smc_lgr_unregister_conn(struct smc_connection * conn)113 static void __smc_lgr_unregister_conn(struct smc_connection *conn)
114 {
115 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
116 struct smc_link_group *lgr = conn->lgr;
117
118 rb_erase(&conn->alert_node, &lgr->conns_all);
119 lgr->conns_num--;
120 conn->alert_token_local = 0;
121 sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */
122 }
123
124 /* Unregister connection from lgr
125 */
smc_lgr_unregister_conn(struct smc_connection * conn)126 static void smc_lgr_unregister_conn(struct smc_connection *conn)
127 {
128 struct smc_link_group *lgr = conn->lgr;
129
130 if (!lgr)
131 return;
132 write_lock_bh(&lgr->conns_lock);
133 if (conn->alert_token_local) {
134 __smc_lgr_unregister_conn(conn);
135 }
136 write_unlock_bh(&lgr->conns_lock);
137 }
138
139 /* Send delete link, either as client to request the initiation
140 * of the DELETE LINK sequence from server; or as server to
141 * initiate the delete processing. See smc_llc_rx_delete_link().
142 */
smc_link_send_delete(struct smc_link * lnk)143 static int smc_link_send_delete(struct smc_link *lnk)
144 {
145 if (lnk->state == SMC_LNK_ACTIVE &&
146 !smc_llc_send_delete_link(lnk, SMC_LLC_REQ, true)) {
147 smc_llc_link_deleting(lnk);
148 return 0;
149 }
150 return -ENOTCONN;
151 }
152
153 static void smc_lgr_free(struct smc_link_group *lgr);
154
smc_lgr_free_work(struct work_struct * work)155 static void smc_lgr_free_work(struct work_struct *work)
156 {
157 struct smc_link_group *lgr = container_of(to_delayed_work(work),
158 struct smc_link_group,
159 free_work);
160 bool conns;
161
162 spin_lock_bh(&smc_lgr_list.lock);
163 read_lock_bh(&lgr->conns_lock);
164 conns = RB_EMPTY_ROOT(&lgr->conns_all);
165 read_unlock_bh(&lgr->conns_lock);
166 if (!conns) { /* number of lgr connections is no longer zero */
167 spin_unlock_bh(&smc_lgr_list.lock);
168 return;
169 }
170 if (!list_empty(&lgr->list))
171 list_del_init(&lgr->list); /* remove from smc_lgr_list */
172 spin_unlock_bh(&smc_lgr_list.lock);
173
174 if (!lgr->is_smcd && !lgr->terminating) {
175 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
176
177 /* try to send del link msg, on error free lgr immediately */
178 if (lnk->state == SMC_LNK_ACTIVE &&
179 !smc_link_send_delete(lnk)) {
180 /* reschedule in case we never receive a response */
181 smc_lgr_schedule_free_work(lgr);
182 return;
183 }
184 }
185
186 if (!delayed_work_pending(&lgr->free_work)) {
187 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
188
189 if (!lgr->is_smcd && lnk->state != SMC_LNK_INACTIVE)
190 smc_llc_link_inactive(lnk);
191 if (lgr->is_smcd)
192 smc_ism_signal_shutdown(lgr);
193 smc_lgr_free(lgr);
194 }
195 }
196
197 /* create a new SMC link group */
smc_lgr_create(struct smc_sock * smc,struct smc_init_info * ini)198 static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini)
199 {
200 struct smc_link_group *lgr;
201 struct smc_link *lnk;
202 u8 rndvec[3];
203 int rc = 0;
204 int i;
205
206 if (ini->is_smcd && ini->vlan_id) {
207 if (smc_ism_get_vlan(ini->ism_dev, ini->vlan_id)) {
208 rc = SMC_CLC_DECL_ISMVLANERR;
209 goto out;
210 }
211 }
212
213 lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);
214 if (!lgr) {
215 rc = SMC_CLC_DECL_MEM;
216 goto ism_put_vlan;
217 }
218 lgr->is_smcd = ini->is_smcd;
219 lgr->sync_err = 0;
220 lgr->vlan_id = ini->vlan_id;
221 rwlock_init(&lgr->sndbufs_lock);
222 rwlock_init(&lgr->rmbs_lock);
223 rwlock_init(&lgr->conns_lock);
224 for (i = 0; i < SMC_RMBE_SIZES; i++) {
225 INIT_LIST_HEAD(&lgr->sndbufs[i]);
226 INIT_LIST_HEAD(&lgr->rmbs[i]);
227 }
228 smc_lgr_list.num += SMC_LGR_NUM_INCR;
229 memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE);
230 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);
231 lgr->conns_all = RB_ROOT;
232 if (ini->is_smcd) {
233 /* SMC-D specific settings */
234 get_device(&ini->ism_dev->dev);
235 lgr->peer_gid = ini->ism_gid;
236 lgr->smcd = ini->ism_dev;
237 } else {
238 /* SMC-R specific settings */
239 get_device(&ini->ib_dev->ibdev->dev);
240 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
241 memcpy(lgr->peer_systemid, ini->ib_lcl->id_for_peer,
242 SMC_SYSTEMID_LEN);
243
244 lnk = &lgr->lnk[SMC_SINGLE_LINK];
245 /* initialize link */
246 lnk->state = SMC_LNK_ACTIVATING;
247 lnk->link_id = SMC_SINGLE_LINK;
248 lnk->smcibdev = ini->ib_dev;
249 lnk->ibport = ini->ib_port;
250 lnk->path_mtu =
251 ini->ib_dev->pattr[ini->ib_port - 1].active_mtu;
252 if (!ini->ib_dev->initialized)
253 smc_ib_setup_per_ibdev(ini->ib_dev);
254 get_random_bytes(rndvec, sizeof(rndvec));
255 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +
256 (rndvec[2] << 16);
257 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,
258 ini->vlan_id, lnk->gid,
259 &lnk->sgid_index);
260 if (rc)
261 goto free_lgr;
262 rc = smc_llc_link_init(lnk);
263 if (rc)
264 goto free_lgr;
265 rc = smc_wr_alloc_link_mem(lnk);
266 if (rc)
267 goto clear_llc_lnk;
268 rc = smc_ib_create_protection_domain(lnk);
269 if (rc)
270 goto free_link_mem;
271 rc = smc_ib_create_queue_pair(lnk);
272 if (rc)
273 goto dealloc_pd;
274 rc = smc_wr_create_link(lnk);
275 if (rc)
276 goto destroy_qp;
277 }
278 smc->conn.lgr = lgr;
279 spin_lock_bh(&smc_lgr_list.lock);
280 list_add(&lgr->list, &smc_lgr_list.list);
281 spin_unlock_bh(&smc_lgr_list.lock);
282 return 0;
283
284 destroy_qp:
285 smc_ib_destroy_queue_pair(lnk);
286 dealloc_pd:
287 smc_ib_dealloc_protection_domain(lnk);
288 free_link_mem:
289 smc_wr_free_link_mem(lnk);
290 clear_llc_lnk:
291 smc_llc_link_clear(lnk);
292 free_lgr:
293 kfree(lgr);
294 ism_put_vlan:
295 if (ini->is_smcd && ini->vlan_id)
296 smc_ism_put_vlan(ini->ism_dev, ini->vlan_id);
297 out:
298 if (rc < 0) {
299 if (rc == -ENOMEM)
300 rc = SMC_CLC_DECL_MEM;
301 else
302 rc = SMC_CLC_DECL_INTERR;
303 }
304 return rc;
305 }
306
smc_buf_unuse(struct smc_connection * conn,struct smc_link_group * lgr)307 static void smc_buf_unuse(struct smc_connection *conn,
308 struct smc_link_group *lgr)
309 {
310 if (conn->sndbuf_desc)
311 conn->sndbuf_desc->used = 0;
312 if (conn->rmb_desc) {
313 if (!conn->rmb_desc->regerr) {
314 if (!lgr->is_smcd) {
315 /* unregister rmb with peer */
316 smc_llc_do_delete_rkey(
317 &lgr->lnk[SMC_SINGLE_LINK],
318 conn->rmb_desc);
319 }
320 conn->rmb_desc->used = 0;
321 } else {
322 /* buf registration failed, reuse not possible */
323 write_lock_bh(&lgr->rmbs_lock);
324 list_del(&conn->rmb_desc->list);
325 write_unlock_bh(&lgr->rmbs_lock);
326
327 smc_buf_free(lgr, true, conn->rmb_desc);
328 }
329 }
330 }
331
332 /* remove a finished connection from its link group */
smc_conn_free(struct smc_connection * conn)333 void smc_conn_free(struct smc_connection *conn)
334 {
335 struct smc_link_group *lgr = conn->lgr;
336
337 if (!lgr)
338 return;
339 if (lgr->is_smcd) {
340 smc_ism_unset_conn(conn);
341 tasklet_kill(&conn->rx_tsklet);
342 } else {
343 smc_cdc_tx_dismiss_slots(conn);
344 }
345 smc_buf_unuse(conn, lgr); /* allow buffer reuse */
346 smc_lgr_unregister_conn(conn);
347 conn->lgr = NULL;
348
349 if (!lgr->conns_num)
350 smc_lgr_schedule_free_work(lgr);
351 }
352
smc_link_clear(struct smc_link * lnk)353 static void smc_link_clear(struct smc_link *lnk)
354 {
355 lnk->peer_qpn = 0;
356 smc_llc_link_clear(lnk);
357 smc_ib_modify_qp_reset(lnk);
358 smc_wr_free_link(lnk);
359 smc_ib_destroy_queue_pair(lnk);
360 smc_ib_dealloc_protection_domain(lnk);
361 smc_wr_free_link_mem(lnk);
362 }
363
smcr_buf_free(struct smc_link_group * lgr,bool is_rmb,struct smc_buf_desc * buf_desc)364 static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb,
365 struct smc_buf_desc *buf_desc)
366 {
367 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
368
369 if (is_rmb) {
370 if (buf_desc->mr_rx[SMC_SINGLE_LINK])
371 smc_ib_put_memory_region(
372 buf_desc->mr_rx[SMC_SINGLE_LINK]);
373 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
374 DMA_FROM_DEVICE);
375 } else {
376 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
377 DMA_TO_DEVICE);
378 }
379 sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]);
380 if (buf_desc->pages)
381 __free_pages(buf_desc->pages, buf_desc->order);
382 kfree(buf_desc);
383 }
384
smcd_buf_free(struct smc_link_group * lgr,bool is_dmb,struct smc_buf_desc * buf_desc)385 static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb,
386 struct smc_buf_desc *buf_desc)
387 {
388 if (is_dmb) {
389 /* restore original buf len */
390 buf_desc->len += sizeof(struct smcd_cdc_msg);
391 smc_ism_unregister_dmb(lgr->smcd, buf_desc);
392 } else {
393 kfree(buf_desc->cpu_addr);
394 }
395 kfree(buf_desc);
396 }
397
smc_buf_free(struct smc_link_group * lgr,bool is_rmb,struct smc_buf_desc * buf_desc)398 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
399 struct smc_buf_desc *buf_desc)
400 {
401 if (lgr->is_smcd)
402 smcd_buf_free(lgr, is_rmb, buf_desc);
403 else
404 smcr_buf_free(lgr, is_rmb, buf_desc);
405 }
406
__smc_lgr_free_bufs(struct smc_link_group * lgr,bool is_rmb)407 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb)
408 {
409 struct smc_buf_desc *buf_desc, *bf_desc;
410 struct list_head *buf_list;
411 int i;
412
413 for (i = 0; i < SMC_RMBE_SIZES; i++) {
414 if (is_rmb)
415 buf_list = &lgr->rmbs[i];
416 else
417 buf_list = &lgr->sndbufs[i];
418 list_for_each_entry_safe(buf_desc, bf_desc, buf_list,
419 list) {
420 list_del(&buf_desc->list);
421 smc_buf_free(lgr, is_rmb, buf_desc);
422 }
423 }
424 }
425
smc_lgr_free_bufs(struct smc_link_group * lgr)426 static void smc_lgr_free_bufs(struct smc_link_group *lgr)
427 {
428 /* free send buffers */
429 __smc_lgr_free_bufs(lgr, false);
430 /* free rmbs */
431 __smc_lgr_free_bufs(lgr, true);
432 }
433
434 /* remove a link group */
smc_lgr_free(struct smc_link_group * lgr)435 static void smc_lgr_free(struct smc_link_group *lgr)
436 {
437 smc_lgr_free_bufs(lgr);
438 if (lgr->is_smcd) {
439 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id);
440 put_device(&lgr->smcd->dev);
441 } else {
442 smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]);
443 put_device(&lgr->lnk[SMC_SINGLE_LINK].smcibdev->ibdev->dev);
444 }
445 kfree(lgr);
446 }
447
smc_lgr_forget(struct smc_link_group * lgr)448 void smc_lgr_forget(struct smc_link_group *lgr)
449 {
450 spin_lock_bh(&smc_lgr_list.lock);
451 /* do not use this link group for new connections */
452 if (!list_empty(&lgr->list))
453 list_del_init(&lgr->list);
454 spin_unlock_bh(&smc_lgr_list.lock);
455 }
456
457 /* terminate linkgroup abnormally */
__smc_lgr_terminate(struct smc_link_group * lgr)458 static void __smc_lgr_terminate(struct smc_link_group *lgr)
459 {
460 struct smc_connection *conn;
461 struct smc_sock *smc;
462 struct rb_node *node;
463
464 if (lgr->terminating)
465 return; /* lgr already terminating */
466 lgr->terminating = 1;
467 if (!list_empty(&lgr->list)) /* forget lgr */
468 list_del_init(&lgr->list);
469 if (!lgr->is_smcd)
470 smc_llc_link_inactive(&lgr->lnk[SMC_SINGLE_LINK]);
471
472 write_lock_bh(&lgr->conns_lock);
473 node = rb_first(&lgr->conns_all);
474 while (node) {
475 conn = rb_entry(node, struct smc_connection, alert_node);
476 smc = container_of(conn, struct smc_sock, conn);
477 sock_hold(&smc->sk); /* sock_put in close work */
478 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
479 __smc_lgr_unregister_conn(conn);
480 conn->lgr = NULL;
481 write_unlock_bh(&lgr->conns_lock);
482 if (!schedule_work(&conn->close_work))
483 sock_put(&smc->sk);
484 write_lock_bh(&lgr->conns_lock);
485 node = rb_first(&lgr->conns_all);
486 }
487 write_unlock_bh(&lgr->conns_lock);
488 if (!lgr->is_smcd)
489 wake_up(&lgr->lnk[SMC_SINGLE_LINK].wr_reg_wait);
490 smc_lgr_schedule_free_work(lgr);
491 }
492
smc_lgr_terminate(struct smc_link_group * lgr)493 void smc_lgr_terminate(struct smc_link_group *lgr)
494 {
495 spin_lock_bh(&smc_lgr_list.lock);
496 __smc_lgr_terminate(lgr);
497 spin_unlock_bh(&smc_lgr_list.lock);
498 }
499
500 /* Called when IB port is terminated */
smc_port_terminate(struct smc_ib_device * smcibdev,u8 ibport)501 void smc_port_terminate(struct smc_ib_device *smcibdev, u8 ibport)
502 {
503 struct smc_link_group *lgr, *l;
504
505 spin_lock_bh(&smc_lgr_list.lock);
506 list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {
507 if (!lgr->is_smcd &&
508 lgr->lnk[SMC_SINGLE_LINK].smcibdev == smcibdev &&
509 lgr->lnk[SMC_SINGLE_LINK].ibport == ibport)
510 __smc_lgr_terminate(lgr);
511 }
512 spin_unlock_bh(&smc_lgr_list.lock);
513 }
514
515 /* Called when SMC-D device is terminated or peer is lost */
smc_smcd_terminate(struct smcd_dev * dev,u64 peer_gid,unsigned short vlan)516 void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid, unsigned short vlan)
517 {
518 struct smc_link_group *lgr, *l;
519 LIST_HEAD(lgr_free_list);
520
521 /* run common cleanup function and build free list */
522 spin_lock_bh(&smc_lgr_list.lock);
523 list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {
524 if (lgr->is_smcd && lgr->smcd == dev &&
525 (!peer_gid || lgr->peer_gid == peer_gid) &&
526 (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) {
527 __smc_lgr_terminate(lgr);
528 list_move(&lgr->list, &lgr_free_list);
529 }
530 }
531 spin_unlock_bh(&smc_lgr_list.lock);
532
533 /* cancel the regular free workers and actually free lgrs */
534 list_for_each_entry_safe(lgr, l, &lgr_free_list, list) {
535 list_del_init(&lgr->list);
536 cancel_delayed_work_sync(&lgr->free_work);
537 if (!peer_gid && vlan == VLAN_VID_MASK) /* dev terminated? */
538 smc_ism_signal_shutdown(lgr);
539 smc_lgr_free(lgr);
540 }
541 }
542
543 /* Determine vlan of internal TCP socket.
544 * @vlan_id: address to store the determined vlan id into
545 */
smc_vlan_by_tcpsk(struct socket * clcsock,struct smc_init_info * ini)546 int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini)
547 {
548 struct dst_entry *dst = sk_dst_get(clcsock->sk);
549 struct net_device *ndev;
550 int i, nest_lvl, rc = 0;
551
552 ini->vlan_id = 0;
553 if (!dst) {
554 rc = -ENOTCONN;
555 goto out;
556 }
557 if (!dst->dev) {
558 rc = -ENODEV;
559 goto out_rel;
560 }
561
562 ndev = dst->dev;
563 if (is_vlan_dev(ndev)) {
564 ini->vlan_id = vlan_dev_vlan_id(ndev);
565 goto out_rel;
566 }
567
568 rtnl_lock();
569 nest_lvl = ndev->lower_level;
570 for (i = 0; i < nest_lvl; i++) {
571 struct list_head *lower = &ndev->adj_list.lower;
572
573 if (list_empty(lower))
574 break;
575 lower = lower->next;
576 ndev = (struct net_device *)netdev_lower_get_next(ndev, &lower);
577 if (is_vlan_dev(ndev)) {
578 ini->vlan_id = vlan_dev_vlan_id(ndev);
579 break;
580 }
581 }
582 rtnl_unlock();
583
584 out_rel:
585 dst_release(dst);
586 out:
587 return rc;
588 }
589
smcr_lgr_match(struct smc_link_group * lgr,struct smc_clc_msg_local * lcl,enum smc_lgr_role role,u32 clcqpn)590 static bool smcr_lgr_match(struct smc_link_group *lgr,
591 struct smc_clc_msg_local *lcl,
592 enum smc_lgr_role role, u32 clcqpn)
593 {
594 return !memcmp(lgr->peer_systemid, lcl->id_for_peer,
595 SMC_SYSTEMID_LEN) &&
596 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid,
597 SMC_GID_SIZE) &&
598 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac,
599 sizeof(lcl->mac)) &&
600 lgr->role == role &&
601 (lgr->role == SMC_SERV ||
602 lgr->lnk[SMC_SINGLE_LINK].peer_qpn == clcqpn);
603 }
604
smcd_lgr_match(struct smc_link_group * lgr,struct smcd_dev * smcismdev,u64 peer_gid)605 static bool smcd_lgr_match(struct smc_link_group *lgr,
606 struct smcd_dev *smcismdev, u64 peer_gid)
607 {
608 return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev;
609 }
610
611 /* create a new SMC connection (and a new link group if necessary) */
smc_conn_create(struct smc_sock * smc,struct smc_init_info * ini)612 int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini)
613 {
614 struct smc_connection *conn = &smc->conn;
615 struct smc_link_group *lgr;
616 enum smc_lgr_role role;
617 int rc = 0;
618
619 ini->cln_first_contact = SMC_FIRST_CONTACT;
620 role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
621 if (role == SMC_CLNT && ini->srv_first_contact)
622 /* create new link group as well */
623 goto create;
624
625 /* determine if an existing link group can be reused */
626 spin_lock_bh(&smc_lgr_list.lock);
627 list_for_each_entry(lgr, &smc_lgr_list.list, list) {
628 write_lock_bh(&lgr->conns_lock);
629 if ((ini->is_smcd ?
630 smcd_lgr_match(lgr, ini->ism_dev, ini->ism_gid) :
631 smcr_lgr_match(lgr, ini->ib_lcl, role, ini->ib_clcqpn)) &&
632 !lgr->sync_err &&
633 lgr->vlan_id == ini->vlan_id &&
634 (role == SMC_CLNT ||
635 (lgr->conns_num < SMC_RMBS_PER_LGR_MAX &&
636 !bitmap_full(lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX)))) {
637 /* link group found */
638 ini->cln_first_contact = SMC_REUSE_CONTACT;
639 conn->lgr = lgr;
640 smc_lgr_register_conn(conn); /* add smc conn to lgr */
641 if (delayed_work_pending(&lgr->free_work))
642 cancel_delayed_work(&lgr->free_work);
643 write_unlock_bh(&lgr->conns_lock);
644 break;
645 }
646 write_unlock_bh(&lgr->conns_lock);
647 }
648 spin_unlock_bh(&smc_lgr_list.lock);
649
650 if (role == SMC_CLNT && !ini->srv_first_contact &&
651 ini->cln_first_contact == SMC_FIRST_CONTACT) {
652 /* Server reuses a link group, but Client wants to start
653 * a new one
654 * send out_of_sync decline, reason synchr. error
655 */
656 return SMC_CLC_DECL_SYNCERR;
657 }
658
659 create:
660 if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
661 rc = smc_lgr_create(smc, ini);
662 if (rc)
663 goto out;
664 lgr = conn->lgr;
665 write_lock_bh(&lgr->conns_lock);
666 smc_lgr_register_conn(conn); /* add smc conn to lgr */
667 write_unlock_bh(&lgr->conns_lock);
668 }
669 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
670 conn->local_tx_ctrl.len = SMC_WR_TX_SIZE;
671 conn->urg_state = SMC_URG_READ;
672 if (ini->is_smcd) {
673 conn->rx_off = sizeof(struct smcd_cdc_msg);
674 smcd_cdc_rx_init(conn); /* init tasklet for this conn */
675 }
676 #ifndef KERNEL_HAS_ATOMIC64
677 spin_lock_init(&conn->acurs_lock);
678 #endif
679
680 out:
681 return rc;
682 }
683
684 /* convert the RMB size into the compressed notation - minimum 16K.
685 * In contrast to plain ilog2, this rounds towards the next power of 2,
686 * so the socket application gets at least its desired sndbuf / rcvbuf size.
687 */
smc_compress_bufsize(int size)688 static u8 smc_compress_bufsize(int size)
689 {
690 u8 compressed;
691
692 if (size <= SMC_BUF_MIN_SIZE)
693 return 0;
694
695 size = (size - 1) >> 14;
696 compressed = ilog2(size) + 1;
697 if (compressed >= SMC_RMBE_SIZES)
698 compressed = SMC_RMBE_SIZES - 1;
699 return compressed;
700 }
701
702 /* convert the RMB size from compressed notation into integer */
smc_uncompress_bufsize(u8 compressed)703 int smc_uncompress_bufsize(u8 compressed)
704 {
705 u32 size;
706
707 size = 0x00000001 << (((int)compressed) + 14);
708 return (int)size;
709 }
710
711 /* try to reuse a sndbuf or rmb description slot for a certain
712 * buffer size; if not available, return NULL
713 */
smc_buf_get_slot(int compressed_bufsize,rwlock_t * lock,struct list_head * buf_list)714 static struct smc_buf_desc *smc_buf_get_slot(int compressed_bufsize,
715 rwlock_t *lock,
716 struct list_head *buf_list)
717 {
718 struct smc_buf_desc *buf_slot;
719
720 read_lock_bh(lock);
721 list_for_each_entry(buf_slot, buf_list, list) {
722 if (cmpxchg(&buf_slot->used, 0, 1) == 0) {
723 read_unlock_bh(lock);
724 return buf_slot;
725 }
726 }
727 read_unlock_bh(lock);
728 return NULL;
729 }
730
731 /* one of the conditions for announcing a receiver's current window size is
732 * that it "results in a minimum increase in the window size of 10% of the
733 * receive buffer space" [RFC7609]
734 */
smc_rmb_wnd_update_limit(int rmbe_size)735 static inline int smc_rmb_wnd_update_limit(int rmbe_size)
736 {
737 return max_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
738 }
739
smcr_new_buf_create(struct smc_link_group * lgr,bool is_rmb,int bufsize)740 static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr,
741 bool is_rmb, int bufsize)
742 {
743 struct smc_buf_desc *buf_desc;
744 struct smc_link *lnk;
745 int rc;
746
747 /* try to alloc a new buffer */
748 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
749 if (!buf_desc)
750 return ERR_PTR(-ENOMEM);
751
752 buf_desc->order = get_order(bufsize);
753 buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN |
754 __GFP_NOMEMALLOC | __GFP_COMP |
755 __GFP_NORETRY | __GFP_ZERO,
756 buf_desc->order);
757 if (!buf_desc->pages) {
758 kfree(buf_desc);
759 return ERR_PTR(-EAGAIN);
760 }
761 buf_desc->cpu_addr = (void *)page_address(buf_desc->pages);
762
763 /* build the sg table from the pages */
764 lnk = &lgr->lnk[SMC_SINGLE_LINK];
765 rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1,
766 GFP_KERNEL);
767 if (rc) {
768 smc_buf_free(lgr, is_rmb, buf_desc);
769 return ERR_PTR(rc);
770 }
771 sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl,
772 buf_desc->cpu_addr, bufsize);
773
774 /* map sg table to DMA address */
775 rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc,
776 is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
777 /* SMC protocol depends on mapping to one DMA address only */
778 if (rc != 1) {
779 smc_buf_free(lgr, is_rmb, buf_desc);
780 return ERR_PTR(-EAGAIN);
781 }
782
783 /* create a new memory region for the RMB */
784 if (is_rmb) {
785 rc = smc_ib_get_memory_region(lnk->roce_pd,
786 IB_ACCESS_REMOTE_WRITE |
787 IB_ACCESS_LOCAL_WRITE,
788 buf_desc);
789 if (rc) {
790 smc_buf_free(lgr, is_rmb, buf_desc);
791 return ERR_PTR(rc);
792 }
793 }
794
795 buf_desc->len = bufsize;
796 return buf_desc;
797 }
798
799 #define SMCD_DMBE_SIZES 6 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */
800
smcd_new_buf_create(struct smc_link_group * lgr,bool is_dmb,int bufsize)801 static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr,
802 bool is_dmb, int bufsize)
803 {
804 struct smc_buf_desc *buf_desc;
805 int rc;
806
807 if (smc_compress_bufsize(bufsize) > SMCD_DMBE_SIZES)
808 return ERR_PTR(-EAGAIN);
809
810 /* try to alloc a new DMB */
811 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
812 if (!buf_desc)
813 return ERR_PTR(-ENOMEM);
814 if (is_dmb) {
815 rc = smc_ism_register_dmb(lgr, bufsize, buf_desc);
816 if (rc) {
817 kfree(buf_desc);
818 return ERR_PTR(-EAGAIN);
819 }
820 buf_desc->pages = virt_to_page(buf_desc->cpu_addr);
821 /* CDC header stored in buf. So, pretend it was smaller */
822 buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg);
823 } else {
824 buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL |
825 __GFP_NOWARN | __GFP_NORETRY |
826 __GFP_NOMEMALLOC);
827 if (!buf_desc->cpu_addr) {
828 kfree(buf_desc);
829 return ERR_PTR(-EAGAIN);
830 }
831 buf_desc->len = bufsize;
832 }
833 return buf_desc;
834 }
835
__smc_buf_create(struct smc_sock * smc,bool is_smcd,bool is_rmb)836 static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
837 {
838 struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM);
839 struct smc_connection *conn = &smc->conn;
840 struct smc_link_group *lgr = conn->lgr;
841 struct list_head *buf_list;
842 int bufsize, bufsize_short;
843 int sk_buf_size;
844 rwlock_t *lock;
845
846 if (is_rmb)
847 /* use socket recv buffer size (w/o overhead) as start value */
848 sk_buf_size = smc->sk.sk_rcvbuf / 2;
849 else
850 /* use socket send buffer size (w/o overhead) as start value */
851 sk_buf_size = smc->sk.sk_sndbuf / 2;
852
853 for (bufsize_short = smc_compress_bufsize(sk_buf_size);
854 bufsize_short >= 0; bufsize_short--) {
855
856 if (is_rmb) {
857 lock = &lgr->rmbs_lock;
858 buf_list = &lgr->rmbs[bufsize_short];
859 } else {
860 lock = &lgr->sndbufs_lock;
861 buf_list = &lgr->sndbufs[bufsize_short];
862 }
863 bufsize = smc_uncompress_bufsize(bufsize_short);
864 if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC)
865 continue;
866
867 /* check for reusable slot in the link group */
868 buf_desc = smc_buf_get_slot(bufsize_short, lock, buf_list);
869 if (buf_desc) {
870 memset(buf_desc->cpu_addr, 0, bufsize);
871 break; /* found reusable slot */
872 }
873
874 if (is_smcd)
875 buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize);
876 else
877 buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize);
878
879 if (PTR_ERR(buf_desc) == -ENOMEM)
880 break;
881 if (IS_ERR(buf_desc))
882 continue;
883
884 buf_desc->used = 1;
885 write_lock_bh(lock);
886 list_add(&buf_desc->list, buf_list);
887 write_unlock_bh(lock);
888 break; /* found */
889 }
890
891 if (IS_ERR(buf_desc))
892 return -ENOMEM;
893
894 if (is_rmb) {
895 conn->rmb_desc = buf_desc;
896 conn->rmbe_size_short = bufsize_short;
897 smc->sk.sk_rcvbuf = bufsize * 2;
898 atomic_set(&conn->bytes_to_rcv, 0);
899 conn->rmbe_update_limit =
900 smc_rmb_wnd_update_limit(buf_desc->len);
901 if (is_smcd)
902 smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */
903 } else {
904 conn->sndbuf_desc = buf_desc;
905 smc->sk.sk_sndbuf = bufsize * 2;
906 atomic_set(&conn->sndbuf_space, bufsize);
907 }
908 return 0;
909 }
910
smc_sndbuf_sync_sg_for_cpu(struct smc_connection * conn)911 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn)
912 {
913 struct smc_link_group *lgr = conn->lgr;
914
915 if (!conn->lgr || conn->lgr->is_smcd)
916 return;
917 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
918 conn->sndbuf_desc, DMA_TO_DEVICE);
919 }
920
smc_sndbuf_sync_sg_for_device(struct smc_connection * conn)921 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn)
922 {
923 struct smc_link_group *lgr = conn->lgr;
924
925 if (!conn->lgr || conn->lgr->is_smcd)
926 return;
927 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
928 conn->sndbuf_desc, DMA_TO_DEVICE);
929 }
930
smc_rmb_sync_sg_for_cpu(struct smc_connection * conn)931 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
932 {
933 struct smc_link_group *lgr = conn->lgr;
934
935 if (!conn->lgr || conn->lgr->is_smcd)
936 return;
937 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
938 conn->rmb_desc, DMA_FROM_DEVICE);
939 }
940
smc_rmb_sync_sg_for_device(struct smc_connection * conn)941 void smc_rmb_sync_sg_for_device(struct smc_connection *conn)
942 {
943 struct smc_link_group *lgr = conn->lgr;
944
945 if (!conn->lgr || conn->lgr->is_smcd)
946 return;
947 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
948 conn->rmb_desc, DMA_FROM_DEVICE);
949 }
950
951 /* create the send and receive buffer for an SMC socket;
952 * receive buffers are called RMBs;
953 * (even though the SMC protocol allows more than one RMB-element per RMB,
954 * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
955 * extra RMB for every connection in a link group
956 */
smc_buf_create(struct smc_sock * smc,bool is_smcd)957 int smc_buf_create(struct smc_sock *smc, bool is_smcd)
958 {
959 int rc;
960
961 /* create send buffer */
962 rc = __smc_buf_create(smc, is_smcd, false);
963 if (rc)
964 return rc;
965 /* create rmb */
966 rc = __smc_buf_create(smc, is_smcd, true);
967 if (rc)
968 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc);
969 return rc;
970 }
971
smc_rmb_reserve_rtoken_idx(struct smc_link_group * lgr)972 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
973 {
974 int i;
975
976 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {
977 if (!test_and_set_bit(i, lgr->rtokens_used_mask))
978 return i;
979 }
980 return -ENOSPC;
981 }
982
983 /* add a new rtoken from peer */
smc_rtoken_add(struct smc_link_group * lgr,__be64 nw_vaddr,__be32 nw_rkey)984 int smc_rtoken_add(struct smc_link_group *lgr, __be64 nw_vaddr, __be32 nw_rkey)
985 {
986 u64 dma_addr = be64_to_cpu(nw_vaddr);
987 u32 rkey = ntohl(nw_rkey);
988 int i;
989
990 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
991 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) &&
992 (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) &&
993 test_bit(i, lgr->rtokens_used_mask)) {
994 /* already in list */
995 return i;
996 }
997 }
998 i = smc_rmb_reserve_rtoken_idx(lgr);
999 if (i < 0)
1000 return i;
1001 lgr->rtokens[i][SMC_SINGLE_LINK].rkey = rkey;
1002 lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = dma_addr;
1003 return i;
1004 }
1005
1006 /* delete an rtoken */
smc_rtoken_delete(struct smc_link_group * lgr,__be32 nw_rkey)1007 int smc_rtoken_delete(struct smc_link_group *lgr, __be32 nw_rkey)
1008 {
1009 u32 rkey = ntohl(nw_rkey);
1010 int i;
1011
1012 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
1013 if (lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey &&
1014 test_bit(i, lgr->rtokens_used_mask)) {
1015 lgr->rtokens[i][SMC_SINGLE_LINK].rkey = 0;
1016 lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = 0;
1017
1018 clear_bit(i, lgr->rtokens_used_mask);
1019 return 0;
1020 }
1021 }
1022 return -ENOENT;
1023 }
1024
1025 /* save rkey and dma_addr received from peer during clc handshake */
smc_rmb_rtoken_handling(struct smc_connection * conn,struct smc_clc_msg_accept_confirm * clc)1026 int smc_rmb_rtoken_handling(struct smc_connection *conn,
1027 struct smc_clc_msg_accept_confirm *clc)
1028 {
1029 conn->rtoken_idx = smc_rtoken_add(conn->lgr, clc->rmb_dma_addr,
1030 clc->rmb_rkey);
1031 if (conn->rtoken_idx < 0)
1032 return conn->rtoken_idx;
1033 return 0;
1034 }
1035
1036 /* Called (from smc_exit) when module is removed */
smc_core_exit(void)1037 void smc_core_exit(void)
1038 {
1039 struct smc_link_group *lgr, *lg;
1040 LIST_HEAD(lgr_freeing_list);
1041
1042 spin_lock_bh(&smc_lgr_list.lock);
1043 if (!list_empty(&smc_lgr_list.list))
1044 list_splice_init(&smc_lgr_list.list, &lgr_freeing_list);
1045 spin_unlock_bh(&smc_lgr_list.lock);
1046 list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) {
1047 list_del_init(&lgr->list);
1048 if (!lgr->is_smcd) {
1049 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
1050
1051 if (lnk->state == SMC_LNK_ACTIVE)
1052 smc_llc_send_delete_link(lnk, SMC_LLC_REQ,
1053 false);
1054 smc_llc_link_inactive(lnk);
1055 }
1056 cancel_delayed_work_sync(&lgr->free_work);
1057 if (lgr->is_smcd)
1058 smc_ism_signal_shutdown(lgr);
1059 smc_lgr_free(lgr); /* free link group */
1060 }
1061 }
1062